From: "Daniel P. Berrange" <berrange(a)redhat.com&gt; Currently the QEMU driver has three ways of setting up cgroups. It either skips them entirely (if non-root), or uses systemd-machined, or uses cgroups directly. This change adds ability to configure the mechanism for registering resources between all these options explicitly. via <resource backend="none|cgroupfs|machined"/> It is further possible to register directly with systemd and bypass machined. We don't support this but systemd-nsspawn does and we ought to consider this at some point. This would involve a new "systemd" backend type alongside "machined". Signed-off-by: Daniel P. Berrange <berrange(a)redhat.com&gt; --- docs/formatdomain.html.in | 24 +++++++++++- docs/schemas/domaincommon.rng | 17 +++++++-- src/conf/domain_conf.c | 46 ++++++++++++++++++----- src/conf/domain_conf.h | 13 +++++++ src/libvirt_private.syms | 2 + src/lxc/lxc_cgroup.c | 8 ++++ src/lxc/lxc_process.c | 1 + src/qemu/qemu_cgroup.c | 14 +++++++ tests/lxcxml2xmldata/lxc-capabilities.xml | 2 +- tests/lxcxml2xmldata/lxc-idmap.xml | 2 +- 10 files changed, 113 insertions(+), 16 deletions(-) diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index 0d229386eb..a016e789f1 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -1445,7 +1445,7 @@ </p> <pre> ... -&lt;resource&gt; +&lt;resource backend='none|cgroupfs|machined'&gt; &lt;partition&gt;/virtualmachines/production&lt;/partition&gt; &lt;/resource&gt; ... @@ -1455,8 +1455,30 @@ Resource partitions are currently supported by the QEMU and LXC drivers, which map partition paths to cgroups directories, in all mounted controllers. <span class="since">Since 1.0.5</span> + There is a choice of implementations to use for resource partitions + controlled via the optional <code>backend</code> attribute. + <span class="since">Since 6.2.0</span>. It accepts the values </p> + <ul> + <dt>none</dt> + <dd>Resource management in libvirt is disabled, with the APIs + returning an error indicating the functionality is not available. + The QEMU will will remain in whatever cgroup the libvirt daemon + was in. On systemd hosts, this will result in QEMU being + terminated at the same time as the privileged libvirt management + daemon which launched them.</dd> + <dt>cgroupfs</dt> + <dd>Cgroups will be directly created via the cgroups virtual filesystem. + This is not recommended for use in scenarios where systemd is in + charge of the cgroup hierarchy, unless the resource partition points + to a subtree that systemd has delegated administrative for.</dd> + <dt>machined</dt> + <dd>Systemd machined will be called to indirectly create cgroups. + This is recommended for any host where systemd is managing the + cgroup hierarchy.</dd> + </ul> + <h3><a id="elementsCPU">CPU model and topology</a></h3> <p> diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index 6805420451..29ffc3a3cf 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -1126,9 +1126,20 @@ <define name="respartition"> <element name="resource"> - <element name="partition"> - <ref name="absFilePath"/> - </element> + <optional> + <attribute name="backend"> + <choice> + <value>none</value> + <value>cgroupfs</value> + <value>machined</value> + </choice> + </attribute> + </optional> + <optional> + <element name="partition"> + <ref name="absFilePath"/> + </element> + </optional> </element> </define> diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index e0432fc47d..ae512283d0 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -1244,6 +1244,13 @@ VIR_ENUM_IMPL(virDomainOsDefFirmware, "efi", ); +VIR_ENUM_IMPL(virDomainResourceBackend, + VIR_DOMAIN_RESOURCE_BACKEND_LAST, + "default", + "none", + "cgroupfs", + "machined"); + /* Internal mapping: subset of block job types that can be present in * <mirror> XML (remaining types are not two-phase). */ VIR_ENUM_DECL(virDomainBlockJob); @@ -19100,17 +19107,24 @@ virDomainResourceDefParse(xmlNodePtr node, { VIR_XPATH_NODE_AUTORESTORE(ctxt); virDomainResourceDefPtr def = NULL; + g_autofree char *reg = NULL; ctxt->node = node; if (VIR_ALLOC(def) < 0) goto error; - /* Find out what type of virtualization to use */ - if (!(def->partition = virXPathString("string(./partition)", ctxt))) { - virReportError(VIR_ERR_INTERNAL_ERROR, - "%s", _("missing resource partition attribute")); - goto error; + def->partition = virXPathString("string(./partition)", ctxt); + + reg = virXMLPropString(node, "backend"); + if (reg != NULL) { + if ((def->backend = virDomainResourceBackendTypeFromString(reg)) <= 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + "%s", _("Invalid backend attribute")); + goto error; + } + } else { + def->backend = VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT; } return def; @@ -27983,11 +27997,23 @@ static void virDomainResourceDefFormat(virBufferPtr buf, virDomainResourceDefPtr def) { - virBufferAddLit(buf, "<resource>\n"); - virBufferAdjustIndent(buf, 2); - virBufferEscapeString(buf, "<partition>%s</partition>\n", def->partition); - virBufferAdjustIndent(buf, -2); - virBufferAddLit(buf, "</resource>\n"); + if (def->backend == VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT && + def->partition == NULL) + return; + + virBufferAddLit(buf, "<resource"); + if (def->backend != VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT) + virBufferAsprintf(buf, " backend='%s'", virDomainResourceBackendTypeToString(def->backend)); + + if (def->partition) { + virBufferAddLit(buf, ">\n"); + virBufferAdjustIndent(buf, 2); + virBufferEscapeString(buf, "<partition>%s</partition>\n", def->partition); + virBufferAdjustIndent(buf, -2); + virBufferAddLit(buf, "</resource>\n"); + } else { + virBufferAddLit(buf, "/>\n"); + } } diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 73bd097cf8..4bfda29dee 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2228,7 +2228,19 @@ struct _virDomainPanicDef { void virBlkioDeviceArrayClear(virBlkioDevicePtr deviceWeights, int ndevices); +typedef enum { + VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT, + VIR_DOMAIN_RESOURCE_BACKEND_NONE, + VIR_DOMAIN_RESOURCE_BACKEND_CGROUPFS, + VIR_DOMAIN_RESOURCE_BACKEND_MACHINED, + + VIR_DOMAIN_RESOURCE_BACKEND_LAST, +} virDomainResourceBackend; + +typedef struct _virDomainResourceDef virDomainResourceDef; +typedef virDomainResourceDef *virDomainResourceDefPtr; struct _virDomainResourceDef { + int backend; /* enum virDomainResourceBackend */ char *partition; }; @@ -3525,6 +3537,7 @@ VIR_ENUM_DECL(virDomainIOMMUModel); VIR_ENUM_DECL(virDomainVsockModel); VIR_ENUM_DECL(virDomainShmemModel); VIR_ENUM_DECL(virDomainLaunchSecurity); +VIR_ENUM_DECL(virDomainResourceBackend); /* from libvirt.h */ VIR_ENUM_DECL(virDomainState); VIR_ENUM_DECL(virDomainNostateReason); diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 6b305bdd0e..6e5cc201ff 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -575,6 +575,8 @@ virDomainRedirdevBusTypeToString; virDomainRedirdevDefFind; virDomainRedirdevDefFree; virDomainRedirdevDefRemove; +virDomainResourceBackendTypeFromString; +virDomainResourceBackendTypeToString; virDomainRNGBackendTypeToString; virDomainRNGDefFree; virDomainRNGFind; diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index 7df723a4da..326d33981c 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -392,6 +392,14 @@ virCgroupPtr virLXCCgroupCreate(virDomainDefPtr def, if (!machineName) goto cleanup; + if (def->resource->backend != VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Resource backend '%s' not available"), + virDomainResourceBackendTypeToString( + def->resource->backend)); + goto cleanup; + } + if (def->resource->partition[0] != '/') { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Resource partition '%s' must start with '/'"), diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c index 5199f3806e..4ec3cc5619 100644 --- a/src/lxc/lxc_process.c +++ b/src/lxc/lxc_process.c @@ -1260,6 +1260,7 @@ int virLXCProcessStart(virConnectPtr conn, if (VIR_ALLOC(res) < 0) goto cleanup; + res->backend = VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT; res->partition = g_strdup("/machine"); vm->def->resource = res; diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index c0e30f6152..c407431f6b 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -940,11 +940,20 @@ qemuInitCgroup(virDomainObjPtr vm, if (VIR_ALLOC(res) < 0) goto cleanup; + res->backend = VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT; res->partition = g_strdup("/machine"); vm->def->resource = res; } + if (vm->def->resource->backend != VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Resource backend '%s' not available"), + virDomainResourceBackendTypeToString( + vm->def->resource->backend)); + goto cleanup; + } + if (vm->def->resource->partition[0] != '/') { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Resource partition '%s' must start with '/'"), @@ -1061,6 +1070,11 @@ qemuConnectCgroup(virDomainObjPtr vm) virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(priv->driver); int ret = -1; + if (vm->def->resource && + vm->def->resource->backend == VIR_DOMAIN_RESOURCE_BACKEND_NONE) { + goto done; + } + if (!virQEMUDriverIsPrivileged(priv->driver)) goto done; diff --git a/tests/lxcxml2xmldata/lxc-capabilities.xml b/tests/lxcxml2xmldata/lxc-capabilities.xml index 04d64e3e41..335fdf8b91 100644 --- a/tests/lxcxml2xmldata/lxc-capabilities.xml +++ b/tests/lxcxml2xmldata/lxc-capabilities.xml @@ -4,7 +4,7 @@ <memory unit='KiB'>1048576</memory> <currentMemory unit='KiB'>1048576</currentMemory> <vcpu placement='static'>1</vcpu> - <resource> + <resource backend='cgroupfs'> <partition>/machine</partition> </resource> <os> diff --git a/tests/lxcxml2xmldata/lxc-idmap.xml b/tests/lxcxml2xmldata/lxc-idmap.xml index b477636c30..d618d69706 100644 --- a/tests/lxcxml2xmldata/lxc-idmap.xml +++ b/tests/lxcxml2xmldata/lxc-idmap.xml @@ -4,7 +4,7 @@ <memory unit='KiB'>1048576</memory> <currentMemory unit='KiB'>1048576</currentMemory> <vcpu placement='static'>1</vcpu> - <resource> + <resource backend='machined'> <partition>/machine</partition> </resource> <os> -- 2.24.1

Currently the QEMU driver chooses between systemd machined and direct cgroups usage of privileged, and does not use either when unprivileged. This wires up support for the new backend choice introduced by the earlier commits, allowing apps to override the default logic in the driver when privileged. This reverts commit c32a7de7d8f81384b17dbe529c6d3b3ac13c631d. --- src/qemu/qemu_cgroup.c | 68 ++++++++++++++++++++++++++++++++--------- src/qemu/qemu_command.c | 9 +++--- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index cd7c381185..a1b53f6628 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -917,6 +917,46 @@ qemuSetupCpuCgroup(virDomainObjPtr vm) } +static int qemuGetCgroupMode(virDomainObjPtr vm, + virDomainResourceBackend backend, + virCgroupRegister *cgreg) +{ + qemuDomainObjPrivatePtr priv = vm->privateData; + bool avail = virQEMUDriverIsPrivileged(priv->driver) && + virCgroupAvailable(); + + switch (backend) { + case VIR_DOMAIN_RESOURCE_BACKEND_NONE: + return 0; + case VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT: + if (!avail) + return 0; + *cgreg = VIR_CGROUP_REGISTER_DEFAULT; + break; + case VIR_DOMAIN_RESOURCE_BACKEND_MACHINED: + if (!avail) + goto unsupported; + *cgreg = VIR_CGROUP_REGISTER_MACHINED; + break; + case VIR_DOMAIN_RESOURCE_BACKEND_CGROUPFS: + if (!avail) + goto unsupported; + *cgreg = VIR_CGROUP_REGISTER_DIRECT; + break; + case VIR_DOMAIN_RESOURCE_BACKEND_LAST: + default: + virReportEnumRangeError(virDomainResourceBackend, backend); + } + + return 1; + + unsupported: + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Resource backend '%s' not available"), + virDomainResourceBackendTypeToString(backend)); + return -1; +} + static int qemuInitCgroup(virDomainObjPtr vm, size_t nnicindexes, @@ -925,11 +965,17 @@ qemuInitCgroup(virDomainObjPtr vm, int ret = -1; qemuDomainObjPrivatePtr priv = vm->privateData; virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(priv->driver); + virCgroupRegister reg; + int rv; - if (!virQEMUDriverIsPrivileged(priv->driver)) - goto done; - - if (!virCgroupAvailable()) + rv = qemuGetCgroupMode(vm, + vm->def->resource ? + vm->def->resource->backend : + VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT, + &reg); + if (rv < 0) + goto cleanup; + if (rv == 0) goto done; virCgroupFree(&priv->cgroup); @@ -941,18 +987,12 @@ qemuInitCgroup(virDomainObjPtr vm, goto cleanup; res->backend = VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT; - res->partition = g_strdup("/machine"); - vm->def->resource = res; } - if (vm->def->resource->backend != VIR_DOMAIN_RESOURCE_BACKEND_DEFAULT) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("Resource backend '%s' not available"), - virDomainResourceBackendTypeToString( - vm->def->resource->backend)); - goto cleanup; - } + if (vm->def->resource->backend != VIR_DOMAIN_RESOURCE_BACKEND_NONE && + !vm->def->resource->partition) + vm->def->resource->partition = g_strdup("/machine"); if (vm->def->resource->partition[0] != '/') { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, @@ -968,7 +1008,7 @@ qemuInitCgroup(virDomainObjPtr vm, vm->pid, false, nnicindexes, nicindexes, - VIR_CGROUP_REGISTER_DEFAULT, + &reg, vm->def->resource->partition, cfg->cgroupControllers, cfg->maxThreadsPerProc, diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 9790c92cf8..eb1c3f6e12 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -9662,7 +9662,8 @@ qemuBuildCommandLineValidate(virQEMUDriverPtr driver, int spice = 0; int egl_headless = 0; - if (!virQEMUDriverIsPrivileged(driver)) { + if (!virQEMUDriverIsPrivileged(driver) || + (def->resource && def->resource->backend == VIR_DOMAIN_RESOURCE_BACKEND_NONE)) { /* If we have no cgroups then we can have no tunings that * require them */ @@ -9670,13 +9671,13 @@ qemuBuildCommandLineValidate(virQEMUDriverPtr driver, virMemoryLimitIsSet(def->mem.soft_limit) || virMemoryLimitIsSet(def->mem.swap_hard_limit)) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("Memory tuning is not available in session mode")); + _("Memory tuning is not available without cgroups")); return -1; } if (def->blkio.weight) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("Block I/O tuning is not available in session mode")); + _("Block I/O tuning is not available without cgroups")); return -1; } @@ -9686,7 +9687,7 @@ qemuBuildCommandLineValidate(virQEMUDriverPtr driver, def->cputune.emulator_quota || def->cputune.iothread_period || def->cputune.iothread_quota) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("CPU tuning is not available in session mode")); + _("CPU tuning is not available without cgroups")); return -1; } } -- 2.24.1