[libvirt] [PATCH v1 0/6] disable cgroup cpuset
The reason to disable cgroup cpuset by default is: currently, sub-directories of cgroup cpuset won't be auto-updated if cpu hot-plug happens. For example, if we now have 0-3,5-7 pcpus online, the value of /sys/fs/cgroup/cpuset/libvirt/qemu/mydom/cpuset.cpus is 0-3,5-7, then pcpu 4 is hot-plugged, but the value of cpuset.cpus is still 0-3,5-7. This will cause a problem that we can't pin to a hot-plugged pcpu. Users can still enable cpuset by editing qemu.conf. Hu Tao (6): cgroup: Add a flag VIR_CGROUP_DISABLE_CPUSET create cgroup controllers for driver according to config create cgroup controllers for domain according to config create cgroup controllers for vcpu according to config create cgroup controllers for emulator according to config disable cgroup cpuset by default src/lxc/lxc_cgroup.c | 4 +-- src/lxc/lxc_driver.c | 22 ++++++------ src/lxc/lxc_process.c | 4 +-- src/qemu/qemu.conf | 2 +- src/qemu/qemu_cgroup.c | 15 ++++---- src/qemu/qemu_conf.c | 1 - src/qemu/qemu_driver.c | 92 +++++++++++++++++++++++++++++------------------ src/qemu/qemu_hotplug.c | 9 +++-- src/qemu/qemu_migration.c | 2 +- src/util/cgroup.c | 49 +++++++++++++++++++------ src/util/cgroup.h | 12 ++++--- 11 files changed, 136 insertions(+), 76 deletions(-) -- 1.7.11.7
Add a flag VIR_CGROUP_DISABLE_CPUSET to disable cgroup cpuset. This flag inhibits making of directory under cpuset. --- src/util/cgroup.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/util/cgroup.c b/src/util/cgroup.c index 3f7b5f7..2f64f5d 100644 --- a/src/util/cgroup.c +++ b/src/util/cgroup.c @@ -72,6 +72,7 @@ typedef enum { */ VIR_CGROUP_VCPU = 1 << 1, /* create subdir only under the cgroup cpu, * cpuacct and cpuset if possible. */ + VIR_CGROUP_DISABLE_CPUSET = 1 << 2, } virCgroupFlags; /** @@ -540,6 +541,12 @@ static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group, if (!group->controllers[i].mountPoint) continue; + if ((flags & VIR_CGROUP_DISABLE_CPUSET) && + i == VIR_CGROUP_CONTROLLER_CPUSET) { + group->controllers[i].mountPoint = NULL; + continue; + } + /* We need to control cpu bandwidth for each vcpu now */ if ((flags & VIR_CGROUP_VCPU) && (i != VIR_CGROUP_CONTROLLER_CPU && -- 1.7.11.7
Add a parameter to virCgroupForDriver to indicate which cgroup controllers to create. The value of the parameter is read from user config. --- src/lxc/lxc_cgroup.c | 2 +- src/lxc/lxc_driver.c | 2 +- src/qemu/qemu_driver.c | 12 ++++++------ src/util/cgroup.c | 12 +++++++++--- src/util/cgroup.h | 3 ++- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index bdfaa54..a5e2e97 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -226,7 +226,7 @@ int virLXCCgroupSetup(virDomainDefPtr def) virCgroupPtr cgroup = NULL; int rc = -1; - rc = virCgroupForDriver("lxc", &driver, 1, 0); + rc = virCgroupForDriver("lxc", &driver, 1, 0, ~0); if (rc != 0) { /* Skip all if no driver cgroup is configured */ if (rc == -ENXIO || rc == -ENOENT) diff --git a/src/lxc/lxc_driver.c b/src/lxc/lxc_driver.c index 991b593..5401fe7 100644 --- a/src/lxc/lxc_driver.c +++ b/src/lxc/lxc_driver.c @@ -1443,7 +1443,7 @@ static int lxcStartup(int privileged) lxc_driver->log_libvirtd = 0; /* by default log to container logfile */ lxc_driver->have_netns = lxcCheckNetNsSupport(); - rc = virCgroupForDriver("lxc", &lxc_driver->cgroup, privileged, 1); + rc = virCgroupForDriver("lxc", &lxc_driver->cgroup, privileged, 1, ~0); if (rc < 0) { char buf[1024] ATTRIBUTE_UNUSED; VIR_DEBUG("Unable to create cgroup for LXC driver: %s", diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 7b8eec6..007fea9 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -740,17 +740,17 @@ qemudStartup(int privileged) { VIR_FREE(base); - rc = virCgroupForDriver("qemu", &qemu_driver->cgroup, privileged, 1); - if (rc < 0) { - VIR_INFO("Unable to create cgroup for driver: %s", - virStrerror(-rc, ebuf, sizeof(ebuf))); - } - if (qemudLoadDriverConfig(qemu_driver, driverConf) < 0) { goto error; } VIR_FREE(driverConf); + rc = virCgroupForDriver("qemu", &qemu_driver->cgroup, privileged, 1, qemu_driver->cgroupControllers); + if (rc < 0) { + VIR_INFO("Unable to create cgroup for driver: %s", + virStrerror(-rc, ebuf, sizeof(ebuf))); + } + /* Allocate bitmap for remote display port reservations. We cannot * do this before the config is loaded properly, since the port * numbers are configurable now */ diff --git a/src/util/cgroup.c b/src/util/cgroup.c index 2f64f5d..02fc24a 100644 --- a/src/util/cgroup.c +++ b/src/util/cgroup.c @@ -932,11 +932,13 @@ cleanup: int virCgroupForDriver(const char *name, virCgroupPtr *group, int privileged, - int create) + int create, + int controllers) { int rc; char *path = NULL; virCgroupPtr rootgrp = NULL; + unsigned flags = VIR_CGROUP_NONE; rc = virCgroupAppRoot(privileged, &rootgrp, create); if (rc != 0) @@ -947,11 +949,14 @@ int virCgroupForDriver(const char *name, goto out; } + if (!(controllers & (1UL << VIR_CGROUP_CONTROLLER_CPUSET))) + flags |= VIR_CGROUP_DISABLE_CPUSET; + rc = virCgroupNew(path, group); VIR_FREE(path); if (rc == 0) { - rc = virCgroupMakeGroup(rootgrp, *group, create, VIR_CGROUP_NONE); + rc = virCgroupMakeGroup(rootgrp, *group, create, flags); if (rc != 0) virCgroupFree(group); } @@ -965,7 +970,8 @@ out: int virCgroupForDriver(const char *name ATTRIBUTE_UNUSED, virCgroupPtr *group ATTRIBUTE_UNUSED, int privileged ATTRIBUTE_UNUSED, - int create ATTRIBUTE_UNUSED) + int create ATTRIBUTE_UNUSED, + int controllers) { /* Claim no support */ return -ENXIO; diff --git a/src/util/cgroup.h b/src/util/cgroup.h index 38fa4b7..f0945f1 100644 --- a/src/util/cgroup.h +++ b/src/util/cgroup.h @@ -45,7 +45,8 @@ VIR_ENUM_DECL(virCgroupController); int virCgroupForDriver(const char *name, virCgroupPtr *group, int privileged, - int create); + int create, + int controllers); int virCgroupForDomain(virCgroupPtr driver, const char *name, -- 1.7.11.7
Add a parameter to virCgroupForDomain to indicate which cgroup controllers to create for domain. The value of the parameter is read from user config. --- src/lxc/lxc_cgroup.c | 2 +- src/lxc/lxc_driver.c | 20 +++++++++--------- src/lxc/lxc_process.c | 4 ++-- src/qemu/qemu_cgroup.c | 10 ++++----- src/qemu/qemu_driver.c | 53 ++++++++++++++++++++++++++++++----------------- src/qemu/qemu_hotplug.c | 9 +++++--- src/qemu/qemu_migration.c | 2 +- src/util/cgroup.c | 9 ++++++-- src/util/cgroup.h | 3 ++- 9 files changed, 68 insertions(+), 44 deletions(-) diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index a5e2e97..b636db7 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -237,7 +237,7 @@ int virLXCCgroupSetup(virDomainDefPtr def) return rc; } - rc = virCgroupForDomain(driver, def->name, &cgroup, 1); + rc = virCgroupForDomain(driver, def->name, &cgroup, 1, ~0); if (rc != 0) { virReportSystemError(-rc, _("Unable to create cgroup for domain %s"), diff --git a/src/lxc/lxc_driver.c b/src/lxc/lxc_driver.c index 5401fe7..8a08404 100644 --- a/src/lxc/lxc_driver.c +++ b/src/lxc/lxc_driver.c @@ -541,7 +541,7 @@ static int lxcDomainGetInfo(virDomainPtr dom, info->cpuTime = 0; info->memory = vm->def->mem.cur_balloon; } else { - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, ~0) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to get cgroup for %s"), vm->def->name); goto cleanup; @@ -734,7 +734,7 @@ static int lxcDomainSetMemory(virDomainPtr dom, unsigned long newmem) { goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, ~0) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to get cgroup for %s"), vm->def->name); goto cleanup; @@ -791,7 +791,7 @@ lxcDomainSetMemoryParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, ~0) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; @@ -868,7 +868,7 @@ lxcDomainGetMemoryParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, ~0) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to get cgroup for %s"), vm->def->name); goto cleanup; @@ -1810,7 +1810,7 @@ lxcSetSchedulerParametersFlags(virDomainPtr dom, "%s", _("cgroup CPU controller is not mounted")); goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, ~0) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); @@ -1954,7 +1954,7 @@ lxcGetSchedulerParametersFlags(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, ~0) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; @@ -2059,7 +2059,7 @@ lxcDomainSetBlkioParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, ~0) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; @@ -2164,7 +2164,7 @@ lxcDomainGetBlkioParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, ~0) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; @@ -2398,7 +2398,7 @@ static int lxcFreezeContainer(virLXCDriverPtr driver, virDomainObjPtr vm) virCgroupPtr cgroup = NULL; if (!(driver->cgroup && - virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) == 0)) + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, ~0) == 0)) return -1; /* From here on, we know that cgroup != NULL. */ @@ -2536,7 +2536,7 @@ static int lxcUnfreezeContainer(virLXCDriverPtr driver, virDomainObjPtr vm) virCgroupPtr cgroup = NULL; if (!(driver->cgroup && - virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) == 0)) + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, ~0) == 0)) return -1; ret = virCgroupSetFreezerState(cgroup, "THAWED"); diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c index 079bc3a..b35ed0e 100644 --- a/src/lxc/lxc_process.c +++ b/src/lxc/lxc_process.c @@ -266,7 +266,7 @@ static void virLXCProcessCleanup(virLXCDriverPtr driver, virDomainConfVMNWFilterTeardown(vm); if (driver->cgroup && - virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) == 0) { + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, ~0) == 0) { virCgroupRemove(cgroup); virCgroupFree(&cgroup); } @@ -699,7 +699,7 @@ int virLXCProcessStop(virLXCDriverPtr driver, VIR_FREE(vm->def->seclabels[0]->imagelabel); } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) == 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, ~0) == 0) { rc = virCgroupKillPainfully(group); if (rc < 0) { virReportSystemError(-rc, "%s", diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 288187c..ba54a68 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -203,7 +203,7 @@ int qemuSetupCgroup(struct qemud_driver *driver, if (driver->cgroup == NULL) return 0; /* Not supported, so claim success */ - rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 1); + rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 1, driver->cgroupControllers); if (rc != 0) { virReportSystemError(-rc, _("Unable to create cgroup for %s"), @@ -558,7 +558,7 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm) if (driver->cgroup == NULL) return 0; - rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0); + rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, driver->cgroupControllers); if (rc != 0) { virReportSystemError(-rc, _("Unable to find cgroup for %s"), @@ -661,7 +661,7 @@ int qemuSetupCgroupForEmulator(struct qemud_driver *driver, if (driver->cgroup == NULL) return 0; /* Not supported, so claim success */ - rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0); + rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, driver->cgroupControllers); if (rc != 0) { virReportSystemError(-rc, _("Unable to find cgroup for %s"), @@ -748,7 +748,7 @@ int qemuRemoveCgroup(struct qemud_driver *driver, if (driver->cgroup == NULL) return 0; /* Not supported, so claim success */ - rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0); + rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, driver->cgroupControllers); if (rc != 0) { if (!quiet) virReportError(VIR_ERR_INTERNAL_ERROR, @@ -772,7 +772,7 @@ int qemuAddToCgroup(struct qemud_driver *driver, if (driver->cgroup == NULL) return 0; /* Not supported, so claim success */ - rc = virCgroupForDomain(driver->cgroup, def->name, &cgroup, 0); + rc = virCgroupForDomain(driver->cgroup, def->name, &cgroup, 0, driver->cgroupControllers); if (rc != 0) { virReportSystemError(-rc, _("unable to find cgroup for domain %s"), diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 007fea9..cc532c7 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -3709,7 +3709,7 @@ static int qemudDomainHotplugVcpus(struct qemud_driver *driver, } cgroup_available = (virCgroupForDomain(driver->cgroup, vm->def->name, - &cgroup, 0) == 0); + &cgroup, 0, driver->cgroupControllers) == 0); if (nvcpus > oldvcpus) { for (i = oldvcpus; i < nvcpus; i++) { @@ -4035,7 +4035,8 @@ qemudDomainPinVcpuFlags(virDomainPtr dom, /* Configure the corresponding cpuset cgroup before set affinity. */ if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) { - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup_dom, 0) == 0 && + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup_dom, 0, + driver->cgroupControllers) == 0 && virCgroupForVcpu(cgroup_dom, vcpu, &cgroup_vcpu, 0) == 0 && qemuSetupCgroupVcpuPin(cgroup_vcpu, newVcpuPin, newVcpuPinNum, vcpu) < 0) { virReportError(VIR_ERR_OPERATION_INVALID, @@ -4301,7 +4302,7 @@ qemudDomainPinEmulator(virDomainPtr dom, * If no cgroup for domain or hypervisor exists, do nothing. */ if (virCgroupForDomain(driver->cgroup, vm->def->name, - &cgroup_dom, 0) == 0) { + &cgroup_dom, 0, driver->cgroupControllers) == 0) { if (virCgroupForEmulator(cgroup_dom, &cgroup_emulator, 0) == 0) { if (qemuSetupCgroupEmulatorPin(cgroup_emulator, newVcpuPin[0]->cpumask) < 0) { @@ -5888,7 +5889,8 @@ qemuDomainAttachDeviceDiskLive(virConnectPtr conn, goto end; if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES)) { - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0)) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers)) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); @@ -6119,8 +6121,8 @@ qemuDomainChangeDiskMediaLive(virDomainObjPtr vm, goto end; if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES)) { - if (virCgroupForDomain(driver->cgroup, - vm->def->name, &cgroup, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); @@ -6984,7 +6986,8 @@ qemuDomainSetBlkioParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); @@ -7145,7 +7148,8 @@ qemuDomainGetBlkioParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; @@ -7334,7 +7338,8 @@ qemuDomainSetMemoryParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; @@ -7491,7 +7496,8 @@ qemuDomainGetMemoryParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; @@ -7653,7 +7659,8 @@ qemuDomainSetNumaParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); @@ -7809,7 +7816,8 @@ qemuDomainGetNumaParameters(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); @@ -8014,7 +8022,8 @@ qemuSetSchedulerParametersFlags(virDomainPtr dom, "%s", _("cgroup CPU controller is not mounted")); goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); @@ -8301,7 +8310,8 @@ qemuGetSchedulerParametersFlags(virDomainPtr dom, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; @@ -11029,7 +11039,8 @@ qemuDomainSnapshotCreateDiskActive(struct qemud_driver *driver, } if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES) && - virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0)) { + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers)) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); @@ -12748,7 +12759,8 @@ qemuDomainBlockPivot(virConnectPtr conn, * we know for sure that there is a backing chain. */ if (disk->mirrorFormat && disk->mirrorFormat != VIR_STORAGE_FILE_RAW && qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES) && - virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) < 0) { + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); @@ -13054,7 +13066,8 @@ qemuDomainBlockCopy(virDomainPtr dom, const char *path, goto cleanup; } if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES) && - virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) < 0) { + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); @@ -13338,7 +13351,8 @@ qemuDomainBlockCommit(virDomainPtr dom, const char *path, const char *base, * operation succeeds, but doing that requires tracking the * operation in XML across libvirtd restarts. */ if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES) && - virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) < 0) { + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); @@ -14270,7 +14284,8 @@ qemuDomainGetCPUStats(virDomainPtr domain, goto cleanup; } - if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &group, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find cgroup for domain %s"), vm->def->name); goto cleanup; diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index ae8381e..b955c5b 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -1097,7 +1097,8 @@ int qemuDomainAttachHostUsbDevice(struct qemud_driver *driver, usbDevice *usb; qemuCgroupData data; - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); @@ -1946,7 +1947,8 @@ int qemuDomainDetachPciDiskDevice(struct qemud_driver *driver, } if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES)) { - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); @@ -2057,7 +2059,8 @@ int qemuDomainDetachDiskDevice(struct qemud_driver *driver, } if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES)) { - if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) != 0) { + if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0, + driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 5f8a9c5..e36bece 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -3510,7 +3510,7 @@ qemuMigrationToFile(struct qemud_driver *driver, virDomainObjPtr vm, if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES)) { if (virCgroupForDomain(driver->cgroup, vm->def->name, - &cgroup, 0) != 0) { + &cgroup, 0, driver->cgroupControllers) != 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Unable to find cgroup for %s"), vm->def->name); diff --git a/src/util/cgroup.c b/src/util/cgroup.c index 02fc24a..d07a66a 100644 --- a/src/util/cgroup.c +++ b/src/util/cgroup.c @@ -992,14 +992,19 @@ int virCgroupForDriver(const char *name ATTRIBUTE_UNUSED, int virCgroupForDomain(virCgroupPtr driver, const char *name, virCgroupPtr *group, - int create) + int create, + int controllers) { + unsigned flags = VIR_CGROUP_MEM_HIERACHY; int rc; char *path; if (driver == NULL) return -EINVAL; + if (!(controllers & (1UL << VIR_CGROUP_CONTROLLER_CPUSET))) + flags |= VIR_CGROUP_DISABLE_CPUSET; + if (virAsprintf(&path, "%s/%s", driver->path, name) < 0) return -ENOMEM; @@ -1017,7 +1022,7 @@ int virCgroupForDomain(virCgroupPtr driver, * a group for driver, is to avoid overhead to track * cumulative usage that we don't need. */ - rc = virCgroupMakeGroup(driver, *group, create, VIR_CGROUP_MEM_HIERACHY); + rc = virCgroupMakeGroup(driver, *group, create, flags); if (rc != 0) virCgroupFree(group); } diff --git a/src/util/cgroup.h b/src/util/cgroup.h index f0945f1..b121bca 100644 --- a/src/util/cgroup.h +++ b/src/util/cgroup.h @@ -51,7 +51,8 @@ int virCgroupForDriver(const char *name, int virCgroupForDomain(virCgroupPtr driver, const char *name, virCgroupPtr *group, - int create); + int create, + int controllers); int virCgroupForVcpu(virCgroupPtr driver, int vcpuid, -- 1.7.11.7
Add a parameter to virCgroupForVcpu to indicate which cgroup controllers to create. The value of the parameter is read from user config. --- src/qemu/qemu_cgroup.c | 3 ++- src/qemu/qemu_driver.c | 18 ++++++++++++------ src/util/cgroup.c | 9 +++++++-- src/util/cgroup.h | 3 ++- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index ba54a68..04a2c08 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -576,7 +576,8 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm) } for (i = 0; i < priv->nvcpupids; i++) { - rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 1); + rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 1, + driver->cgroupControllers); if (rc < 0) { virReportSystemError(-rc, _("Unable to create vcpu cgroup for %s(vcpu:" diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index cc532c7..dcaa693 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -3716,7 +3716,8 @@ static int qemudDomainHotplugVcpus(struct qemud_driver *driver, if (cgroup_available) { int rv = -1; /* Create cgroup for the onlined vcpu */ - rv = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 1); + rv = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 1, + driver->cgroupControllers); if (rv < 0) { virReportSystemError(-rv, _("Unable to create vcpu cgroup for %s(vcpu:" @@ -3790,7 +3791,8 @@ static int qemudDomainHotplugVcpus(struct qemud_driver *driver, if (cgroup_available) { int rv = -1; - rv = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 0); + rv = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 0, + driver->cgroupControllers); if (rv < 0) { virReportSystemError(-rv, _("Unable to access vcpu cgroup for %s(vcpu:" @@ -4037,7 +4039,8 @@ qemudDomainPinVcpuFlags(virDomainPtr dom, if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) { if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup_dom, 0, driver->cgroupControllers) == 0 && - virCgroupForVcpu(cgroup_dom, vcpu, &cgroup_vcpu, 0) == 0 && + virCgroupForVcpu(cgroup_dom, vcpu, &cgroup_vcpu, 0, + driver->cgroupControllers) == 0 && qemuSetupCgroupVcpuPin(cgroup_vcpu, newVcpuPin, newVcpuPinNum, vcpu) < 0) { virReportError(VIR_ERR_OPERATION_INVALID, _("failed to set cpuset.cpus in cgroup" @@ -7897,7 +7900,8 @@ qemuSetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup, */ if (priv->nvcpupids != 0 && priv->vcpupids[0] != vm->pid) { for (i = 0; i < priv->nvcpupids; i++) { - rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 0); + rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 0, + qemu_driver->cgroupControllers); if (rc < 0) { virReportSystemError(-rc, _("Unable to find vcpu cgroup for %s(vcpu:" @@ -8188,7 +8192,8 @@ qemuGetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup, } /* get period and quota for vcpu0 */ - rc = virCgroupForVcpu(cgroup, 0, &cgroup_vcpu, 0); + rc = virCgroupForVcpu(cgroup, 0, &cgroup_vcpu, 0, + qemu_driver->cgroupControllers); if (!cgroup_vcpu) { virReportSystemError(-rc, _("Unable to find vcpu cgroup for %s(vcpu: 0)"), @@ -14110,7 +14115,8 @@ getSumVcpuPercpuStats(virCgroupPtr group, unsigned long long tmp; int j; - if (virCgroupForVcpu(group, i, &group_vcpu, 0) < 0) { + if (virCgroupForVcpu(group, i, &group_vcpu, 0, + qemu_driver->cgroupControllers) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("error accessing cgroup cpuacct for vcpu")); goto cleanup; diff --git a/src/util/cgroup.c b/src/util/cgroup.c index d07a66a..b43933e 100644 --- a/src/util/cgroup.c +++ b/src/util/cgroup.c @@ -1052,14 +1052,19 @@ int virCgroupForDomain(virCgroupPtr driver ATTRIBUTE_UNUSED, int virCgroupForVcpu(virCgroupPtr driver, int vcpuid, virCgroupPtr *group, - int create) + int create, + int controllers) { + unsigned flags = VIR_CGROUP_VCPU; int rc; char *path; if (driver == NULL) return -EINVAL; + if (!(controllers & (1UL << VIR_CGROUP_CONTROLLER_CPUSET))) + flags |= VIR_CGROUP_DISABLE_CPUSET; + if (virAsprintf(&path, "%s/vcpu%d", driver->path, vcpuid) < 0) return -ENOMEM; @@ -1067,7 +1072,7 @@ int virCgroupForVcpu(virCgroupPtr driver, VIR_FREE(path); if (rc == 0) { - rc = virCgroupMakeGroup(driver, *group, create, VIR_CGROUP_VCPU); + rc = virCgroupMakeGroup(driver, *group, create, flags); if (rc != 0) virCgroupFree(group); } diff --git a/src/util/cgroup.h b/src/util/cgroup.h index b121bca..dc0811a 100644 --- a/src/util/cgroup.h +++ b/src/util/cgroup.h @@ -57,7 +57,8 @@ int virCgroupForDomain(virCgroupPtr driver, int virCgroupForVcpu(virCgroupPtr driver, int vcpuid, virCgroupPtr *group, - int create); + int create, + int controllers); int virCgroupForEmulator(virCgroupPtr driver, virCgroupPtr *group, -- 1.7.11.7
Add a parameter to virCgroupForEmulator to indicate which cgroup controllers to create. The value of the parameter is read from user config. --- src/qemu/qemu_cgroup.c | 2 +- src/qemu/qemu_driver.c | 9 ++++++--- src/util/cgroup.c | 12 +++++++++--- src/util/cgroup.h | 3 ++- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 04a2c08..69320e4 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -670,7 +670,7 @@ int qemuSetupCgroupForEmulator(struct qemud_driver *driver, goto cleanup; } - rc = virCgroupForEmulator(cgroup, &cgroup_emulator, 1); + rc = virCgroupForEmulator(cgroup, &cgroup_emulator, 1, driver->cgroupControllers); if (rc < 0) { virReportSystemError(-rc, _("Unable to create emulator cgroup for %s"), diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index dcaa693..e9985e8 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -4306,7 +4306,8 @@ qemudDomainPinEmulator(virDomainPtr dom, */ if (virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup_dom, 0, driver->cgroupControllers) == 0) { - if (virCgroupForEmulator(cgroup_dom, &cgroup_emulator, 0) == 0) { + if (virCgroupForEmulator(cgroup_dom, &cgroup_emulator, 0, + driver->cgroupControllers) == 0) { if (qemuSetupCgroupEmulatorPin(cgroup_emulator, newVcpuPin[0]->cpumask) < 0) { virReportError(VIR_ERR_OPERATION_INVALID, "%s", @@ -7939,7 +7940,8 @@ qemuSetEmulatorBandwidthLive(virDomainObjPtr vm, virCgroupPtr cgroup, return 0; } - rc = virCgroupForEmulator(cgroup, &cgroup_emulator, 0); + rc = virCgroupForEmulator(cgroup, &cgroup_emulator, 0, + qemu_driver->cgroupControllers); if (rc < 0) { virReportSystemError(-rc, _("Unable to find emulator cgroup for %s"), @@ -8231,7 +8233,8 @@ qemuGetEmulatorBandwidthLive(virDomainObjPtr vm, virCgroupPtr cgroup, } /* get period and quota for emulator */ - rc = virCgroupForEmulator(cgroup, &cgroup_emulator, 0); + rc = virCgroupForEmulator(cgroup, &cgroup_emulator, 0, + qemu_driver->cgroupControllers); if (!cgroup_emulator) { virReportSystemError(-rc, _("Unable to find emulator cgroup for %s"), diff --git a/src/util/cgroup.c b/src/util/cgroup.c index b43933e..d069bf5 100644 --- a/src/util/cgroup.c +++ b/src/util/cgroup.c @@ -1100,14 +1100,19 @@ int virCgroupForVcpu(virCgroupPtr driver ATTRIBUTE_UNUSED, #if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R int virCgroupForEmulator(virCgroupPtr driver, virCgroupPtr *group, - int create) + int create, + int controllers) { + unsigned flags = VIR_CGROUP_VCPU; int rc; char *path; if (driver == NULL) return -EINVAL; + if (!(controllers & (1UL << VIR_CGROUP_CONTROLLER_CPUSET))) + flags |= VIR_CGROUP_DISABLE_CPUSET; + if (virAsprintf(&path, "%s/emulator", driver->path) < 0) return -ENOMEM; @@ -1115,7 +1120,7 @@ int virCgroupForEmulator(virCgroupPtr driver, VIR_FREE(path); if (rc == 0) { - rc = virCgroupMakeGroup(driver, *group, create, VIR_CGROUP_VCPU); + rc = virCgroupMakeGroup(driver, *group, create, flags); if (rc != 0) virCgroupFree(group); } @@ -1125,7 +1130,8 @@ int virCgroupForEmulator(virCgroupPtr driver, #else int virCgroupForEmulator(virCgroupPtr driver ATTRIBUTE_UNUSED, virCgroupPtr *group ATTRIBUTE_UNUSED, - int create ATTRIBUTE_UNUSED) + int create ATTRIBUTE_UNUSED, + int controllers ATTRIBUTE_UNUSED) { return -ENXIO; } diff --git a/src/util/cgroup.h b/src/util/cgroup.h index dc0811a..e3bdc9b 100644 --- a/src/util/cgroup.h +++ b/src/util/cgroup.h @@ -62,7 +62,8 @@ int virCgroupForVcpu(virCgroupPtr driver, int virCgroupForEmulator(virCgroupPtr driver, virCgroupPtr *group, - int create); + int create, + int controllers); int virCgroupPathOfController(virCgroupPtr group, int controller, -- 1.7.11.7
The reason to disable cgroup cpuset by default is: currently, sub-directories of cgroup cpuset won't be auto-updated if cpu hot-plug happens. For example, if we now have 0-3,5-7 pcpus online, the value of /sys/fs/cgroup/cpuset/libvirt/qemu/mydom/cpuset.cpus is 0-3,5-7, then pcpu 4 is hot-plugged, but the value of cpuset.cpus is still 0-3,5-7. This will cause a problem that we can't pin to a hot-plugged pcpu. Users can still enable cpuset by editing qemu.conf. --- src/qemu/qemu.conf | 2 +- src/qemu/qemu_conf.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf index dd853c8..b853852 100644 --- a/src/qemu/qemu.conf +++ b/src/qemu/qemu.conf @@ -226,7 +226,7 @@ # can be mounted in different locations. libvirt will detect # where they are located. # -#cgroup_controllers = [ "cpu", "devices", "memory", "blkio", "cpuset", "cpuacct" ] +#cgroup_controllers = [ "cpu", "devices", "memory", "blkio", "cpuacct" ] # This is the basic set of devices allowed / required by # all virtual machines. diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c index dc4d680..92c1864 100644 --- a/src/qemu/qemu_conf.c +++ b/src/qemu/qemu_conf.c @@ -402,7 +402,6 @@ int qemudLoadDriverConfig(struct qemud_driver *driver, (1 << VIR_CGROUP_CONTROLLER_DEVICES) | (1 << VIR_CGROUP_CONTROLLER_MEMORY) | (1 << VIR_CGROUP_CONTROLLER_BLKIO) | - (1 << VIR_CGROUP_CONTROLLER_CPUSET) | (1 << VIR_CGROUP_CONTROLLER_CPUACCT); } for (i = 0 ; i < VIR_CGROUP_CONTROLLER_LAST ; i++) { -- 1.7.11.7
ping... On Wed, Nov 07, 2012 at 06:38:40PM +0800, Hu Tao wrote:
The reason to disable cgroup cpuset by default is: currently, sub-directories of cgroup cpuset won't be auto-updated if cpu hot-plug happens. For example, if we now have 0-3,5-7 pcpus online, the value of /sys/fs/cgroup/cpuset/libvirt/qemu/mydom/cpuset.cpus is 0-3,5-7, then pcpu 4 is hot-plugged, but the value of cpuset.cpus is still 0-3,5-7. This will cause a problem that we can't pin to a hot-plugged pcpu.
Users can still enable cpuset by editing qemu.conf.
Hu Tao (6): cgroup: Add a flag VIR_CGROUP_DISABLE_CPUSET create cgroup controllers for driver according to config create cgroup controllers for domain according to config create cgroup controllers for vcpu according to config create cgroup controllers for emulator according to config disable cgroup cpuset by default
src/lxc/lxc_cgroup.c | 4 +-- src/lxc/lxc_driver.c | 22 ++++++------ src/lxc/lxc_process.c | 4 +-- src/qemu/qemu.conf | 2 +- src/qemu/qemu_cgroup.c | 15 ++++---- src/qemu/qemu_conf.c | 1 - src/qemu/qemu_driver.c | 92 +++++++++++++++++++++++++++++------------------ src/qemu/qemu_hotplug.c | 9 +++-- src/qemu/qemu_migration.c | 2 +- src/util/cgroup.c | 49 +++++++++++++++++++------ src/util/cgroup.h | 12 ++++--- 11 files changed, 136 insertions(+), 76 deletions(-)
-- 1.7.11.7
-- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
ping... On Wed, Nov 07, 2012 at 06:38:40PM +0800, Hu Tao wrote:
The reason to disable cgroup cpuset by default is: currently, sub-directories of cgroup cpuset won't be auto-updated if cpu hot-plug happens. For example, if we now have 0-3,5-7 pcpus online, the value of /sys/fs/cgroup/cpuset/libvirt/qemu/mydom/cpuset.cpus is 0-3,5-7, then pcpu 4 is hot-plugged, but the value of cpuset.cpus is still 0-3,5-7. This will cause a problem that we can't pin to a hot-plugged pcpu.
Users can still enable cpuset by editing qemu.conf.
Hu Tao (6): cgroup: Add a flag VIR_CGROUP_DISABLE_CPUSET create cgroup controllers for driver according to config create cgroup controllers for domain according to config create cgroup controllers for vcpu according to config create cgroup controllers for emulator according to config disable cgroup cpuset by default
src/lxc/lxc_cgroup.c | 4 +-- src/lxc/lxc_driver.c | 22 ++++++------ src/lxc/lxc_process.c | 4 +-- src/qemu/qemu.conf | 2 +- src/qemu/qemu_cgroup.c | 15 ++++---- src/qemu/qemu_conf.c | 1 - src/qemu/qemu_driver.c | 92 +++++++++++++++++++++++++++++------------------ src/qemu/qemu_hotplug.c | 9 +++-- src/qemu/qemu_migration.c | 2 +- src/util/cgroup.c | 49 +++++++++++++++++++------ src/util/cgroup.h | 12 ++++--- 11 files changed, 136 insertions(+), 76 deletions(-)
-- 1.7.11.7
-- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On 2012年11月07日 18:38, Hu Tao wrote:
The reason to disable cgroup cpuset by default is: currently, sub-directories of cgroup cpuset won't be auto-updated if cpu hot-plug happens. For example, if we now have 0-3,5-7 pcpus online, the value of /sys/fs/cgroup/cpuset/libvirt/qemu/mydom/cpuset.cpus is 0-3,5-7, then pcpu 4 is hot-plugged, but the value of cpuset.cpus is still 0-3,5-7. This will cause a problem that we can't pin to a hot-plugged pcpu
IMHO it's a kernel bug and should be fixed in kernel instead.
Users can still enable cpuset by editing qemu.conf.
And per users can enable/disable the cpuset cgroup in qemu.conf, why do we need to disable it by default? On one hand, it has to be enabled again once the kernel bug is fixed; On the other hand, disabling it affects many functions, it's not deserved to disable it with the only problem (actually the libvirt pinning just behaves correctly, as the hotplug pCPU is not visible yet, the root cause is in kernel) on pinning from my p.o.v.
Hu Tao (6): cgroup: Add a flag VIR_CGROUP_DISABLE_CPUSET create cgroup controllers for driver according to config create cgroup controllers for domain according to config create cgroup controllers for vcpu according to config create cgroup controllers for emulator according to config disable cgroup cpuset by default
src/lxc/lxc_cgroup.c | 4 +-- src/lxc/lxc_driver.c | 22 ++++++------ src/lxc/lxc_process.c | 4 +-- src/qemu/qemu.conf | 2 +- src/qemu/qemu_cgroup.c | 15 ++++---- src/qemu/qemu_conf.c | 1 - src/qemu/qemu_driver.c | 92 +++++++++++++++++++++++++++++------------------ src/qemu/qemu_hotplug.c | 9 +++-- src/qemu/qemu_migration.c | 2 +- src/util/cgroup.c | 49 +++++++++++++++++++------ src/util/cgroup.h | 12 ++++--- 11 files changed, 136 insertions(+), 76 deletions(-)
On Thu, Dec 13, 2012 at 12:21:29PM +0800, Osier Yang wrote:
On 2012年11月07日 18:38, Hu Tao wrote:
The reason to disable cgroup cpuset by default is: currently, sub-directories of cgroup cpuset won't be auto-updated if cpu hot-plug happens. For example, if we now have 0-3,5-7 pcpus online, the value of /sys/fs/cgroup/cpuset/libvirt/qemu/mydom/cpuset.cpus is 0-3,5-7, then pcpu 4 is hot-plugged, but the value of cpuset.cpus is still 0-3,5-7. This will cause a problem that we can't pin to a hot-plugged pcpu
IMHO it's a kernel bug and should be fixed in kernel instead.
The problem was discussed before, see http://www.redhat.com/archives/libvir-list/2012-September/msg00165.html http://www.redhat.com/archives/libvir-list/2012-September/msg00221.html
Users can still enable cpuset by editing qemu.conf.
And per users can enable/disable the cpuset cgroup in qemu.conf, why do we need to disable it by default? On one hand, it has to be enabled again once the kernel bug is fixed; On the other hand, disabling it affects many functions, it's not deserved to disable it with the only problem (actually the libvirt pinning just behaves correctly, as the hotplug pCPU is not visible yet, the root cause is in kernel) on pinning from my p.o.v.
Hu Tao (6): cgroup: Add a flag VIR_CGROUP_DISABLE_CPUSET create cgroup controllers for driver according to config create cgroup controllers for domain according to config create cgroup controllers for vcpu according to config create cgroup controllers for emulator according to config disable cgroup cpuset by default
src/lxc/lxc_cgroup.c | 4 +-- src/lxc/lxc_driver.c | 22 ++++++------ src/lxc/lxc_process.c | 4 +-- src/qemu/qemu.conf | 2 +- src/qemu/qemu_cgroup.c | 15 ++++---- src/qemu/qemu_conf.c | 1 - src/qemu/qemu_driver.c | 92 +++++++++++++++++++++++++++++------------------ src/qemu/qemu_hotplug.c | 9 +++-- src/qemu/qemu_migration.c | 2 +- src/util/cgroup.c | 49 +++++++++++++++++++------ src/util/cgroup.h | 12 ++++--- 11 files changed, 136 insertions(+), 76 deletions(-)
On 2012年12月13日 14:59, Hu Tao wrote:
On Thu, Dec 13, 2012 at 12:21:29PM +0800, Osier Yang wrote:
On 2012年11月07日 18:38, Hu Tao wrote:
The reason to disable cgroup cpuset by default is: currently, sub-directories of cgroup cpuset won't be auto-updated if cpu hot-plug happens. For example, if we now have 0-3,5-7 pcpus online, the value of /sys/fs/cgroup/cpuset/libvirt/qemu/mydom/cpuset.cpus is 0-3,5-7, then pcpu 4 is hot-plugged, but the value of cpuset.cpus is still 0-3,5-7. This will cause a problem that we can't pin to a hot-plugged pcpu
IMHO it's a kernel bug and should be fixed in kernel instead.
The problem was discussed before, see
http://www.redhat.com/archives/libvir-list/2012-September/msg00165.html http://www.redhat.com/archives/libvir-list/2012-September/msg00221.html
Okay, read through the threads. It's fine for emulatorpin and vcpupin, because they can still use sched_setaffinity. But APIs like domain{Get,Set}NumaParameters can't be used anymore. Will document the problem somewhere and clarify it's not recommended to use be enough? Or do we really want to disable it by default, one can disable any controller by changing the default configurations in qemu.conf as long as the bug is fixed: https://www.redhat.com/archives/libvir-list/2012-December/msg00774.html
Users can still enable cpuset by editing qemu.conf.
And per users can enable/disable the cpuset cgroup in qemu.conf, why do we need to disable it by default? On one hand, it has to be enabled again once the kernel bug is fixed; On the other hand, disabling it affects many functions, it's not deserved to disable it with the only problem (actually the libvirt pinning just behaves correctly, as the hotplug pCPU is not visible yet, the root cause is in kernel) on pinning from my p.o.v.
Hu Tao (6): cgroup: Add a flag VIR_CGROUP_DISABLE_CPUSET create cgroup controllers for driver according to config create cgroup controllers for domain according to config create cgroup controllers for vcpu according to config create cgroup controllers for emulator according to config disable cgroup cpuset by default
src/lxc/lxc_cgroup.c | 4 +-- src/lxc/lxc_driver.c | 22 ++++++------ src/lxc/lxc_process.c | 4 +-- src/qemu/qemu.conf | 2 +- src/qemu/qemu_cgroup.c | 15 ++++---- src/qemu/qemu_conf.c | 1 - src/qemu/qemu_driver.c | 92 +++++++++++++++++++++++++++++------------------ src/qemu/qemu_hotplug.c | 9 +++-- src/qemu/qemu_migration.c | 2 +- src/util/cgroup.c | 49 +++++++++++++++++++------ src/util/cgroup.h | 12 ++++--- 11 files changed, 136 insertions(+), 76 deletions(-)
participants (2)
-
Hu Tao -
Osier Yang