[libvirt] [PATCH v5 0/7] Per domain bandwidth settings
We decide to make a global per domain bandwidth setting as were discussed in mailing list earlier. This patchset implements hierarchy top level cpu.cfs_period_us and cpu.cfs_quota_us control knob. I've named this parameters as global_period and global_quota. Changes in v2: add XML validation test Changes in v3: remove unneccessary cgroup copying Changes in v4: fix little rebase error Changes in v5: rebase to version 1.3.1 Alexander Burluka (7): Add global period definitions Add global quota parameter necessary definitions Add error checking on global quota and period Add global_period and global_quota XML validation test Rename qemuSetupCgroupVcpuBW to qemuSetupBandwidthCgroup Implement qemuSetupGlobalCpuCgroup Implement handling of per-domain bandwidth settings docs/schemas/domaincommon.rng | 10 +++ include/libvirt/libvirt-domain.h | 32 ++++++++ src/conf/domain_conf.c | 37 +++++++++ src/conf/domain_conf.h | 2 + src/qemu/qemu_cgroup.c | 68 ++++++++++++++-- src/qemu/qemu_cgroup.h | 7 +- src/qemu/qemu_command.c | 3 +- src/qemu/qemu_driver.c | 102 ++++++++++++++++++++++-- src/qemu/qemu_process.c | 4 + tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 + 10 files changed, 251 insertions(+), 16 deletions(-) -- 1.8.3.1
This parameter represents top level period cgroup that limits whole domain enforcement period for a quota Signed-off-by: Alexander Burluka <aburluka@virtuozzo.com> --- docs/schemas/domaincommon.rng | 5 +++++ include/libvirt/libvirt-domain.h | 16 ++++++++++++++++ src/conf/domain_conf.c | 18 ++++++++++++++++++ src/conf/domain_conf.h | 1 + 4 files changed, 40 insertions(+) diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index 5deb17b..aa7eae9 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -670,6 +670,11 @@ </element> </optional> <optional> + <element name="global_period"> + <ref name="cpuperiod"/> + </element> + </optional> + <optional> <element name="period"> <ref name="cpuperiod"/> </element> diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index d26faa5..cb30313 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -312,6 +312,14 @@ typedef enum { # define VIR_DOMAIN_SCHEDULER_CPU_SHARES "cpu_shares" /** + * VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD: + * + * Macro represents the enforcement period for a quota, in microseconds, + * for whole domain, when using the posix scheduler, as a ullong. + */ +# define VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD "global_period" + +/** * VIR_DOMAIN_SCHEDULER_VCPU_PERIOD: * * Macro represents the enforcement period for a quota, in microseconds, @@ -3318,6 +3326,14 @@ typedef void (*virConnectDomainEventDeviceAddedCallback)(virConnectPtr conn, # define VIR_DOMAIN_TUNABLE_CPU_CPU_SHARES "cputune.cpu_shares" /** + * VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD: + * + * Macro represents the enforcement period for a quota, in microseconds, + * for whole domain, when using the posix scheduler, as VIR_TYPED_PARAM_ULLONG. + */ +# define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD "cputune.global_period" + +/** * VIR_DOMAIN_TUNABLE_CPU_VCPU_PERIOD: * * Macro represents the enforcement period for a quota, in microseconds, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index a9706b0..5ee53c2 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -15059,6 +15059,21 @@ virDomainDefParseXML(xmlDocPtr xml, goto error; } + if (virXPathULongLong("string(./cputune/global_period[1])", ctxt, + &def->cputune.global_period) < -1) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("can't parse cputune global period value")); + goto error; + } + + if (def->cputune.global_period > 0 && + (def->cputune.global_period < 1000 || def->cputune.global_period > 1000000)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Value of cputune global period must be in range " + "[1000, 1000000]")); + goto error; + } + if (virXPathULongLong("string(./cputune/emulator_period[1])", ctxt, &def->cputune.emulator_period) < -1) { virReportError(VIR_ERR_XML_ERROR, "%s", @@ -21818,6 +21833,9 @@ virDomainDefFormatInternal(virDomainDefPtr def, if (def->cputune.quota) virBufferAsprintf(&childrenBuf, "<quota>%lld</quota>\n", def->cputune.quota); + if (def->cputune.global_period) + virBufferAsprintf(&childrenBuf, "<global_period>%llu</global_period>\n", + def->cputune.global_period); if (def->cputune.emulator_period) virBufferAsprintf(&childrenBuf, "<emulator_period>%llu" diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 0141009..7c43e7d 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2135,6 +2135,7 @@ struct _virDomainCputune { bool sharesSpecified; unsigned long long period; long long quota; + unsigned long long global_period; unsigned long long emulator_period; long long emulator_quota; size_t nvcpupin; -- 1.8.3.1
This parameter controls the maximum bandwidth to be used within a period for whole domain. Signed-off-by: Alexander Burluka <aburluka@virtuozzo.com> --- docs/schemas/domaincommon.rng | 5 +++++ include/libvirt/libvirt-domain.h | 16 ++++++++++++++++ src/conf/domain_conf.c | 19 +++++++++++++++++++ src/conf/domain_conf.h | 1 + 4 files changed, 41 insertions(+) diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index aa7eae9..17653e1 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -675,6 +675,11 @@ </element> </optional> <optional> + <element name="global_quota"> + <ref name="cpuquota"/> + </element> + </optional> + <optional> <element name="period"> <ref name="cpuperiod"/> </element> diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index cb30313..7d3d8e8 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -320,6 +320,14 @@ typedef enum { # define VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD "global_period" /** + * VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA: + * + * Macro represents the maximum bandwidth to be used within a period for + * whole domain, when using the posix scheduler, as an llong. + */ +# define VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA "global_quota" + +/** * VIR_DOMAIN_SCHEDULER_VCPU_PERIOD: * * Macro represents the enforcement period for a quota, in microseconds, @@ -3334,6 +3342,14 @@ typedef void (*virConnectDomainEventDeviceAddedCallback)(virConnectPtr conn, # define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD "cputune.global_period" /** + * VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA: + * + * Macro represents the maximum bandwidth to be used within a period for + * whole domain, when using the posix scheduler, as VIR_TYPED_PARAM_LLONG. + */ +# define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA "cputune.global_quota" + +/** * VIR_DOMAIN_TUNABLE_CPU_VCPU_PERIOD: * * Macro represents the enforcement period for a quota, in microseconds, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 5ee53c2..7e7a69c 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -15074,6 +15074,22 @@ virDomainDefParseXML(xmlDocPtr xml, goto error; } + if (virXPathLongLong("string(./cputune/global_quota[1])", ctxt, + &def->cputune.global_quota) < -1) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("can't parse cputune global quota value")); + goto error; + } + + if (def->cputune.global_quota > 0 && + (def->cputune.global_quota < 1000 || + def->cputune.global_quota > 18446744073709551LL)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Value of cputune global quota must be in range " + "[1000, 18446744073709551]")); + goto error; + } + if (virXPathULongLong("string(./cputune/emulator_period[1])", ctxt, &def->cputune.emulator_period) < -1) { virReportError(VIR_ERR_XML_ERROR, "%s", @@ -21836,6 +21852,9 @@ virDomainDefFormatInternal(virDomainDefPtr def, if (def->cputune.global_period) virBufferAsprintf(&childrenBuf, "<global_period>%llu</global_period>\n", def->cputune.global_period); + if (def->cputune.global_quota) + virBufferAsprintf(&childrenBuf, "<global_quota>%lld</global_quota>\n", + def->cputune.global_quota); if (def->cputune.emulator_period) virBufferAsprintf(&childrenBuf, "<emulator_period>%llu" diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 7c43e7d..bac82b3 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2136,6 +2136,7 @@ struct _virDomainCputune { unsigned long long period; long long quota; unsigned long long global_period; + long long global_quota; unsigned long long emulator_period; long long emulator_quota; size_t nvcpupin; -- 1.8.3.1
Signed-off-by: Alexander Burluka <aburluka@virtuozzo.com> --- src/qemu/qemu_command.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 5d3ab3a..087e9ad 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -9397,7 +9397,8 @@ qemuBuildCommandLine(virConnectPtr conn, } if (def->cputune.sharesSpecified || def->cputune.period || - def->cputune.quota || def->cputune.emulator_period || + def->cputune.quota || def->cputune.global_period || + def->cputune.global_quota || def->cputune.emulator_period || def->cputune.emulator_quota) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("CPU tuning is not available in session mode")); -- 1.8.3.1
--- tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml b/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml index 813d201..a056089 100644 --- a/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml +++ b/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml @@ -8,6 +8,8 @@ <shares>2048</shares> <period>1000000</period> <quota>-1</quota> + <global_period>1000000</global_period> + <global_quota>-1</global_quota> <vcpupin vcpu='0' cpuset='0'/> <vcpupin vcpu='1' cpuset='1'/> <emulatorpin cpuset='1'/> -- 1.8.3.1
This rename is required to reuse this function in per-domain bandwidth setup routine Signed-off-by: Alexander Burluka <aburluka@virtuozzo.com> --- src/qemu/qemu_cgroup.c | 14 +++++++------- src/qemu/qemu_cgroup.h | 6 +++--- src/qemu/qemu_driver.c | 5 ++--- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index e41f461..452d71d 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -944,9 +944,9 @@ qemuSetupCgroup(virQEMUDriverPtr driver, } int -qemuSetupCgroupVcpuBW(virCgroupPtr cgroup, - unsigned long long period, - long long quota) +qemuSetupBandwidthCgroup(virCgroupPtr cgroup, + unsigned long long period, + long long quota) { unsigned long long old_period; @@ -1053,7 +1053,7 @@ qemuSetupCgroupForVcpu(virDomainObjPtr vm) goto cleanup; if (period || quota) { - if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0) + if (qemuSetupBandwidthCgroup(cgroup_vcpu, period, quota) < 0) goto cleanup; } @@ -1152,8 +1152,8 @@ qemuSetupCgroupForEmulator(virDomainObjPtr vm) if (period || quota) { if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) && - qemuSetupCgroupVcpuBW(cgroup_emulator, period, - quota) < 0) + qemuSetupBandwidthCgroup(cgroup_emulator, period, + quota) < 0) goto cleanup; } @@ -1217,7 +1217,7 @@ qemuSetupCgroupForIOThreads(virDomainObjPtr vm) goto cleanup; if (period || quota) { - if (qemuSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0) + if (qemuSetupBandwidthCgroup(cgroup_iothread, period, quota) < 0) goto cleanup; } diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h index 2bcf071..17da920 100644 --- a/src/qemu/qemu_cgroup.h +++ b/src/qemu/qemu_cgroup.h @@ -49,9 +49,9 @@ int qemuSetupCgroup(virQEMUDriverPtr driver, size_t nnicindexes, int *nicindexes); int qemuSetupCpusetMems(virDomainObjPtr vm); -int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup, - unsigned long long period, - long long quota); +int qemuSetupBandwidthCgroup(virCgroupPtr cgroup, + unsigned long long period, + long long quota); int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask); int qemuSetupCgroupForVcpu(virDomainObjPtr vm); int qemuSetupCgroupForIOThreads(virDomainObjPtr vm); diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 8ccf68b..48aeab6 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -10251,8 +10251,7 @@ qemuSetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup, if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_VCPU, i, false, &cgroup_vcpu) < 0) goto cleanup; - - if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0) + if (qemuSetupBandwidthCgroup(cgroup_vcpu, period, quota) < 0) goto cleanup; virCgroupFree(&cgroup_vcpu); @@ -10279,7 +10278,7 @@ qemuSetEmulatorBandwidthLive(virCgroupPtr cgroup, false, &cgroup_emulator) < 0) goto cleanup; - if (qemuSetupCgroupVcpuBW(cgroup_emulator, period, quota) < 0) + if (qemuSetupBandwidthCgroup(cgroup_emulator, period, quota) < 0) goto cleanup; virCgroupFree(&cgroup_emulator); -- 1.8.3.1
This functions setups per-domain cpu bandwidth parameters Signed-off-by: Alexander Burluka <aburluka@virtuozzo.com> --- src/qemu/qemu_cgroup.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_cgroup.h | 1 + src/qemu/qemu_process.c | 4 ++++ 3 files changed, 59 insertions(+) diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 452d71d..1638bfe 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -1002,6 +1002,60 @@ qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, } int +qemuSetupGlobalCpuCgroup(virDomainObjPtr vm) +{ + qemuDomainObjPrivatePtr priv = vm->privateData; + unsigned long long period = vm->def->cputune.global_period; + long long quota = vm->def->cputune.global_quota; + char *mem_mask = NULL; + virDomainNumatuneMemMode mem_mode; + + if ((period || quota) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + return -1; + } + + /* + * If CPU cgroup controller is not initialized here, then we need + * neither period nor quota settings. And if CPUSET controller is + * not initialized either, then there's nothing to do anyway. + */ + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + /* We are trying to setup cgroups for CPU pinning, which can also be done + * with virProcessSetAffinity, thus the lack of cgroups is not fatal here. + */ + if (priv->cgroup == NULL) + return 0; + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && + virDomainNumatuneMaybeFormatNodeset(vm->def->numa, + priv->autoNodeset, + &mem_mask, -1) < 0) + goto cleanup; + + if (period || quota) { + if (qemuSetupBandwidthCgroup(priv->cgroup, period, quota) < 0) + goto cleanup; + } + + VIR_FREE(mem_mask); + + return 0; + + cleanup: + VIR_FREE(mem_mask); + + return -1; +} + + +int qemuSetupCgroupForVcpu(virDomainObjPtr vm) { virCgroupPtr cgroup_vcpu = NULL; diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h index 17da920..75f9eb7 100644 --- a/src/qemu/qemu_cgroup.h +++ b/src/qemu/qemu_cgroup.h @@ -54,6 +54,7 @@ int qemuSetupBandwidthCgroup(virCgroupPtr cgroup, long long quota); int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask); int qemuSetupCgroupForVcpu(virDomainObjPtr vm); +int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm); int qemuSetupCgroupForIOThreads(virDomainObjPtr vm); int qemuSetupCgroupForEmulator(virDomainObjPtr vm); int qemuRemoveCgroup(virQEMUDriverPtr driver, virDomainObjPtr vm); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 05cbda2..96c7a6a 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -4982,6 +4982,10 @@ qemuProcessLaunch(virConnectPtr conn, if (qemuProcessDetectIOThreadPIDs(driver, vm, asyncJob) < 0) goto cleanup; + VIR_DEBUG("Setting global CPU cgroup (if required)"); + if (qemuSetupGlobalCpuCgroup(vm) < 0) + goto cleanup; + VIR_DEBUG("Setting cgroup for each VCPU (if required)"); if (qemuSetupCgroupForVcpu(vm) < 0) goto cleanup; -- 1.8.3.1
On Mon, 2016-01-18 at 13:08 +0300, Alexander Burluka wrote:
This functions setups per-domain cpu bandwidth parameters
Signed-off-by: Alexander Burluka <aburluka@virtuozzo.com> --- src/qemu/qemu_cgroup.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_cgroup.h | 1 + src/qemu/qemu_process.c | 4 ++++ 3 files changed, 59 insertions(+)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 452d71d..1638bfe 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -1002,6 +1002,60 @@ qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, } int +qemuSetupGlobalCpuCgroup(virDomainObjPtr vm) +{ + qemuDomainObjPrivatePtr priv = vm->privateData; + unsigned long long period = vm->def->cputune.global_period; + long long quota = vm->def->cputune.global_quota; + char *mem_mask = NULL; + virDomainNumatuneMemMode mem_mode; + + if ((period || quota) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + return -1; + } + + /* + * If CPU cgroup controller is not initialized here, then we need + * neither period nor quota settings. And if CPUSET controller is + * not initialized either, then there's nothing to do anyway. + */ + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + /* We are trying to setup cgroups for CPU pinning, which can also be done + * with virProcessSetAffinity, thus the lack of cgroups is not fatal here. + */ + if (priv->cgroup == NULL) + return 0; + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && + virDomainNumatuneMaybeFormatNodeset(vm->def->numa, + priv->autoNodeset, + &mem_mask, -1) < 0) + goto cleanup; + + if (period || quota) { + if (qemuSetupBandwidthCgroup(priv->cgroup, period, quota) < 0) + goto cleanup; + } + + VIR_FREE(mem_mask); + + return 0; + + cleanup:
This path is aclually error, not cleanup. But I think it should prevent from pushing the patch, since it can be simply fixed in patch file.
+ VIR_FREE(mem_mask); + + return -1; +} + + +int qemuSetupCgroupForVcpu(virDomainObjPtr vm) { virCgroupPtr cgroup_vcpu = NULL; diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h index 17da920..75f9eb7 100644 --- a/src/qemu/qemu_cgroup.h +++ b/src/qemu/qemu_cgroup.h @@ -54,6 +54,7 @@ int qemuSetupBandwidthCgroup(virCgroupPtr cgroup, long long quota); int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask); int qemuSetupCgroupForVcpu(virDomainObjPtr vm); +int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm); int qemuSetupCgroupForIOThreads(virDomainObjPtr vm); int qemuSetupCgroupForEmulator(virDomainObjPtr vm); int qemuRemoveCgroup(virQEMUDriverPtr driver, virDomainObjPtr vm); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 05cbda2..96c7a6a 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -4982,6 +4982,10 @@ qemuProcessLaunch(virConnectPtr conn, if (qemuProcessDetectIOThreadPIDs(driver, vm, asyncJob) < 0) goto cleanup; + VIR_DEBUG("Setting global CPU cgroup (if required)"); + if (qemuSetupGlobalCpuCgroup(vm) < 0) + goto cleanup; + VIR_DEBUG("Setting cgroup for each VCPU (if required)"); if (qemuSetupCgroupForVcpu(vm) < 0) goto cleanup;
18.01.2016 13:08, Alexander Burluka пишет:
This functions setups per-domain cpu bandwidth parameters
Signed-off-by: Alexander Burluka <aburluka@virtuozzo.com> --- src/qemu/qemu_cgroup.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_cgroup.h | 1 + src/qemu/qemu_process.c | 4 ++++ 3 files changed, 59 insertions(+)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 452d71d..1638bfe 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -1002,6 +1002,60 @@ qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, }
int +qemuSetupGlobalCpuCgroup(virDomainObjPtr vm) +{ + qemuDomainObjPrivatePtr priv = vm->privateData; + unsigned long long period = vm->def->cputune.global_period; + long long quota = vm->def->cputune.global_quota; + char *mem_mask = NULL; + virDomainNumatuneMemMode mem_mode; + + if ((period || quota) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + return -1; + } + + /* + * If CPU cgroup controller is not initialized here, then we need + * neither period nor quota settings. And if CPUSET controller is + * not initialized either, then there's nothing to do anyway. + */ + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + /* We are trying to setup cgroups for CPU pinning, which can also be done + * with virProcessSetAffinity, thus the lack of cgroups is not fatal here. + */ + if (priv->cgroup == NULL) + return 0; This check is unnecessary because virCgroupHasController already has it.
+ + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && + virDomainNumatuneMaybeFormatNodeset(vm->def->numa, + priv->autoNodeset, + &mem_mask, -1) < 0) + goto cleanup; I'd like to see the reason in error message here similar to error path at the beginning of the function.
+ + if (period || quota) { + if (qemuSetupBandwidthCgroup(priv->cgroup, period, quota) < 0) + goto cleanup; + } + + VIR_FREE(mem_mask); + + return 0; + + cleanup: + VIR_FREE(mem_mask); + + return -1; +} + + +int qemuSetupCgroupForVcpu(virDomainObjPtr vm) { virCgroupPtr cgroup_vcpu = NULL; diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h index 17da920..75f9eb7 100644 --- a/src/qemu/qemu_cgroup.h +++ b/src/qemu/qemu_cgroup.h @@ -54,6 +54,7 @@ int qemuSetupBandwidthCgroup(virCgroupPtr cgroup, long long quota); int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask); int qemuSetupCgroupForVcpu(virDomainObjPtr vm); +int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm); int qemuSetupCgroupForIOThreads(virDomainObjPtr vm); int qemuSetupCgroupForEmulator(virDomainObjPtr vm); int qemuRemoveCgroup(virQEMUDriverPtr driver, virDomainObjPtr vm); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 05cbda2..96c7a6a 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -4982,6 +4982,10 @@ qemuProcessLaunch(virConnectPtr conn, if (qemuProcessDetectIOThreadPIDs(driver, vm, asyncJob) < 0) goto cleanup;
+ VIR_DEBUG("Setting global CPU cgroup (if required)"); + if (qemuSetupGlobalCpuCgroup(vm) < 0) + goto cleanup; + VIR_DEBUG("Setting cgroup for each VCPU (if required)"); if (qemuSetupCgroupForVcpu(vm) < 0) goto cleanup;
Signed-off-by: Alexander Burluka <aburluka@virtuozzo.com> --- src/qemu/qemu_driver.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 48aeab6..1932d7f 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -8906,7 +8906,7 @@ static char *qemuDomainGetSchedulerType(virDomainPtr dom, /* Domain not running, thus no cgroups - return defaults */ if (!virDomainObjIsActive(vm)) { if (nparams) - *nparams = 5; + *nparams = 7; ignore_value(VIR_STRDUP(ret, "posix")); goto cleanup; } @@ -8919,7 +8919,7 @@ static char *qemuDomainGetSchedulerType(virDomainPtr dom, if (nparams) { if (virCgroupSupportsCpuBW(priv->cgroup)) - *nparams = 5; + *nparams = 7; else *nparams = 1; } @@ -10234,6 +10234,19 @@ qemuDomainGetNumaParameters(virDomainPtr dom, } static int +qemuSetGlobalBWLive(virCgroupPtr cgroup, unsigned long long period, + long long quota) +{ + if (period == 0 && quota == 0) + return 0; + + if (qemuSetupBandwidthCgroup(cgroup, period, quota) < 0) + return -1; + + return 0; +} + +static int qemuSetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup, unsigned long long period, long long quota) { @@ -10329,6 +10342,10 @@ qemuDomainSetSchedulerParametersFlags(virDomainPtr dom, VIR_TYPED_PARAM_ULLONG, VIR_DOMAIN_SCHEDULER_VCPU_QUOTA, VIR_TYPED_PARAM_LLONG, + VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD, + VIR_TYPED_PARAM_ULLONG, + VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA, + VIR_TYPED_PARAM_LLONG, VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD, VIR_TYPED_PARAM_ULLONG, VIR_DOMAIN_SCHEDULER_EMULATOR_QUOTA, @@ -10445,6 +10462,46 @@ qemuDomainSetSchedulerParametersFlags(virDomainPtr dom, if (flags & VIR_DOMAIN_AFFECT_CONFIG) vmdef->cputune.quota = value_l; + } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD)) { + SCHED_RANGE_CHECK(value_ul, VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD, + QEMU_SCHED_MIN_PERIOD, QEMU_SCHED_MAX_PERIOD); + + if (flags & VIR_DOMAIN_AFFECT_LIVE && value_ul) { + if ((rc = qemuSetGlobalBWLive(priv->cgroup, value_ul, 0))) + goto endjob; + + vm->def->cputune.global_period = value_ul; + + if (virTypedParamsAddULLong(&eventParams, &eventNparams, + &eventMaxNparams, + VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD, + value_ul) < 0) + goto endjob; + } + + if (flags & VIR_DOMAIN_AFFECT_CONFIG) + vmdef->cputune.period = params[i].value.ul; + + } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA)) { + SCHED_RANGE_CHECK(value_l, VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA, + QEMU_SCHED_MIN_QUOTA, QEMU_SCHED_MAX_QUOTA); + + if (flags & VIR_DOMAIN_AFFECT_LIVE && value_l) { + if ((rc = qemuSetGlobalBWLive(priv->cgroup, 0, value_l))) + goto endjob; + + vm->def->cputune.global_quota = value_l; + + if (virTypedParamsAddLLong(&eventParams, &eventNparams, + &eventMaxNparams, + VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA, + value_l) < 0) + goto endjob; + } + + if (flags & VIR_DOMAIN_AFFECT_CONFIG) + vmdef->cputune.global_quota = value_l; + } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD)) { SCHED_RANGE_CHECK(value_ul, VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD, QEMU_SCHED_MIN_PERIOD, QEMU_SCHED_MAX_PERIOD); @@ -10609,6 +10666,16 @@ qemuGetEmulatorBandwidthLive(virCgroupPtr cgroup, } static int +qemuGetGlobalBWLive(virCgroupPtr cgroup, unsigned long long *period, + long long *quota) +{ + if (qemuGetVcpuBWLive(cgroup, period, quota) < 0) + return -1; + + return 0; +} + +static int qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, virTypedParameterPtr params, int *nparams, @@ -10619,6 +10686,8 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, unsigned long long shares; unsigned long long period; long long quota; + unsigned long long global_period; + long long global_quota; unsigned long long emulator_period; long long emulator_quota; int ret = -1; @@ -10665,6 +10734,8 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, if (*nparams > 1) { period = persistentDef->cputune.period; quota = persistentDef->cputune.quota; + global_period = persistentDef->cputune.global_period; + global_quota = persistentDef->cputune.global_quota; emulator_period = persistentDef->cputune.emulator_period; emulator_quota = persistentDef->cputune.emulator_quota; cpu_bw_status = true; /* Allow copy of data to params[] */ @@ -10694,6 +10765,12 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, goto cleanup; } + if (*nparams > 5 && cpu_bw_status) { + rc = qemuGetGlobalBWLive(priv->cgroup, &global_period, &global_quota); + if (rc != 0) + goto cleanup; + } + out: if (virTypedParameterAssign(¶ms[0], VIR_DOMAIN_SCHEDULER_CPU_SHARES, VIR_TYPED_PARAM_ULLONG, shares) < 0) @@ -10734,6 +10811,22 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, goto cleanup; saved_nparams++; } + + if (*nparams > saved_nparams) { + if (virTypedParameterAssign(¶ms[5], + VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD, + VIR_TYPED_PARAM_ULLONG, global_period) < 0) + goto cleanup; + saved_nparams++; + } + + if (*nparams > saved_nparams) { + if (virTypedParameterAssign(¶ms[6], + VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA, + VIR_TYPED_PARAM_LLONG, global_quota) < 0) + goto cleanup; + saved_nparams++; + } } *nparams = saved_nparams; -- 1.8.3.1
On Mon, 2016-01-18 at 13:08 +0300, Alexander Burluka wrote:
We decide to make a global per domain bandwidth setting as were discussed in mailing list earlier. This patchset implements hierarchy top level cpu.cfs_period_us and cpu.cfs_quota_us control knob. I've named this parameters as global_period and global_quota.
All issues have been fixed, so ACK from me to the series.
Changes in v2: add XML validation test Changes in v3: remove unneccessary cgroup copying Changes in v4: fix little rebase error Changes in v5: rebase to version 1.3.1
Alexander Burluka (7): Add global period definitions Add global quota parameter necessary definitions Add error checking on global quota and period Add global_period and global_quota XML validation test Rename qemuSetupCgroupVcpuBW to qemuSetupBandwidthCgroup Implement qemuSetupGlobalCpuCgroup Implement handling of per-domain bandwidth settings
docs/schemas/domaincommon.rng | 10 +++ include/libvirt/libvirt-domain.h | 32 ++++++++ src/conf/domain_conf.c | 37 +++++++++ src/conf/domain_conf.h | 2 + src/qemu/qemu_cgroup.c | 68 ++++++++++++++-- src/qemu/qemu_cgroup.h | 7 +- src/qemu/qemu_command.c | 3 +- src/qemu/qemu_driver.c | 102 ++++++++++++++++++++++-- src/qemu/qemu_process.c | 4 + tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 + 10 files changed, 251 insertions(+), 16 deletions(-)
Hi! Unfortunately I've no reply on this patchset from community and would like to remind about it. Thank you! On 01/18/2016 01:08 PM, Alexander Burluka wrote:
We decide to make a global per domain bandwidth setting as were discussed in mailing list earlier. This patchset implements hierarchy top level cpu.cfs_period_us and cpu.cfs_quota_us control knob. I've named this parameters as global_period and global_quota.
Changes in v2: add XML validation test Changes in v3: remove unneccessary cgroup copying Changes in v4: fix little rebase error Changes in v5: rebase to version 1.3.1
Alexander Burluka (7): Add global period definitions Add global quota parameter necessary definitions Add error checking on global quota and period Add global_period and global_quota XML validation test Rename qemuSetupCgroupVcpuBW to qemuSetupBandwidthCgroup Implement qemuSetupGlobalCpuCgroup Implement handling of per-domain bandwidth settings
docs/schemas/domaincommon.rng | 10 +++ include/libvirt/libvirt-domain.h | 32 ++++++++ src/conf/domain_conf.c | 37 +++++++++ src/conf/domain_conf.h | 2 + src/qemu/qemu_cgroup.c | 68 ++++++++++++++-- src/qemu/qemu_cgroup.h | 7 +- src/qemu/qemu_command.c | 3 +- src/qemu/qemu_driver.c | 102 ++++++++++++++++++++++-- src/qemu/qemu_process.c | 4 + tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 + 10 files changed, 251 insertions(+), 16 deletions(-)
-- Regards, Alexander Burluka
participants (3)
-
Alexander Burluka -
Dmitry Guryanov -
Maxim Nestratov