This parameter represents top level period cgroup that limits whole domain enforcement period for a quota Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com&gt; --- docs/schemas/domaincommon.rng | 5 +++++ include/libvirt/libvirt-domain.h | 16 ++++++++++++++++ src/conf/domain_conf.c | 18 ++++++++++++++++++ src/conf/domain_conf.h | 1 + 4 files changed, 40 insertions(+) diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index 5deb17b..aa7eae9 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -670,6 +670,11 @@ </element> </optional> <optional> + <element name="global_period"> + <ref name="cpuperiod"/> + </element> + </optional> + <optional> <element name="period"> <ref name="cpuperiod"/> </element> diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 65f1618..3ec982c 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -312,6 +312,14 @@ typedef enum { # define VIR_DOMAIN_SCHEDULER_CPU_SHARES "cpu_shares" /** + * VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD: + * + * Macro represents the enforcement period for a quota, in microseconds, + * for whole domain, when using the posix scheduler, as a ullong. + */ +# define VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD "global_period" + +/** * VIR_DOMAIN_SCHEDULER_VCPU_PERIOD: * * Macro represents the enforcement period for a quota, in microseconds, @@ -3339,6 +3347,14 @@ typedef void (*virConnectDomainEventMigrationIterationCallback)(virConnectPtr co # define VIR_DOMAIN_TUNABLE_CPU_CPU_SHARES "cputune.cpu_shares" /** + * VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD: + * + * Macro represents the enforcement period for a quota, in microseconds, + * for whole domain, when using the posix scheduler, as VIR_TYPED_PARAM_ULLONG. + */ +# define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD "cputune.global_period" + +/** * VIR_DOMAIN_TUNABLE_CPU_VCPU_PERIOD: * * Macro represents the enforcement period for a quota, in microseconds, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 67415fa..1eb474a 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -15097,6 +15097,21 @@ virDomainDefParseXML(xmlDocPtr xml, goto error; } + if (virXPathULongLong("string(./cputune/global_period[1])", ctxt, + &def->cputune.global_period) < -1) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("can't parse cputune global period value")); + goto error; + } + + if (def->cputune.global_period > 0 && + (def->cputune.global_period < 1000 || def->cputune.global_period > 1000000)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Value of cputune global period must be in range " + "[1000, 1000000]")); + goto error; + } + if (virXPathULongLong("string(./cputune/emulator_period[1])", ctxt, &def->cputune.emulator_period) < -1) { virReportError(VIR_ERR_XML_ERROR, "%s", @@ -21597,6 +21612,9 @@ virDomainCputuneDefFormat(virBufferPtr buf, if (def->cputune.quota) virBufferAsprintf(&childrenBuf, "<quota>%lld</quota>\n", def->cputune.quota); + if (def->cputune.global_period) + virBufferAsprintf(&childrenBuf, "<global_period>%llu</global_period>\n", + def->cputune.global_period); if (def->cputune.emulator_period) virBufferAsprintf(&childrenBuf, "<emulator_period>%llu" diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 230b23d..a71d761 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2109,6 +2109,7 @@ struct _virDomainCputune { bool sharesSpecified; unsigned long long period; long long quota; + unsigned long long global_period; unsigned long long emulator_period; long long emulator_quota; virBitmapPtr emulatorpin; -- 1.8.3.1

This parameter controls the maximum bandwidth to be used within a period for whole domain. Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com&gt; --- docs/schemas/domaincommon.rng | 5 +++++ include/libvirt/libvirt-domain.h | 16 ++++++++++++++++ src/conf/domain_conf.c | 19 +++++++++++++++++++ src/conf/domain_conf.h | 1 + 4 files changed, 41 insertions(+) diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index aa7eae9..17653e1 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -675,6 +675,11 @@ </element> </optional> <optional> + <element name="global_quota"> + <ref name="cpuquota"/> + </element> + </optional> + <optional> <element name="period"> <ref name="cpuperiod"/> </element> diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 3ec982c..79c25df 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -320,6 +320,14 @@ typedef enum { # define VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD "global_period" /** + * VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA: + * + * Macro represents the maximum bandwidth to be used within a period for + * whole domain, when using the posix scheduler, as an llong. + */ +# define VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA "global_quota" + +/** * VIR_DOMAIN_SCHEDULER_VCPU_PERIOD: * * Macro represents the enforcement period for a quota, in microseconds, @@ -3355,6 +3363,14 @@ typedef void (*virConnectDomainEventMigrationIterationCallback)(virConnectPtr co # define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD "cputune.global_period" /** + * VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA: + * + * Macro represents the maximum bandwidth to be used within a period for + * whole domain, when using the posix scheduler, as VIR_TYPED_PARAM_LLONG. + */ +# define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA "cputune.global_quota" + +/** * VIR_DOMAIN_TUNABLE_CPU_VCPU_PERIOD: * * Macro represents the enforcement period for a quota, in microseconds, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 1eb474a..620cc47 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -15112,6 +15112,22 @@ virDomainDefParseXML(xmlDocPtr xml, goto error; } + if (virXPathLongLong("string(./cputune/global_quota[1])", ctxt, + &def->cputune.global_quota) < -1) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("can't parse cputune global quota value")); + goto error; + } + + if (def->cputune.global_quota > 0 && + (def->cputune.global_quota < 1000 || + def->cputune.global_quota > 18446744073709551LL)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Value of cputune global quota must be in range " + "[1000, 18446744073709551]")); + goto error; + } + if (virXPathULongLong("string(./cputune/emulator_period[1])", ctxt, &def->cputune.emulator_period) < -1) { virReportError(VIR_ERR_XML_ERROR, "%s", @@ -21615,6 +21631,9 @@ virDomainCputuneDefFormat(virBufferPtr buf, if (def->cputune.global_period) virBufferAsprintf(&childrenBuf, "<global_period>%llu</global_period>\n", def->cputune.global_period); + if (def->cputune.global_quota) + virBufferAsprintf(&childrenBuf, "<global_quota>%lld</global_quota>\n", + def->cputune.global_quota); if (def->cputune.emulator_period) virBufferAsprintf(&childrenBuf, "<emulator_period>%llu" diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index a71d761..4babfb4 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2110,6 +2110,7 @@ struct _virDomainCputune { unsigned long long period; long long quota; unsigned long long global_period; + long long global_quota; unsigned long long emulator_period; long long emulator_quota; virBitmapPtr emulatorpin; -- 1.8.3.1

Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com&gt; --- src/qemu/qemu_command.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 7a8ae73..d2217b0 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -9174,7 +9174,8 @@ qemuBuildCommandLine(virConnectPtr conn, } if (def->cputune.sharesSpecified || def->cputune.period || - def->cputune.quota || def->cputune.emulator_period || + def->cputune.quota || def->cputune.global_period || + def->cputune.global_quota || def->cputune.emulator_period || def->cputune.emulator_quota) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("CPU tuning is not available in session mode")); -- 1.8.3.1

Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com&gt; --- tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 ++ tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml b/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml index 951fef9..9595903 100644 --- a/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml +++ b/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml @@ -8,6 +8,8 @@ <shares>2048</shares> <period>1000000</period> <quota>-1</quota> + <global_period>1000000</global_period> + <global_quota>-1</global_quota> <vcpupin vcpu='0' cpuset='0'/> <vcpupin vcpu='1' cpuset='1'/> <emulatorpin cpuset='1'/> diff --git a/tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml b/tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml index de9c331..baf1779 100644 --- a/tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml +++ b/tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml @@ -8,6 +8,8 @@ <shares>2048</shares> <period>1000000</period> <quota>-1</quota> + <global_period>1000000</global_period> + <global_quota>-1</global_quota> <vcpupin vcpu='0' cpuset='0'/> <vcpupin vcpu='1' cpuset='1'/> <emulatorpin cpuset='1'/> -- 1.8.3.1

This functions setups per-domain cpu bandwidth parameters Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com&gt; --- src/qemu/qemu_cgroup.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_cgroup.h | 1 + src/qemu/qemu_process.c | 4 ++++ 3 files changed, 54 insertions(+) diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 4b5cf36..4240b20 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -1020,6 +1020,55 @@ qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, int +qemuSetupGlobalCpuCgroup(virDomainObjPtr vm) +{ + qemuDomainObjPrivatePtr priv = vm->privateData; + unsigned long long period = vm->def->cputune.global_period; + long long quota = vm->def->cputune.global_quota; + char *mem_mask = NULL; + virDomainNumatuneMemMode mem_mode; + + if ((period || quota) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + return -1; + } + + /* + * If CPU cgroup controller is not initialized here, then we need + * neither period nor quota settings. And if CPUSET controller is + * not initialized either, then there's nothing to do anyway. + */ + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && + virDomainNumatuneMaybeFormatNodeset(vm->def->numa, + priv->autoNodeset, + &mem_mask, -1) < 0) + goto cleanup; + + if (period || quota) { + if (qemuSetupCgroupVcpuBW(priv->cgroup, period, quota) < 0) + goto cleanup; + } + + VIR_FREE(mem_mask); + + return 0; + + cleanup: + VIR_FREE(mem_mask); + + return -1; +} + + +int qemuSetupCgroupForEmulator(virDomainObjPtr vm) { virBitmapPtr cpumask = NULL; diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h index 021bfe6..93328ea 100644 --- a/src/qemu/qemu_cgroup.h +++ b/src/qemu/qemu_cgroup.h @@ -53,6 +53,7 @@ int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup, unsigned long long period, long long quota); int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask); +int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm); int qemuSetupCgroupForEmulator(virDomainObjPtr vm); int qemuRemoveCgroup(virDomainObjPtr vm); int qemuAddToCgroup(virDomainObjPtr vm); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 23238df..7070140 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -5118,6 +5118,10 @@ qemuProcessLaunch(virConnectPtr conn, if (qemuProcessDetectIOThreadPIDs(driver, vm, asyncJob) < 0) goto cleanup; + VIR_DEBUG("Setting global CPU cgroup (if required)"); + if (qemuSetupGlobalCpuCgroup(vm) < 0) + goto cleanup; + VIR_DEBUG("Setting vCPU tuning/settings"); if (qemuProcessSetupVcpus(vm) < 0) goto cleanup; -- 1.8.3.1

Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com&gt; --- src/qemu/qemu_driver.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 2bbc724..d211a32 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -8820,7 +8820,7 @@ static char *qemuDomainGetSchedulerType(virDomainPtr dom, /* Domain not running, thus no cgroups - return defaults */ if (!virDomainObjIsActive(vm)) { if (nparams) - *nparams = 5; + *nparams = 7; ignore_value(VIR_STRDUP(ret, "posix")); goto cleanup; } @@ -8833,7 +8833,7 @@ static char *qemuDomainGetSchedulerType(virDomainPtr dom, if (nparams) { if (virCgroupSupportsCpuBW(priv->cgroup)) - *nparams = 5; + *nparams = 7; else *nparams = 1; } @@ -10153,6 +10153,19 @@ qemuDomainGetNumaParameters(virDomainPtr dom, } static int +qemuSetGlobalBWLive(virCgroupPtr cgroup, unsigned long long period, + long long quota) +{ + if (period == 0 && quota == 0) + return 0; + + if (qemuSetupCgroupVcpuBW(cgroup, period, quota) < 0) + return -1; + + return 0; +} + +static int qemuSetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup, unsigned long long period, long long quota) { @@ -10253,6 +10266,10 @@ qemuDomainSetSchedulerParametersFlags(virDomainPtr dom, VIR_TYPED_PARAM_ULLONG, VIR_DOMAIN_SCHEDULER_VCPU_QUOTA, VIR_TYPED_PARAM_LLONG, + VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD, + VIR_TYPED_PARAM_ULLONG, + VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA, + VIR_TYPED_PARAM_LLONG, VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD, VIR_TYPED_PARAM_ULLONG, VIR_DOMAIN_SCHEDULER_EMULATOR_QUOTA, @@ -10369,6 +10386,46 @@ qemuDomainSetSchedulerParametersFlags(virDomainPtr dom, if (flags & VIR_DOMAIN_AFFECT_CONFIG) vmdef->cputune.quota = value_l; + } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD)) { + SCHED_RANGE_CHECK(value_ul, VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD, + QEMU_SCHED_MIN_PERIOD, QEMU_SCHED_MAX_PERIOD); + + if (flags & VIR_DOMAIN_AFFECT_LIVE && value_ul) { + if ((rc = qemuSetGlobalBWLive(priv->cgroup, value_ul, 0))) + goto endjob; + + vm->def->cputune.global_period = value_ul; + + if (virTypedParamsAddULLong(&eventParams, &eventNparams, + &eventMaxNparams, + VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD, + value_ul) < 0) + goto endjob; + } + + if (flags & VIR_DOMAIN_AFFECT_CONFIG) + vmdef->cputune.period = params[i].value.ul; + + } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA)) { + SCHED_RANGE_CHECK(value_l, VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA, + QEMU_SCHED_MIN_QUOTA, QEMU_SCHED_MAX_QUOTA); + + if (flags & VIR_DOMAIN_AFFECT_LIVE && value_l) { + if ((rc = qemuSetGlobalBWLive(priv->cgroup, 0, value_l))) + goto endjob; + + vm->def->cputune.global_quota = value_l; + + if (virTypedParamsAddLLong(&eventParams, &eventNparams, + &eventMaxNparams, + VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA, + value_l) < 0) + goto endjob; + } + + if (flags & VIR_DOMAIN_AFFECT_CONFIG) + vmdef->cputune.global_quota = value_l; + } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD)) { SCHED_RANGE_CHECK(value_ul, VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD, QEMU_SCHED_MIN_PERIOD, QEMU_SCHED_MAX_PERIOD); @@ -10533,6 +10590,16 @@ qemuGetEmulatorBandwidthLive(virCgroupPtr cgroup, } static int +qemuGetGlobalBWLive(virCgroupPtr cgroup, unsigned long long *period, + long long *quota) +{ + if (qemuGetVcpuBWLive(cgroup, period, quota) < 0) + return -1; + + return 0; +} + +static int qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, virTypedParameterPtr params, int *nparams, @@ -10543,6 +10610,8 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, unsigned long long shares; unsigned long long period; long long quota; + unsigned long long global_period; + long long global_quota; unsigned long long emulator_period; long long emulator_quota; int ret = -1; @@ -10589,6 +10658,8 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, if (*nparams > 1) { period = persistentDef->cputune.period; quota = persistentDef->cputune.quota; + global_period = persistentDef->cputune.global_period; + global_quota = persistentDef->cputune.global_quota; emulator_period = persistentDef->cputune.emulator_period; emulator_quota = persistentDef->cputune.emulator_quota; cpu_bw_status = true; /* Allow copy of data to params[] */ @@ -10618,6 +10689,12 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, goto cleanup; } + if (*nparams > 5 && cpu_bw_status) { + rc = qemuGetGlobalBWLive(priv->cgroup, &global_period, &global_quota); + if (rc != 0) + goto cleanup; + } + out: if (virTypedParameterAssign(&params[0], VIR_DOMAIN_SCHEDULER_CPU_SHARES, VIR_TYPED_PARAM_ULLONG, shares) < 0) @@ -10658,6 +10735,22 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, goto cleanup; saved_nparams++; } + + if (*nparams > saved_nparams) { + if (virTypedParameterAssign(&params[5], + VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD, + VIR_TYPED_PARAM_ULLONG, global_period) < 0) + goto cleanup; + saved_nparams++; + } + + if (*nparams > saved_nparams) { + if (virTypedParameterAssign(&params[6], + VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA, + VIR_TYPED_PARAM_LLONG, global_quota) < 0) + goto cleanup; + saved_nparams++; + } } *nparams = saved_nparams; -- 1.8.3.1