[libvirt] [PATCH 0/2] Add ability to set specific IOThread scheduler quota/period
https://bugzilla.redhat.com/show_bug.cgi?id=1356937 John Ferlan (2): conf: Add IOThread quota and period scheduler/cputune defs qemu: Add support to get/set IOThread period and quota cgroup values docs/formatdomain.html.in | 25 ++++ docs/schemas/domaincommon.rng | 10 ++ include/libvirt/libvirt-domain.h | 32 ++++++ src/conf/domain_conf.c | 42 +++++++ src/conf/domain_conf.h | 2 + src/qemu/qemu_command.c | 3 +- src/qemu/qemu_driver.c | 127 ++++++++++++++++++++- src/qemu/qemu_process.c | 4 +- tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 + .../qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml | 2 + tools/virsh.pod | 9 +- 11 files changed, 249 insertions(+), 9 deletions(-) -- 2.5.5
https://bugzilla.redhat.com/show_bug.cgi?id=1356937 Add the definitions to allow for viewing/setting cgroup period and quota limits for IOThreads. This is similar to the work done for emulator quota and period by commit ids 'b65dafa' and 'e051c482'. Being able to view/set the IOThread specific values is related to more recent changes adding global period (commmit id '4d92d58f') and global quota (commit id '55ecdae') definitions and qemu support (commit id '4e17ff79' and 'fbcbd1b2'). With a global setting though, if somehow the IOThread value in the cgroup hierarchy was set "outside of libvirt" to a value that is incompatible with the global value. Allowing control over IOThread specific values provides the capability to alter the IOThread values as necessary. Signed-off-by: John Ferlan <jferlan@redhat.com> --- docs/formatdomain.html.in | 25 +++++++++++++ docs/schemas/domaincommon.rng | 10 ++++++ include/libvirt/libvirt-domain.h | 32 +++++++++++++++++ src/conf/domain_conf.c | 42 ++++++++++++++++++++++ src/conf/domain_conf.h | 2 ++ tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 ++ .../qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml | 2 ++ tools/virsh.pod | 9 ++--- 8 files changed, 120 insertions(+), 4 deletions(-) diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index 59a8bb9..253379e 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -616,6 +616,8 @@ <quota>-1</quota> <emulator_period>1000000</emulator_period> <emulator_quota>-1</emulator_quota> + <iothread_period>1000000</iothread_period> + <iothread_quota>-1</iothread_quota> <vcpusched vcpus='0-4,^3' scheduler='fifo' priority='1'/> <iothreadsched iothreads='2' scheduler='batch'/> </cputune> @@ -720,6 +722,29 @@ <span class="since">Only QEMU driver support since 0.10.0</span> </dd> + <dt><code>iothread_period</code></dt> + <dd> + The optional <code>iothread_period</code> element specifies the + enforcement interval(unit: microseconds) for IOThreads. Within + <code>iothread_period</code>, each IOThread of the domain will + not be allowed to consume more than <code>iothread_quota</code> + worth of runtime. The value should be in range [1000, 1000000]. + An iothread_period with value 0 means no value. + <span class="since">Only QEMU driver support since 2.1.0</span> + </dd> + <dt><code>iothread_quota</code></dt> + <dd> + The optional <code>iothread_quota</code> element specifies the maximum + allowed bandwidth(unit: microseconds) for IOThreads. A domain with + <code>iothread_quota</code> as any negative value indicates that the + domain IOThreads have infinite bandwidth, which means that it is + not bandwidth controlled. The value should be in range + [1000, 18446744073709551] or less than 0. An <code>iothread_quota</code> + with value 0 means no value. You can use this feature to ensure that + all IOThreads run at the same speed. + <span class="since">Only QEMU driver support since 2.1.0</span> + </dd> + <dt><code>vcpusched</code> and <code>iothreadsched</code></dt> <dd> The optional <code>vcpusched</code> elements specifies the scheduler diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index 741f268..7fedc36 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -731,6 +731,16 @@ <ref name="cpuquota"/> </element> </optional> + <optional> + <element name="iothread_period"> + <ref name="cpuperiod"/> + </element> + </optional> + <optional> + <element name="iothread_quota"> + <ref name="cpuquota"/> + </element> + </optional> <zeroOrMore> <element name="vcpupin"> <attribute name="vcpu"> diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 7ea93aa..af3ecae 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -365,6 +365,22 @@ typedef enum { # define VIR_DOMAIN_SCHEDULER_EMULATOR_QUOTA "emulator_quota" /** + * VIR_DOMAIN_SCHEDULER_IOTHREAD_PERIOD: + * + * Macro represents the enforcement period for a quota, in microseconds, + * for IOThreads only, when using the posix scheduler, as a ullong. + */ +# define VIR_DOMAIN_SCHEDULER_IOTHREAD_PERIOD "iothread_period" + +/** + * VIR_DOMAIN_SCHEDULER_IOTHREAD_QUOTA: + * + * Macro represents the maximum bandwidth to be used within a period for + * IOThreads only, when using the posix scheduler, as an llong. + */ +# define VIR_DOMAIN_SCHEDULER_IOTHREAD_QUOTA "iothread_quota" + +/** * VIR_DOMAIN_SCHEDULER_WEIGHT: * * Macro represents the relative weight, when using the credit @@ -3604,6 +3620,22 @@ typedef void (*virConnectDomainEventJobCompletedCallback)(virConnectPtr conn, # define VIR_DOMAIN_TUNABLE_CPU_EMULATOR_QUOTA "cputune.emulator_quota" /** + * VIR_DOMAIN_TUNABLE_CPU_IOTHREAD_PERIOD: + * + * Macro represents the enforcement period for a quota, in microseconds, for + * iothreads only, when using the posix scheduler, as VIR_TYPED_PARAM_ULLONG. + */ +# define VIR_DOMAIN_TUNABLE_CPU_IOTHREAD_PERIOD "cputune.iothread_period" + +/** + * VIR_DOMAIN_TUNABLE_CPU_IOTHREAD_QUOTA: + * + * Macro represents the maximum bandwidth to be used within a period for + * iothreads only, when using the posix scheduler, as VIR_TYPED_PARAM_LLONG. + */ +# define VIR_DOMAIN_TUNABLE_CPU_IOTHREAD_QUOTA "cputune.iothread_quota" + +/** * VIR_DOMAIN_TUNABLE_BLKDEV_DISK: * * Macro represents the name of guest disk for which the values are updated, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 6dfcf81..0e0f6a8 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -16183,6 +16183,38 @@ virDomainDefParseXML(xmlDocPtr xml, goto error; } + if (virXPathULongLong("string(./cputune/iothread_period[1])", ctxt, + &def->cputune.iothread_period) < -1) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("can't parse cputune iothread period value")); + goto error; + } + + if (def->cputune.iothread_period > 0 && + (def->cputune.iothread_period < 1000 || + def->cputune.iothread_period > 1000000)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Value of cputune iothread_period must be in range " + "[1000, 1000000]")); + goto error; + } + + if (virXPathLongLong("string(./cputune/iothread_quota[1])", ctxt, + &def->cputune.iothread_quota) < -1) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("can't parse cputune iothread quota value")); + goto error; + } + + if (def->cputune.iothread_quota > 0 && + (def->cputune.iothread_quota < 1000 || + def->cputune.iothread_quota > 18446744073709551LL)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Value of cputune iothread_quota must be in range " + "[1000, 18446744073709551]")); + goto error; + } + if ((n = virXPathNodeSet("./cputune/vcpupin", ctxt, &nodes)) < 0) goto error; @@ -22864,6 +22896,16 @@ virDomainCputuneDefFormat(virBufferPtr buf, "</emulator_quota>\n", def->cputune.emulator_quota); + if (def->cputune.iothread_period) + virBufferAsprintf(&childrenBuf, "<iothread_period>%llu" + "</iothread_period>\n", + def->cputune.iothread_period); + + if (def->cputune.iothread_quota) + virBufferAsprintf(&childrenBuf, "<iothread_quota>%lld" + "</iothread_quota>\n", + def->cputune.iothread_quota); + for (i = 0; i < def->maxvcpus; i++) { char *cpumask; virDomainVcpuDefPtr vcpu = def->vcpus[i]; diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 00041c9..7a9664d 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2031,6 +2031,8 @@ struct _virDomainCputune { long long global_quota; unsigned long long emulator_period; long long emulator_quota; + unsigned long long iothread_period; + long long iothread_quota; virBitmapPtr emulatorpin; }; diff --git a/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml b/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml index 9595903..e6ef51d 100644 --- a/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml +++ b/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml @@ -10,6 +10,8 @@ <quota>-1</quota> <global_period>1000000</global_period> <global_quota>-1</global_quota> + <iothread_period>1000000</iothread_period> + <iothread_quota>-1</iothread_quota> <vcpupin vcpu='0' cpuset='0'/> <vcpupin vcpu='1' cpuset='1'/> <emulatorpin cpuset='1'/> diff --git a/tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml b/tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml index baf1779..7a1f50d 100644 --- a/tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml +++ b/tests/qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml @@ -10,6 +10,8 @@ <quota>-1</quota> <global_period>1000000</global_period> <global_quota>-1</global_quota> + <iothread_period>1000000</iothread_period> + <iothread_quota>-1</iothread_quota> <vcpupin vcpu='0' cpuset='0'/> <vcpupin vcpu='1' cpuset='1'/> <emulatorpin cpuset='1'/> diff --git a/tools/virsh.pod b/tools/virsh.pod index d7cd10e..f0793b3 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -1912,7 +1912,7 @@ available for each hypervisor are: LXC (posix scheduler) : cpu_shares, vcpu_period, vcpu_quota QEMU/KVM (posix scheduler): cpu_shares, vcpu_period, vcpu_quota, -emulator_period, emulator_quota +emulator_period, emulator_quota, iothread_quota, iothread_period Xen (credit scheduler): weight, cap @@ -1930,9 +1930,10 @@ values 0 and 1 are automatically converted to a minimal value of 2. B<Note>: The weight and cap parameters are defined only for the XEN_CREDIT scheduler and are now I<DEPRECATED>. -B<Note>: The vcpu_period/emulator_period parameters have a valid value range -of 1000-1000000 or 0, and the vcpu_quota/emulator_quota parameters have a -valid value range of 1000-18446744073709551 or less than 0. The value 0 for +B<Note>: The vcpu_period, emulator_period, and iothread_period parameters +have a valid value range of 1000-1000000 or 0, and the vcpu_quota, +emulator_quota, and iothread_quota parameters have a valid value range of +1000-18446744073709551 or less than 0. The value 0 for either parameter is the same as not specifying that parameter. =item B<screenshot> I<domain> [I<imagefilepath>] [I<--screen> B<screenID>] -- 2.5.5
https://bugzilla.redhat.com/show_bug.cgi?id=1356937 Add support for IOThread quota/bandwidth and period parameters for non session mode. If in session mode, then error out. Uses all the same places where {vcpu|emulator|global}_{period|quota} are adjusted and adds the iothread values. Signed-off-by: John Ferlan <jferlan@redhat.com> --- src/qemu/qemu_command.c | 3 +- src/qemu/qemu_driver.c | 127 +++++++++++++++++++++++++++++++++++++++++++++++- src/qemu/qemu_process.c | 4 +- 3 files changed, 129 insertions(+), 5 deletions(-) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 4558b9f..52b23bb 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -9068,7 +9068,8 @@ qemuBuildCommandLineValidate(virQEMUDriverPtr driver, if (def->cputune.sharesSpecified || def->cputune.period || def->cputune.quota || def->cputune.global_period || def->cputune.global_quota || def->cputune.emulator_period || - def->cputune.emulator_quota) { + def->cputune.emulator_quota || def->cputune.iothread_period || + def->cputune.iothread_quota) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("CPU tuning is not available in session mode")); return -1; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 0501003..7645051 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -8581,7 +8581,7 @@ static char *qemuDomainGetSchedulerType(virDomainPtr dom, /* Domain not running, thus no cgroups - return defaults */ if (!virDomainObjIsActive(vm)) { if (nparams) - *nparams = 7; + *nparams = 9; ignore_value(VIR_STRDUP(ret, "posix")); goto cleanup; } @@ -8594,7 +8594,7 @@ static char *qemuDomainGetSchedulerType(virDomainPtr dom, if (nparams) { if (virCgroupSupportsCpuBW(priv->cgroup)) - *nparams = 7; + *nparams = 9; else *nparams = 1; } @@ -9817,6 +9817,40 @@ qemuSetEmulatorBandwidthLive(virCgroupPtr cgroup, return -1; } + +static int +qemuSetIOThreadsBWLive(virDomainObjPtr vm, virCgroupPtr cgroup, + unsigned long long period, long long quota) +{ + size_t i; + virCgroupPtr cgroup_iothread = NULL; + + if (period == 0 && quota == 0) + return 0; + + if (!vm->def->niothreadids) + return 0; + + for (i = 0; i < vm->def->niothreadids; i++) { + if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_IOTHREAD, + vm->def->iothreadids[i]->iothread_id, + false, &cgroup_iothread) < 0) + goto cleanup; + + if (qemuSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0) + goto cleanup; + + virCgroupFree(&cgroup_iothread); + } + + return 0; + + cleanup: + virCgroupFree(&cgroup_iothread); + return -1; +} + + #define SCHED_RANGE_CHECK(VAR, NAME, MIN, MAX) \ if (((VAR) > 0 && (VAR) < (MIN)) || (VAR) > (MAX)) { \ virReportError(VIR_ERR_INVALID_ARG, \ @@ -9867,6 +9901,10 @@ qemuDomainSetSchedulerParametersFlags(virDomainPtr dom, VIR_TYPED_PARAM_ULLONG, VIR_DOMAIN_SCHEDULER_EMULATOR_QUOTA, VIR_TYPED_PARAM_LLONG, + VIR_DOMAIN_SCHEDULER_IOTHREAD_PERIOD, + VIR_TYPED_PARAM_ULLONG, + VIR_DOMAIN_SCHEDULER_IOTHREAD_QUOTA, + VIR_TYPED_PARAM_LLONG, NULL) < 0) return -1; @@ -10059,6 +10097,46 @@ qemuDomainSetSchedulerParametersFlags(virDomainPtr dom, if (persistentDef) persistentDefCopy->cputune.emulator_quota = value_l; + + } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_IOTHREAD_PERIOD)) { + SCHED_RANGE_CHECK(value_ul, VIR_DOMAIN_SCHEDULER_IOTHREAD_PERIOD, + QEMU_SCHED_MIN_PERIOD, QEMU_SCHED_MAX_PERIOD); + + if (def && value_ul) { + if ((rc = qemuSetIOThreadsBWLive(vm, priv->cgroup, value_ul, 0))) + goto endjob; + + def->cputune.iothread_period = value_ul; + + if (virTypedParamsAddULLong(&eventParams, &eventNparams, + &eventMaxNparams, + VIR_DOMAIN_TUNABLE_CPU_IOTHREAD_PERIOD, + value_ul) < 0) + goto endjob; + } + + if (persistentDef) + persistentDefCopy->cputune.iothread_period = params[i].value.ul; + + } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_IOTHREAD_QUOTA)) { + SCHED_RANGE_CHECK(value_l, VIR_DOMAIN_SCHEDULER_IOTHREAD_QUOTA, + QEMU_SCHED_MIN_QUOTA, QEMU_SCHED_MAX_QUOTA); + + if (def && value_l) { + if ((rc = qemuSetIOThreadsBWLive(vm, priv->cgroup, 0, value_l))) + goto endjob; + + def->cputune.iothread_quota = value_l; + + if (virTypedParamsAddLLong(&eventParams, &eventNparams, + &eventMaxNparams, + VIR_DOMAIN_TUNABLE_CPU_IOTHREAD_QUOTA, + value_l) < 0) + goto endjob; + } + + if (persistentDef) + persistentDefCopy->cputune.iothread_quota = value_l; } } @@ -10182,6 +10260,43 @@ qemuGetEmulatorBandwidthLive(virCgroupPtr cgroup, } static int +qemuGetIOThreadsBWLive(virDomainObjPtr vm, + unsigned long long *period, long long *quota) +{ + virCgroupPtr cgroup_iothread = NULL; + qemuDomainObjPrivatePtr priv = NULL; + int rc; + int ret = -1; + + priv = vm->privateData; + if (!vm->def->niothreadids) { + /* We do not create sub dir for each iothread */ + if ((rc = qemuGetVcpuBWLive(priv->cgroup, period, quota)) < 0) + goto cleanup; + + goto out; + } + + /* get period and quota for the "first" IOThread */ + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD, + vm->def->iothreadids[0]->iothread_id, + false, &cgroup_iothread) < 0) + goto cleanup; + + rc = qemuGetVcpuBWLive(cgroup_iothread, period, quota); + if (rc < 0) + goto cleanup; + + out: + ret = 0; + + cleanup: + virCgroupFree(&cgroup_iothread); + return ret; +} + + +static int qemuGetGlobalBWLive(virCgroupPtr cgroup, unsigned long long *period, long long *quota) { @@ -10256,6 +10371,11 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, qemuGetGlobalBWLive(priv->cgroup, &data.global_period, &data.global_quota) < 0) goto cleanup; + + if (maxparams > 7 && + qemuGetIOThreadsBWLive(vm, &data.iothread_period, + &data.iothread_quota) < 0) + goto cleanup; } else { cpu_bw_status = false; } @@ -10280,6 +10400,9 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom, QEMU_SCHED_ASSIGN(global_period, GLOBAL_PERIOD, ULLONG); QEMU_SCHED_ASSIGN(global_quota, GLOBAL_QUOTA, LLONG); + + QEMU_SCHED_ASSIGN(iothread_period, IOTHREAD_PERIOD, ULLONG); + QEMU_SCHED_ASSIGN(iothread_quota, IOTHREAD_QUOTA, LLONG); } #undef QEMU_SCHED_ASSIGN diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index c7639c1..74443db 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -4692,8 +4692,8 @@ qemuProcessSetupIOThread(virDomainObjPtr vm, VIR_CGROUP_THREAD_IOTHREAD, iothread->iothread_id, iothread->cpumask, - vm->def->cputune.period, - vm->def->cputune.quota, + vm->def->cputune.iothread_period, + vm->def->cputune.iothread_quota, &iothread->sched); } -- 2.5.5
ping? Thanks - John On 07/25/2016 05:42 PM, John Ferlan wrote:
https://bugzilla.redhat.com/show_bug.cgi?id=1356937
John Ferlan (2): conf: Add IOThread quota and period scheduler/cputune defs qemu: Add support to get/set IOThread period and quota cgroup values
docs/formatdomain.html.in | 25 ++++ docs/schemas/domaincommon.rng | 10 ++ include/libvirt/libvirt-domain.h | 32 ++++++ src/conf/domain_conf.c | 42 +++++++ src/conf/domain_conf.h | 2 + src/qemu/qemu_command.c | 3 +- src/qemu/qemu_driver.c | 127 ++++++++++++++++++++- src/qemu/qemu_process.c | 4 +- tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 + .../qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml | 2 + tools/virsh.pod | 9 +- 11 files changed, 249 insertions(+), 9 deletions(-)
On 25.07.2016 23:42, John Ferlan wrote:
https://bugzilla.redhat.com/show_bug.cgi?id=1356937
John Ferlan (2): conf: Add IOThread quota and period scheduler/cputune defs qemu: Add support to get/set IOThread period and quota cgroup values
docs/formatdomain.html.in | 25 ++++ docs/schemas/domaincommon.rng | 10 ++ include/libvirt/libvirt-domain.h | 32 ++++++ src/conf/domain_conf.c | 42 +++++++ src/conf/domain_conf.h | 2 + src/qemu/qemu_command.c | 3 +- src/qemu/qemu_driver.c | 127 ++++++++++++++++++++- src/qemu/qemu_process.c | 4 +- tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 + .../qemuxml2xmloutdata/qemuxml2xmlout-cputune.xml | 2 + tools/virsh.pod | 9 +- 11 files changed, 249 insertions(+), 9 deletions(-)
ACK to both. Michal
participants (2)
-
John Ferlan -
Michal Privoznik