[libvirt] [PATCH 0/5] Add support for setting the emulator scheduler parameters
I/O threads and vCPU threads already support setting schedulers, but until now it was impossible to do so for the main QEMU thread (emulator thread in the libvirt naming). This is, however, requested for some very specific scenarios, for example when vCPU threads are running at such priority that could starve the main thread. Martin Kletzander (5): conf: Parse common scheduler attributes in separate function conf: Format thread IDs optionally docs: Mention iothreadsched element in the docs and reword conf: Add support for emulatorsched qemu: Add support for emulatorsched docs/formatdomain.html.in | 25 +++-- docs/news.xml | 12 +++ docs/schemas/domaincommon.rng | 5 + src/conf/domain_conf.c | 136 +++++++++++++++++------- src/conf/domain_conf.h | 1 + src/qemu/qemu_process.c | 2 +- tests/genericxml2xmlindata/cputune.xml | 37 +++++++ tests/genericxml2xmloutdata/cputune.xml | 40 +++++++ tests/genericxml2xmltest.c | 2 + 9 files changed, 212 insertions(+), 48 deletions(-) create mode 100644 tests/genericxml2xmlindata/cputune.xml create mode 100644 tests/genericxml2xmloutdata/cputune.xml -- 2.21.0
This will become useful later when parsing emulatorsched parameters which don't need the rest of the current function. Signed-off-by: Martin Kletzander <mkletzan@redhat.com> --- src/conf/domain_conf.c | 70 +++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index b969a9f6e566..15838c2a23f5 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -18441,45 +18441,24 @@ virDomainLoaderDefParseXML(xmlNodePtr node, } -static virBitmapPtr -virDomainSchedulerParse(xmlNodePtr node, - const char *name, - virProcessSchedPolicy *policy, - int *priority) +static int +virDomainSchedulerParseCommonAttrs(xmlNodePtr node, + virProcessSchedPolicy *policy, + int *priority) { - virBitmapPtr ret = NULL; int pol = 0; VIR_AUTOFREE(char *) tmp = NULL; - if (!(tmp = virXMLPropString(node, name))) { - virReportError(VIR_ERR_XML_ERROR, - _("Missing attribute '%s' in element '%sched'"), - name, name); - goto error; - } - - if (virBitmapParse(tmp, &ret, VIR_DOMAIN_CPUMASK_LEN) < 0) - goto error; - - if (virBitmapIsAllClear(ret)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("'%s' scheduler bitmap '%s' is empty"), - name, tmp); - goto error; - } - - VIR_FREE(tmp); - if (!(tmp = virXMLPropString(node, "scheduler"))) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Missing scheduler attribute")); - goto error; + return -1; } if ((pol = virProcessSchedPolicyTypeFromString(tmp)) <= 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Invalid scheduler attribute: '%s'"), tmp); - goto error; + return -1; } *policy = pol; @@ -18490,16 +18469,49 @@ virDomainSchedulerParse(xmlNodePtr node, if (!(tmp = virXMLPropString(node, "priority"))) { virReportError(VIR_ERR_XML_ERROR, "%s", _("Missing scheduler priority")); - goto error; + return -1; } if (virStrToLong_i(tmp, NULL, 10, priority) < 0) { virReportError(VIR_ERR_XML_ERROR, "%s", _("Invalid value for element priority")); - goto error; + return -1; } } + return 0; +} + + +static virBitmapPtr +virDomainSchedulerParse(xmlNodePtr node, + const char *name, + virProcessSchedPolicy *policy, + int *priority) +{ + virBitmapPtr ret = NULL; + VIR_AUTOFREE(char *) tmp = NULL; + + if (!(tmp = virXMLPropString(node, name))) { + virReportError(VIR_ERR_XML_ERROR, + _("Missing attribute '%s' in element '%sched'"), + name, name); + goto error; + } + + if (virBitmapParse(tmp, &ret, VIR_DOMAIN_CPUMASK_LEN) < 0) + goto error; + + if (virBitmapIsAllClear(ret)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("'%s' scheduler bitmap '%s' is empty"), + name, tmp); + goto error; + } + + if (virDomainSchedulerParseCommonAttrs(node, policy, priority) < 0) + goto error; + return ret; error: -- 2.21.0
This will be used later when we want to format emulator scheduler parameters which don't apply for multiple threads. Signed-off-by: Martin Kletzander <mkletzan@redhat.com> squash! conf: Do not format negative thread IDs Signed-off-by: Martin Kletzander <mkletzan@redhat.com> --- src/conf/domain_conf.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 15838c2a23f5..e7b8b51aad0e 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -27205,22 +27205,25 @@ static void virDomainSchedulerFormat(virBufferPtr buf, const char *name, virDomainThreadSchedParamPtr sched, - size_t id) + size_t id, + bool multiple_threads) { switch (sched->policy) { case VIR_PROC_POLICY_BATCH: case VIR_PROC_POLICY_IDLE: - virBufferAsprintf(buf, "<%ssched " - "%ss='%zu' scheduler='%s'/>\n", - name, name, id, + virBufferAsprintf(buf, "<%ssched", name); + if (multiple_threads) + virBufferAsprintf(buf, " %ss='%zu'", name, id); + virBufferAsprintf(buf, " scheduler='%s'/>\n", virProcessSchedPolicyTypeToString(sched->policy)); break; case VIR_PROC_POLICY_RR: case VIR_PROC_POLICY_FIFO: - virBufferAsprintf(buf, "<%ssched " - "%ss='%zu' scheduler='%s' priority='%d'/>\n", - name, name, id, + virBufferAsprintf(buf, "<%ssched", name); + if (multiple_threads) + virBufferAsprintf(buf, " %ss='%zu'", name, id); + virBufferAsprintf(buf, " scheduler='%s' priority='%d'/>\n", virProcessSchedPolicyTypeToString(sched->policy), sched->priority); break; @@ -27489,13 +27492,14 @@ virDomainCputuneDefFormat(virBufferPtr buf, for (i = 0; i < def->maxvcpus; i++) { virDomainSchedulerFormat(&childrenBuf, "vcpu", - &def->vcpus[i]->sched, i); + &def->vcpus[i]->sched, i, true); } for (i = 0; i < def->niothreadids; i++) { virDomainSchedulerFormat(&childrenBuf, "iothread", &def->iothreadids[i]->sched, - def->iothreadids[i]->iothread_id); + def->iothreadids[i]->iothread_id, + true); } for (i = 0; i < def->nresctrls; i++) -- 2.21.0
Just one missing occurrence of iothreadsched fixed plus some rewording for this to make more sense for the readers. Signed-off-by: Martin Kletzander <mkletzan@redhat.com> --- docs/formatdomain.html.in | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index 0e3799061d93..af4b88c60984 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -937,14 +937,16 @@ <dt><code>vcpusched</code> and <code>iothreadsched</code></dt> <dd> - The optional <code>vcpusched</code> elements specifies the scheduler - type (values <code>batch</code>, <code>idle</code>, <code>fifo</code>, - <code>rr</code>) for particular vCPU/IOThread threads (based on - <code>vcpus</code> and <code>iothreads</code>, leaving out - <code>vcpus</code>/<code>iothreads</code> sets the default). Valid - <code>vcpus</code> values start at 0 through one less than the - number of vCPU's defined for the domain. Valid <code>iothreads</code> - values are described in the <code>iothreadids</code> + The optional <code>vcpusched</code> and <code>iothreadsched</code> + elements specify the scheduler type + (values <code>batch</code>, <code>idle</code>, <code>fifo</code>, + <code>rr</code>) for particular vCPU and IOThread threads respecively. + The attributes <code>vcpus</code> and <code>iothreads</code> select + which vCPUs/IOThreads this setting applies to, leaving them out sets the + default. Valid <code>vcpus</code> values start at 0 through one less + than the number of vCPU's defined for the + domain. Valid <code>iothreads</code> values are described in + the <code>iothreadids</code> <a href="#elementsIOThreadsAllocation"><code>description</code></a>. If no <code>iothreadids</code> are defined, then libvirt numbers IOThreads from 1 to the number of <code>iothreads</code> available -- 2.21.0
Signed-off-by: Martin Kletzander <mkletzan@redhat.com> --- docs/formatdomain.html.in | 17 ++++++---- docs/schemas/domaincommon.rng | 5 +++ src/conf/domain_conf.c | 44 +++++++++++++++++++++++++ src/conf/domain_conf.h | 1 + tests/genericxml2xmlindata/cputune.xml | 37 +++++++++++++++++++++ tests/genericxml2xmloutdata/cputune.xml | 40 ++++++++++++++++++++++ tests/genericxml2xmltest.c | 2 ++ 7 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 tests/genericxml2xmlindata/cputune.xml create mode 100644 tests/genericxml2xmloutdata/cputune.xml diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index af4b88c60984..74d32e2a9b8c 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -935,15 +935,19 @@ <span class="since">Only QEMU driver support since 2.1.0</span> </dd> - <dt><code>vcpusched</code> and <code>iothreadsched</code></dt> + <dt><code>vcpusched</code>, <code>iothreadsched</code> + and <code>emulatorsched</code></dt> <dd> - The optional <code>vcpusched</code> and <code>iothreadsched</code> - elements specify the scheduler type + The optional + <code>vcpusched</code>, <code>iothreadsched</code> + and <code>emulatorsched</code> elements specify the scheduler type (values <code>batch</code>, <code>idle</code>, <code>fifo</code>, - <code>rr</code>) for particular vCPU and IOThread threads respecively. - The attributes <code>vcpus</code> and <code>iothreads</code> select + <code>rr</code>) for particular vCPU, IOThread and emulator threads + respecively. For <code>vcpusched</code> and <code>iothreadsched</code> + the attributes <code>vcpus</code> and <code>iothreads</code> select which vCPUs/IOThreads this setting applies to, leaving them out sets the - default. Valid <code>vcpus</code> values start at 0 through one less + default. The element <code>emulatorsched</code> does not have that + attribute. Valid <code>vcpus</code> values start at 0 through one less than the number of vCPU's defined for the domain. Valid <code>iothreads</code> values are described in the <code>iothreadids</code> @@ -955,6 +959,7 @@ well (and is ignored for non-real-time ones). The value range for the priority depends on the host kernel (usually 1-99). <span class="since">Since 1.2.13</span> + <code>emulatorsched</code> <span class="since">since 5.3.0</span> </dd> <dt><code>cachetune</code><span class="since">Since 4.1.0</span></dt> diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index 623ef28719df..111b85c36fd3 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -967,6 +967,11 @@ <ref name="schedparam"/> </element> </zeroOrMore> + <optional> + <element name="emulatorsched"> + <ref name="schedparam"/> + </element> + </optional> <zeroOrMore> <element name="cachetune"> <attribute name="vcpus"> diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index e7b8b51aad0e..17e8975be399 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -3396,6 +3396,7 @@ void virDomainDefFree(virDomainDefPtr def) virDomainIOThreadIDDefArrayFree(def->iothreadids, def->niothreadids); virBitmapFree(def->cputune.emulatorpin); + VIR_FREE(def->cputune.emulatorsched); virDomainNumaFree(def->numa); @@ -18483,6 +18484,25 @@ virDomainSchedulerParseCommonAttrs(xmlNodePtr node, } +static int +virDomainEmulatorSchedParse(xmlNodePtr node, + virDomainDefPtr def) +{ + VIR_AUTOFREE(virDomainThreadSchedParamPtr) sched = NULL; + + if (VIR_ALLOC(sched) < 0) + return -1; + + if (virDomainSchedulerParseCommonAttrs(node, + &sched->policy, + &sched->priority) < 0) + return -1; + + VIR_STEAL_PTR(def->cputune.emulatorsched, sched); + return 0; +} + + static virBitmapPtr virDomainSchedulerParse(xmlNodePtr node, const char *name, @@ -19913,6 +19933,25 @@ virDomainDefParseXML(xmlDocPtr xml, } VIR_FREE(nodes); + if ((n = virXPathNodeSet("./cputune/emulatorsched", ctxt, &nodes)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("cannot extract emulatorsched nodes")); + goto error; + } + + if (n) { + if (n > 1) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("only one emulatorsched is supported")); + VIR_FREE(nodes); + goto error; + } + + if (virDomainEmulatorSchedParse(nodes[0], def) < 0) + goto error; + } + VIR_FREE(nodes); + if ((n = virXPathNodeSet("./cputune/cachetune", ctxt, &nodes)) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("cannot extract cachetune nodes")); @@ -27490,6 +27529,11 @@ virDomainCputuneDefFormat(virBufferPtr buf, VIR_FREE(cpumask); } + if (def->cputune.emulatorsched) { + virDomainSchedulerFormat(&childrenBuf, "emulator", + def->cputune.emulatorsched, 0, false); + } + for (i = 0; i < def->maxvcpus; i++) { virDomainSchedulerFormat(&childrenBuf, "vcpu", &def->vcpus[i]->sched, i, true); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 12eb71c1977e..988ef90f1189 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2159,6 +2159,7 @@ struct _virDomainCputune { unsigned long long iothread_period; long long iothread_quota; virBitmapPtr emulatorpin; + virDomainThreadSchedParamPtr emulatorsched; }; diff --git a/tests/genericxml2xmlindata/cputune.xml b/tests/genericxml2xmlindata/cputune.xml new file mode 100644 index 000000000000..999c8cef73cd --- /dev/null +++ b/tests/genericxml2xmlindata/cputune.xml @@ -0,0 +1,37 @@ +<domain type='qemu'> + <name>QEMUGuest1</name> + <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid> + <memory unit='KiB'>219100</memory> + <currentMemory unit='KiB'>219100</currentMemory> + <vcpu placement='static'>4</vcpu> + <iothreads>4</iothreads> + <cputune> + <shares>2048</shares> + <period>1000000</period> + <quota>-1</quota> + <vcpupin vcpu='0' cpuset='0'/> + <vcpupin vcpu='1' cpuset='1'/> + <emulatorpin cpuset='1'/> + <emulatorsched scheduler='fifo' priority='2'/> + <vcpusched vcpus='0-1' scheduler='fifo' priority='1'/> + <vcpusched vcpus='1-3,^1' scheduler='idle'/> + <iothreadsched iothreads='1,3' scheduler='batch'/> + <iothreadsched iothreads='2' scheduler='rr' priority='99'/> + </cputune> + <os> + <type arch='x86_64' machine='q35'>hvm</type> + <boot dev='hd'/> + </os> + <features> + <smm state='on'> + <tseg unit='MiB'>48</tseg> + </smm> + </features> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-x86_64</emulator> + </devices> +</domain> diff --git a/tests/genericxml2xmloutdata/cputune.xml b/tests/genericxml2xmloutdata/cputune.xml new file mode 100644 index 000000000000..f9f99a4846e6 --- /dev/null +++ b/tests/genericxml2xmloutdata/cputune.xml @@ -0,0 +1,40 @@ +<domain type='qemu'> + <name>QEMUGuest1</name> + <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid> + <memory unit='KiB'>219100</memory> + <currentMemory unit='KiB'>219100</currentMemory> + <vcpu placement='static'>4</vcpu> + <iothreads>4</iothreads> + <cputune> + <shares>2048</shares> + <period>1000000</period> + <quota>-1</quota> + <vcpupin vcpu='0' cpuset='0'/> + <vcpupin vcpu='1' cpuset='1'/> + <emulatorpin cpuset='1'/> + <emulatorsched scheduler='fifo' priority='2'/> + <vcpusched vcpus='0' scheduler='fifo' priority='1'/> + <vcpusched vcpus='1' scheduler='fifo' priority='1'/> + <vcpusched vcpus='2' scheduler='idle'/> + <vcpusched vcpus='3' scheduler='idle'/> + <iothreadsched iothreads='1' scheduler='batch'/> + <iothreadsched iothreads='2' scheduler='rr' priority='99'/> + <iothreadsched iothreads='3' scheduler='batch'/> + </cputune> + <os> + <type arch='x86_64' machine='q35'>hvm</type> + <boot dev='hd'/> + </os> + <features> + <smm state='on'> + <tseg unit='MiB'>48</tseg> + </smm> + </features> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-x86_64</emulator> + </devices> +</domain> diff --git a/tests/genericxml2xmltest.c b/tests/genericxml2xmltest.c index 4393d444641d..1840d6e6a8f9 100644 --- a/tests/genericxml2xmltest.c +++ b/tests/genericxml2xmltest.c @@ -149,6 +149,8 @@ mymain(void) DO_TEST("launch-security-sev"); + DO_TEST_DIFFERENT("cputune"); + virObjectUnref(caps); virObjectUnref(xmlopt); -- 2.21.0
This helps in a scenarios where vCPUs run with a priority that is so high they might starve the emulator thread. And it also fits with the rest of the settings. Signed-off-by: Martin Kletzander <mkletzan@redhat.com> --- docs/news.xml | 12 ++++++++++++ src/qemu/qemu_process.c | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/news.xml b/docs/news.xml index 86c77346940a..84de507b0e9a 100644 --- a/docs/news.xml +++ b/docs/news.xml @@ -35,6 +35,18 @@ <libvirt> <release version="v5.3.0" date="unreleased"> <section title="New features"> + <change> + <summary> + qemu: Add support for setting the emulator scheduler parameters + </summary> + <description> + I/O threads and vCPU threads already support setting schedulers, but + until now it was impossible to do so for the main QEMU thread + (emulator thread in the libvirt naming). This is, however, requested + for some very specific scenarios, for example when vCPU threads are + running at such priority that could starve the main thread. + </description> + </change> </section> <section title="Improvements"> <change> diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index f773aa89b78d..55f4074ea146 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -2683,7 +2683,7 @@ qemuProcessSetupEmulator(virDomainObjPtr vm) 0, vm->def->cputune.emulatorpin, vm->def->cputune.emulator_period, vm->def->cputune.emulator_quota, - NULL); + vm->def->cputune.emulatorsched); } -- 2.21.0
On Mon, Apr 15, 2019 at 01:49:46PM +0200, Martin Kletzander wrote:
This helps in a scenarios where vCPUs run with a priority that is so high they might starve the emulator thread. And it also fits with the rest of the settings.
Signed-off-by: Martin Kletzander <mkletzan@redhat.com> --- docs/news.xml | 12 ++++++++++++ src/qemu/qemu_process.c | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-)
v2 with the news.xml split on the way...
participants (1)
-
Martin Kletzander