On Tue, Aug 02, 2022 at 10:13:40PM +0800, ~hyman wrote: We've generally tried to avoid adding single purpose APIs for tunable parameters, instead using APIs with virTypedParameter arrays to allow bulk updates. I note that we don't appear to have any mechanism currently to set the VCPU scheduler tunables either Perhaps we should have a more general virDomainSetVCPUTuneParameters(virDomainPtr domain, int vcpu, virTypedParameterPtr params, unsigned int params, unsigned int flags); With regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

From: Hyman Huang(黄勇) <yong.huang(a)smartx.com&gt; The upper limit (megabyte/s) of the dirty page rate configured by the user can be tracked by the XML. To allow this, add the following XML: <domain> ... <vcpu current='2'>3</vcpu> <vcpus> <vcpu id='0' hotpluggable='no' dirty_limit='10' order='1'.../> <vcpu id='1' hotpluggable='yes' dirty_limit='10' order='2'.../> </vcpus> ... The "dirty_limit" attribute in "vcpu" sub-element within "vcpus" element allows to set an upper limit for the individual vCPU. The value can be set dynamically by limit-dirty-page-rate API. Note that the dirty limit feature is based on the dirty-ring feature, so it requires dirty-ring size configuration in XML. Signed-off-by: Hyman Huang(黄勇) <yong.huang(a)smartx.com&gt; --- docs/formatdomain.rst | 7 ++++++- src/conf/domain_conf.c | 26 ++++++++++++++++++++++++ src/conf/domain_conf.h | 8 ++++++++ src/conf/domain_validate.c | 33 +++++++++++++++++++++++++++++++ src/conf/schemas/domaincommon.rng | 5 +++++ 5 files changed, 78 insertions(+), 1 deletion(-) diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index bc469e5f9f..337b7ec9cc 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -649,7 +649,7 @@ CPU Allocation ... <vcpu placement='static' cpuset="1-4,^3,6" current="1">2</vcpu> <vcpus> - <vcpu id='0' enabled='yes' hotpluggable='no' order='1'/> + <vcpu id='0' enabled='yes' hotpluggable='no' order='1' dirty_limit='64'/> <vcpu id='1' enabled='no' hotpluggable='yes'/> </vcpus> ... @@ -715,6 +715,11 @@ CPU Allocation be enabled and non-hotpluggable. On PPC64 along with it vCPUs that are in the same core need to be enabled as well. All non-hotpluggable CPUs present at boot need to be grouped after vCPU 0. :since:`Since 2.2.0 (QEMU only)` + ``dirty_limit`` :since:`Since 9.7.0 (QEMU and KVM only)` + The optional attribute ``dirty_limit`` allows to set an upper limit (MB/s) + of the dirty page rate for the vCPU. User can change the upper limit value + dynamically by using ``limit-dirty-page-rate`` API. Require ``dirty-ring`` + size configured. IOThreads Allocation diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 2c8727de54..aff0dba84a 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -17088,6 +17088,7 @@ virDomainVcpuParse(virDomainDef *def, virDomainXMLOption *xmlopt) { int n; + int rv; xmlNodePtr vcpuNode; size_t i; unsigned int maxvcpus; @@ -17175,6 +17176,13 @@ virDomainVcpuParse(virDomainDef *def, if (virXMLPropUInt(nodes[i], "order", 10, VIR_XML_PROP_NONE, &vcpu->order) < 0) return -1; + + if ((rv = virXMLPropULongLong(nodes[i], "dirty_limit", 10, VIR_XML_PROP_NONNEGATIVE, + &vcpu->dirty_limit)) < 0) { + return -1; + } else if (rv > 0) { + vcpu->dirtyLimitSet = true; + } } } else { if (virDomainDefSetVcpus(def, vcpus) < 0) @@ -21219,6 +21227,20 @@ virDomainDefVcpuCheckAbiStability(virDomainDef *src, i); return false; } + + if (svcpu->dirtyLimitSet != dvcpu->dirtyLimitSet) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Dirty limit state of vCPU '%1$zu' differs between source and destination definitions"), + i); + return false; + } + + if (svcpu->dirty_limit != dvcpu->dirty_limit) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Dirty limit of vCPU '%1$zu' differs between source and destination definitions"), + i); + return false; + } } return true; @@ -26822,6 +26844,10 @@ virDomainCpuDefFormat(virBuffer *buf, if (vcpu->order != 0) virBufferAsprintf(buf, " order='%d'", vcpu->order); + if (vcpu->dirtyLimitSet) { + virBufferAsprintf(buf, " dirty_limit='%llu'", vcpu->dirty_limit); + } + virBufferAddLit(buf, "/>\n"); } virBufferAdjustIndent(buf, -2); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index ca195a52d2..0bca85bec6 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2812,6 +2812,14 @@ struct _virDomainVcpuDef { virDomainThreadSchedParam sched; virObject *privateData; + + /* set to true if the dirty page rate upper limit for + * the virtual CPU is configured + * */ + bool dirtyLimitSet; + + /* dirty page rate upper limit */ + unsigned long long dirty_limit; }; struct _virDomainBlkiotune { diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index b050f21edb..2df9db3467 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -1785,6 +1785,36 @@ virDomainDefValidateIOThreads(const virDomainDef *def) return 0; } +static int +virDomainDefHasDirtyLimitStartupVcpus(const virDomainDef *def) +{ + size_t maxvcpus = virDomainDefGetVcpusMax(def); + virDomainVcpuDef *vcpu; + size_t i; + + for (i = 0; i < maxvcpus; i++) { + vcpu = def->vcpus[i]; + + if (vcpu->dirtyLimitSet && (vcpu->dirty_limit != 0)) + return true; + } + + return false; +} + +static int +virDomainDefDirtyLimitValidate(const virDomainDef *def) +{ + if (virDomainDefHasDirtyLimitStartupVcpus(def)) { + if (def->kvm_features->dirty_ring_size == 0) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("Dirty limit requires dirty-ring size configuration")); + return -1; + } + } + + return 0; +} static int virDomainDefValidateInternal(const virDomainDef *def, @@ -1841,6 +1871,9 @@ virDomainDefValidateInternal(const virDomainDef *def, if (virDomainDefValidateIOThreads(def) < 0) return -1; + if (virDomainDefDirtyLimitValidate(def) < 0) + return -1; + return 0; } diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index 2f9ba31c0a..2e6a744eb4 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -844,6 +844,11 @@ <ref name="unsignedInt"/> </attribute> </optional> + <optional> + <attribute name="dirty_limit"> + <ref name="unsignedLong"/> + </attribute> + </optional> </element> </zeroOrMore> </element> -- 2.38.5

Thanks Daniel for the comments ! On Wed, Sep 6, 2023 at 4:48 PM Daniel P. Berrangé <berrange(a)redhat.com&gt; wrote: To be completely honest, I haven't given the scenarios anythought. This method has been utilized up till now by migration to throttle the guest within QEMU. The most likely situation, in my opinion, is that some virtual machines on a host are using up all of the memory bandwidth, either intentionally or accidentally, while other virtual machines perform worse when writing to memory. Currently, the "calc-dirty-rate" API can be used by upper-level apps to peek at the dirty page rate of the vCPU. If more details regarding setting values were required, we would add them to the comments. Guest memory writes would take longer than usual to complete. Single vCPU. Since this issue needs to be discussed, I'm inclined to offer a per-vCPU API and let the top APP make the final decision. Yong -- Best regards

On Wed, Sep 6, 2023 at 7:24 PM Daniel P. Berrangé <berrange(a)redhat.com&gt; wrote: Ok, that works for me. Thanks to the comment, this api is more graceful and I'll try it in the next version. Thanks, Yong -- Best regards

From: Hyman Huang(黄勇) <yong.huang(a)smartx.com&gt; Implement qemuDomainSetVcpuDirtyLimit, which can be used to set or cancel the upper limit of the dirty page rate for virtual CPUs. Signed-off-by: Hyman Huang(黄勇) <yong.huang(a)smartx.com&gt; --- src/qemu/qemu_driver.c | 131 +++++++++++++++++++++++++++++++++++ src/qemu/qemu_monitor.c | 13 ++++ src/qemu/qemu_monitor.h | 5 ++ src/qemu/qemu_monitor_json.c | 43 ++++++++++++ src/qemu/qemu_monitor_json.h | 5 ++ 5 files changed, 197 insertions(+) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 0d4da937b0..64d97c0fba 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -19876,6 +19876,136 @@ qemuDomainFDAssociate(virDomainPtr domain, return ret; } +static void +qemuDomainSetDirtyLimit(virDomainVcpuDef *vcpu, + unsigned long long rate) +{ + if (rate > 0) { + vcpu->dirtyLimitSet = true; + vcpu->dirty_limit = rate; + } else { + vcpu->dirtyLimitSet = false; + vcpu->dirty_limit = 0; + } +} + +static void +qemuDomainSetVcpuDirtyLimitConfig(virDomainDef *def, + int vcpu, + unsigned long long rate) +{ + def->individualvcpus = true; + + if (vcpu == -1) { + size_t maxvcpus = virDomainDefGetVcpusMax(def); + size_t i; + for (i = 0; i < maxvcpus; i++) { + qemuDomainSetDirtyLimit(virDomainDefGetVcpu(def, i), rate); + } + } else { + qemuDomainSetDirtyLimit(virDomainDefGetVcpu(def, vcpu), rate); + } +} + +static int +qemuDomainSetVcpuDirtyLimitInternal(virQEMUDriver *driver, + virDomainObj *vm, + virDomainDef *def, + virDomainDef *persistentDef, + int vcpu, + unsigned long long rate) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + qemuDomainObjPrivate *priv = vm->privateData; + + VIR_DEBUG("vcpu %d, rate %llu", vcpu, rate); + if (def) { + qemuDomainObjEnterMonitor(vm); + if (qemuMonitorSetVcpuDirtyLimit(priv->mon, vcpu, rate) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to set dirty page rate limit")); + qemuDomainObjExitMonitor(vm); + return -1; + } + qemuDomainObjExitMonitor(vm); + qemuDomainSetVcpuDirtyLimitConfig(def, vcpu, rate); + } + + if (persistentDef) { + qemuDomainSetVcpuDirtyLimitConfig(persistentDef, vcpu, rate); + if (virDomainDefSave(persistentDef, driver->xmlopt, cfg->configDir) < 0) + return -1; + } + + return 0; +} + +static int +qemuDomainSetVcpuDirtyLimit(virDomainPtr domain, + int vcpu, + unsigned long long rate, + unsigned int flags) +{ + virQEMUDriver *driver = domain->conn->privateData; + virDomainObj *vm = NULL; + qemuDomainObjPrivate *priv; + virDomainDef *def = NULL; + virDomainDef *persistentDef = NULL; + int ret = -1; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG, -1); + + if (!(vm = qemuDomainObjFromDomain(domain))) + return -1; + + if (virDomainSetVcpuDirtyLimitEnsureACL(domain->conn, vm->def, flags) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0) + goto cleanup; + + if (virDomainObjGetDefs(vm, flags, &def, &persistentDef) < 0) + goto endjob; + + if (persistentDef) { + if (vcpu >= 0 && vcpu >= (int)virDomainDefGetVcpusMax(persistentDef)) { + virReportError(VIR_ERR_INVALID_ARG, + _("vcpu %1$d is not present in persistent config"), + vcpu); + goto endjob; + } + } + + if (def) { + if (virDomainObjCheckActive(vm) < 0) + goto endjob; + + priv = vm->privateData; + if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_VCPU_DIRTY_LIMIT)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("QEMU does not support setting dirty page rate limit")); + goto endjob; + } + + if (vcpu >= 0 && vcpu >= (int)virDomainDefGetVcpusMax(def)) { + virReportError(VIR_ERR_INVALID_ARG, + _("vcpu %1$d is not present in live config"), + vcpu); + goto endjob; + } + } + + ret = qemuDomainSetVcpuDirtyLimitInternal(driver, vm, def, persistentDef, + vcpu, rate); + + endjob: + virDomainObjEndJob(vm); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -20126,6 +20256,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ .domainFDAssociate = qemuDomainFDAssociate, /* 9.0.0 */ + .domainSetVcpuDirtyLimit = qemuDomainSetVcpuDirtyLimit, /* 9.7.0 */ }; diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index 7053539c7d..90bc0e62c9 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -4500,3 +4500,16 @@ qemuMonitorGetStatsByQOMPath(virJSONValue *arr, return NULL; } + + +int +qemuMonitorSetVcpuDirtyLimit(qemuMonitor *mon, + int vcpu, + unsigned long long rate) +{ + VIR_DEBUG("set vcpu %d dirty page rate limit %llu", vcpu, rate); + + QEMU_CHECK_MONITOR(mon); + + return qemuMonitorJSONSetVcpuDirtyLimit(mon, vcpu, rate); +} diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 6c590933aa..07a05365cf 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -1579,3 +1579,8 @@ qemuMonitorExtractQueryStats(virJSONValue *info); virJSONValue * qemuMonitorGetStatsByQOMPath(virJSONValue *arr, char *qom_path); + +int +qemuMonitorSetVcpuDirtyLimit(qemuMonitor *mon, + int vcpu, + unsigned long long rate); diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index 5b9edadcf7..c8f6069566 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -8852,3 +8852,46 @@ qemuMonitorJSONQueryStats(qemuMonitor *mon, return virJSONValueObjectStealArray(reply, "return"); } + +/** + * qemuMonitorJSONSetVcpuDirtyLimit: + * @mon: monitor object + * @vcpu: virtual cpu index to be set, -1 affects all virtual CPUs + * @rate: dirty page rate upper limit to be set, use 0 to disable + * and a positive value to enable + * + * Returns -1 on failure. + */ +int +qemuMonitorJSONSetVcpuDirtyLimit(qemuMonitor *mon, + int vcpu, + unsigned long long rate) +{ + g_autoptr(virJSONValue) cmd = NULL; + g_autoptr(virJSONValue) reply = NULL; + + if (rate != 0) { + /* set the vcpu dirty page rate limit */ + if (!(cmd = qemuMonitorJSONMakeCommand("set-vcpu-dirty-limit", + "k:cpu-index", vcpu, + "U:dirty-rate", rate, + NULL))) { + return -1; + } + } else { + /* cancel the vcpu dirty page rate limit */ + if (!(cmd = qemuMonitorJSONMakeCommand("cancel-vcpu-dirty-limit", + "k:cpu-index", vcpu, + NULL))) { + return -1; + } + } + + if (qemuMonitorJSONCommand(mon, cmd, &reply) < 0) + return -1; + + if (qemuMonitorJSONCheckError(cmd, reply) < 0) + return -1; + + return 0; +} diff --git a/src/qemu/qemu_monitor_json.h b/src/qemu/qemu_monitor_json.h index 06023b98ea..89f61b3052 100644 --- a/src/qemu/qemu_monitor_json.h +++ b/src/qemu/qemu_monitor_json.h @@ -825,3 +825,8 @@ qemuMonitorJSONQueryStats(qemuMonitor *mon, qemuMonitorQueryStatsTargetType target, char **vcpus, GPtrArray *providers); + +int +qemuMonitorJSONSetVcpuDirtyLimit(qemuMonitor *mon, + int vcpu, + unsigned long long rate); -- 2.38.5

From: Hyman Huang(黄勇) <yong.huang(a)smartx.com&gt; Export virDomainDefHasDirtyLimitStartupVcpus as a util function, which could be used in qemu_process.c file for the next commit. Signed-off-by: Hyman Huang(黄勇) <yong.huang(a)smartx.com&gt; --- src/conf/domain_validate.c | 2 +- src/conf/domain_validate.h | 2 ++ src/libvirt_private.syms | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index 2df9db3467..04c18462da 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -1785,7 +1785,7 @@ virDomainDefValidateIOThreads(const virDomainDef *def) return 0; } -static int +int virDomainDefHasDirtyLimitStartupVcpus(const virDomainDef *def) { size_t maxvcpus = virDomainDefGetVcpusMax(def); diff --git a/src/conf/domain_validate.h b/src/conf/domain_validate.h index fc441cef5b..ccec3663cc 100644 --- a/src/conf/domain_validate.h +++ b/src/conf/domain_validate.h @@ -47,3 +47,5 @@ int virDomainDiskDefSourceLUNValidate(const virStorageSource *src); int virDomainDefOSValidate(const virDomainDef *def, virDomainXMLOption *xmlopt); + +int virDomainDefHasDirtyLimitStartupVcpus(const virDomainDef *def); diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index f8cda2f773..32656dea7a 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -793,6 +793,7 @@ virDomainDefPostParse; # conf/domain_validate.h virDomainActualNetDefValidate; +virDomainDefHasDirtyLimitStartupVcpus; virDomainDefOSValidate; virDomainDefValidate; virDomainDeviceValidateAliasForHotplug; -- 2.38.5

From: Hyman Huang(黄勇) <yong.huang(a)smartx.com&gt; Introduce command 'virsh domstats --dirtylimit' for reporting dirty page rate upper limit (megabyte/s) infomation. The info is listed as follows: Domain: 'vm' dirtylimit.vcpu.0.limit=10 dirtylimit.vcpu.0.current=16 dirtylimit.vcpu.1.limit=10 dirtylimit.vcpu.1.current=0 ... Signed-off-by: Hyman Huang(黄勇) <yong.huang(a)smartx.com&gt; --- docs/manpages/virsh.rst | 12 ++++++++++-- tools/virsh-domain-monitor.c | 7 +++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index 33a8f3b9fe..5bd06cb3a5 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -2317,7 +2317,7 @@ domstats domstats [--raw] [--enforce] [--backing] [--nowait] [--state] [--cpu-total] [--balloon] [--vcpu] [--interface] [--block] [--perf] [--iothread] [--memory] [--dirtyrate] [--vm] - [[--list-active] [--list-inactive] + [--dirtylimit] [[--list-active] [--list-inactive] [--list-persistent] [--list-transient] [--list-running]y [--list-paused] [--list-shutoff] [--list-other]] | [domain ...] @@ -2336,7 +2336,7 @@ The individual statistics groups are selectable via specific flags. By default all supported statistics groups are returned. Supported statistics groups flags are: *--state*, *--cpu-total*, *--balloon*, *--vcpu*, *--interface*, *--block*, *--perf*, *--iothread*, *--memory*, -*--dirtyrate*, *--vm*. +*--dirtyrate*, *--vm*, *--dirtylimit*. Note that - depending on the hypervisor type and version or the domain state - not all of the following statistics may be returned. @@ -2579,6 +2579,14 @@ not available for statistical purposes. The *--vm* option enables reporting of hypervisor-specific statistics. Naming and meaning of the fields is entirely hypervisor dependent. + +*--dirtylimit* returns: + +* ``dirtylimit.vcpu.<num>.limit`` - the upper limit of dirty page rate for a + virtual CPU in megabyte/s +* ``dirtylimit.vcpu.<num>.current`` - the dirty page rate for a virtual CPU + currently in megabyte/s + The statistics in this group have the following naming scheme: ``vm.$NAME.$TYPE`` diff --git a/tools/virsh-domain-monitor.c b/tools/virsh-domain-monitor.c index 89fdc7a050..efa2609719 100644 --- a/tools/virsh-domain-monitor.c +++ b/tools/virsh-domain-monitor.c @@ -2063,6 +2063,10 @@ static const vshCmdOptDef opts_domstats[] = { .type = VSH_OT_BOOL, .help = N_("report hypervisor-specific statistics"), }, + {.name = "dirtylimit", + .type = VSH_OT_BOOL, + .help = N_("report domain dirty page rate upper limit infomation"), + }, {.name = "list-active", .type = VSH_OT_BOOL, .help = N_("list only active domains"), @@ -2187,6 +2191,9 @@ cmdDomstats(vshControl *ctl, const vshCmd *cmd) if (vshCommandOptBool(cmd, "vm")) stats |= VIR_DOMAIN_STATS_VM; + if (vshCommandOptBool(cmd, "dirtylimit")) + stats |= VIR_DOMAIN_STATS_DIRTYLIMIT; + if (vshCommandOptBool(cmd, "list-active")) flags |= VIR_CONNECT_GET_ALL_DOMAINS_STATS_ACTIVE; -- 2.38.5