4:08 a.m.
We decide to make a global per domain bandwidth setting
as were discussed in mailing list earlier.
This patchset implements hierarchy top level cpu.cfs_period_us
and cpu.cfs_quota_us control knob. I've named this parameters
as global_period and global_quota.
Changes in v2: add XML validation test
Changes in v3: remove unneccessary cgroup copying
Changes in v4: fix little rebase error
Changes in v5: rebase to version 1.3.1
Alexander Burluka (7):
Add global period definitions
Add global quota parameter necessary definitions
Add error checking on global quota and period
Add global_period and global_quota XML validation test
Rename qemuSetupCgroupVcpuBW to qemuSetupBandwidthCgroup
Implement qemuSetupGlobalCpuCgroup
Implement handling of per-domain bandwidth settings
docs/schemas/domaincommon.rng | 10 +++
include/libvirt/libvirt-domain.h | 32 ++++++++
src/conf/domain_conf.c | 37 +++++++++
src/conf/domain_conf.h | 2 +
src/qemu/qemu_cgroup.c | 68 ++++++++++++++--
src/qemu/qemu_cgroup.h | 7 +-
src/qemu/qemu_command.c | 3 +-
src/qemu/qemu_driver.c | 102 ++++++++++++++++++++++--
src/qemu/qemu_process.c | 4 +
tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 +
10 files changed, 251 insertions(+), 16 deletions(-)
--
1.8.3.1
4:08 a.m.
New subject: [libvirt] [PATCH v5 1/7] Add global period definitions
This parameter represents top level period cgroup
that limits whole domain enforcement period for a quota
Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com>
---
docs/schemas/domaincommon.rng | 5 +++++
include/libvirt/libvirt-domain.h | 16 ++++++++++++++++
src/conf/domain_conf.c | 18 ++++++++++++++++++
src/conf/domain_conf.h | 1 +
4 files changed, 40 insertions(+)
diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
index 5deb17b..aa7eae9 100644
--- a/docs/schemas/domaincommon.rng
+++ b/docs/schemas/domaincommon.rng
@@ -670,6 +670,11 @@
</element>
</optional>
<optional>
+ <element name="global_period">
+ <ref name="cpuperiod"/>
+ </element>
+ </optional>
+ <optional>
<element name="period">
<ref name="cpuperiod"/>
</element>
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h
index d26faa5..cb30313 100644
--- a/include/libvirt/libvirt-domain.h
+++ b/include/libvirt/libvirt-domain.h
@@ -312,6 +312,14 @@ typedef enum {
# define VIR_DOMAIN_SCHEDULER_CPU_SHARES "cpu_shares"
/**
+ * VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD:
+ *
+ * Macro represents the enforcement period for a quota, in microseconds,
+ * for whole domain, when using the posix scheduler, as a ullong.
+ */
+# define VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD "global_period"
+
+/**
* VIR_DOMAIN_SCHEDULER_VCPU_PERIOD:
*
* Macro represents the enforcement period for a quota, in microseconds,
@@ -3318,6 +3326,14 @@ typedef void
(*virConnectDomainEventDeviceAddedCallback)(virConnectPtr conn,
# define VIR_DOMAIN_TUNABLE_CPU_CPU_SHARES "cputune.cpu_shares"
/**
+ * VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD:
+ *
+ * Macro represents the enforcement period for a quota, in microseconds,
+ * for whole domain, when using the posix scheduler, as VIR_TYPED_PARAM_ULLONG.
+ */
+# define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD "cputune.global_period"
+
+/**
* VIR_DOMAIN_TUNABLE_CPU_VCPU_PERIOD:
*
* Macro represents the enforcement period for a quota, in microseconds,
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index a9706b0..5ee53c2 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -15059,6 +15059,21 @@ virDomainDefParseXML(xmlDocPtr xml,
goto error;
}
+ if (virXPathULongLong("string(./cputune/global_period[1])", ctxt,
+ &def->cputune.global_period) < -1) {
+ virReportError(VIR_ERR_XML_ERROR, "%s",
+ _("can't parse cputune global period value"));
+ goto error;
+ }
+
+ if (def->cputune.global_period > 0 &&
+ (def->cputune.global_period < 1000 || def->cputune.global_period >
1000000)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("Value of cputune global period must be in range "
+ "[1000, 1000000]"));
+ goto error;
+ }
+
if (virXPathULongLong("string(./cputune/emulator_period[1])", ctxt,
&def->cputune.emulator_period) < -1) {
virReportError(VIR_ERR_XML_ERROR, "%s",
@@ -21818,6 +21833,9 @@ virDomainDefFormatInternal(virDomainDefPtr def,
if (def->cputune.quota)
virBufferAsprintf(&childrenBuf,
"<quota>%lld</quota>\n",
def->cputune.quota);
+ if (def->cputune.global_period)
+ virBufferAsprintf(&childrenBuf,
"<global_period>%llu</global_period>\n",
+ def->cputune.global_period);
if (def->cputune.emulator_period)
virBufferAsprintf(&childrenBuf, "<emulator_period>%llu"
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 0141009..7c43e7d 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -2135,6 +2135,7 @@ struct _virDomainCputune {
bool sharesSpecified;
unsigned long long period;
long long quota;
+ unsigned long long global_period;
unsigned long long emulator_period;
long long emulator_quota;
size_t nvcpupin;
--
1.8.3.1
4:08 a.m.
New subject: [libvirt] [PATCH v5 2/7] Add global quota parameter necessary definitions
This parameter controls the maximum bandwidth to be used
within a period for whole domain.
Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com>
---
docs/schemas/domaincommon.rng | 5 +++++
include/libvirt/libvirt-domain.h | 16 ++++++++++++++++
src/conf/domain_conf.c | 19 +++++++++++++++++++
src/conf/domain_conf.h | 1 +
4 files changed, 41 insertions(+)
diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
index aa7eae9..17653e1 100644
--- a/docs/schemas/domaincommon.rng
+++ b/docs/schemas/domaincommon.rng
@@ -675,6 +675,11 @@
</element>
</optional>
<optional>
+ <element name="global_quota">
+ <ref name="cpuquota"/>
+ </element>
+ </optional>
+ <optional>
<element name="period">
<ref name="cpuperiod"/>
</element>
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h
index cb30313..7d3d8e8 100644
--- a/include/libvirt/libvirt-domain.h
+++ b/include/libvirt/libvirt-domain.h
@@ -320,6 +320,14 @@ typedef enum {
# define VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD "global_period"
/**
+ * VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA:
+ *
+ * Macro represents the maximum bandwidth to be used within a period for
+ * whole domain, when using the posix scheduler, as an llong.
+ */
+# define VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA "global_quota"
+
+/**
* VIR_DOMAIN_SCHEDULER_VCPU_PERIOD:
*
* Macro represents the enforcement period for a quota, in microseconds,
@@ -3334,6 +3342,14 @@ typedef void
(*virConnectDomainEventDeviceAddedCallback)(virConnectPtr conn,
# define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD "cputune.global_period"
/**
+ * VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA:
+ *
+ * Macro represents the maximum bandwidth to be used within a period for
+ * whole domain, when using the posix scheduler, as VIR_TYPED_PARAM_LLONG.
+ */
+# define VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA "cputune.global_quota"
+
+/**
* VIR_DOMAIN_TUNABLE_CPU_VCPU_PERIOD:
*
* Macro represents the enforcement period for a quota, in microseconds,
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 5ee53c2..7e7a69c 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -15074,6 +15074,22 @@ virDomainDefParseXML(xmlDocPtr xml,
goto error;
}
+ if (virXPathLongLong("string(./cputune/global_quota[1])", ctxt,
+ &def->cputune.global_quota) < -1) {
+ virReportError(VIR_ERR_XML_ERROR, "%s",
+ _("can't parse cputune global quota value"));
+ goto error;
+ }
+
+ if (def->cputune.global_quota > 0 &&
+ (def->cputune.global_quota < 1000 ||
+ def->cputune.global_quota > 18446744073709551LL)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("Value of cputune global quota must be in range "
+ "[1000, 18446744073709551]"));
+ goto error;
+ }
+
if (virXPathULongLong("string(./cputune/emulator_period[1])", ctxt,
&def->cputune.emulator_period) < -1) {
virReportError(VIR_ERR_XML_ERROR, "%s",
@@ -21836,6 +21852,9 @@ virDomainDefFormatInternal(virDomainDefPtr def,
if (def->cputune.global_period)
virBufferAsprintf(&childrenBuf,
"<global_period>%llu</global_period>\n",
def->cputune.global_period);
+ if (def->cputune.global_quota)
+ virBufferAsprintf(&childrenBuf,
"<global_quota>%lld</global_quota>\n",
+ def->cputune.global_quota);
if (def->cputune.emulator_period)
virBufferAsprintf(&childrenBuf, "<emulator_period>%llu"
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 7c43e7d..bac82b3 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -2136,6 +2136,7 @@ struct _virDomainCputune {
unsigned long long period;
long long quota;
unsigned long long global_period;
+ long long global_quota;
unsigned long long emulator_period;
long long emulator_quota;
size_t nvcpupin;
--
1.8.3.1
4:08 a.m.
New subject: [libvirt] [PATCH v5 3/7] Add error checking on global quota and period
Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com>
---
src/qemu/qemu_command.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 5d3ab3a..087e9ad 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -9397,7 +9397,8 @@ qemuBuildCommandLine(virConnectPtr conn,
}
if (def->cputune.sharesSpecified || def->cputune.period ||
- def->cputune.quota || def->cputune.emulator_period ||
+ def->cputune.quota || def->cputune.global_period ||
+ def->cputune.global_quota || def->cputune.emulator_period ||
def->cputune.emulator_quota) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("CPU tuning is not available in session mode"));
--
1.8.3.1
4:08 a.m.
New subject: [libvirt] [PATCH v5 4/7] Add global_period and global_quota XML validation test
---
tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml
b/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml
index 813d201..a056089 100644
--- a/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml
+++ b/tests/qemuxml2argvdata/qemuxml2argv-cputune.xml
@@ -8,6 +8,8 @@
<shares>2048</shares>
<period>1000000</period>
<quota>-1</quota>
+ <global_period>1000000</global_period>
+ <global_quota>-1</global_quota>
<vcpupin vcpu='0' cpuset='0'/>
<vcpupin vcpu='1' cpuset='1'/>
<emulatorpin cpuset='1'/>
--
1.8.3.1
4:08 a.m.
New subject: [libvirt] [PATCH v5 5/7] Rename qemuSetupCgroupVcpuBW to qemuSetupBandwidthCgroup
This rename is required to reuse this function in
per-domain bandwidth setup routine
Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com>
---
src/qemu/qemu_cgroup.c | 14 +++++++-------
src/qemu/qemu_cgroup.h | 6 +++---
src/qemu/qemu_driver.c | 5 ++---
3 files changed, 12 insertions(+), 13 deletions(-)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index e41f461..452d71d 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -944,9 +944,9 @@ qemuSetupCgroup(virQEMUDriverPtr driver,
}
int
-qemuSetupCgroupVcpuBW(virCgroupPtr cgroup,
- unsigned long long period,
- long long quota)
+qemuSetupBandwidthCgroup(virCgroupPtr cgroup,
+ unsigned long long period,
+ long long quota)
{
unsigned long long old_period;
@@ -1053,7 +1053,7 @@ qemuSetupCgroupForVcpu(virDomainObjPtr vm)
goto cleanup;
if (period || quota) {
- if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
+ if (qemuSetupBandwidthCgroup(cgroup_vcpu, period, quota) < 0)
goto cleanup;
}
@@ -1152,8 +1152,8 @@ qemuSetupCgroupForEmulator(virDomainObjPtr vm)
if (period || quota) {
if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)
&&
- qemuSetupCgroupVcpuBW(cgroup_emulator, period,
- quota) < 0)
+ qemuSetupBandwidthCgroup(cgroup_emulator, period,
+ quota) < 0)
goto cleanup;
}
@@ -1217,7 +1217,7 @@ qemuSetupCgroupForIOThreads(virDomainObjPtr vm)
goto cleanup;
if (period || quota) {
- if (qemuSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0)
+ if (qemuSetupBandwidthCgroup(cgroup_iothread, period, quota) < 0)
goto cleanup;
}
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index 2bcf071..17da920 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -49,9 +49,9 @@ int qemuSetupCgroup(virQEMUDriverPtr driver,
size_t nnicindexes,
int *nicindexes);
int qemuSetupCpusetMems(virDomainObjPtr vm);
-int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup,
- unsigned long long period,
- long long quota);
+int qemuSetupBandwidthCgroup(virCgroupPtr cgroup,
+ unsigned long long period,
+ long long quota);
int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask);
int qemuSetupCgroupForVcpu(virDomainObjPtr vm);
int qemuSetupCgroupForIOThreads(virDomainObjPtr vm);
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 8ccf68b..48aeab6 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -10251,8 +10251,7 @@ qemuSetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup,
if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_VCPU, i,
false, &cgroup_vcpu) < 0)
goto cleanup;
-
- if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
+ if (qemuSetupBandwidthCgroup(cgroup_vcpu, period, quota) < 0)
goto cleanup;
virCgroupFree(&cgroup_vcpu);
@@ -10279,7 +10278,7 @@ qemuSetEmulatorBandwidthLive(virCgroupPtr cgroup,
false, &cgroup_emulator) < 0)
goto cleanup;
- if (qemuSetupCgroupVcpuBW(cgroup_emulator, period, quota) < 0)
+ if (qemuSetupBandwidthCgroup(cgroup_emulator, period, quota) < 0)
goto cleanup;
virCgroupFree(&cgroup_emulator);
--
1.8.3.1
4:08 a.m.
New subject: [libvirt] [PATCH v5 6/7] Implement qemuSetupGlobalCpuCgroup
This functions setups per-domain cpu bandwidth parameters
Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com>
---
src/qemu/qemu_cgroup.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_cgroup.h | 1 +
src/qemu/qemu_process.c | 4 ++++
3 files changed, 59 insertions(+)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 452d71d..1638bfe 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -1002,6 +1002,60 @@ qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup,
}
int
+qemuSetupGlobalCpuCgroup(virDomainObjPtr vm)
+{
+ qemuDomainObjPrivatePtr priv = vm->privateData;
+ unsigned long long period = vm->def->cputune.global_period;
+ long long quota = vm->def->cputune.global_quota;
+ char *mem_mask = NULL;
+ virDomainNumatuneMemMode mem_mode;
+
+ if ((period || quota) &&
+ !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("cgroup cpu is required for scheduler tuning"));
+ return -1;
+ }
+
+ /*
+ * If CPU cgroup controller is not initialized here, then we need
+ * neither period nor quota settings. And if CPUSET controller is
+ * not initialized either, then there's nothing to do anyway.
+ */
+ if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
+ !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
+ return 0;
+
+ /* We are trying to setup cgroups for CPU pinning, which can also be done
+ * with virProcessSetAffinity, thus the lack of cgroups is not fatal here.
+ */
+ if (priv->cgroup == NULL)
+ return 0;
+
+ if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 &&
+ mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
+ virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
+ priv->autoNodeset,
+ &mem_mask, -1) < 0)
+ goto cleanup;
+
+ if (period || quota) {
+ if (qemuSetupBandwidthCgroup(priv->cgroup, period, quota) < 0)
+ goto cleanup;
+ }
+
+ VIR_FREE(mem_mask);
+
+ return 0;
+
+ cleanup:
+ VIR_FREE(mem_mask);
+
+ return -1;
+}
+
+
+int
qemuSetupCgroupForVcpu(virDomainObjPtr vm)
{
virCgroupPtr cgroup_vcpu = NULL;
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index 17da920..75f9eb7 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -54,6 +54,7 @@ int qemuSetupBandwidthCgroup(virCgroupPtr cgroup,
long long quota);
int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask);
int qemuSetupCgroupForVcpu(virDomainObjPtr vm);
+int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm);
int qemuSetupCgroupForIOThreads(virDomainObjPtr vm);
int qemuSetupCgroupForEmulator(virDomainObjPtr vm);
int qemuRemoveCgroup(virQEMUDriverPtr driver, virDomainObjPtr vm);
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 05cbda2..96c7a6a 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -4982,6 +4982,10 @@ qemuProcessLaunch(virConnectPtr conn,
if (qemuProcessDetectIOThreadPIDs(driver, vm, asyncJob) < 0)
goto cleanup;
+ VIR_DEBUG("Setting global CPU cgroup (if required)");
+ if (qemuSetupGlobalCpuCgroup(vm) < 0)
+ goto cleanup;
+
VIR_DEBUG("Setting cgroup for each VCPU (if required)");
if (qemuSetupCgroupForVcpu(vm) < 0)
goto cleanup;
--
1.8.3.1
6:05 a.m.
New subject: [libvirt] [PATCH v5 6/7] Implement qemuSetupGlobalCpuCgroup
On Mon, 2016-01-18 at 13:08 +0300, Alexander Burluka wrote:
This functions setups per-domain cpu bandwidth parameters
Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com>
---
src/qemu/qemu_cgroup.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_cgroup.h | 1 +
src/qemu/qemu_process.c | 4 ++++
3 files changed, 59 insertions(+)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 452d71d..1638bfe 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -1002,6 +1002,60 @@ qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup,
}
int
+qemuSetupGlobalCpuCgroup(virDomainObjPtr vm)
+{
+ qemuDomainObjPrivatePtr priv = vm->privateData;
+ unsigned long long period = vm->def->cputune.global_period;
+ long long quota = vm->def->cputune.global_quota;
+ char *mem_mask = NULL;
+ virDomainNumatuneMemMode mem_mode;
+
+ if ((period || quota) &&
+ !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("cgroup cpu is required for scheduler tuning"));
+ return -1;
+ }
+
+ /*
+ * If CPU cgroup controller is not initialized here, then we need
+ * neither period nor quota settings. And if CPUSET controller is
+ * not initialized either, then there's nothing to do anyway.
+ */
+ if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
+ !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
+ return 0;
+
+ /* We are trying to setup cgroups for CPU pinning, which can also be done
+ * with virProcessSetAffinity, thus the lack of cgroups is not fatal here.
+ */
+ if (priv->cgroup == NULL)
+ return 0;
+
+ if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0
&&
+ mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
+ virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
+ priv->autoNodeset,
+ &mem_mask, -1) < 0)
+ goto cleanup;
+
+ if (period || quota) {
+ if (qemuSetupBandwidthCgroup(priv->cgroup, period, quota) < 0)
+ goto cleanup;
+ }
+
+ VIR_FREE(mem_mask);
+
+ return 0;
+
+ cleanup:
This path is aclually error, not cleanup. But I think it should prevent from
pushing the patch, since it can be simply fixed in patch file.
+ VIR_FREE(mem_mask);
+
+ return -1;
+}
+
+
+int
qemuSetupCgroupForVcpu(virDomainObjPtr vm)
{
virCgroupPtr cgroup_vcpu = NULL;
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index 17da920..75f9eb7 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -54,6 +54,7 @@ int qemuSetupBandwidthCgroup(virCgroupPtr cgroup,
long long quota);
int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask);
int qemuSetupCgroupForVcpu(virDomainObjPtr vm);
+int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm);
int qemuSetupCgroupForIOThreads(virDomainObjPtr vm);
int qemuSetupCgroupForEmulator(virDomainObjPtr vm);
int qemuRemoveCgroup(virQEMUDriverPtr driver, virDomainObjPtr vm);
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 05cbda2..96c7a6a 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -4982,6 +4982,10 @@ qemuProcessLaunch(virConnectPtr conn,
if (qemuProcessDetectIOThreadPIDs(driver, vm, asyncJob) < 0)
goto cleanup;
+ VIR_DEBUG("Setting global CPU cgroup (if required)");
+ if (qemuSetupGlobalCpuCgroup(vm) < 0)
+ goto cleanup;
+
VIR_DEBUG("Setting cgroup for each VCPU (if required)");
if (qemuSetupCgroupForVcpu(vm) < 0)
goto cleanup;
2:55 a.m.
New subject: [libvirt] [PATCH v5 6/7] Implement qemuSetupGlobalCpuCgroup
18.01.2016 13:08, Alexander Burluka пишет:
This functions setups per-domain cpu bandwidth parameters
Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com>
---
src/qemu/qemu_cgroup.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_cgroup.h | 1 +
src/qemu/qemu_process.c | 4 ++++
3 files changed, 59 insertions(+)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 452d71d..1638bfe 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -1002,6 +1002,60 @@ qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup,
}
int
+qemuSetupGlobalCpuCgroup(virDomainObjPtr vm)
+{
+ qemuDomainObjPrivatePtr priv = vm->privateData;
+ unsigned long long period = vm->def->cputune.global_period;
+ long long quota = vm->def->cputune.global_quota;
+ char *mem_mask = NULL;
+ virDomainNumatuneMemMode mem_mode;
+
+ if ((period || quota) &&
+ !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("cgroup cpu is required for scheduler tuning"));
+ return -1;
+ }
+
+ /*
+ * If CPU cgroup controller is not initialized here, then we need
+ * neither period nor quota settings. And if CPUSET controller is
+ * not initialized either, then there's nothing to do anyway.
+ */
+ if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
+ !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
+ return 0;
+
+ /* We are trying to setup cgroups for CPU pinning, which can also be done
+ * with virProcessSetAffinity, thus the lack of cgroups is not fatal here.
+ */
+ if (priv->cgroup == NULL)
+ return 0;
This check is unnecessary because virCgroupHasController already
has it.
+
+ if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0
&&
+ mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
+ virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
+ priv->autoNodeset,
+ &mem_mask, -1) < 0)
+ goto cleanup;
I'd like to see the reason in error message here similar
to error path
at the beginning of the function.
+
+ if (period || quota) {
+ if (qemuSetupBandwidthCgroup(priv->cgroup, period, quota) < 0)
+ goto cleanup;
+ }
+
+ VIR_FREE(mem_mask);
+
+ return 0;
+
+ cleanup:
+ VIR_FREE(mem_mask);
+
+ return -1;
+}
+
+
+int
qemuSetupCgroupForVcpu(virDomainObjPtr vm)
{
virCgroupPtr cgroup_vcpu = NULL;
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index 17da920..75f9eb7 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -54,6 +54,7 @@ int qemuSetupBandwidthCgroup(virCgroupPtr cgroup,
long long quota);
int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask);
int qemuSetupCgroupForVcpu(virDomainObjPtr vm);
+int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm);
int qemuSetupCgroupForIOThreads(virDomainObjPtr vm);
int qemuSetupCgroupForEmulator(virDomainObjPtr vm);
int qemuRemoveCgroup(virQEMUDriverPtr driver, virDomainObjPtr vm);
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 05cbda2..96c7a6a 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -4982,6 +4982,10 @@ qemuProcessLaunch(virConnectPtr conn,
if (qemuProcessDetectIOThreadPIDs(driver, vm, asyncJob) < 0)
goto cleanup;
+ VIR_DEBUG("Setting global CPU cgroup (if required)");
+ if (qemuSetupGlobalCpuCgroup(vm) < 0)
+ goto cleanup;
+
VIR_DEBUG("Setting cgroup for each VCPU (if required)");
if (qemuSetupCgroupForVcpu(vm) < 0)
goto cleanup;
4:08 a.m.
New subject: [libvirt] [PATCH v5 7/7] Implement handling of per-domain bandwidth settings
Signed-off-by: Alexander Burluka <aburluka(a)virtuozzo.com>
---
src/qemu/qemu_driver.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 95 insertions(+), 2 deletions(-)
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 48aeab6..1932d7f 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -8906,7 +8906,7 @@ static char *qemuDomainGetSchedulerType(virDomainPtr dom,
/* Domain not running, thus no cgroups - return defaults */
if (!virDomainObjIsActive(vm)) {
if (nparams)
- *nparams = 5;
+ *nparams = 7;
ignore_value(VIR_STRDUP(ret, "posix"));
goto cleanup;
}
@@ -8919,7 +8919,7 @@ static char *qemuDomainGetSchedulerType(virDomainPtr dom,
if (nparams) {
if (virCgroupSupportsCpuBW(priv->cgroup))
- *nparams = 5;
+ *nparams = 7;
else
*nparams = 1;
}
@@ -10234,6 +10234,19 @@ qemuDomainGetNumaParameters(virDomainPtr dom,
}
static int
+qemuSetGlobalBWLive(virCgroupPtr cgroup, unsigned long long period,
+ long long quota)
+{
+ if (period == 0 && quota == 0)
+ return 0;
+
+ if (qemuSetupBandwidthCgroup(cgroup, period, quota) < 0)
+ return -1;
+
+ return 0;
+}
+
+static int
qemuSetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup,
unsigned long long period, long long quota)
{
@@ -10329,6 +10342,10 @@ qemuDomainSetSchedulerParametersFlags(virDomainPtr dom,
VIR_TYPED_PARAM_ULLONG,
VIR_DOMAIN_SCHEDULER_VCPU_QUOTA,
VIR_TYPED_PARAM_LLONG,
+ VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD,
+ VIR_TYPED_PARAM_ULLONG,
+ VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA,
+ VIR_TYPED_PARAM_LLONG,
VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD,
VIR_TYPED_PARAM_ULLONG,
VIR_DOMAIN_SCHEDULER_EMULATOR_QUOTA,
@@ -10445,6 +10462,46 @@ qemuDomainSetSchedulerParametersFlags(virDomainPtr dom,
if (flags & VIR_DOMAIN_AFFECT_CONFIG)
vmdef->cputune.quota = value_l;
+ } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD)) {
+ SCHED_RANGE_CHECK(value_ul, VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD,
+ QEMU_SCHED_MIN_PERIOD, QEMU_SCHED_MAX_PERIOD);
+
+ if (flags & VIR_DOMAIN_AFFECT_LIVE && value_ul) {
+ if ((rc = qemuSetGlobalBWLive(priv->cgroup, value_ul, 0)))
+ goto endjob;
+
+ vm->def->cputune.global_period = value_ul;
+
+ if (virTypedParamsAddULLong(&eventParams, &eventNparams,
+ &eventMaxNparams,
+ VIR_DOMAIN_TUNABLE_CPU_GLOBAL_PERIOD,
+ value_ul) < 0)
+ goto endjob;
+ }
+
+ if (flags & VIR_DOMAIN_AFFECT_CONFIG)
+ vmdef->cputune.period = params[i].value.ul;
+
+ } else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA)) {
+ SCHED_RANGE_CHECK(value_l, VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA,
+ QEMU_SCHED_MIN_QUOTA, QEMU_SCHED_MAX_QUOTA);
+
+ if (flags & VIR_DOMAIN_AFFECT_LIVE && value_l) {
+ if ((rc = qemuSetGlobalBWLive(priv->cgroup, 0, value_l)))
+ goto endjob;
+
+ vm->def->cputune.global_quota = value_l;
+
+ if (virTypedParamsAddLLong(&eventParams, &eventNparams,
+ &eventMaxNparams,
+ VIR_DOMAIN_TUNABLE_CPU_GLOBAL_QUOTA,
+ value_l) < 0)
+ goto endjob;
+ }
+
+ if (flags & VIR_DOMAIN_AFFECT_CONFIG)
+ vmdef->cputune.global_quota = value_l;
+
} else if (STREQ(param->field, VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD)) {
SCHED_RANGE_CHECK(value_ul, VIR_DOMAIN_SCHEDULER_EMULATOR_PERIOD,
QEMU_SCHED_MIN_PERIOD, QEMU_SCHED_MAX_PERIOD);
@@ -10609,6 +10666,16 @@ qemuGetEmulatorBandwidthLive(virCgroupPtr cgroup,
}
static int
+qemuGetGlobalBWLive(virCgroupPtr cgroup, unsigned long long *period,
+ long long *quota)
+{
+ if (qemuGetVcpuBWLive(cgroup, period, quota) < 0)
+ return -1;
+
+ return 0;
+}
+
+static int
qemuDomainGetSchedulerParametersFlags(virDomainPtr dom,
virTypedParameterPtr params,
int *nparams,
@@ -10619,6 +10686,8 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom,
unsigned long long shares;
unsigned long long period;
long long quota;
+ unsigned long long global_period;
+ long long global_quota;
unsigned long long emulator_period;
long long emulator_quota;
int ret = -1;
@@ -10665,6 +10734,8 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom,
if (*nparams > 1) {
period = persistentDef->cputune.period;
quota = persistentDef->cputune.quota;
+ global_period = persistentDef->cputune.global_period;
+ global_quota = persistentDef->cputune.global_quota;
emulator_period = persistentDef->cputune.emulator_period;
emulator_quota = persistentDef->cputune.emulator_quota;
cpu_bw_status = true; /* Allow copy of data to params[] */
@@ -10694,6 +10765,12 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom,
goto cleanup;
}
+ if (*nparams > 5 && cpu_bw_status) {
+ rc = qemuGetGlobalBWLive(priv->cgroup, &global_period,
&global_quota);
+ if (rc != 0)
+ goto cleanup;
+ }
+
out:
if (virTypedParameterAssign(¶ms[0], VIR_DOMAIN_SCHEDULER_CPU_SHARES,
VIR_TYPED_PARAM_ULLONG, shares) < 0)
@@ -10734,6 +10811,22 @@ qemuDomainGetSchedulerParametersFlags(virDomainPtr dom,
goto cleanup;
saved_nparams++;
}
+
+ if (*nparams > saved_nparams) {
+ if (virTypedParameterAssign(¶ms[5],
+ VIR_DOMAIN_SCHEDULER_GLOBAL_PERIOD,
+ VIR_TYPED_PARAM_ULLONG, global_period) < 0)
+ goto cleanup;
+ saved_nparams++;
+ }
+
+ if (*nparams > saved_nparams) {
+ if (virTypedParameterAssign(¶ms[6],
+ VIR_DOMAIN_SCHEDULER_GLOBAL_QUOTA,
+ VIR_TYPED_PARAM_LLONG, global_quota) < 0)
+ goto cleanup;
+ saved_nparams++;
+ }
}
*nparams = saved_nparams;
--
1.8.3.1
6:48 a.m.
On Mon, 2016-01-18 at 13:08 +0300, Alexander Burluka wrote:
We decide to make a global per domain bandwidth setting
as were discussed in mailing list earlier.
This patchset implements hierarchy top level cpu.cfs_period_us
and cpu.cfs_quota_us control knob. I've named this parameters
as global_period and global_quota.
All issues have been fixed, so ACK from me to the series.
Changes in v2: add XML validation test
Changes in v3: remove unneccessary cgroup copying
Changes in v4: fix little rebase error
Changes in v5: rebase to version 1.3.1
Alexander Burluka (7):
Add global period definitions
Add global quota parameter necessary definitions
Add error checking on global quota and period
Add global_period and global_quota XML validation test
Rename qemuSetupCgroupVcpuBW to qemuSetupBandwidthCgroup
Implement qemuSetupGlobalCpuCgroup
Implement handling of per-domain bandwidth settings
docs/schemas/domaincommon.rng | 10 +++
include/libvirt/libvirt-domain.h | 32 ++++++++
src/conf/domain_conf.c | 37 +++++++++
src/conf/domain_conf.h | 2 +
src/qemu/qemu_cgroup.c | 68 ++++++++++++++--
src/qemu/qemu_cgroup.h | 7 +-
src/qemu/qemu_command.c | 3 +-
src/qemu/qemu_driver.c | 102 ++++++++++++++++++++++--
src/qemu/qemu_process.c | 4 +
tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 +
10 files changed, 251 insertions(+), 16 deletions(-)
5:55 a.m.
New subject: [libvirt] [PATCH v5 0/7] Per domain bandwidth settings
Hi!
Unfortunately I've no reply on this patchset from community and would
like to remind about it.
Thank you!
On 01/18/2016 01:08 PM, Alexander Burluka wrote:
We decide to make a global per domain bandwidth setting
as were discussed in mailing list earlier.
This patchset implements hierarchy top level cpu.cfs_period_us
and cpu.cfs_quota_us control knob. I've named this parameters
as global_period and global_quota.
Changes in v2: add XML validation test
Changes in v3: remove unneccessary cgroup copying
Changes in v4: fix little rebase error
Changes in v5: rebase to version 1.3.1
Alexander Burluka (7):
Add global period definitions
Add global quota parameter necessary definitions
Add error checking on global quota and period
Add global_period and global_quota XML validation test
Rename qemuSetupCgroupVcpuBW to qemuSetupBandwidthCgroup
Implement qemuSetupGlobalCpuCgroup
Implement handling of per-domain bandwidth settings
docs/schemas/domaincommon.rng | 10 +++
include/libvirt/libvirt-domain.h | 32 ++++++++
src/conf/domain_conf.c | 37 +++++++++
src/conf/domain_conf.h | 2 +
src/qemu/qemu_cgroup.c | 68 ++++++++++++++--
src/qemu/qemu_cgroup.h | 7 +-
src/qemu/qemu_command.c | 3 +-
src/qemu/qemu_driver.c | 102 ++++++++++++++++++++++--
src/qemu/qemu_process.c | 4 +
tests/qemuxml2argvdata/qemuxml2argv-cputune.xml | 2 +
10 files changed, 251 insertions(+), 16 deletions(-)
--
Regards,
Alexander Burluka
3253
days inactive
3261
days old
11 comments
3 participants
participants (3)
-
Alexander Burluka -
Dmitry Guryanov -
Maxim Nestratov