10:30 p.m.
From: Hu Tao <hutao(a)cn.fujitsu.com>
Implement setting/getting per-device blkio weights in qemu,
using the cgroups blkio.weight_device tunable.
---
src/libvirt_private.syms | 1 +
src/qemu/qemu_cgroup.c | 20 ++
src/qemu/qemu_driver.c | 218 +++++++++++++++++++-
src/util/cgroup.c | 51 +++++
src/util/cgroup.h | 4 +
.../qemuxml2argv-blkiotune-device.args | 4 +
tests/qemuxml2argvtest.c | 1 +
tests/qemuxml2xmltest.c | 1 +
8 files changed, 295 insertions(+), 5 deletions(-)
create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-blkiotune-device.args
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index d78fd53..3575fe0 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -89,6 +89,7 @@ virCgroupKillRecursive;
virCgroupMounted;
virCgroupPathOfController;
virCgroupRemove;
+virCgroupSetBlkioDeviceWeight;
virCgroupSetBlkioWeight;
virCgroupSetCpuShares;
virCgroupSetCpuCfsPeriod;
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 2a10bd2..eda4c66 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -312,6 +312,26 @@ int qemuSetupCgroup(struct qemud_driver *driver,
}
}
+ if (vm->def->blkio.ndevices) {
+ if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_BLKIO)) {
+ for (i = 0; i < vm->def->blkio.ndevices; i++) {
+ virBlkioDeviceWeightPtr dw = &vm->def->blkio.devices[i];
+ rc = virCgroupSetBlkioDeviceWeight(cgroup, dw->path,
+ dw->weight);
+ if (rc != 0) {
+ virReportSystemError(-rc,
+ _("Unable to set io device weight "
+ "for domain %s"),
+ vm->def->name);
+ goto cleanup;
+ }
+ }
+ } else {
+ qemuReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Block I/O tuning is not available on this
host"));
+ }
+ }
+
if (vm->def->mem.hard_limit != 0 ||
vm->def->mem.soft_limit != 0 ||
vm->def->mem.swap_hard_limit != 0) {
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index b0ce115..43f4041 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -112,7 +112,7 @@
# define KVM_CAP_NR_VCPUS 9 /* returns max vcpus per vm */
#endif
-#define QEMU_NB_BLKIO_PARAM 1
+#define QEMU_NB_BLKIO_PARAM 2
static void processWatchdogEvent(void *data, void *opaque);
@@ -5885,6 +5885,105 @@ cleanup:
return ret;
}
+/* deviceWeightStr in the form of /device/path,weight,/device/path,weight
+ * for example, /dev/disk/by-path/pci-0000:00:1f.2-scsi-0:0:0:0,800
+ */
+static int
+parseBlkioWeightDeviceStr(char *deviceWeightStr,
+ virBlkioDeviceWeightPtr *dw, int *size,
+ virCgroupPtr cgroup)
+{
+ char *temp;
+ int ndevices = 0;
+ int nsep = 0;
+ int i;
+ virBlkioDeviceWeightPtr result = NULL;
+
+ temp = deviceWeightStr;
+ while (temp) {
+ temp = strchr(temp, ',');
+ if (temp) {
+ temp++;
+ nsep++;
+ }
+ }
+
+ /* A valid string must have even number of fields, hence an odd
+ * number of commas. */
+ if (!(nsep & 1))
+ goto error;
+
+ ndevices = (nsep + 1) / 2;
+
+ if (VIR_ALLOC_N(result, ndevices) < 0) {
+ virReportOOMError();
+ return -1;
+ }
+
+ i = 0;
+ temp = deviceWeightStr;
+ while (temp) {
+ char *p = temp;
+
+ /* device path */
+ p = strchr(p, ',');
+ if (!p)
+ goto error;
+
+ result[i].path = strndup(temp, p - temp);
+ if (!result[i].path) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ /* weight */
+ temp = p + 1;
+
+ if (virStrToLong_ui(temp, &p, 10, &result[i].weight) < 0)
+ goto error;
+
+ if (cgroup) {
+ int rc = virCgroupSetBlkioDeviceWeight(cgroup,
+ result[i].path,
+ result[i].weight);
+ if (rc < 0) {
+ virReportSystemError(-rc,
+ _("Unable to set io device weight "
+ "for path %s"),
+ result[i].path);
+ goto cleanup;
+ }
+ }
+
+ /* 0-weight entries only affect cgroups, they don't stick in xml */
+ if (result[i].weight)
+ i++;
+ else
+ VIR_FREE(result[i].path);
+ if (*p == '\0')
+ break;
+ else if (*p != ',')
+ goto error;
+ temp = p + 1;
+ }
+
+ if (!i)
+ VIR_FREE(result);
+
+ *dw = result;
+ *size = i;
+
+ return 0;
+
+error:
+ qemuReportError(VIR_ERR_INVALID_ARG,
+ _("unable to parse %s"), deviceWeightStr);
+cleanup:
+ virBlkioDeviceWeightArrayClear(result, ndevices);
+ VIR_FREE(result);
+ return -1;
+}
+
static int qemuDomainSetBlkioParameters(virDomainPtr dom,
virTypedParameterPtr params,
int nparams,
@@ -5951,10 +6050,10 @@ static int qemuDomainSetBlkioParameters(virDomainPtr dom,
ret = 0;
if (flags & VIR_DOMAIN_AFFECT_LIVE) {
for (i = 0; i < nparams; i++) {
+ int rc;
virTypedParameterPtr param = ¶ms[i];
if (STREQ(param->field, VIR_DOMAIN_BLKIO_WEIGHT)) {
- int rc;
if (param->type != VIR_TYPED_PARAM_UINT) {
qemuReportError(VIR_ERR_INVALID_ARG, "%s",
_("invalid type for blkio weight tunable,
expected a 'unsigned int'"));
@@ -5975,6 +6074,28 @@ static int qemuDomainSetBlkioParameters(virDomainPtr dom,
_("unable to set blkio weight
tunable"));
ret = -1;
}
+ } else if (STREQ(param->field, VIR_DOMAIN_BLKIO_DEVICE_WEIGHT)) {
+ int ndevices;
+ virBlkioDeviceWeightPtr devices = NULL;
+ if (param->type != VIR_TYPED_PARAM_STRING) {
+ qemuReportError(VIR_ERR_INVALID_ARG, "%s",
+ _("invalid type for device_weight tunable,
"
+ "expected a 'char *'"));
+ ret = -1;
+ continue;
+ }
+
+ if (parseBlkioWeightDeviceStr(params[i].value.s,
+ &devices,
+ &ndevices, group) < 0) {
+ ret = -1;
+ continue;
+ }
+ virBlkioDeviceWeightArrayClear(vm->def->blkio.devices,
+ vm->def->blkio.ndevices);
+ VIR_FREE(vm->def->blkio.devices);
+ vm->def->blkio.devices = devices;
+ vm->def->blkio.ndevices = ndevices;
} else {
qemuReportError(VIR_ERR_INVALID_ARG,
_("Parameter `%s' not supported"),
param->field);
@@ -6007,9 +6128,31 @@ static int qemuDomainSetBlkioParameters(virDomainPtr dom,
}
persistentDef->blkio.weight = params[i].value.ui;
+ } else if (STREQ(param->field, VIR_DOMAIN_BLKIO_DEVICE_WEIGHT)) {
+ virBlkioDeviceWeightPtr devices = NULL;
+ int ndevices;
+ if (param->type != VIR_TYPED_PARAM_STRING) {
+ qemuReportError(VIR_ERR_INVALID_ARG, "%s",
+ _("invalid type for device_weight tunable,
"
+ "expected a 'char *'"));
+ ret = -1;
+ continue;
+ }
+ if (parseBlkioWeightDeviceStr(params[i].value.s,
+ &devices,
+ &ndevices, NULL) < 0) {
+ ret = -1;
+ continue;
+ }
+ virBlkioDeviceWeightArrayClear(persistentDef->blkio.devices,
+ persistentDef->blkio.ndevices);
+ VIR_FREE(persistentDef->blkio.devices);
+ persistentDef->blkio.devices = devices;
+ persistentDef->blkio.ndevices = ndevices;
} else {
qemuReportError(VIR_ERR_INVALID_ARG,
- _("Parameter `%s' not supported"),
param->field);
+ _("Parameter `%s' not supported"),
+ param->field);
ret = -1;
}
}
@@ -6032,7 +6175,7 @@ static int qemuDomainGetBlkioParameters(virDomainPtr dom,
unsigned int flags)
{
struct qemud_driver *driver = dom->conn->privateData;
- int i;
+ int i, j;
virCgroupPtr group = NULL;
virDomainObjPtr vm = NULL;
virDomainDefPtr persistentDef = NULL;
@@ -6046,7 +6189,9 @@ static int qemuDomainGetBlkioParameters(virDomainPtr dom,
VIR_TYPED_PARAM_STRING_OKAY, -1);
qemuDriverLock(driver);
- /* We don't return strings, and thus trivially support this flag. */
+ /* We blindly return a string, and let libvirt.c and
+ * remote_driver.c do the filtering on behalf of older clients
+ * that can't parse it. */
flags &= ~VIR_TYPED_PARAM_STRING_OKAY;
vm = virDomainFindByUUID(&driver->domains, dom->uuid);
@@ -6125,6 +6270,37 @@ static int qemuDomainGetBlkioParameters(virDomainPtr dom,
}
param->value.ui = val;
break;
+ case 1: /* blkiotune.device_weight */
+ if (vm->def->blkio.ndevices > 0) {
+ virBuffer buf = VIR_BUFFER_INITIALIZER;
+ for (j = 0; j < vm->def->blkio.ndevices; j++) {
+ if (j)
+ virBufferAddChar(&buf, ',');
+ virBufferAsprintf(&buf, "%s,%u",
+ vm->def->blkio.devices[j].path,
+ vm->def->blkio.devices[j].weight);
+ }
+ if (virBufferError(&buf)) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ param->value.s = virBufferContentAndReset(&buf);
+ } else {
+ param->value.s = strdup("");
+ if (!param->value.s) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ }
+ param->type = VIR_TYPED_PARAM_STRING;
+ if (virStrcpyStatic(param->field,
+ VIR_DOMAIN_BLKIO_DEVICE_WEIGHT) == NULL) {
+ qemuReportError(VIR_ERR_INTERNAL_ERROR,
+ _("Field name '%s' too long"),
+ VIR_DOMAIN_BLKIO_DEVICE_WEIGHT);
+ goto cleanup;
+ }
+ break;
default:
break;
@@ -6149,6 +6325,38 @@ static int qemuDomainGetBlkioParameters(virDomainPtr dom,
param->value.ui = persistentDef->blkio.weight;
break;
+ case 1: /* blkiotune.device_weight */
+ if (persistentDef->blkio.ndevices > 0) {
+ virBuffer buf = VIR_BUFFER_INITIALIZER;
+ for (j = 0; j < persistentDef->blkio.ndevices; j++) {
+ if (j)
+ virBufferAddChar(&buf, ',');
+ virBufferAsprintf(&buf, "%s,%u",
+ persistentDef->blkio.devices[j].path,
+ persistentDef->blkio.devices[j].weight);
+ }
+ if (virBufferError(&buf)) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ param->value.s = virBufferContentAndReset(&buf);
+ } else {
+ param->value.s = strdup("");
+ if (!param->value.s) {
+ virReportOOMError();
+ goto cleanup;
+ }
+ }
+ param->type = VIR_TYPED_PARAM_STRING;
+ if (virStrcpyStatic(param->field,
+ VIR_DOMAIN_BLKIO_DEVICE_WEIGHT) == NULL) {
+ qemuReportError(VIR_ERR_INTERNAL_ERROR,
+ _("Field name '%s' too long"),
+ VIR_DOMAIN_BLKIO_DEVICE_WEIGHT);
+ goto cleanup;
+ }
+ break;
+
default:
break;
/* should not hit here */
diff --git a/src/util/cgroup.c b/src/util/cgroup.c
index c8d1f33..496f9ff 100644
--- a/src/util/cgroup.c
+++ b/src/util/cgroup.c
@@ -982,6 +982,57 @@ int virCgroupGetBlkioWeight(virCgroupPtr group, unsigned int
*weight)
}
/**
+ * virCgroupSetBlkioDeviceWeight:
+ *
+ * @group: The cgroup to change io device weight device for
+ * @path: The device with a weight to alter
+ * @weight: The new device weight (100-1000), or 0 to clear
+ *
+ * device_weight is treated as a write-only parameter, so
+ * there isn't a getter counterpart.
+ *
+ * Returns: 0 on success, -errno on failure
+ */
+#if defined(major) && defined(minor)
+int virCgroupSetBlkioDeviceWeight(virCgroupPtr group,
+ const char *path,
+ unsigned int weight)
+{
+ char *str;
+ struct stat sb;
+ int ret;
+
+ if (weight && (weight > 1000 || weight < 100))
+ return -EINVAL;
+
+ if (stat(path, &sb) < 0)
+ return -errno;
+
+ if (!S_ISBLK(sb.st_mode))
+ return -EINVAL;
+
+ if (virAsprintf(&str, "%d:%d %d", major(sb.st_rdev),
minor(sb.st_rdev),
+ weight) < 0)
+ return -errno;
+
+ ret = virCgroupSetValueStr(group,
+ VIR_CGROUP_CONTROLLER_BLKIO,
+ "blkio.weight_device",
+ str);
+ VIR_FREE(str);
+ return ret;
+}
+#else
+ int
+ virCgroupSetBlkioDeviceWeight(virCgroupPtr group ATTRIBUTE_UNUSED,
+ const char *path ATTRIBUTE_UNUSED,
+ unsigned int weight ATTRIBUTE_UNUSED)
+{
+ return -ENOSYS;
+}
+#endif
+
+/**
* virCgroupSetMemory:
*
* @group: The cgroup to change memory for
diff --git a/src/util/cgroup.h b/src/util/cgroup.h
index d190bb3..70dd392 100644
--- a/src/util/cgroup.h
+++ b/src/util/cgroup.h
@@ -55,6 +55,10 @@ int virCgroupAddTask(virCgroupPtr group, pid_t pid);
int virCgroupSetBlkioWeight(virCgroupPtr group, unsigned int weight);
int virCgroupGetBlkioWeight(virCgroupPtr group, unsigned int *weight);
+int virCgroupSetBlkioDeviceWeight(virCgroupPtr group,
+ const char *path,
+ unsigned int weight);
+
int virCgroupSetMemory(virCgroupPtr group, unsigned long long kb);
int virCgroupGetMemoryUsage(virCgroupPtr group, unsigned long *kb);
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-blkiotune-device.args
b/tests/qemuxml2argvdata/qemuxml2argv-blkiotune-device.args
new file mode 100644
index 0000000..651793d
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-blkiotune-device.args
@@ -0,0 +1,4 @@
+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test /usr/bin/qemu -S -M \
+pc -m 214 -smp 1 -name QEMUGuest1 -nographic -monitor unix:/tmp/test-monitor,\
+server,nowait -no-acpi -boot c -hda /dev/HostVG/QEMUGuest1 -net none -serial \
+none -parallel none -usb
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index d9a6e8d..c75b68c 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -583,6 +583,7 @@ mymain(void)
DO_TEST("memtune", false, QEMU_CAPS_NAME);
DO_TEST("blkiotune", false, QEMU_CAPS_NAME);
+ DO_TEST("blkiotune-device", false, QEMU_CAPS_NAME);
DO_TEST("cputune", false, QEMU_CAPS_NAME);
DO_TEST("numatune-memory", false, NONE);
diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c
index 3f37520..32445fd 100644
--- a/tests/qemuxml2xmltest.c
+++ b/tests/qemuxml2xmltest.c
@@ -184,6 +184,7 @@ mymain(void)
DO_TEST("encrypted-disk");
DO_TEST("memtune");
DO_TEST("blkiotune");
+ DO_TEST("blkiotune-device");
DO_TEST("cputune");
DO_TEST("smp");
--
1.7.7.1