[libvirt] [PATCH 0/9] support multi-thread compress migration.
These series patches support multi-thread compress during live migration. Eli Qiao (4): Add test cases for qemuMonitorJSONGetMigrationParameter remote: Add support for set and get multil thread migration parameters qemu_driver: Add support to set/get migration parameters. virsh: Add set and get multi-thread migration parameters commands ShaoHe Feng (5): qemu_migration: Add support for mutil-thread compressed migration enable qemu: Add monitor API for get/set migration parameters set multi-thread compress params for Migrate3 during live migration virsh: add multi-thread migration option for live migrate command Implement the public APIs for multi-thread compress parameters. daemon/remote.c | 62 +++++++++++ include/libvirt/libvirt-domain.h | 31 ++++++ src/driver-hypervisor.h | 14 +++ src/libvirt-domain.c | 110 +++++++++++++++++++ src/libvirt_public.syms | 5 + src/qemu/qemu_domain.h | 3 + src/qemu/qemu_driver.c | 186 ++++++++++++++++++++++++++++++++ src/qemu/qemu_migration.c | 105 ++++++++++++++++++ src/qemu/qemu_migration.h | 18 +++- src/qemu/qemu_monitor.c | 40 ++++++- src/qemu/qemu_monitor.h | 11 ++ src/qemu/qemu_monitor_json.c | 93 ++++++++++++++++ src/qemu/qemu_monitor_json.h | 9 ++ src/qemu/qemu_monitor_text.c | 95 +++++++++++++++++ src/qemu/qemu_monitor_text.h | 10 ++ src/remote/remote_driver.c | 54 ++++++++++ src/remote/remote_protocol.x | 30 +++++- src/remote_protocol-structs | 26 +++++ tests/qemumonitorjsontest.c | 53 +++++++++ tools/virsh-domain.c | 225 ++++++++++++++++++++++++++++++++++++++- tools/virsh.pod | 36 +++++-- 21 files changed, 1206 insertions(+), 10 deletions(-) -- 2.1.4
We need to set the mutil-thread compress capability as true to enable it. Signed-off-by: Eli Qiao <liyong.qiao@intel.com> Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> --- src/qemu/qemu_migration.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_migration.h | 9 +++++++- src/qemu/qemu_monitor.c | 2 +- src/qemu/qemu_monitor.h | 1 + 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index f7432e8..3322f72 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2234,6 +2234,52 @@ qemuMigrationSetCompression(virQEMUDriverPtr driver, return ret; } +int +qemuMigrationSetMultiThreadCompression(virQEMUDriverPtr driver, + virDomainObjPtr vm, + bool state, + qemuDomainAsyncJob job) +{ + qemuDomainObjPrivatePtr priv = vm->privateData; + int ret = -1; + + if (qemuDomainObjEnterMonitorAsync(driver, vm, job) < 0) + return -1; + + ret = qemuMonitorGetMigrationCapability( + priv->mon, + QEMU_MONITOR_MIGRATION_CAPS_MT_COMPRESS); + + if (ret < 0) { + goto cleanup; + } else if (ret == 0 && !state) { + /* Unsupported but we want it off anyway */ + goto cleanup; + } else if (ret == 0) { + if (job == QEMU_ASYNC_JOB_MIGRATION_IN) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("Multi-thread compressed migration is not supported by " + "target QEMU binary")); + } else { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("Multi-thread compressed migration is not supported by " + "source QEMU binary")); + } + ret = -1; + goto cleanup; + } + + ret = qemuMonitorSetMigrationCapability( + priv->mon, + QEMU_MONITOR_MIGRATION_CAPS_MT_COMPRESS, + state); + + cleanup: + if (qemuDomainObjExitMonitor(driver, vm) < 0) + ret = -1; + return ret; +} + static int qemuMigrationSetAutoConverge(virQEMUDriverPtr driver, virDomainObjPtr vm, @@ -3143,6 +3189,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) goto stop; + if (qemuMigrationSetMultiThreadCompression(driver, vm, + flags & VIR_MIGRATE_MT_COMPRESSED, + QEMU_ASYNC_JOB_MIGRATION_IN) < 0) + goto stop; + if (STREQ_NULLABLE(protocol, "rdma") && virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) { goto stop; @@ -3980,6 +4031,11 @@ qemuMigrationRun(virQEMUDriverPtr driver, QEMU_ASYNC_JOB_MIGRATION_OUT) < 0) goto cleanup; + if (qemuMigrationSetMultiThreadCompression(driver, vm, + flags & VIR_MIGRATE_MT_COMPRESSED, + QEMU_ASYNC_JOB_MIGRATION_OUT) < 0) + goto cleanup; + if (qemuMigrationSetAutoConverge(driver, vm, flags & VIR_MIGRATE_AUTO_CONVERGE, QEMU_ASYNC_JOB_MIGRATION_OUT) < 0) diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 1726455..065623b 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -41,7 +41,8 @@ VIR_MIGRATE_COMPRESSED | \ VIR_MIGRATE_ABORT_ON_ERROR | \ VIR_MIGRATE_AUTO_CONVERGE | \ - VIR_MIGRATE_RDMA_PIN_ALL) + VIR_MIGRATE_RDMA_PIN_ALL | \ + VIR_MIGRATE_MT_COMPRESSED) /* All supported migration parameters and their types. */ # define QEMU_MIGRATION_PARAMETERS \ @@ -177,4 +178,10 @@ int qemuMigrationToFile(virQEMUDriverPtr driver, virDomainObjPtr vm, ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(5) ATTRIBUTE_RETURN_CHECK; +int +qemuMigrationSetMultiThreadCompression(virQEMUDriverPtr driver, + virDomainObjPtr vm, + bool state, + qemuDomainAsyncJob job); + #endif /* __QEMU_MIGRATION_H__ */ diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index f959b74..18abfee 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -163,7 +163,7 @@ VIR_ENUM_IMPL(qemuMonitorMigrationStatus, VIR_ENUM_IMPL(qemuMonitorMigrationCaps, QEMU_MONITOR_MIGRATION_CAPS_LAST, - "xbzrle", "auto-converge", "rdma-pin-all") + "xbzrle", "auto-converge", "rdma-pin-all", "compress") VIR_ENUM_IMPL(qemuMonitorVMStatus, QEMU_MONITOR_VM_STATUS_LAST, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 76380a0..52d0be7 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -499,6 +499,7 @@ typedef enum { QEMU_MONITOR_MIGRATION_CAPS_XBZRLE, QEMU_MONITOR_MIGRATION_CAPS_AUTO_CONVERGE, QEMU_MONITOR_MIGRATION_CAPS_RDMA_PIN_ALL, + QEMU_MONITOR_MIGRATION_CAPS_MT_COMPRESS, QEMU_MONITOR_MIGRATION_CAPS_LAST } qemuMonitorMigrationCaps; -- 2.1.4
Add monitor API to tune and query the parameters used in live migration. * src/qemu/qemu_monitor.c, src/qemu/qemu_monitor.h, src/qemu/qemu_monitor_json.c, src/qemu/qemu_monitor_json.h, src/qemu/qemu_monitor_text.c, src/qemu/qemu_monitor_text.h: Add monitor API for get/set migration parameters Signed-off-by: Eli Qiao <liyong.qiao@intel.com> Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> --- src/qemu/qemu_monitor.c | 38 ++++++++++++++++++ src/qemu/qemu_monitor.h | 10 +++++ src/qemu/qemu_monitor_json.c | 93 +++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_monitor_json.h | 9 +++++ src/qemu/qemu_monitor_text.c | 95 ++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_monitor_text.h | 10 +++++ 6 files changed, 255 insertions(+) diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index 18abfee..1ab8dc4 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -2176,6 +2176,44 @@ qemuMonitorSetMigrationCacheSize(qemuMonitorPtr mon, int +qemuMonitorGetMigrationParameters(qemuMonitorPtr mon, + unsigned int *level, + unsigned int *threads, + unsigned int *dthreads) +{ + VIR_DEBUG("level=%p threads=%p dthreads=%p", level, threads, dthreads); + + QEMU_CHECK_MONITOR_JSON(mon); + + if (mon->json) + return qemuMonitorJSONGetMigrationParameters(mon, level, + threads, dthreads); + else + return qemuMonitorTextGetMigrationParameters(mon, level, + threads, dthreads); +} + + +int +qemuMonitorSetMigrationParameters(qemuMonitorPtr mon, + int level, + int threads, + int dthreads) +{ + VIR_DEBUG("level=%d threads=%d dthreads=%d", level, threads, dthreads); + + QEMU_CHECK_MONITOR_JSON(mon); + + if (mon->json) + return qemuMonitorJSONSetMigrationParameters(mon, level, + threads, dthreads); + else + return qemuMonitorTextSetMigrationParameters(mon, level, + threads, dthreads); +} + + +int qemuMonitorGetMigrationStatus(qemuMonitorPtr mon, qemuMonitorMigrationStatusPtr status) { diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 52d0be7..37157ac 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -438,6 +438,16 @@ int qemuMonitorGetMigrationCacheSize(qemuMonitorPtr mon, int qemuMonitorSetMigrationCacheSize(qemuMonitorPtr mon, unsigned long long cacheSize); +int qemuMonitorGetMigrationParameters(qemuMonitorPtr mon, + unsigned int *level, + unsigned int *threads, + unsigned int *dthreads); + +int qemuMonitorSetMigrationParameters(qemuMonitorPtr mon, + int level, + int threads, + int dthreads); + enum { QEMU_MONITOR_MIGRATION_STATUS_INACTIVE, QEMU_MONITOR_MIGRATION_STATUS_ACTIVE, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index e281140..5eaf553 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -2469,6 +2469,99 @@ qemuMonitorJSONSetMigrationCacheSize(qemuMonitorPtr mon, } +int +qemuMonitorJSONGetMigrationParameters(qemuMonitorPtr mon, + unsigned int *level, + unsigned int *threads, + unsigned int *dthreads) +{ + int ret = -1; + virJSONValuePtr result; + virJSONValuePtr cmd; + virJSONValuePtr reply = NULL; + + *level = 0; + *threads = 0; + *dthreads = 0; + + cmd = qemuMonitorJSONMakeCommand("query-migrate-parameters", NULL); + if (!cmd) + return -1; + + ret = qemuMonitorJSONCommand(mon, cmd, &reply); + + if (ret == 0) + ret = qemuMonitorJSONCheckError(cmd, reply); + + if (ret < 0) + goto cleanup; + + if (!(result = virJSONValueObjectGet(reply, "return"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("query-migrate-parameters reply was missing " + "'return' data")); + goto cleanup; + } + + virJSONValueObjectGetNumberUint(result, "compress-level", level); + virJSONValueObjectGetNumberUint(result, "compress-threads", threads); + virJSONValueObjectGetNumberUint(result, "decompress-threads", dthreads); + + ret = 0; + cleanup: + virJSONValueFree(cmd); + virJSONValueFree(reply); + return ret; +} + + +int qemuMonitorJSONSetMigrationParameters(qemuMonitorPtr mon, + int level, + int threads, + int dthreads) +{ + int ret = -1; + virJSONValuePtr cmd = NULL; + virJSONValuePtr arguments = NULL; + virJSONValuePtr reply = NULL; + + if ((level < 0) && (threads < 0) && (dthreads < 0)) + return 1; + + if (!(arguments = virJSONValueNewObject())) + return -1; + + if (level > 0) + if (virJSONValueObjectAdd(arguments, "U:compress-level", level, NULL) < 0) + goto error; + + if (threads > 0) + if (virJSONValueObjectAdd(arguments, "U:compress-threads", threads, NULL) < 0) + goto error; + + if (dthreads > 0) + if (virJSONValueObjectAdd(arguments, "U:decompress-threads", dthreads, NULL) < 0) + goto error; + + if (!(cmd = qemuMonitorJSONMakeCommand("migrate-set-parameters", NULL))) + goto error; + + if (arguments) + if (virJSONValueObjectAppend(cmd, "arguments", arguments) < 0) + goto error; + + ret = qemuMonitorJSONCommand(mon, cmd, &reply); + + if (ret == 0) + ret = qemuMonitorJSONCheckError(cmd, reply); + + error: + virJSONValueFree(arguments); + virJSONValueFree(cmd); + virJSONValueFree(reply); + return ret; +} + static int qemuMonitorJSONGetMigrationStatusReply(virJSONValuePtr reply, qemuMonitorMigrationStatusPtr status) diff --git a/src/qemu/qemu_monitor_json.h b/src/qemu/qemu_monitor_json.h index 43adc90..c524040 100644 --- a/src/qemu/qemu_monitor_json.h +++ b/src/qemu/qemu_monitor_json.h @@ -126,6 +126,15 @@ int qemuMonitorJSONGetMigrationCacheSize(qemuMonitorPtr mon, int qemuMonitorJSONSetMigrationCacheSize(qemuMonitorPtr mon, unsigned long long cacheSize); +int qemuMonitorJSONGetMigrationParameters(qemuMonitorPtr mon, + unsigned int *level, + unsigned int *threads, + unsigned int *dthreads); +int qemuMonitorJSONSetMigrationParameters(qemuMonitorPtr mon, + int level, + int threads, + int dthreads); + int qemuMonitorJSONGetMigrationStatus(qemuMonitorPtr mon, qemuMonitorMigrationStatusPtr status); diff --git a/src/qemu/qemu_monitor_text.c b/src/qemu/qemu_monitor_text.c index f26bc78..3c6d341 100644 --- a/src/qemu/qemu_monitor_text.c +++ b/src/qemu/qemu_monitor_text.c @@ -1353,6 +1353,101 @@ int qemuMonitorTextSetMigrationDowntime(qemuMonitorPtr mon, } +#define MIGRATION_PARAMETER_PRIFIX "parameters: " +#define MIGRATION_PARAMETER_COMPRESS_LEVEL "compress-level: " +#define MIGRATION_PARAMETER_COMPRESS_THREADS "compress-threads: " +#define MIGRATION_PARAMETER_DECOMPRESS_THREADS "decompress-threads: " + +int qemuMonitorTextGetMigrationParameters(qemuMonitorPtr mon, + unsigned int *level, + unsigned int *threads, + unsigned int *dthreads) +{ + char *reply = NULL; + int ret = -1; + char *tmp; + char *end; + const char *cmd_name = "info migrate_parameters"; + + if (qemuMonitorHMPCommand(mon, cmd_name, &reply) < 0) + goto cleanup; + + if (qemuMonitorTextCommandNotFound(cmd_name, reply)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("Command '%s' is not found"), cmd_name); + goto cleanup; + } + + if ((tmp = strstr(reply, MIGRATION_PARAMETER_PRIFIX)) != NULL) { + tmp += strlen(MIGRATION_PARAMETER_PRIFIX); + tmp += strlen(MIGRATION_PARAMETER_COMPRESS_LEVEL); + end = strchr(tmp, ' '); + if (virStrToLong_ui(tmp, &end, 10, level) == -1) { + VIR_WARN("Unable to parse compress level '%s'", tmp); + return -1; + } + tmp = end + 1 + strlen(MIGRATION_PARAMETER_COMPRESS_THREADS); + end = strchr(tmp, ' '); + if (virStrToLong_ui(tmp, &end, 10, threads) == -1) { + VIR_WARN("Unable to parse compress threads'%s'", tmp); + return -1; + } + tmp = end + 1 + strlen(MIGRATION_PARAMETER_DECOMPRESS_THREADS); + end = strchr(tmp, '\0'); + if (virStrToLong_ui(tmp, &end, 10, dthreads) == -1) { + VIR_WARN("Unable to parse compress threads'%s'", tmp); + return -1; + } + ret = 0; + } + + cleanup: + VIR_FREE(reply); + return ret; +} + +int qemuMonitorTextSetMigrationParameters(qemuMonitorPtr mon, + int level, + int threads, + int dthreads) +{ + char *cmd = NULL; + char *info = NULL; + int ret = -1; + + if (level > 0) { + if (virAsprintf(&cmd, "migrate_set_parameter compress-level %d", + level) < 0) + goto cleanup; + + if (qemuMonitorHMPCommand(mon, cmd, &info) < 0) + goto cleanup; + } + if (threads > 0) { + if (virAsprintf(&cmd, "migrate_set_parameter compress-threads %d", + threads) < 0) + goto cleanup; + + if (qemuMonitorHMPCommand(mon, cmd, &info) < 0) + goto cleanup; + } + if (dthreads >= 0) { + if (virAsprintf(&cmd, "migrate_set_parameter compress-dthreads %d", + dthreads) < 0) + goto cleanup; + + if (qemuMonitorHMPCommand(mon, cmd, &info) < 0) + goto cleanup; + } + ret = 0; + + cleanup: + VIR_FREE(info); + VIR_FREE(cmd); + return ret; +} + + #define MIGRATION_PREFIX "Migration status: " #define MIGRATION_TRANSFER_PREFIX "transferred ram: " #define MIGRATION_REMAINING_PREFIX "remaining ram: " diff --git a/src/qemu/qemu_monitor_text.h b/src/qemu/qemu_monitor_text.h index 40edc9a..dd8984f 100644 --- a/src/qemu/qemu_monitor_text.h +++ b/src/qemu/qemu_monitor_text.h @@ -109,6 +109,16 @@ int qemuMonitorTextSetMigrationDowntime(qemuMonitorPtr mon, int qemuMonitorTextGetMigrationStatus(qemuMonitorPtr mon, qemuMonitorMigrationStatusPtr status); +int qemuMonitorTextGetMigrationParameters(qemuMonitorPtr mon, + unsigned int *level, + unsigned int *threads, + unsigned int *dthreads); + +int qemuMonitorTextSetMigrationParameters(qemuMonitorPtr mon, + int level, + int threads, + int dthreads); + int qemuMonitorTextMigrate(qemuMonitorPtr mon, unsigned int flags, const char *uri); -- 2.1.4
From: Eli Qiao <liyong.qiao@intel.com> This patch add test case for qemuMonitorJSONGetMigrationParameter. It will verify if qemu monitor json value can be parsed correctly. Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> Signed-off-by: Eli Qiao <liyong.qiao@intel.com> --- tests/qemumonitorjsontest.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/qemumonitorjsontest.c b/tests/qemumonitorjsontest.c index 0f82fd8..70c28c5 100644 --- a/tests/qemumonitorjsontest.c +++ b/tests/qemumonitorjsontest.c @@ -1620,6 +1620,59 @@ testQemuMonitorJSONqemuMonitorJSONGetBlockStatsInfo(const void *data) return ret; } +tatic int +testQemuMonitorJSONqemuMonitorJSONGetMigrationParameters(const void *data) +{ + virDomainXMLOptionPtr xmlopt = (virDomainXMLOptionPtr)data; + qemuMonitorTestPtr test = qemuMonitorTestNewSimple(true, xmlopt); + int ret = -1; + unsigned int level; + unsigned int threads; + unsigned int dthreads; + + if (!test) + return -1; + + if (qemuMonitorTestAddItem(test, "query-migrate-parameters", + "{" + " \"return\": {" + " \"decompress-threads\": 2," + " \"compress-threads\": 8," + " \"compress-level\": 1" + " }" + "}") < 0) { + goto cleanup; + } + + if (qemuMonitorJSONGetMigrationParameters(qemuMonitorTestGetMonitor(test), + &level, &threads, &dthreads) < 0) + goto cleanup; + if (dthreads != 2) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "Invalid decompress-threads: %d, expected 2", + dthreads); + goto cleanup; + } + if (threads != 8) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "Invalid decompress-threads: %d, expected 8", + threads); + goto cleanup; + } + if (level != 1) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "Invalid decompress-threads: %d, expected 1", + level); + goto cleanup; + } + ret = 0; + + cleanup: + qemuMonitorTestFree(test); + return ret; +} + + static int testQemuMonitorJSONqemuMonitorJSONGetMigrationCacheSize(const void *data) { -- 2.1.4
just support setting these params for Migrate3Full protocal. Signed-off-by: Eli Qiao <liyong.qiao@intel.com> Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> --- include/libvirt/libvirt-domain.h | 24 ++++++++++++++++++++ src/qemu/qemu_domain.h | 3 +++ src/qemu/qemu_driver.c | 28 +++++++++++++++++++++++ src/qemu/qemu_migration.c | 49 ++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_migration.h | 9 ++++++++ 5 files changed, 113 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index d851225..1c6ab16 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -2791,6 +2791,30 @@ int virDomainAbortJob(virDomainPtr dom); /** + * VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL: + * + * .domainMigrateGetParameters field: level of compression when live migration, + * as VIR_TYPED_PARAM_UINT. + */ +# define VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL "migrate.compress-level" + +/** + * VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS: + * + * .domainMigrateGetParameters field: numbers of compression threads when live + * migration, as VIR_TYPED_PARAM_UINT. + */ +# define VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS "migrate.compress-threads" + +/** + * VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS: + * + * .domainMigrateGetParameters field: numbers of decompression threads when + * live migration, as VIR_TYPED_PARAM_UINT. + */ +# define VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS "migrate.decompress-threads" + +/** * virConnectDomainEventGenericCallback: * @conn: the connection pointer * @dom: the domain pointer diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index a6df199..53af176 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -48,6 +48,9 @@ # define QEMU_DOMAIN_MIG_BANDWIDTH_MAX (INT64_MAX / (1024 * 1024)) # endif +# define QEMU_DOMAIN_MIG_PARAMETERS_LEVEL_MAX 9 +# define QEMU_DOMAIN_MIG_PARAMETERS_THREADS_MAX 255 + # define JOB_MASK(job) (1 << (job - 1)) # define QEMU_JOB_DEFAULT_MASK \ (JOB_MASK(QEMU_JOB_QUERY) | \ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index d1b00a2..498bcac 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -12599,6 +12599,7 @@ qemuDomainMigrateBegin3Params(virDomainPtr domain, { const char *xmlin = NULL; const char *dname = NULL; + virQEMUDriverPtr driver = domain->conn->privateData; virDomainObjPtr vm; virCheckFlags(QEMU_MIGRATION_FLAGS, NULL); @@ -12620,6 +12621,10 @@ qemuDomainMigrateBegin3Params(virDomainPtr domain, virDomainObjEndAPI(&vm); return NULL; } + if (qemuMigrationPrepareExtraParams(driver, vm, params, nparams) < 0) { + virDomainObjEndAPI(&vm); + return NULL; + } return qemuMigrationBegin(domain->conn, vm, xmlin, dname, cookieout, cookieoutlen, flags); @@ -12688,11 +12693,14 @@ qemuDomainMigratePrepare3Params(virConnectPtr dconn, virQEMUDriverPtr driver = dconn->privateData; virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver); virDomainDefPtr def = NULL; + virDomainObjPtr vm = NULL; const char *dom_xml = NULL; const char *dname = NULL; const char *uri_in = NULL; const char *listenAddress = cfg->migrationAddress; char *origname = NULL; + unsigned char uuid[VIR_UUID_BUFLEN]; + char *uuidstr = NULL; int ret = -1; virCheckFlagsGoto(QEMU_MIGRATION_FLAGS, cleanup); @@ -12726,6 +12734,8 @@ qemuDomainMigratePrepare3Params(virConnectPtr dconn, if (!(def = qemuMigrationPrepareDef(driver, dom_xml, dname, &origname))) goto cleanup; + memcpy(uuid, def->uuid, VIR_UUID_BUFLEN); + if (virDomainMigratePrepare3ParamsEnsureACL(dconn, def) < 0) goto cleanup; @@ -12734,6 +12744,24 @@ qemuDomainMigratePrepare3Params(virConnectPtr dconn, cookieout, cookieoutlen, uri_in, uri_out, &def, origname, listenAddress, flags); + if (ret < 0) + goto cleanup; + + if (!(vm = virDomainObjListFindByUUID(driver->domains, uuid))) { + virUUIDFormat(uuid, uuidstr); + virReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s'"), uuidstr); + goto cleanup; + } + /* only domain start, we can set these prarams for it. */ + ret = qemuMigrationPrepareExtraParams(driver, vm, params, nparams); + if (ret < 0){ + /* we need to stop the vm and realse the resource. */ + virDomainAuditStart(vm, "migrated", false); + qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, 0); + qemuMigrationJobFinish(driver, vm); + } + virObjectUnlock(vm); cleanup: VIR_FREE(origname); diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 3322f72..4ca4db2 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -3290,6 +3290,55 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, /* + * First version just prepare the multi-thread compress params. + * we can extend more params here for live migration + */ +int +qemuMigrationPrepareExtraParams(virQEMUDriverPtr driver, + virDomainObjPtr vm, + virTypedParameterPtr params, + int nparams) +{ + qemuDomainObjPrivatePtr priv = NULL; + int level = -1; + int threads = -1; + int dthreads = -1; + int ret = -1; + + if (virTypedParamsGetInt(params, nparams, + VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL, &level) < 0) + return -1; + if (virTypedParamsGetInt(params, nparams, + VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS, &threads) < 0) + return -1; + if (virTypedParamsGetInt(params, nparams, + VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS, &dthreads) < 0) + return -1; + + if (level > QEMU_DOMAIN_MIG_PARAMETERS_LEVEL_MAX) { + virReportError(VIR_ERR_OVERFLOW, + _("level must be less than %d"), + QEMU_DOMAIN_MIG_PARAMETERS_LEVEL_MAX + 1); + return -1; + } + if ((threads > QEMU_DOMAIN_MIG_PARAMETERS_THREADS_MAX) || + (dthreads > QEMU_DOMAIN_MIG_PARAMETERS_THREADS_MAX)) { + virReportError(VIR_ERR_OVERFLOW, _("both compress and decompress " + "threads number must be less than %d"), + QEMU_DOMAIN_MIG_PARAMETERS_THREADS_MAX + 1); + return -1; + } + + priv = vm->privateData; + qemuDomainObjEnterMonitor(driver, vm); + ret = qemuMonitorSetMigrationParameters(priv->mon, level, threads, dthreads); + if (qemuDomainObjExitMonitor(driver, vm) < 0) + return -1; + return ret; +} + + +/* * This version starts an empty VM listening on a localhost TCP port, and * sets up the corresponding virStream to handle the incoming data. */ diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 065623b..54b6a32 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -52,6 +52,9 @@ VIR_MIGRATE_PARAM_BANDWIDTH, VIR_TYPED_PARAM_ULLONG, \ VIR_MIGRATE_PARAM_GRAPHICS_URI, VIR_TYPED_PARAM_STRING, \ VIR_MIGRATE_PARAM_LISTEN_ADDRESS, VIR_TYPED_PARAM_STRING, \ + VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL, VIR_TYPED_PARAM_INT, \ + VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS, VIR_TYPED_PARAM_INT, \ + VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS,VIR_TYPED_PARAM_INT, \ NULL @@ -131,6 +134,12 @@ int qemuMigrationPrepareDirect(virQEMUDriverPtr driver, const char *listenAddress, unsigned long flags); +int +qemuMigrationPrepareExtraParams(virQEMUDriverPtr driver, + virDomainObjPtr vm, + virTypedParameterPtr params, + int nparams); + int qemuMigrationPerform(virQEMUDriverPtr driver, virConnectPtr conn, virDomainObjPtr vm, -- 2.1.4
Signed-off-by: Eli Qiao <liyong.qiao@intel.com> Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> --- include/libvirt/libvirt-domain.h | 1 + tools/virsh-domain.c | 54 ++++++++++++++++++++++++++++++++++++++++ tools/virsh.pod | 22 +++++++++++----- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 1c6ab16..29b2b22 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -659,6 +659,7 @@ typedef enum { VIR_MIGRATE_ABORT_ON_ERROR = (1 << 12), /* abort migration on I/O errors happened during migration */ VIR_MIGRATE_AUTO_CONVERGE = (1 << 13), /* force convergence */ VIR_MIGRATE_RDMA_PIN_ALL = (1 << 14), /* RDMA memory pinning */ + VIR_MIGRATE_MT_COMPRESSED = (1 << 15), /* multiple threads compression during migration */ } virDomainMigrateFlags; diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 91a1ca2..b5713cc 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -9853,6 +9853,10 @@ static const vshCmdOptDef opts_migrate[] = { .type = VSH_OT_BOOL, .help = N_("compress repeated pages during live migration") }, + {.name = "multi-thread-compress", + .type = VSH_OT_BOOL, + .help = N_("enable multi-thread compression during live migration") + }, {.name = "auto-converge", .type = VSH_OT_BOOL, .help = N_("force convergence during live migration") @@ -9889,6 +9893,21 @@ static const vshCmdOptDef opts_migrate[] = { .type = VSH_OT_STRING, .help = N_("filename containing updated XML for the target") }, + {.name = "compress-level", + .type = VSH_OT_INT, + .help = N_("compres level, from 0-9, 0 means use default level. " + "'--multi-thread-compress' option will set as true with this option.") + }, + {.name = "compress-threads", + .type = VSH_OT_INT, + .help = N_("compres thread count for multi-thread migration, from 0-255. " + "'--multi-thread-compress' option will set as true with this option.") + }, + {.name = "decompress-threads", + .type = VSH_OT_INT, + .help = N_("decompres thread count for multi-thread migration, from 0-255. " + "'--multi-thread-compress' option will set as true with this option.") + }, {.name = NULL} }; @@ -9907,6 +9926,8 @@ doMigrate(void *opaque) virTypedParameterPtr params = NULL; int nparams = 0; int maxparams = 0; + int value = 0; + int rv; virConnectPtr dconn = data->dconn; sigemptyset(&sigmask); @@ -9948,6 +9969,36 @@ doMigrate(void *opaque) VIR_MIGRATE_PARAM_DEST_NAME, opt) < 0) goto save_error; + if ((rv = vshCommandOptInt(cmd, "compress-level", &value)) < 0) { + goto save_error; + } else if (rv > 0) { + if (virTypedParamsAddInt(¶ms, &nparams, &maxparams, + VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL, + value) < 0) + goto save_error; + flags |= VIR_MIGRATE_MT_COMPRESSED; + } + + if ((rv = vshCommandOptInt(cmd, "compress-threads", &value)) < 0) { + goto save_error; + } else if (rv > 0) { + if (virTypedParamsAddInt(¶ms, &nparams, &maxparams, + VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS, + value) < 0) + goto save_error; + flags |= VIR_MIGRATE_MT_COMPRESSED; + } + + if ((rv = vshCommandOptInt(cmd, "decompress-threads", &value)) < 0) { + goto save_error; + } else if (rv > 0) { + if (virTypedParamsAddInt(¶ms, &nparams, &maxparams, + VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS, + value) < 0) + goto save_error; + flags |= VIR_MIGRATE_MT_COMPRESSED; + } + if (vshCommandOptStringReq(ctl, cmd, "xml", &opt) < 0) goto out; if (opt) { @@ -9996,6 +10047,9 @@ doMigrate(void *opaque) if (vshCommandOptBool(cmd, "compressed")) flags |= VIR_MIGRATE_COMPRESSED; + if (vshCommandOptBool(cmd, "multi-thread-compress")) + flags |= VIR_MIGRATE_MT_COMPRESSED; + if (vshCommandOptBool(cmd, "auto-converge")) flags |= VIR_MIGRATE_AUTO_CONVERGE; diff --git a/tools/virsh.pod b/tools/virsh.pod index d588e5a..b81fd01 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -1518,9 +1518,11 @@ to the I<uri> namespace is displayed instead of being modified. =item B<migrate> [I<--live>] [I<--offline>] [I<--direct>] [I<--p2p> [I<--tunnelled>]] [I<--persistent>] [I<--undefinesource>] [I<--suspend>] [I<--copy-storage-all>] [I<--copy-storage-inc>] [I<--change-protection>] [I<--unsafe>] [I<--verbose>] -[I<--compressed>] [I<--abort-on-error>] [I<--auto-converge>] -I<domain> I<desturi> [I<migrateuri>] [I<graphicsuri>] [I<listen-address>] -[I<dname>] [I<--timeout> B<seconds>] [I<--xml> B<file>] +[I<--compressed>] [I<--multi-thread-compress>] [I<--abort-on-error>] +[I<--auto-converge>] I<domain> I<desturi> [I<migrateuri>] [I<graphicsuri>] +[I<listen-address>] [I<dname>] [I<--timeout> B<seconds>] [I<--xml> B<file>] +[I<--compress-level> B<number>] [I<--compress-threads> B<number>] +[I<--decompress-threads> B<number>] Migrate domain to another host. Add I<--live> for live migration; <--p2p> for peer-2-peer migration; I<--direct> for direct migration; or I<--tunnelled> @@ -1542,9 +1544,10 @@ is implicitly enabled when supported by the hypervisor, but can be explicitly used to reject the migration if the hypervisor lacks change protection support. I<--verbose> displays the progress of migration. I<--compressed> activates compression of memory pages that have to be transferred repeatedly -during live migration. I<--abort-on-error> cancels the migration if a soft -error (for example I/O error) happens during the migration. I<--auto-converge> -forces convergence during live migration. +during live migration. I<--multi-thread-compress> activates multi-thread +compression during live migration. I<--abort-on-error> cancels the migration +if a soft error (for example I/O error) happens during the migration. +I<--auto-converge> forces convergence during live migration. B<Note>: Individual hypervisors usually do not support all possible types of migration. For example, QEMU does not support direct migration. @@ -1564,6 +1567,13 @@ the destination to supply a larger set of changes to any host-specific portions of the domain XML, such as accounting for naming differences between source and destination in accessing underlying storage. +I<--compress-level> is interpreted as an unsigned int value from 0-9. O means +hypervisor choose an default value. Specifying a negative value results in an +essentially unlimited value being provided to the hypervisor. The hypervisor +can choose whether to reject the value or convert it to the maximum value +allowed. I<--compress-threads> is interpreted as an unsigned int value. +I<--decompress-threads> is interpreted as an unsigned int value. + I<--timeout> B<seconds> forces guest to suspend when live migration exceeds that many seconds, and then the migration will complete offline. It can only be used with I<--live>. -- 2.1.4
Define helper function virdomainMigrateSetParameters and virdomainMigrateGetParameters, which allow to or get multi-thread compress parameters. include/libvirt/libvirt-domain.h * Define virdomainMigrateSetParameters, virdomainMigrateGetParameters src/driver-hypervisor.h: * Define domainMigrateSetParameters, domainMigrateGetParameters src/libvirt-domain.c: * Implement virdomainMigrateSetParameters * Implement virdomainMigrateGetParameters src/libvirt_public.syms: * Export the new symbols Signed-off-by: Eli Qiao <liyong.qiao@intel.com> Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> --- include/libvirt/libvirt-domain.h | 10 +++- src/driver-hypervisor.h | 14 +++++ src/libvirt-domain.c | 110 +++++++++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 ++ 4 files changed, 137 insertions(+), 2 deletions(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 29b2b22..092d4f1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -799,6 +799,14 @@ int virDomainMigrateGetMaxSpeed(virDomainPtr domain, unsigned long *bandwidth, unsigned int flags); +int virDomainMigrateSetParameters(virDomainPtr dom, + virTypedParameterPtr params, + int nparams, unsigned int flags); + +int virDomainMigrateGetParameters(virDomainPtr dom, + virTypedParameterPtr params, + int *nparams, unsigned int flags); + char * virConnectGetDomainCapabilities(virConnectPtr conn, const char *emulatorbin, const char *arch, @@ -2789,8 +2797,6 @@ int virDomainAbortJob(virDomainPtr dom); */ # define VIR_DOMAIN_JOB_COMPRESSION_OVERFLOW "compression_overflow" - - /** * VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL: * diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 3275343..7e5f619 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -712,6 +712,18 @@ typedef int unsigned int flags); typedef int +(*virDrvDomainMigrateSetParameters)(virDomainPtr dom, + virTypedParameterPtr params, + int nparams, + unsigned int flags); + +typedef int +(*virDrvDomainMigrateGetParameters)(virDomainPtr dom, + virTypedParameterPtr params, + int *nparams, + unsigned int flags); + +typedef int (*virDrvConnectDomainEventRegisterAny)(virConnectPtr conn, virDomainPtr dom, int eventID, @@ -1359,6 +1371,8 @@ struct _virHypervisorDriver { virDrvDomainMigrateSetCompressionCache domainMigrateSetCompressionCache; virDrvDomainMigrateGetMaxSpeed domainMigrateGetMaxSpeed; virDrvDomainMigrateSetMaxSpeed domainMigrateSetMaxSpeed; + virDrvDomainMigrateSetParameters domainMigrateSetParameters; + virDrvDomainMigrateGetParameters domainMigrateGetParameters; virDrvConnectDomainEventRegisterAny connectDomainEventRegisterAny; virDrvConnectDomainEventDeregisterAny connectDomainEventDeregisterAny; virDrvDomainManagedSave domainManagedSave; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 2edba1a..c704af7 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -9165,6 +9165,116 @@ virDomainMigrateGetMaxSpeed(virDomainPtr domain, /** + * virDomainMigrateSetParameters: + * @dom: a domain object + * @params: pointer to memory parameter objects + * @nparams: number of live mirgration compress parameter (this value can be + * the same or less than the number of parameters supported) + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * The parameters which will be used when doing live migration if we enable + * multi-thread compression. Currently it supports compress-level, + * compress-threads, decompress-threads. Not all hypervisors + * will support multi-thread compression. + * + * Returns 0 in case of success, -1 otherwise. + */ +int +virDomainMigrateSetParameters(virDomainPtr dom, + virTypedParameterPtr params, + int nparams, unsigned int flags) +{ + virConnectPtr conn; + + VIR_DOMAIN_DEBUG(dom, "miagrateion set parameters " + "params=%p, nparams=%d, flags=%x", params, nparams, flags); + VIR_TYPED_PARAMS_DEBUG(params, nparams); + + virResetLastError(); + + virCheckDomainReturn(dom, -1); + conn = dom->conn; + + virCheckReadOnlyGoto(conn->flags, error); + virCheckPositiveArgGoto(nparams, error); + virCheckNonNullArgGoto(params, error); + + if (virTypedParameterValidateSet(dom->conn, params, nparams) < 0) + goto error; + + if (conn->driver->domainMigrateSetParameters) { + if (conn->driver->domainMigrateSetParameters( + dom, params, nparams, flags) < 0) + goto error; + return 0; + } + + virReportUnsupportedError(); + + error: + virDispatchError(conn); + return -1; +} + + +/** + * virDomainMigrateGetParameters: + * @dom: a domain object + * @params: pointer to memory parameter object + * (return value, allocated by the caller) + * @nparams: number of live mirgration compress parameter (this value can be + * the same or less than the number of parameters supported) + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Get parameters which will be used when doing live migration if we enable + * multi-thread compression. Currently it supports compress-level, + * compress-threads, decompress-threads. Not all hypervisors will + * support multi-thread compression. + * + * Returns 0 in case of success, -1 otherwise. + */ +int +virDomainMigrateGetParameters(virDomainPtr dom, + virTypedParameterPtr params, + int *nparams, unsigned int flags) +{ + virConnectPtr conn; + + VIR_DOMAIN_DEBUG(dom, "miagrateion get parameters " + "params=%p, nparams=%d, flags=%x", + params, (nparams) ? *nparams : -1, flags); + + virResetLastError(); + + virCheckDomainReturn(dom, -1); + conn = dom->conn; + virCheckNonNullArgGoto(nparams, error); + virCheckNonNegativeArgGoto(*nparams, error); + if (*nparams != 0) + virCheckNonNullArgGoto(params, error); + + if (VIR_DRV_SUPPORTS_FEATURE(dom->conn->driver, dom->conn, + VIR_DRV_FEATURE_TYPED_PARAM_STRING)) + flags |= VIR_TYPED_PARAM_STRING_OKAY; + + + virCheckReadOnlyGoto(conn->flags, error); + + if (conn->driver->domainMigrateGetParameters) { + if (conn->driver->domainMigrateGetParameters( + dom, params, nparams, flags) < 0) + goto error; + return 0; + } + + virReportUnsupportedError(); + error: + virDispatchError(conn); + return -1; +} + + +/** * virConnectDomainEventRegisterAny: * @conn: pointer to the connection * @dom: pointer to the domain diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index 716dd2f..b7a3cad 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -715,4 +715,9 @@ LIBVIRT_1.2.16 { virDomainSetUserPassword; } LIBVIRT_1.2.15; +LIBVIRT_1.2.17 { + global: + virDomainMigrateSetParameters; + virDomainMigrateGetParameters; +} LIBVIRT_1.2.16; # .... define new API here using predicted next version number .... -- 2.1.4
From: Eli Qiao <liyong.qiao@intel.com> Add remote support for the set/get migration parameters API's Signed-off-by: Eli Qiao <liyong.qiao@intel.com> Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> --- daemon/remote.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ src/remote/remote_driver.c | 54 ++++++++++++++++++++++++++++++++++++++ src/remote/remote_protocol.x | 30 ++++++++++++++++++++- src/remote_protocol-structs | 26 +++++++++++++++++++ 4 files changed, 171 insertions(+), 1 deletion(-) diff --git a/daemon/remote.c b/daemon/remote.c index e259a76..7e4838f 100644 --- a/daemon/remote.c +++ b/daemon/remote.c @@ -5642,6 +5642,68 @@ remoteDispatchDomainMigrateFinish3Params(virNetServerPtr server ATTRIBUTE_UNUSED static int +remoteDispatchDomainMigrateGetParameters(virNetServerPtr server ATTRIBUTE_UNUSED, + virNetServerClientPtr client ATTRIBUTE_UNUSED, + virNetMessagePtr hdr ATTRIBUTE_UNUSED, + virNetMessageErrorPtr rerr, + remote_domain_migrate_get_parameters_args *args, + remote_domain_migrate_get_parameters_ret *ret) +{ + virDomainPtr dom = NULL; + int rv = -1; + virTypedParameterPtr params = NULL; + int nparams = 0; + struct daemonClientPrivate *priv = + virNetServerClientGetPrivateData(client); + + if (!priv->conn) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("connection not open")); + goto cleanup; + } + + if (args->nparams > REMOTE_DOMAIN_MIGRATE_PARAM_LIST_MAX) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("nparams too large")); + goto cleanup; + } + + if (args->nparams && VIR_ALLOC_N(params, args->nparams) < 0) + goto cleanup; + nparams = args->nparams; + + if (!(dom = get_nonnull_domain(priv->conn, args->dom))) + goto cleanup; + + if (virDomainMigrateGetParameters(dom, params, &nparams, args->flags) < 0) + goto cleanup; + + /* In this case, we need to send back the number of parameters + * supported + */ + if (args->nparams == 0) { + ret->nparams = nparams; + goto success; + } + + /* Serialise the migration parameters. */ + if (remoteSerializeTypedParameters(params, nparams, + &ret->params.params_val, + &ret->params.params_len, + args->flags) < 0) + goto cleanup; + + success: + rv = 0; + + cleanup: + if (rv < 0) + virNetMessageSaveError(rerr); + virTypedParamsFree(params, nparams); + virObjectUnref(dom); + return rv; +} + + +static int remoteDispatchDomainMigrateConfirm3Params(virNetServerPtr server ATTRIBUTE_UNUSED, virNetServerClientPtr client ATTRIBUTE_UNUSED, virNetMessagePtr msg ATTRIBUTE_UNUSED, diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index dd8dab6..4ed87dc 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -7442,6 +7442,58 @@ remoteDomainMigrateConfirm3Params(virDomainPtr domain, } +static int remoteDomainMigrateGetParameters(virDomainPtr domain, + virTypedParameterPtr params, + int *nparams, + unsigned int flags) +{ + int rv = -1; + remote_domain_migrate_get_parameters_args args; + remote_domain_migrate_get_parameters_ret ret; + struct private_data *priv = domain->conn->privateData; + + remoteDriverLock(priv); + make_nonnull_domain(&args.dom, domain); + args.nparams = *nparams; + args.flags = flags; + + memset(&ret, 0, sizeof(ret)); + + if (call(domain->conn, priv, 0, REMOTE_PROC_DOMAIN_MIGRATE_GET_PARAMETERS, + (xdrproc_t) xdr_remote_domain_migrate_get_parameters_args, + (char *) &args, + (xdrproc_t) xdr_remote_domain_migrate_get_parameters_ret, + (char *) &ret) == -1) { + goto done; + } + + /* Handle the case when the caller does not know the number of parameters + * and is asking for the number of parameters supported + */ + if (*nparams == 0) { + *nparams = ret.nparams; + rv = 0; + goto cleanup; + } + + if (remoteDeserializeTypedParameters(ret.params.params_val, + ret.params.params_len, + REMOTE_DOMAIN_MIGRATE_PARAM_LIST_MAX, + ¶ms, + nparams) < 0) + goto cleanup; + + rv = 0; + + cleanup: + xdr_free((xdrproc_t) xdr_remote_domain_migrate_get_parameters_ret, + (char *) &ret); + done: + remoteDriverUnlock(priv); + return rv; +} + + static virDomainPtr remoteDomainCreateXMLWithFiles(virConnectPtr conn, const char *xml_desc, unsigned int nfiles, int *files, unsigned int flags) @@ -8307,6 +8359,8 @@ static virHypervisorDriver hypervisor_driver = { .domainMigrateSetMaxDowntime = remoteDomainMigrateSetMaxDowntime, /* 0.8.0 */ .domainMigrateGetCompressionCache = remoteDomainMigrateGetCompressionCache, /* 1.0.3 */ .domainMigrateSetCompressionCache = remoteDomainMigrateSetCompressionCache, /* 1.0.3 */ + .domainMigrateGetParameters = remoteDomainMigrateGetParameters, /* 1.2.17 */ + .domainMigrateSetParameters = remoteDomainMigrateSetParameters, /* 1.2.17 */ .domainMigrateSetMaxSpeed = remoteDomainMigrateSetMaxSpeed, /* 0.9.0 */ .domainMigrateGetMaxSpeed = remoteDomainMigrateGetMaxSpeed, /* 0.9.5 */ .connectDomainEventRegisterAny = remoteConnectDomainEventRegisterAny, /* 0.8.0 */ diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 9f1be6b..659ea3b 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -2342,6 +2342,22 @@ struct remote_domain_migrate_get_max_speed_ret { unsigned hyper bandwidth; /* insert@1 */ }; +struct remote_domain_migrate_set_parameters_args { + remote_nonnull_domain dom; + remote_typed_param params<REMOTE_DOMAIN_MIGRATE_PARAM_LIST_MAX>; + unsigned int flags; +}; + +struct remote_domain_migrate_get_parameters_args { + remote_nonnull_domain dom; + int nparams; + unsigned int flags; +}; + +struct remote_domain_migrate_get_parameters_ret { + remote_typed_param params<REMOTE_DOMAIN_MIGRATE_PARAM_LIST_MAX>; + int nparams; +}; struct remote_connect_domain_event_register_any_args { int eventID; @@ -5696,5 +5712,17 @@ enum remote_procedure { * @generate:both * @acl: domain:set_password */ - REMOTE_PROC_DOMAIN_SET_USER_PASSWORD = 357 + REMOTE_PROC_DOMAIN_SET_USER_PASSWORD = 357, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_SET_PARAMETERS = 358, + + /** + * @generate: none + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_GET_PARAMETERS = 359 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index 48c3bd8..fd39ae8 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -1775,6 +1775,30 @@ struct remote_domain_migrate_get_max_speed_args { struct remote_domain_migrate_get_max_speed_ret { uint64_t bandwidth; }; + +struct remote_domain_migrate_set_parameters_args { + remote_nonnull_domain dom; + struct { + u_int params_len; + remote_typed_param * params_val; + } params; + u_int flags; +}; + +struct remote_domain_migrate_get_parameters_args { + remote_nonnull_domain dom; + int nparams; + u_int flags; +}; + +struct remote_domain_migrate_get_parameters_ret { + struct { + u_int params_len; + remote_typed_param * params_val; + } params; + int nparams; +}; + struct remote_connect_domain_event_register_any_args { int eventID; }; @@ -3042,4 +3066,6 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_ADD_IOTHREAD = 355, REMOTE_PROC_DOMAIN_DEL_IOTHREAD = 356, REMOTE_PROC_DOMAIN_SET_USER_PASSWORD = 357, + REMOTE_PROC_DOMAIN_MIGRATE_SET_PARAMETERS = 358, + REMOTE_PROC_DOMAIN_MIGRATE_GET_PARAMETERS = 359, }; -- 2.1.4
From: Eli Qiao <liyong.qiao@intel.com> Add qemuDomainMigrateGetParameters and qemuDomainMigrateSetParameters in order to set or get multi-thread compress parameters. Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> Signed-off-by: Eli Qiao <liyong.qiao@intel.com> --- src/qemu/qemu_driver.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 498bcac..16a8b44 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -110,6 +110,8 @@ VIR_LOG_INIT("qemu.qemu_driver"); #define QEMU_NB_BLOCK_IO_TUNE_PARAM 6 #define QEMU_NB_BLOCK_IO_TUNE_PARAM_MAX 13 +#define QEMU_NB_MIGRATE_COMPRESS_PARAM 3 + #define QEMU_NB_NUMA_PARAM 2 #define QEMU_SCHED_MIN_PERIOD 1000LL @@ -13745,6 +13747,160 @@ qemuDomainMigrateGetMaxSpeed(virDomainPtr dom, return ret; } +static int +qemuDomainMigrateSetParameters(virDomainPtr dom, + virTypedParameterPtr params, + int nparams, + unsigned int flags) +{ + size_t i; + virQEMUDriverPtr driver = dom->conn->privateData; + virDomainObjPtr vm; + qemuDomainObjPrivatePtr priv; + int ret = -1; + int level = -1; + int threads = -1; + int dthreads = -1; + + virCheckFlags(VIR_TYPED_PARAM_STRING_OKAY, -1); + + if (!(vm = qemuDomObjFromDomain(dom))) + goto cleanup; + + if (virTypedParamsValidate(params, nparams, + VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL, + VIR_TYPED_PARAM_INT, + VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS, + VIR_TYPED_PARAM_INT, + VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS, + VIR_TYPED_PARAM_INT, + NULL) < 0) + goto cleanup; + + if (virDomainMigrateSetParametersEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + priv = vm->privateData; + + for (i = 0; i < nparams; i++) { + virTypedParameterPtr param = ¶ms[i]; + + if (STREQ(param->field, VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL)) { + level = params[i].value.i; + } else if (STREQ(param->field, VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS)) { + threads = params[i].value.i; + } else if (STREQ(param->field, VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS)) { + dthreads = params[i].value.i; + } + } + + if (level > QEMU_DOMAIN_MIG_PARAMETERS_LEVEL_MAX) { + virReportError(VIR_ERR_OVERFLOW, + _("level must be less than %d"), + QEMU_DOMAIN_MIG_PARAMETERS_LEVEL_MAX + 1); + goto cleanup; + } else if (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_IN) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("compress level is invalid for destination " + "hypervisor during live migration")); + goto cleanup; + } + + if ((threads > QEMU_DOMAIN_MIG_PARAMETERS_THREADS_MAX) || + (dthreads > QEMU_DOMAIN_MIG_PARAMETERS_THREADS_MAX)) { + virReportError(VIR_ERR_OVERFLOW, _("both compress and decompress " + "threads number must be less than %d"), + QEMU_DOMAIN_MIG_PARAMETERS_THREADS_MAX + 1); + goto cleanup; + } else if ((priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_IN) || + (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("neither compress nor decompress threads number " + "setting is valid during live migration")); + goto cleanup; + } + + VIR_DEBUG("Setting migration multi-thread compress parameters: " + "level is %d, compress threads is %d, " + "decompress threads is %d", level, threads, dthreads); + + qemuDomainObjEnterMonitor(driver, vm); + ret = qemuMonitorSetMigrationParameters(priv->mon, level, threads, dthreads); + + if (qemuDomainObjExitMonitor(driver, vm) < 0) + ret = -1; + ret = 0; + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + + +static int +qemuDomainMigrateGetParameters(virDomainPtr dom, + virTypedParameterPtr params, + int *nparams, + unsigned int flags) +{ + virDomainObjPtr vm; + virQEMUDriverPtr driver = dom->conn->privateData; + qemuDomainObjPrivatePtr priv; + int ret = -1; + unsigned int level = 0; + unsigned int threads = 0; + unsigned int dthreads = 0; + + + virCheckFlags(VIR_TYPED_PARAM_STRING_OKAY, -1); + + if (!(vm = qemuDomObjFromDomain(dom))) + goto cleanup; + + priv = vm->privateData; + + if (virDomainMigrateGetParametersEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if ((*nparams) == 0) { + *nparams = QEMU_NB_MIGRATE_COMPRESS_PARAM; + ret = 0; + goto cleanup; + } + + qemuDomainObjEnterMonitor(driver, vm); + /* now qemu only support 3 compression parameters, + and compession method such as zlib will support in later version. */ + ret = qemuMonitorGetMigrationParameters(priv->mon, &level, &threads, &dthreads) < 0; + if (qemuDomainObjExitMonitor(driver, vm) < 0) + ret = -1; + if (ret < 0) + goto cleanup; + + if (virTypedParameterAssign(params, + VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL, + VIR_TYPED_PARAM_UINT, + level) < 0) + goto cleanup; + + if (virTypedParameterAssign(params + 1, + VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS, + VIR_TYPED_PARAM_UINT, + threads) < 0) + goto cleanup; + + if (virTypedParameterAssign(params + 2, + VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS, + VIR_TYPED_PARAM_UINT, + dthreads) < 0) + goto cleanup; + + ret = 0; + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} typedef enum { VIR_DISK_CHAIN_NO_ACCESS, @@ -20315,6 +20471,8 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainMigrateSetCompressionCache = qemuDomainMigrateSetCompressionCache, /* 1.0.3 */ .domainMigrateSetMaxSpeed = qemuDomainMigrateSetMaxSpeed, /* 0.9.0 */ .domainMigrateGetMaxSpeed = qemuDomainMigrateGetMaxSpeed, /* 0.9.5 */ + .domainMigrateGetParameters = qemuDomainMigrateGetParameters, /* 1.2.17 */ + .domainMigrateSetParameters = qemuDomainMigrateSetParameters, /* 1.2.17 */ .connectDomainEventRegisterAny = qemuConnectDomainEventRegisterAny, /* 0.8.0 */ .connectDomainEventDeregisterAny = qemuConnectDomainEventDeregisterAny, /* 0.8.0 */ .domainManagedSave = qemuDomainManagedSave, /* 0.8.0 */ -- 2.1.4
From: Eli Qiao <liyong.qiao@intel.com> Add command to allow set and get multi-thread migration parameters for a domain. Signed-off-by: ShaoHe Feng <shaohe.feng@intel.com> Signed-off-by: Eli Qiao <liyong.qiao@intel.com> --- tools/virsh-domain.c | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++- tools/virsh.pod | 14 +++++ 2 files changed, 184 insertions(+), 1 deletion(-) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index b5713cc..11ac27d 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -10301,7 +10301,7 @@ static const vshCmdInfo info_migrate_setspeed[] = { }, {.name = "desc", .data = N_("Set the maximum migration bandwidth (in MiB/s) for a domain " - "which is being migrated to another host.") + "before migrating another host.") }, {.name = NULL} }; @@ -10392,6 +10392,163 @@ cmdMigrateGetMaxSpeed(vshControl *ctl, const vshCmd *cmd) } /* + * "migrate-setparameters" command + */ +static const vshCmdInfo info_migrate_setparameters[] = { + {.name = "help", + .data = N_("Set the multi-thread migration parameters") + }, + {.name = "desc", + .data = N_("Set the multi-thread migration parameters for a domain " + "which is being migrated to another host.") + }, + {.name = NULL} +}; + +static const vshCmdOptDef opts_migrate_setparameters[] = { + {.name = "domain", + .type = VSH_OT_DATA, + .flags = VSH_OFLAG_REQ, + .help = N_("domain name, id or uuid") + }, + {.name = "level", + .type = VSH_OT_INT, + .help = N_("compression level, from 1-9, 0 means use default level") + }, + {.name = "threads", + .type = VSH_OT_INT, + .help = N_("compression thread count for multi-thread migration") + }, + {.name = "dthreads", + .type = VSH_OT_INT, + .help = N_("decompression thread count for multi-thread migration") + }, + {.name = NULL} +}; + +static bool +cmdMigrateSetParameters(vshControl *ctl, const vshCmd *cmd) +{ + virDomainPtr dom = NULL; + virTypedParameterPtr params = NULL; + int flags = 0; + unsigned int level = 0; + unsigned int threads = 0; + unsigned int dthreads = 0; + int nparams = 0; + int maxparams = 0; + bool ret = false; + int rv = 0; + + if (!(dom = vshCommandOptDomain(ctl, cmd, NULL))) + return false; + + if ((rv = vshCommandOptUIntWrap(cmd, "level", &level)) < 0) { + vshError(ctl, + _("Numeric value for <%s> option is malformed or out of range"), + "level"); + goto done; + } else if (rv > 0) { + if (virTypedParamsAddUInt(¶ms, &nparams, &maxparams, + VIR_DOMAIN_MIRGRATE_COMPRESSION_LEVEL, + level) < 0) + goto done; + } + + if ((rv = vshCommandOptUIntWrap(cmd, "threads", &threads)) < 0) { + vshError(ctl, + _("Numeric value for <%s> option is malformed or out of range"), + "threads"); + goto done; + } else if (rv > 0) { + if (virTypedParamsAddUInt(¶ms, &nparams, &maxparams, + VIR_DOMAIN_MIRGRATE_COMPRESSION_THREADS, + threads) < 0) + goto done; + } + + if ((rv = vshCommandOptUIntWrap(cmd, "dthreads", &dthreads)) < 0) { + vshError(ctl, + _("Numeric value for <%s> option is malformed or out of range"), + "dthreads"); + goto done; + } else if (rv > 0) { + if (virTypedParamsAddUInt(¶ms, &nparams, &maxparams, + VIR_DOMAIN_MIRGRATE_DECOMPRESSION_THREADS, + dthreads) < 0) + goto done; + } + + if (virDomainMigrateSetParameters(dom, params, nparams, flags) < 0) + goto done; + + ret = true; + + done: + virDomainFree(dom); + virTypedParamsFree(params, nparams); + return ret; +} + +/* + * "migrate-getparameters" command + */ +static const vshCmdInfo info_migrate_getparameters[] = { + {.name = "help", + .data = N_("Get the mutit-thread migration parameters") + }, + {.name = "desc", + .data = N_("Get the mutit-thread migration parameters for a domain.") + }, + {.name = NULL} +}; + +static const vshCmdOptDef opts_migrate_getparameters[] = { + {.name = "domain", + .type = VSH_OT_DATA, + .flags = VSH_OFLAG_REQ, + .help = N_("domain name, id or uuid") + }, + {.name = NULL} +}; + +static bool +cmdMigrateGetParameters(vshControl *ctl, const vshCmd *cmd) +{ + virDomainPtr dom = NULL; + int flags = 0; + int i = 0; + int nparams = 0; + virTypedParameterPtr params = NULL; + bool ret = false; + + if (!(dom = vshCommandOptDomain(ctl, cmd, NULL))) + return false; + + /* probe the compress param number of migrate. Now it supports 3 params. */ + if (virDomainMigrateGetParameters(dom, NULL, &nparams, 0) < 0) + goto done; + + params = vshCalloc(ctl, nparams, sizeof(*params)); + + if (virDomainMigrateGetParameters(dom, params, &nparams, flags) < 0) + goto done; + + for (i = 0; i < nparams; i++) { + char *str = vshGetTypedParamValue(ctl, ¶ms[i]); + vshPrint(ctl, "%-15s: %s\n", params[i].field, str); + VIR_FREE(str); + } + + ret = true; + + done: + virDomainFree(dom); + virTypedParamsFree(params, nparams); + return ret; +} + +/* * "domdisplay" command */ static const vshCmdInfo info_domdisplay[] = { @@ -13239,6 +13396,18 @@ const vshCmdDef domManagementCmds[] = { .info = info_migrate_getspeed, .flags = 0 }, + {.name = "migrate-setparameters", + .handler = cmdMigrateSetParameters, + .opts = opts_migrate_setparameters, + .info = info_migrate_setparameters, + .flags = 0 + }, + {.name = "migrate-getparameters", + .handler = cmdMigrateGetParameters, + .opts = opts_migrate_getparameters, + .info = info_migrate_getparameters, + .flags = 0 + }, {.name = "numatune", .handler = cmdNumatune, .opts = opts_numatune, diff --git a/tools/virsh.pod b/tools/virsh.pod index b81fd01..c158e8d 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -1685,6 +1685,20 @@ reject the value or convert it to the maximum value allowed. Get the maximum migration bandwidth (in MiB/s) for a domain. +=item B<migrate-setparameters> I<domain> [I<level> B<number>] +[I<threads> B<number>] [I<dthreads> B<number>] + +Set the multi-thread parameters for live migration. I<level> is interpreted as +an unsigned int value from 0-9. O means hypervisor choose an default value. +Specifying a negative value results in an essentially unlimited value being +provided to the hypervisor. The hypervisor can choose whether to reject the +value or convert it to the maximum value allowed. I<threads> is interpreted as +an unsigned int value. I<dthreads> is interpreted as an unsigned int value. + +=item B<migrate-getspeed> I<domain> + +Get the multi-thread migration parameters for a domain. + =item B<numatune> I<domain> [I<--mode> B<mode>] [I<--nodeset> B<nodeset>] [[I<--config>] [I<--live>] | [I<--current>]] -- 2.1.4
participants (1)
-
ShaoHe Feng