[libvirt] [PATCH 0/3] qemu: Add support for parallel migration
This series adds a new VIR_MIGRATE_PARALLEL flag for migration APIs which will ask the hypervisor to use multiple parallel connections for migrating a domain. The number of parallel connections can be set using VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS typed parameter. Jiri Denemark (3): Public API for parallel migration qemu: Add support for parallel migration virsh: Add options for parallel migration include/libvirt/libvirt-domain.h | 14 ++++++++++++++ src/qemu/qemu_migration.c | 9 ++++++--- src/qemu/qemu_migration.h | 5 ++++- src/qemu/qemu_migration_params.c | 18 ++++++++++++++++++ src/qemu/qemu_migration_params.h | 2 ++ tools/virsh-domain.c | 19 +++++++++++++++++++ tools/virsh.pod | 7 +++++++ 7 files changed, 70 insertions(+), 4 deletions(-) -- 2.20.1
This patch adds a new VIR_MIGRATE_PARALLEL flag for migration APIs which will ask the hypervisor to use multiple parallel connections for migrating a domain. The number of parallel connections can be set using VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS typed parameter. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- include/libvirt/libvirt-domain.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index d99f9b690b..b79a817ce5 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -831,6 +831,12 @@ typedef enum { */ VIR_MIGRATE_TLS = (1 << 16), + /* Send memory pages to the destination host through several network + * connections. See VIR_MIGRATE_PARAM_PARALLEL_* parameters for + * configuring the parallel migration. + */ + VIR_MIGRATE_PARALLEL = (1 << 17), + } virDomainMigrateFlags; @@ -1025,6 +1031,14 @@ typedef enum { */ # define VIR_MIGRATE_PARAM_AUTO_CONVERGE_INCREMENT "auto_converge.increment" +/** + * VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS: + * + * virDomainMigrate* params field: number of connections used during parallel + * migration. As VIR_TYPED_PARAM_INT. + */ +# define VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS "parallel.connections" + /* Domain migration. */ virDomainPtr virDomainMigrate (virDomainPtr domain, virConnectPtr dconn, unsigned long flags, const char *dname, -- 2.20.1
The VIR_MIGRATE_PARALLEL flag is implemented using QEMU's multifd migration capability and the corresponding multifd-channels migration parameter. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: QEMU still uses the x- prefix for multifd capability and multifd-channels, but Juan already sent a series of patches to make multifd migration fully supported by dropping the experimental prefix. src/qemu/qemu_migration.c | 9 ++++++--- src/qemu/qemu_migration.h | 5 ++++- src/qemu/qemu_migration_params.c | 18 ++++++++++++++++++ src/qemu/qemu_migration_params.h | 2 ++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 1433b2c2f3..d66313d35e 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -3823,10 +3823,13 @@ qemuMigrationSrcPerformNative(virQEMUDriverPtr driver, } } - if (STRNEQ(uribits->scheme, "rdma")) - spec.destType = MIGRATION_DEST_CONNECT_HOST; - else + /* RDMA and multi-fd migration requires QEMU to connect to the destination + * itself. + */ + if (STREQ(uribits->scheme, "rdma") || (flags & VIR_MIGRATE_PARALLEL)) spec.destType = MIGRATION_DEST_HOST; + else + spec.destType = MIGRATION_DEST_CONNECT_HOST; spec.dest.host.protocol = uribits->scheme; spec.dest.host.name = uribits->server; spec.dest.host.port = uribits->port; diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index ca73d3e467..fea5000a4d 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -57,7 +57,9 @@ VIR_MIGRATE_AUTO_CONVERGE | \ VIR_MIGRATE_RDMA_PIN_ALL | \ VIR_MIGRATE_POSTCOPY | \ - VIR_MIGRATE_TLS) + VIR_MIGRATE_TLS | \ + VIR_MIGRATE_PARALLEL | \ + 0) /* All supported migration parameters and their types. */ # define QEMU_MIGRATION_PARAMETERS \ @@ -80,6 +82,7 @@ VIR_MIGRATE_PARAM_AUTO_CONVERGE_INITIAL, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_AUTO_CONVERGE_INCREMENT, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_BANDWIDTH_POSTCOPY, VIR_TYPED_PARAM_ULLONG, \ + VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, VIR_TYPED_PARAM_INT, \ NULL diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 67070b9d08..7908aa95cd 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -86,6 +86,7 @@ VIR_ENUM_IMPL(qemuMigrationCapability, QEMU_MIGRATION_CAP_LAST, "compress", "pause-before-switchover", "late-block-activate", + "multifd", ); @@ -103,6 +104,7 @@ VIR_ENUM_IMPL(qemuMigrationParam, QEMU_MIGRATION_PARAM_LAST, "block-incremental", "xbzrle-cache-size", "max-postcopy-bandwidth", + "multifd-channels", ); typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem; @@ -151,6 +153,10 @@ static const qemuMigrationParamsFlagMapItem qemuMigrationParamsFlagMap[] = { {VIR_MIGRATE_POSTCOPY, QEMU_MIGRATION_CAP_POSTCOPY, QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION}, + + {VIR_MIGRATE_PARALLEL, + QEMU_MIGRATION_CAP_MULTIFD, + QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION}, }; /* Translation from VIR_MIGRATE_PARAM_* typed parameters to @@ -184,6 +190,10 @@ static const qemuMigrationParamsTPMapItem qemuMigrationParamsTPMap[] = { .unit = 1024 * 1024, /* MB/s */ .param = QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH, .party = QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION}, + + {.typedParam = VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, + .param = QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS, + .party = QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION}, }; static const qemuMigrationParamType qemuMigrationParamTypes[] = { @@ -199,6 +209,7 @@ static const qemuMigrationParamType qemuMigrationParamTypes[] = { [QEMU_MIGRATION_PARAM_BLOCK_INCREMENTAL] = QEMU_MIGRATION_PARAM_TYPE_BOOL, [QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE] = QEMU_MIGRATION_PARAM_TYPE_ULL, [QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH] = QEMU_MIGRATION_PARAM_TYPE_ULL, + [QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS] = QEMU_MIGRATION_PARAM_TYPE_INT, }; verify(ARRAY_CARDINALITY(qemuMigrationParamTypes) == QEMU_MIGRATION_PARAM_LAST); @@ -535,6 +546,13 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, goto error; } + if (migParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set && + !(flags & VIR_MIGRATE_PARALLEL)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Turn parallel migration on to tune it")); + goto error; + } + if (qemuMigrationParamsSetCompression(params, nparams, flags, migParams) < 0) goto error; diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 2460684a00..959fdd5854 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -38,6 +38,7 @@ typedef enum { QEMU_MIGRATION_CAP_COMPRESS, QEMU_MIGRATION_CAP_PAUSE_BEFORE_SWITCHOVER, QEMU_MIGRATION_CAP_LATE_BLOCK_ACTIVATE, + QEMU_MIGRATION_CAP_MULTIFD, QEMU_MIGRATION_CAP_LAST } qemuMigrationCapability; @@ -56,6 +57,7 @@ typedef enum { QEMU_MIGRATION_PARAM_BLOCK_INCREMENTAL, QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE, QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH, + QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS, QEMU_MIGRATION_PARAM_LAST } qemuMigrationParam; -- 2.20.1
On 2/6/19 5:09 PM, Jiri Denemark wrote:
The VIR_MIGRATE_PARALLEL flag is implemented using QEMU's multifd migration capability and the corresponding multifd-channels migration parameter.
Signed-off-by: Jiri Denemark <jdenemar@redhat.com> ---
Notes: QEMU still uses the x- prefix for multifd capability and multifd-channels, but Juan already sent a series of patches to make multifd migration fully supported by dropping the experimental prefix.
src/qemu/qemu_migration.c | 9 ++++++--- src/qemu/qemu_migration.h | 5 ++++- src/qemu/qemu_migration_params.c | 18 ++++++++++++++++++ src/qemu/qemu_migration_params.h | 2 ++ 4 files changed, 30 insertions(+), 4 deletions(-)
I'm unable to apply this patch. Can you please rebase and resend? Michal
Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- tools/virsh-domain.c | 19 +++++++++++++++++++ tools/virsh.pod | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index db0d5d4dcc..94cade5757 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -10561,6 +10561,14 @@ static const vshCmdOptDef opts_migrate[] = { .type = VSH_OT_INT, .help = N_("post-copy migration bandwidth limit in MiB/s") }, + {.name = "parallel", + .type = VSH_OT_BOOL, + .help = N_("enable parallel migration") + }, + {.name = "parallel-connections", + .type = VSH_OT_INT, + .help = N_("number of connections for parallel migration") + }, {.name = NULL} }; @@ -10766,6 +10774,14 @@ doMigrate(void *opaque) goto save_error; } + if (vshCommandOptInt(ctl, cmd, "parallel-connections", &intOpt) < 0) + goto out; + if (intOpt && + virTypedParamsAddInt(¶ms, &nparams, &maxparams, + VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, + intOpt) < 0) + goto save_error; + if (vshCommandOptBool(cmd, "live")) flags |= VIR_MIGRATE_LIVE; if (vshCommandOptBool(cmd, "p2p")) @@ -10814,6 +10830,9 @@ doMigrate(void *opaque) if (vshCommandOptBool(cmd, "tls")) flags |= VIR_MIGRATE_TLS; + if (vshCommandOptBool(cmd, "parallel")) + flags |= VIR_MIGRATE_PARALLEL; + if (flags & VIR_MIGRATE_PEER2PEER || vshCommandOptBool(cmd, "direct")) { if (virDomainMigrateToURI3(dom, desturi, params, nparams, flags) == 0) ret = '0'; diff --git a/tools/virsh.pod b/tools/virsh.pod index 67edb57b14..7604c7536e 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -1904,6 +1904,7 @@ I<domain> I<desturi> [I<migrateuri>] [I<graphicsuri>] [I<listen-address>] [I<dna [I<--comp-xbzrle-cache>] [I<--auto-converge>] [I<auto-converge-initial>] [I<auto-converge-increment>] [I<--persistent-xml> B<file>] [I<--tls>] [I<--postcopy-bandwidth> B<bandwidth>] +[I<--parallel> [I<parallel-connections> B<connections>]] Migrate domain to another host. Add I<--live> for live migration; <--p2p> for peer-2-peer migration; I<--direct> for direct migration; or I<--tunnelled> @@ -1997,6 +1998,12 @@ Providing I<--tls> causes the migration to use the host configured TLS setup the migration of the domain. Usage requires proper TLS setup for both source and target. +I<--parallel> option will cause migration data to be sent over multiple +parallel connections. The number of such connections can be set using +I<parallel-connections>. Parallel connections may help with saturating the +network link between the source and the target and thus speeding up the +migration. + Running migration can be canceled by interrupting virsh (usually using C<Ctrl-C>) or by B<domjobabort> command sent from another virsh instance. -- 2.20.1
participants (2)
-
Jiri Denemark -
Michal Privoznik