Re: [libvirt] [PATCH 5/6] qemu: RDMA migration support

From: "Michael R. Hines" <mrhines(a)us.ibm.com&gt; This patch adds support for RDMA protocol in migration URIs. USAGE: $ virsh migrate --live --migrateuri rdma://hostname domain qemu+ssh://hostname/system Since libvirt runs QEMU in a pretty restricted environment, several files needs to be added to cgroup_device_acl (in qemu.conf) for QEMU to be able to access the host's infiniband hardware. Full documenation of the feature can be found on QEMU wiki: http://wiki.qemu.org/Features/RDMALiveMigration Signed-off-by: Michael R. Hines <mrhines(a)us.ibm.com&gt; Signed-off-by: Jiri Denemark <jdenemar(a)redhat.com&gt; --- Notes: The question is whether the IB devices should be added to cgroup_device_acl by default or not... Version 3: - moved capabilities code into a separate patch - got rid of migration URI hacks - removed hacks that disabled IPv6 with RDMA - moved refactoring into a dedicated patch - documented IB devices which need to be added to cgroup acl in qemu.conf - forbid RDMA migrations unless memory hard limit is set until we have a better plan for setting limits for mlock - set QEMU's RLIMIT_MEMLOCK to memory hard_limit before starting RDMA migration - check if RDMA migration is supported by source QEMU before trying to migrate src/qemu/qemu.conf | 8 ++++++++ src/qemu/qemu_command.c | 8 ++++++++ src/qemu/qemu_migration.c | 39 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf index 79bba36..92ca715 100644 --- a/src/qemu/qemu.conf +++ b/src/qemu/qemu.conf @@ -274,6 +274,14 @@ # "/dev/ptmx", "/dev/kvm", "/dev/kqemu", # "/dev/rtc","/dev/hpet", "/dev/vfio/vfio" #] +# +# RDMA migration requires the following extra files to be added to the list: +# "/dev/infiniband/rdma_cm", +# "/dev/infiniband/issm0", +# "/dev/infiniband/issm1", +# "/dev/infiniband/umad0", +# "/dev/infiniband/umad1", +# "/dev/infiniband/uverbs0" # The default format for Qemu/KVM guest save images is raw; that is, the diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index a892d99..fceed62 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -9399,6 +9399,14 @@ qemuBuildCommandLine(virConnectPtr conn, goto error; } virCommandAddArg(cmd, migrateFrom); + } else if (STRPREFIX(migrateFrom, "rdma")) { + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("incoming RDMA migration is not supported " + "with this QEMU binary")); + goto error; + } + virCommandAddArg(cmd, migrateFrom); } else if (STREQ(migrateFrom, "stdio")) { if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) { virCommandAddArgFormat(cmd, "fd:%d", migrateFd); diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index d0e2653..b59e94d 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -56,6 +56,7 @@ #include "virhook.h" #include "virstring.h" #include "virtypedparam.h" +#include "virprocess.h" #define VIR_FROM_THIS VIR_FROM_QEMU @@ -2653,6 +2654,13 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, QEMU_MIGRATION_COOKIE_NBD))) goto cleanup; + if (STREQ(protocol, "rdma") && !vm->def->mem.hard_limit) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cannot start RDMA migration with no memory hard " + "limit set")); + goto cleanup; + } + if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) goto cleanup; qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PREPARE); @@ -2696,6 +2704,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) goto stop; + if (STREQ(protocol, "rdma") && + virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) { + goto stop; + } + if (mig->lockState) { VIR_DEBUG("Received lockstate %s", mig->lockState); VIR_FREE(priv->lockState); @@ -2926,7 +2939,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, if (!(uri = qemuMigrationParseURI(uri_in, &well_formed_uri))) goto cleanup; - if (STRNEQ(uri->scheme, "tcp")) { + if (STRNEQ(uri->scheme, "tcp") && + STRNEQ(uri->scheme, "rdma")) { virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, _("unsupported scheme %s in migration URI %s"), uri->scheme, uri_in); @@ -3545,6 +3559,11 @@ qemuMigrationRun(virQEMUDriverPtr driver, switch (spec->destType) { case MIGRATION_DEST_HOST: + if (STREQ(spec->dest.host.protocol, "rdma") && + virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) { + qemuDomainObjExitMonitor(driver, vm); + goto cleanup; + } ret = qemuMonitorMigrateToHost(priv->mon, migrate_flags, spec->dest.host.protocol, spec->dest.host.name, @@ -3717,7 +3736,23 @@ static int doNativeMigrate(virQEMUDriverPtr driver, if (!(uribits = qemuMigrationParseURI(uri, NULL))) return -1; - if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) + if (STREQ(uribits->scheme, "rdma")) { + if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("outgoing RDMA migration is not supported " + "with this QEMU binary"));

+ return -1; + } + if (!vm->def->mem.hard_limit) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cannot start RDMA migration with no memory hard " + "limit set")); + return -1;

+ } + } + + if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) && + STRNEQ(uribits->scheme, "rdma")) spec.destType = MIGRATION_DEST_CONNECT_HOST; else spec.destType = MIGRATION_DEST_HOST;