12:47 p.m.
From: "Michael R. Hines" <mrhines(a)us.ibm.com>
RDMA Live migration requires registering memory with the hardware,
and thus QEMU offers a new 'capability' which supports the ability
to pre-register / mlock() the guest memory in advance for higher
RDMA performance before the migration begins.
This patch exposes this capability with the following example usage:
virsh migrate --live --x-rdma-pin-all --migrateuri x-rdma:hostname domain
qemu+tcp://hostname/system
This capability is disabled by default, and thus ommiting it will
cause QEMU to register the memory with the hardware in an on-demand basis.
Signed-off-by: Michael R. Hines <mrhines(a)us.ibm.com>
---
include/libvirt/libvirt.h.in | 1 +
src/qemu/qemu_migration.c | 50 ++++++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_migration.h | 3 ++-
src/qemu/qemu_monitor.c | 2 +-
src/qemu/qemu_monitor.h | 1 +
tools/virsh-domain.c | 7 ++++++
6 files changed, 62 insertions(+), 2 deletions(-)
diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in
index 31fb37e..d21cb74 100644
--- a/include/libvirt/libvirt.h.in
+++ b/include/libvirt/libvirt.h.in
@@ -1193,6 +1193,7 @@ typedef enum {
VIR_MIGRATE_OFFLINE = (1 << 10), /* offline migrate */
VIR_MIGRATE_COMPRESSED = (1 << 11), /* compress data during migration
*/
VIR_MIGRATE_ABORT_ON_ERROR = (1 << 12), /* abort migration on I/O errors
happened during migration */
+ VIR_MIGRATE_X_RDMA_PIN_ALL = (1 << 13), /* RDMA memory pinning */
} virDomainMigrateFlags;
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index de20d23..f5d9b16 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -1574,6 +1574,46 @@ cleanup:
}
static int
+qemuMigrationSetPinAll(virQEMUDriverPtr driver,
+ virDomainObjPtr vm,
+ enum qemuDomainAsyncJob job)
+{
+ qemuDomainObjPrivatePtr priv = vm->privateData;
+ int ret;
+
+ if (qemuDomainObjEnterMonitorAsync(driver, vm, job) < 0)
+ return -1;
+
+ ret = qemuMonitorGetMigrationCapability(
+ priv->mon,
+ QEMU_MONITOR_MIGRATION_CAPS_X_RDMA_PIN_ALL);
+
+ if (ret < 0) {
+ goto cleanup;
+ } else if (ret == 0) {
+ if (job == QEMU_ASYNC_JOB_MIGRATION_IN) {
+ virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
+ _("rdma pinning migration is not supported by "
+ "target QEMU binary"));
+ } else {
+ virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
+ _("rdma pinning migration is not supported by "
+ "source QEMU binary"));
+ }
+ ret = -1;
+ goto cleanup;
+ }
+
+ ret = qemuMonitorSetMigrationCapability(
+ priv->mon,
+ QEMU_MONITOR_MIGRATION_CAPS_X_RDMA_PIN_ALL);
+
+cleanup:
+ qemuDomainObjExitMonitor(driver, vm);
+ return ret;
+}
+
+static int
qemuMigrationWaitForSpice(virQEMUDriverPtr driver,
virDomainObjPtr vm)
{
@@ -2363,6 +2403,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
goto stop;
+ if (flags & VIR_MIGRATE_X_RDMA_PIN_ALL &&
+ qemuMigrationSetPinAll(driver, vm,
+ QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
+ goto stop;
+
if (mig->lockState) {
VIR_DEBUG("Received lockstate %s", mig->lockState);
VIR_FREE(priv->lockState);
@@ -3156,6 +3201,11 @@ qemuMigrationRun(virQEMUDriverPtr driver,
QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
goto cleanup;
+ if (flags & VIR_MIGRATE_X_RDMA_PIN_ALL &&
+ qemuMigrationSetPinAll(driver, vm,
+ QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
+ goto cleanup;
+
if (qemuDomainObjEnterMonitorAsync(driver, vm,
QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
goto cleanup;
diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h
index 0f6c5f7..d08979f 100644
--- a/src/qemu/qemu_migration.h
+++ b/src/qemu/qemu_migration.h
@@ -39,7 +39,8 @@
VIR_MIGRATE_UNSAFE | \
VIR_MIGRATE_OFFLINE | \
VIR_MIGRATE_COMPRESSED | \
- VIR_MIGRATE_ABORT_ON_ERROR)
+ VIR_MIGRATE_ABORT_ON_ERROR | \
+ VIR_MIGRATE_X_RDMA_PIN_ALL)
/* All supported migration parameters and their types. */
# define QEMU_MIGRATION_PARAMETERS \
diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index ce95174..34d6840 100644
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -112,7 +112,7 @@ VIR_ENUM_IMPL(qemuMonitorMigrationStatus,
VIR_ENUM_IMPL(qemuMonitorMigrationCaps,
QEMU_MONITOR_MIGRATION_CAPS_LAST,
- "xbzrle")
+ "xbzrle", "x-rdma-pin-all")
VIR_ENUM_IMPL(qemuMonitorVMStatus,
QEMU_MONITOR_VM_STATUS_LAST,
diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index d722e12..83864f2 100644
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -405,6 +405,7 @@ int qemuMonitorGetSpiceMigrationStatus(qemuMonitorPtr mon,
typedef enum {
QEMU_MONITOR_MIGRATION_CAPS_XBZRLE,
+ QEMU_MONITOR_MIGRATION_CAPS_X_RDMA_PIN_ALL,
QEMU_MONITOR_MIGRATION_CAPS_LAST
} qemuMonitorMigrationCaps;
diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c
index 8cafce4..ee8b423 100644
--- a/tools/virsh-domain.c
+++ b/tools/virsh-domain.c
@@ -8380,6 +8380,10 @@ static const vshCmdOptDef opts_migrate[] = {
.type = VSH_OT_BOOL,
.help = N_("compress repeated pages during live migration")
},
+ {.name = "x-rdma-pin-all",
+ .type = VSH_OT_BOOL,
+ .help = N_("support memory pinning during rdma live migration")
+ },
{.name = "abort-on-error",
.type = VSH_OT_BOOL,
.help = N_("abort on soft errors during migration")
@@ -8513,6 +8517,9 @@ doMigrate(void *opaque)
if (vshCommandOptBool(cmd, "compressed"))
flags |= VIR_MIGRATE_COMPRESSED;
+ if (vshCommandOptBool(cmd, "x-rdma-pin-all"))
+ flags |= VIR_MIGRATE_X_RDMA_PIN_ALL;
+
if (vshCommandOptBool(cmd, "offline")) {
flags |= VIR_MIGRATE_OFFLINE;
}
--
1.7.10.4