On 04/22/2010 11:06 AM, Kenneth Nagin wrote:
Support for live migration between hosts that do not share storage was added to qemu-kvm release 0.12.1. It supports two flags: -b migration without shared storage with full disk copy -i migration without shared storage with incremental copy (same base image shared between source and destination).
In order to support kvm's live migration with non-shared storage I added two migration flags that user can invoke:
VIR_MIGRATE_NON_SHARED_DISK = (1 << 6), /* migration with non-shared storage with full disk copy */ VIR_MIGRATE_NON_SHARED_INC = (1 << 7), /* migration with non-shared storage with incremental copy */ /* (same base image shared between source and destination) */ Likewise I add complementary flags to virsh migrate: non_shared_disk and non_shared_inc
As Daniel B. suggested I also added internal flags to be passed in the "background" parameter to the qemu monitor: typedef enum { QEMU_MONITOR_MIGRATE_BACKGROUND = 1 << 0, QEMU_MONITOR_MIGRATE_NON_SHARED_DISK = 1 << 1, /* migration with non-shared storage with full disk copy */ QEMU_MONITOR_MIGRATE_NON_SHARED_INC = 1 << 2, /* migration with non-shared storage with incremental copy */ QEMU_MONITOR_MIGRATION_FLAGS_LAST };
I updated qemu_driver.c's doNativeMigrate and doTunnelMigrate to map the external flags to the internal flags. I updated qemu_monitor_text's qemuMonitorTextMigrate to map the internal flags to the qemu monitor's command line flags.
Also, qemudDomainSave ends up calling qemuMonitorTextMigrate, but it didn't support the external flags so I assumed that it was not relevant.
I tested the live migration without shared storage (both flags) for native and p2p with and without tunnelling. I also verified that the fix doesn't affect normal migration with shared storage.
Here is the diff patch file from the current git:
(See attached file: libvirt_migration_ns_100422.patch)
-- Kenneth Nagin
Of course, I tell you to resend then forget to review. Sorry :( Applying the patch (to last weeks checkout), there are some compiler warnings: make sure you configure with --enable-compiler-warnings=error. 'make syntax-check' also fails, so please address these. Also, changes are probably needed to qemu_monitor_json.c to enable this feature for future qemu releases. Probably a simple change, but I don't know what it should be. Maybe Danpb can chime in here. Finding a way to post the patch in-line will also probably get better attention: just pasting it into the mail client will probably mangle the patch, I'd recommend git send-email. For the virsh changes, I wouldn't use underscores in cli flag names. Maybe change this to 'copy-storage' and 'increment-storage'. Not too fond of the latter, maybe others have ideas. Other than that, the code generally looks like (except for the compiler and syntax-check warnings). - Cole (Here's the patch inline for the benefit of other reviewers)
diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in index 4addc62..28fabd9 100644 --- a/include/libvirt/libvirt.h.in +++ b/include/libvirt/libvirt.h.in @@ -411,6 +411,10 @@ typedef enum { VIR_MIGRATE_PERSIST_DEST = (1 << 3), /* persist the VM on the destination */ VIR_MIGRATE_UNDEFINE_SOURCE = (1 << 4), /* undefine the VM on the source */ VIR_MIGRATE_PAUSED = (1 << 5), /* pause on remote side */ + VIR_MIGRATE_NON_SHARED_DISK = (1 << 6), /* migration with non-shared storage with full disk copy */ + VIR_MIGRATE_NON_SHARED_INC = (1 << 7), /* migration with non-shared storage with incremental copy */ + /* (same base image shared between source and destination) */ + } virDomainMigrateFlags;
/* Domain migration. */ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 5f4adfd..521638c 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -4871,7 +4871,7 @@ static int qemudDomainSaveFlag(virDomainPtr dom, const char *path, if (header.compressed == QEMUD_SAVE_FORMAT_RAW) { const char *args[] = { "cat", NULL }; qemuDomainObjEnterMonitorWithDriver(driver, vm); - rc = qemuMonitorMigrateToCommand(priv->mon, 1, args, path); + rc = qemuMonitorMigrateToCommand(priv->mon, QEMU_MONITOR_MIGRATE_BACKGROUND, args, path); qemuDomainObjExitMonitorWithDriver(driver, vm); } else { const char *prog = qemudSaveCompressionTypeToString(header.compressed); @@ -4881,7 +4881,7 @@ static int qemudDomainSaveFlag(virDomainPtr dom, const char *path, NULL }; qemuDomainObjEnterMonitorWithDriver(driver, vm); - rc = qemuMonitorMigrateToCommand(priv->mon, 1, args, path); + rc = qemuMonitorMigrateToCommand(priv->mon, QEMU_MONITOR_MIGRATE_BACKGROUND, args, path); qemuDomainObjExitMonitorWithDriver(driver, vm); }
@@ -5173,7 +5173,7 @@ static int qemudDomainCoreDump(virDomainPtr dom, }
qemuDomainObjEnterMonitor(vm); - ret = qemuMonitorMigrateToCommand(priv->mon, 1, args, path); + ret = qemuMonitorMigrateToCommand(priv->mon, QEMU_MONITOR_MIGRATE_BACKGROUND, args, path); qemuDomainObjExitMonitor(vm);
if (ret < 0) @@ -9650,13 +9650,14 @@ cleanup: static int doNativeMigrate(struct qemud_driver *driver, virDomainObjPtr vm, const char *uri, - unsigned long flags ATTRIBUTE_UNUSED, + unsigned long flags , const char *dname ATTRIBUTE_UNUSED, unsigned long resource) { int ret = -1; xmlURIPtr uribits = NULL; qemuDomainObjPrivatePtr priv = vm->privateData; + int background_flags = 0;
/* Issue the migrate command. */ if (STRPREFIX(uri, "tcp:") && !STRPREFIX(uri, "tcp://")) { @@ -9684,7 +9685,13 @@ static int doNativeMigrate(struct qemud_driver *driver, goto cleanup; }
- if (qemuMonitorMigrateToHost(priv->mon, 1, uribits->server, uribits->port) < 0) { + if (flags & VIR_MIGRATE_NON_SHARED_DISK) + background_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_DISK; + + if (flags & VIR_MIGRATE_NON_SHARED_INC) + background_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_INC; + + if (qemuMonitorMigrateToHost(priv->mon, background_flags, uribits->server, uribits->port) < 0) { qemuDomainObjExitMonitorWithDriver(driver, vm); goto cleanup; } @@ -9769,7 +9776,7 @@ static int doTunnelMigrate(virDomainPtr dom, unsigned long long qemuCmdFlags; int status; unsigned long long transferred, remaining, total; - + int background_flags = QEMU_MONITOR_MIGRATE_BACKGROUND; /* * The order of operations is important here to avoid touching * the source VM until we are very sure we can successfully @@ -9854,11 +9861,16 @@ static int doTunnelMigrate(virDomainPtr dom,
/* 3. start migration on source */ qemuDomainObjEnterMonitorWithDriver(driver, vm); - if (qemuCmdFlags & QEMUD_CMD_FLAG_MIGRATE_QEMU_UNIX) - internalret = qemuMonitorMigrateToUnix(priv->mon, 1, unixfile); + if (flags & VIR_MIGRATE_NON_SHARED_DISK) + background_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_DISK; + if (flags & VIR_MIGRATE_NON_SHARED_INC) + background_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_INC; + if (qemuCmdFlags & QEMUD_CMD_FLAG_MIGRATE_QEMU_UNIX){ + internalret = qemuMonitorMigrateToUnix(priv->mon, background_flags, unixfile); + } else if (qemuCmdFlags & QEMUD_CMD_FLAG_MIGRATE_QEMU_EXEC) { const char *args[] = { "nc", "-U", unixfile, NULL }; - internalret = qemuMonitorMigrateToCommand(priv->mon, 1, args, "/dev/null"); + internalret = qemuMonitorMigrateToCommand(priv->mon, background_flags, args, "/dev/null"); } else { internalret = -1; } diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 251233a..40a21f5 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -239,6 +239,13 @@ int qemuMonitorGetMigrationStatus(qemuMonitorPtr mon, unsigned long long *remaining, unsigned long long *total);
+typedef enum { + QEMU_MONITOR_MIGRATE_BACKGROUND = 1 << 0, + QEMU_MONITOR_MIGRATE_NON_SHARED_DISK = 1 << 1, /* migration with non-shared storage with full disk copy */ + QEMU_MONITOR_MIGRATE_NON_SHARED_INC = 1 << 2, /* migration with non-shared storage with incremental copy */ + QEMU_MONITOR_MIGRATION_FLAGS_LAST +}; + int qemuMonitorMigrateToHost(qemuMonitorPtr mon, int background, const char *hostname, diff --git a/src/qemu/qemu_monitor_text.c b/src/qemu/qemu_monitor_text.c index 6ad07b1..136ac12 100644 --- a/src/qemu/qemu_monitor_text.c +++ b/src/qemu/qemu_monitor_text.c @@ -1132,18 +1132,19 @@ static int qemuMonitorTextMigrate(qemuMonitorPtr mon, char *info = NULL; int ret = -1; char *safedest = qemuMonitorEscapeArg(dest); - const char *extra; + //const char *extra; + const char extra[25] = " ";
if (!safedest) { virReportOOMError(); return -1; } - - if (background) - extra = "-d "; - else - extra = " "; - + if (background & QEMU_MONITOR_MIGRATE_BACKGROUND) + strcat(extra," -d"); + if (background & QEMU_MONITOR_MIGRATE_NON_SHARED_DISK) + strcat(extra," -b"); + if (background & QEMU_MONITOR_MIGRATE_NON_SHARED_INC) + strcat(extra," -i"); if (virAsprintf(&cmd, "migrate %s\"%s\"", extra, safedest) < 0) { virReportOOMError(); goto cleanup; diff --git a/tools/virsh.c b/tools/virsh.c index b2a1538..bd8719e 100644 --- a/tools/virsh.c +++ b/tools/virsh.c @@ -2804,6 +2804,8 @@ static const vshCmdOptDef opts_migrate[] = { {"persistent", VSH_OT_BOOL, 0, N_("persist VM on destination")}, {"undefinesource", VSH_OT_BOOL, 0, N_("undefine VM on source")}, {"suspend", VSH_OT_BOOL, 0, N_("do not restart the domain on the destination host")}, + {"non_shared_disk", VSH_OT_BOOL, 0, N_("migration with non-shared storage with full disk copy")}, + {"non_shared_inc", VSH_OT_BOOL, 0, N_("migration with non-shared storage with incremental copy (same base image shared between source and destination)")}, {"domain", VSH_OT_DATA, VSH_OFLAG_REQ, N_("domain name, id or uuid")}, {"desturi", VSH_OT_DATA, VSH_OFLAG_REQ, N_("connection URI of the destination host")}, {"migrateuri", VSH_OT_DATA, 0, N_("migration URI, usually can be omitted")},
@@ -2851,6 +2853,12 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd) if (vshCommandOptBool (cmd, "suspend")) flags |= VIR_MIGRATE_PAUSED;
+ if (vshCommandOptBool (cmd, "non_shared_disk")) + flags |= VIR_MIGRATE_NON_SHARED_DISK; + + if (vshCommandOptBool (cmd, "non_shared_inc")) + flags |= VIR_MIGRATE_NON_SHARED_INC; + if ((flags & VIR_MIGRATE_PEER2PEER) || vshCommandOptBool (cmd, "direct")) { /* For peer2peer migration or direct migration we only expect one URI