During incoming migration the QEMU monitor is not able to be used. The incoming migration code did not keep hold of the job lock because migration is split across multiple API calls. This meant that further monitor commands on the guest would hang until migration finished with no timeout. In this change the qemuDomainMigratePrepare method sets the job flag just before it returns. The qemuDomainMigrateFinish method checks for this job flag & clears it once done. This prevents any use of the monitor between prepare+finish steps. The qemuDomainGetJobInfo method is also updated to refresh the job elapsed time. This means that virsh domjobinfo can return time data during incoming migration * src/qemu/qemu_driver.c: Keep a job active during incoming migration. Refresh job elapsed time when returning job info --- src/qemu/qemu_driver.c | 80 +++++++++++++++++++++++++++++++++++++++++++++--- 1 files changed, 75 insertions(+), 5 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 6274d4c..cad4eed 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -97,7 +97,8 @@ enum qemuDomainJob { QEMU_JOB_NONE = 0, /* Always set to 0 for easy if (jobActive) conditions */ QEMU_JOB_UNSPECIFIED, - QEMU_JOB_MIGRATION, + QEMU_JOB_MIGRATION_OUT, + QEMU_JOB_MIGRATION_IN, }; enum qemuDomainJobSignals { @@ -4356,7 +4357,7 @@ static int qemudDomainSuspend(virDomainPtr dom) { priv = vm->privateData; - if (priv->jobActive == QEMU_JOB_MIGRATION) { + if (priv->jobActive == QEMU_JOB_MIGRATION_OUT) { if (vm->state != VIR_DOMAIN_PAUSED) { VIR_DEBUG("Requesting domain pause on %s", vm->def->name); @@ -10193,6 +10194,14 @@ qemudDomainMigratePrepareTunnel(virConnectPtr dconn, char *unixfile = NULL; unsigned long long qemuCmdFlags; struct qemuStreamMigFile *qemust = NULL; + qemuDomainObjPrivatePtr priv = NULL; + struct timeval now; + + if (gettimeofday(&now, NULL) < 0) { + virReportSystemError(errno, "%s", + _("cannot get time of day")); + return -1; + } qemuDriverLock(driver); if (!dom_xml) { @@ -10237,9 +10246,11 @@ qemudDomainMigratePrepareTunnel(virConnectPtr dconn, goto cleanup; } def = NULL; + priv = vm->privateData; if (qemuDomainObjBeginJobWithDriver(driver, vm) < 0) goto cleanup; + priv->jobActive = QEMU_JOB_MIGRATION_OUT; /* Domain starts inactive, even if the domain XML had an id field. */ vm->def->id = -1; @@ -10315,6 +10326,18 @@ endjob: qemuDomainObjEndJob(vm) == 0) vm = NULL; + /* We set a fake job active which is held across + * API calls until the finish() call. This prevents + * any other APIs being invoked while incoming + * migration is taking place + */ + if (vm && + virDomainObjIsActive(vm)) { + priv->jobActive = QEMU_JOB_MIGRATION_IN; + priv->jobInfo.type = VIR_DOMAIN_JOB_UNBOUNDED; + priv->jobStart = (now.tv_sec * 1000ull) + (now.tv_usec / 1000); + } + cleanup: virDomainDefFree(def); if (unixfile) @@ -10354,6 +10377,14 @@ qemudDomainMigratePrepare2 (virConnectPtr dconn, virDomainEventPtr event = NULL; int ret = -1; int internalret; + qemuDomainObjPrivatePtr priv = NULL; + struct timeval now; + + if (gettimeofday(&now, NULL) < 0) { + virReportSystemError(errno, "%s", + _("cannot get time of day")); + return -1; + } virCheckFlags(VIR_MIGRATE_LIVE | VIR_MIGRATE_PEER2PEER | @@ -10482,9 +10513,11 @@ qemudDomainMigratePrepare2 (virConnectPtr dconn, goto cleanup; } def = NULL; + priv = vm->privateData; if (qemuDomainObjBeginJobWithDriver(driver, vm) < 0) goto cleanup; + priv->jobActive = QEMU_JOB_MIGRATION_OUT; /* Domain starts inactive, even if the domain XML had an id field. */ vm->def->id = -1; @@ -10516,6 +10549,18 @@ endjob: qemuDomainObjEndJob(vm) == 0) vm = NULL; + /* We set a fake job active which is held across + * API calls until the finish() call. This prevents + * any other APIs being invoked while incoming + * migration is taking place + */ + if (vm && + virDomainObjIsActive(vm)) { + priv->jobActive = QEMU_JOB_MIGRATION_IN; + priv->jobInfo.type = VIR_DOMAIN_JOB_UNBOUNDED; + priv->jobStart = (now.tv_sec * 1000ull) + (now.tv_usec / 1000); + } + cleanup: VIR_FREE(hostname); virDomainDefFree(def); @@ -10988,7 +11033,7 @@ qemudDomainMigratePerform (virDomainPtr dom, if (qemuDomainObjBeginJobWithDriver(driver, vm) < 0) goto cleanup; - priv->jobActive = QEMU_JOB_MIGRATION; + priv->jobActive = QEMU_JOB_MIGRATION_OUT; if (!virDomainObjIsActive(vm)) { qemuReportError(VIR_ERR_OPERATION_INVALID, @@ -11077,6 +11122,7 @@ qemudDomainMigrateFinish2 (virConnectPtr dconn, virDomainEventPtr event = NULL; virErrorPtr orig_err; int newVM = 1; + qemuDomainObjPrivatePtr priv = NULL; virCheckFlags(VIR_MIGRATE_LIVE | VIR_MIGRATE_PEER2PEER | @@ -11098,6 +11144,15 @@ qemudDomainMigrateFinish2 (virConnectPtr dconn, goto cleanup; } + priv = vm->privateData; + if (priv->jobActive != QEMU_JOB_MIGRATION_IN) { + qemuReportError(VIR_ERR_NO_DOMAIN, + _("domain '%s' is not processing incoming migration"), dname); + goto cleanup; + } + priv->jobActive = QEMU_JOB_NONE; + memset(&priv->jobInfo, 0, sizeof(priv->jobInfo)); + if (qemuDomainObjBeginJobWithDriver(driver, vm) < 0) goto cleanup; @@ -11140,7 +11195,6 @@ qemudDomainMigrateFinish2 (virConnectPtr dconn, event = NULL; } - qemuDomainObjPrivatePtr priv = vm->privateData; dom = virGetDomain (dconn, vm->def->name, vm->def->uuid); if (!(flags & VIR_MIGRATE_PAUSED)) { @@ -11382,7 +11436,23 @@ static int qemuDomainGetJobInfo(virDomainPtr dom, if (virDomainObjIsActive(vm)) { if (priv->jobActive) { + struct timeval now; + memcpy(info, &priv->jobInfo, sizeof(*info)); + + /* Refresh elapsed time again just to ensure it + * is fully updated. This is primarily for benefit + * of incoming migration which we don't currently + * monitor actively in the background thread + */ + if (gettimeofday(&now, NULL) < 0) { + virReportSystemError(errno, "%s", + _("cannot get time of day")); + goto cleanup; + } + info->timeElapsed = + ((now.tv_sec * 1000ull) + (now.tv_usec / 1000)) - + priv->jobStart; } else { memset(info, 0, sizeof(*info)); info->type = VIR_DOMAIN_JOB_NONE; @@ -11476,7 +11546,7 @@ qemuDomainMigrateSetMaxDowntime(virDomainPtr dom, priv = vm->privateData; - if (priv->jobActive != QEMU_JOB_MIGRATION) { + if (priv->jobActive != QEMU_JOB_MIGRATION_OUT) { qemuReportError(VIR_ERR_OPERATION_INVALID, "%s", _("domain is not being migrated")); goto cleanup; -- 1.6.6.1

The patches for shared storage migration were not correctly written for json mode. Thus the 'blk' and 'inc' parameters were never being set. In addition they didn't set the QEMU_MONITOR_MIGRATE_BACKGROUND so migration was synchronous. Due to multiple bugs in QEMU's JSON impl this wasn't noticed because it treated the sync migration requst as asynchronous anyway. Finally 'background' parameter was converted to take arbitrary flags but not renamed, and not all uses were changed to unsigned int. * src/qemu/qemu_driver.c: Set QEMU_MONITOR_MIGRATE_BACKGROUND in doNativeMigrate * src/qemu/qemu_monitor_json.c: Process QEMU_MONITOR_MIGRATE_NON_SHARED_DISK and QEMU_MONITOR_MIGRATE_NON_SHARED_INC flags * src/qemu/qemu_monitor.c, src/qemu/qemu_monitor.h, src/qemu/qemu_monitor_json.h, src/qemu/qemu_monitor_text.c, src/qemu/qemu_monitor_text.h: change 'int background' to 'unsigned int flags' in migration APIs. Add logging of flags parameter --- src/qemu/qemu_driver.c | 2 +- src/qemu/qemu_monitor.c | 40 ++++++++++++++++++++-------------------- src/qemu/qemu_monitor.h | 8 ++++---- src/qemu/qemu_monitor_json.c | 29 ++++++++++++++++------------- src/qemu/qemu_monitor_json.h | 8 ++++---- src/qemu/qemu_monitor_text.c | 24 ++++++++++++------------ src/qemu/qemu_monitor_text.h | 8 ++++---- 7 files changed, 61 insertions(+), 58 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index cad4eed..c0cd35a 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -10589,7 +10589,7 @@ static int doNativeMigrate(struct qemud_driver *driver, int ret = -1; xmlURIPtr uribits = NULL; qemuDomainObjPrivatePtr priv = vm->privateData; - unsigned int background_flags = 0; + unsigned int background_flags = QEMU_MONITOR_MIGRATE_BACKGROUND; /* Issue the migrate command. */ if (STRPREFIX(uri, "tcp:") && !STRPREFIX(uri, "tcp://")) { diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index ea0351e..f428665 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -1299,13 +1299,13 @@ int qemuMonitorGetMigrationStatus(qemuMonitorPtr mon, int qemuMonitorMigrateToHost(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *hostname, int port) { int ret; - DEBUG("mon=%p hostname=%s port=%d", - mon, hostname, port); + DEBUG("mon=%p hostname=%s port=%d flags=%u", + mon, hostname, port, flags); if (!mon) { qemuReportError(VIR_ERR_INVALID_ARG, "%s", @@ -1314,20 +1314,20 @@ int qemuMonitorMigrateToHost(qemuMonitorPtr mon, } if (mon->json) - ret = qemuMonitorJSONMigrateToHost(mon, background, hostname, port); + ret = qemuMonitorJSONMigrateToHost(mon, flags, hostname, port); else - ret = qemuMonitorTextMigrateToHost(mon, background, hostname, port); + ret = qemuMonitorTextMigrateToHost(mon, flags, hostname, port); return ret; } int qemuMonitorMigrateToCommand(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char * const *argv) { int ret; - DEBUG("mon=%p argv=%p", - mon, argv); + DEBUG("mon=%p argv=%p flags=%u", + mon, argv, flags); if (!mon) { qemuReportError(VIR_ERR_INVALID_ARG, "%s", @@ -1336,21 +1336,21 @@ int qemuMonitorMigrateToCommand(qemuMonitorPtr mon, } if (mon->json) - ret = qemuMonitorJSONMigrateToCommand(mon, background, argv); + ret = qemuMonitorJSONMigrateToCommand(mon, flags, argv); else - ret = qemuMonitorTextMigrateToCommand(mon, background, argv); + ret = qemuMonitorTextMigrateToCommand(mon, flags, argv); return ret; } int qemuMonitorMigrateToFile(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char * const *argv, const char *target, unsigned long long offset) { int ret; - DEBUG("mon=%p argv=%p target=%s offset=%llu", - mon, argv, target, offset); + DEBUG("mon=%p argv=%p target=%s offset=%llu flags=%u", + mon, argv, target, offset, flags); if (!mon) { qemuReportError(VIR_ERR_INVALID_ARG, "%s", @@ -1366,19 +1366,19 @@ int qemuMonitorMigrateToFile(qemuMonitorPtr mon, } if (mon->json) - ret = qemuMonitorJSONMigrateToFile(mon, background, argv, target, offset); + ret = qemuMonitorJSONMigrateToFile(mon, flags, argv, target, offset); else - ret = qemuMonitorTextMigrateToFile(mon, background, argv, target, offset); + ret = qemuMonitorTextMigrateToFile(mon, flags, argv, target, offset); return ret; } int qemuMonitorMigrateToUnix(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *unixfile) { int ret; - DEBUG("mon=%p, unixfile=%s", - mon, unixfile); + DEBUG("mon=%p, unixfile=%s flags=%u", + mon, unixfile, flags); if (!mon) { qemuReportError(VIR_ERR_INVALID_ARG, "%s", @@ -1387,9 +1387,9 @@ int qemuMonitorMigrateToUnix(qemuMonitorPtr mon, } if (mon->json) - ret = qemuMonitorJSONMigrateToUnix(mon, background, unixfile); + ret = qemuMonitorJSONMigrateToUnix(mon, flags, unixfile); else - ret = qemuMonitorTextMigrateToUnix(mon, background, unixfile); + ret = qemuMonitorTextMigrateToUnix(mon, flags, unixfile); return ret; } diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index b6a8f9f..459ccb7 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -256,12 +256,12 @@ typedef enum { } QEMU_MONITOR_MIGRATE; int qemuMonitorMigrateToHost(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *hostname, int port); int qemuMonitorMigrateToCommand(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char * const *argv); /* In general, BS is the smallest fundamental block size we can use to @@ -276,13 +276,13 @@ int qemuMonitorMigrateToCommand(qemuMonitorPtr mon, # define QEMU_MONITOR_MIGRATE_TO_FILE_TRANSFER_SIZE (1024llu * 1024) int qemuMonitorMigrateToFile(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char * const *argv, const char *target, unsigned long long offset); int qemuMonitorMigrateToUnix(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *unixfile); int qemuMonitorMigrateCancel(qemuMonitorPtr mon); diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index 40218aa..fb58963 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -1599,14 +1599,17 @@ int qemuMonitorJSONGetMigrationStatus(qemuMonitorPtr mon, static int qemuMonitorJSONMigrate(qemuMonitorPtr mon, - int background, + unsigned int flags, const char *uri) { int ret; - virJSONValuePtr cmd = qemuMonitorJSONMakeCommand("migrate", - "i:detach", background ? 1 : 0, - "s:uri", uri, - NULL); + virJSONValuePtr cmd = + qemuMonitorJSONMakeCommand("migrate", + "i:detach", flags & QEMU_MONITOR_MIGRATE_BACKGROUND ? 1 : 0, + "i:blk", flags & QEMU_MONITOR_MIGRATE_NON_SHARED_DISK ? 1 : 0, + "i:inc", flags & QEMU_MONITOR_MIGRATE_NON_SHARED_INC ? 1 : 0, + "s:uri", uri, + NULL); virJSONValuePtr reply = NULL; if (!cmd) @@ -1624,7 +1627,7 @@ static int qemuMonitorJSONMigrate(qemuMonitorPtr mon, int qemuMonitorJSONMigrateToHost(qemuMonitorPtr mon, - int background, + unsigned int flags, const char *hostname, int port) { @@ -1636,7 +1639,7 @@ int qemuMonitorJSONMigrateToHost(qemuMonitorPtr mon, return -1; } - ret = qemuMonitorJSONMigrate(mon, background, uri); + ret = qemuMonitorJSONMigrate(mon, flags, uri); VIR_FREE(uri); @@ -1645,7 +1648,7 @@ int qemuMonitorJSONMigrateToHost(qemuMonitorPtr mon, int qemuMonitorJSONMigrateToCommand(qemuMonitorPtr mon, - int background, + unsigned int flags, const char * const *argv) { char *argstr; @@ -1663,7 +1666,7 @@ int qemuMonitorJSONMigrateToCommand(qemuMonitorPtr mon, goto cleanup; } - ret = qemuMonitorJSONMigrate(mon, background, dest); + ret = qemuMonitorJSONMigrate(mon, flags, dest); cleanup: VIR_FREE(argstr); @@ -1672,7 +1675,7 @@ cleanup: } int qemuMonitorJSONMigrateToFile(qemuMonitorPtr mon, - int background, + unsigned int flags, const char * const *argv, const char *target, unsigned long long offset) @@ -1709,7 +1712,7 @@ int qemuMonitorJSONMigrateToFile(qemuMonitorPtr mon, goto cleanup; } - ret = qemuMonitorJSONMigrate(mon, background, dest); + ret = qemuMonitorJSONMigrate(mon, flags, dest); cleanup: VIR_FREE(safe_target); @@ -1719,7 +1722,7 @@ cleanup: } int qemuMonitorJSONMigrateToUnix(qemuMonitorPtr mon, - int background, + unsigned int flags, const char *unixfile) { char *dest = NULL; @@ -1730,7 +1733,7 @@ int qemuMonitorJSONMigrateToUnix(qemuMonitorPtr mon, return -1; } - ret = qemuMonitorJSONMigrate(mon, background, dest); + ret = qemuMonitorJSONMigrate(mon, flags, dest); VIR_FREE(dest); diff --git a/src/qemu/qemu_monitor_json.h b/src/qemu/qemu_monitor_json.h index c578211..6fa8d83 100644 --- a/src/qemu/qemu_monitor_json.h +++ b/src/qemu/qemu_monitor_json.h @@ -97,22 +97,22 @@ int qemuMonitorJSONGetMigrationStatus(qemuMonitorPtr mon, unsigned long long *total); int qemuMonitorJSONMigrateToHost(qemuMonitorPtr mon, - int background, + unsigned int flags, const char *hostname, int port); int qemuMonitorJSONMigrateToCommand(qemuMonitorPtr mon, - int background, + unsigned int flags, const char * const *argv); int qemuMonitorJSONMigrateToFile(qemuMonitorPtr mon, - int background, + unsigned int flags, const char * const *argv, const char *target, unsigned long long offset); int qemuMonitorJSONMigrateToUnix(qemuMonitorPtr mon, - int background, + unsigned int flags, const char *unixfile); int qemuMonitorJSONMigrateCancel(qemuMonitorPtr mon); diff --git a/src/qemu/qemu_monitor_text.c b/src/qemu/qemu_monitor_text.c index c086962..569742a 100644 --- a/src/qemu/qemu_monitor_text.c +++ b/src/qemu/qemu_monitor_text.c @@ -1136,7 +1136,7 @@ cleanup: static int qemuMonitorTextMigrate(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *dest) { char *cmd = NULL; @@ -1151,11 +1151,11 @@ static int qemuMonitorTextMigrate(qemuMonitorPtr mon, return -1; } - if (background & QEMU_MONITOR_MIGRATE_BACKGROUND) + if (flags & QEMU_MONITOR_MIGRATE_BACKGROUND) virBufferAddLit(&extra, " -d"); - if (background & QEMU_MONITOR_MIGRATE_NON_SHARED_DISK) + if (flags & QEMU_MONITOR_MIGRATE_NON_SHARED_DISK) virBufferAddLit(&extra, " -b"); - if (background & QEMU_MONITOR_MIGRATE_NON_SHARED_INC) + if (flags & QEMU_MONITOR_MIGRATE_NON_SHARED_INC) virBufferAddLit(&extra, " -i"); if (virBufferError(&extra)) { virBufferFreeAndReset(&extra); @@ -1202,7 +1202,7 @@ cleanup: } int qemuMonitorTextMigrateToHost(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *hostname, int port) { @@ -1214,7 +1214,7 @@ int qemuMonitorTextMigrateToHost(qemuMonitorPtr mon, return -1; } - ret = qemuMonitorTextMigrate(mon, background, uri); + ret = qemuMonitorTextMigrate(mon, flags, uri); VIR_FREE(uri); @@ -1223,7 +1223,7 @@ int qemuMonitorTextMigrateToHost(qemuMonitorPtr mon, int qemuMonitorTextMigrateToCommand(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char * const *argv) { char *argstr; @@ -1241,7 +1241,7 @@ int qemuMonitorTextMigrateToCommand(qemuMonitorPtr mon, goto cleanup; } - ret = qemuMonitorTextMigrate(mon, background, dest); + ret = qemuMonitorTextMigrate(mon, flags, dest); cleanup: VIR_FREE(argstr); @@ -1250,7 +1250,7 @@ cleanup: } int qemuMonitorTextMigrateToFile(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char * const *argv, const char *target, unsigned long long offset) @@ -1287,7 +1287,7 @@ int qemuMonitorTextMigrateToFile(qemuMonitorPtr mon, goto cleanup; } - ret = qemuMonitorTextMigrate(mon, background, dest); + ret = qemuMonitorTextMigrate(mon, flags, dest); cleanup: VIR_FREE(safe_target); @@ -1297,7 +1297,7 @@ cleanup: } int qemuMonitorTextMigrateToUnix(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *unixfile) { char *dest = NULL; @@ -1308,7 +1308,7 @@ int qemuMonitorTextMigrateToUnix(qemuMonitorPtr mon, return -1; } - ret = qemuMonitorTextMigrate(mon, background, dest); + ret = qemuMonitorTextMigrate(mon, flags, dest); VIR_FREE(dest); diff --git a/src/qemu/qemu_monitor_text.h b/src/qemu/qemu_monitor_text.h index 91eed84..9926d34 100644 --- a/src/qemu/qemu_monitor_text.h +++ b/src/qemu/qemu_monitor_text.h @@ -95,22 +95,22 @@ int qemuMonitorTextGetMigrationStatus(qemuMonitorPtr mon, unsigned long long *total); int qemuMonitorTextMigrateToHost(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *hostname, int port); int qemuMonitorTextMigrateToCommand(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char * const *argv); int qemuMonitorTextMigrateToFile(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char * const *argv, const char *target, unsigned long long offset); int qemuMonitorTextMigrateToUnix(qemuMonitorPtr mon, - unsigned int background, + unsigned int flags, const char *unixfile); int qemuMonitorTextMigrateCancel(qemuMonitorPtr mon); -- 1.6.6.1