[libvirt] [PATCH 1/2] qemu: don't bootup guest in reboot operation if the migration job is running
After an reboot operation, the guest will shutdown first then bootup in a separate thread from the eventloop thread. If we start the migration job, according to the time gap between reboot and migrate command, the system_reset action of bootup operation will probably block from the migration job utill timeouts. In this case, the solution is that we don't bootup the guest on the source server, we pass the task to target machine. --- src/qemu/qemu_process.c | 15 +++++++++++++-- 1 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 1df3637..7c7132a 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -632,14 +632,14 @@ qemuProcessHandleShutdown(qemuMonitorPtr mon ATTRIBUTE_UNUSED, virDomainObjPtr vm) { struct qemud_driver *driver = qemu_driver; - qemuDomainObjPrivatePtr priv; + qemuDomainObjPrivatePtr priv = vm->privateData; + struct qemuDomainJobObj *job = &priv->job; virDomainEventPtr event = NULL; VIR_DEBUG("vm=%p", vm); virDomainObjLock(vm); - priv = vm->privateData; if (priv->gotShutdown) { VIR_DEBUG("Ignoring repeated SHUTDOWN event from domain %s", vm->def->name); @@ -668,6 +668,17 @@ qemuProcessHandleShutdown(qemuMonitorPtr mon ATTRIBUTE_UNUSED, if (priv->agent) qemuAgentNotifyEvent(priv->agent, QEMU_AGENT_EVENT_SHUTDOWN); + /* + * When a vm is in the job of migrating out and meanwhile + * the guest is shutting down for an reboot, we keep the + * shutdown state of the vm on the source server, letting + * the target machine do the booting work after migration. + */ + if (priv->fakeReboot && + job->asyncJob && + job->asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT) + goto unlock; + qemuProcessShutdownOrReboot(driver, vm); unlock: -- 1.7.7.5
When the guest is rebooting, there are two situations to consider. If the guest rebooting finished after migration job, we implement the fakeReboot in cookie passed into target libvirtd server in Prepare phase. The qemu monitor callback will handle it automatically. If the guest rebooting finished before migration job, we stop the guest bootup on the source, and inform the target libvirtd server in the Perfrom phase and let target libvirtd do the bootup work instead. --- src/qemu/qemu_migration.c | 175 ++++++++++++++++++++++++++++++++++++--------- 1 files changed, 140 insertions(+), 35 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index b893fd5..5038d7a 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -70,6 +70,8 @@ enum qemuMigrationCookieFlags { QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS, QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE, QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT, + QEMU_MIGRATION_COOKIE_FLAG_FAKEREBOOT, + QEMU_MIGRATION_COOKIE_FLAG_SHUTDOWN, QEMU_MIGRATION_COOKIE_FLAG_LAST }; @@ -77,12 +79,14 @@ enum qemuMigrationCookieFlags { VIR_ENUM_DECL(qemuMigrationCookieFlag); VIR_ENUM_IMPL(qemuMigrationCookieFlag, QEMU_MIGRATION_COOKIE_FLAG_LAST, - "graphics", "lockstate", "persistent"); + "graphics", "lockstate", "persistent", "fakereboot", "shutdown"); enum qemuMigrationCookieFeatures { QEMU_MIGRATION_COOKIE_GRAPHICS = (1 << QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS), QEMU_MIGRATION_COOKIE_LOCKSTATE = (1 << QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE), QEMU_MIGRATION_COOKIE_PERSISTENT = (1 << QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT), + QEMU_MIGRATION_COOKIE_FAKEREBOOT = (1 << QEMU_MIGRATION_COOKIE_FLAG_FAKEREBOOT), + QEMU_MIGRATION_COOKIE_SHUTDOWN = (1 << QEMU_MIGRATION_COOKIE_FLAG_SHUTDOWN), }; typedef struct _qemuMigrationCookieGraphics qemuMigrationCookieGraphics; @@ -111,6 +115,11 @@ struct _qemuMigrationCookie { unsigned char uuid[VIR_UUID_BUFLEN]; char *name; + /* If (flags & QEMU_MIGRATION_COOKIE_FAKEREBOOT) */ + bool fakeReboot; + /* If (flags & QEMU_MIGRATION_COOKIE_SHUTDOWN) */ + bool gotShutdown; + /* If (flags & QEMU_MIGRATION_COOKIE_LOCKSTATE) */ char *lockState; char *lockDriver; @@ -370,6 +379,43 @@ qemuMigrationCookieAddPersistent(qemuMigrationCookiePtr mig, } +static int +qemuMigrationCookieFakeRebootCheck(qemuMigrationCookiePtr mig, + virDomainObjPtr dom) +{ + qemuDomainObjPrivatePtr priv = dom->privateData; + + if (mig->fakeReboot) { + qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Migration fakereboot value already set")); + return -1; + } + + if (priv->fakeReboot) + mig->fakeReboot = priv->fakeReboot; + + return 0; +} + + +static int +qemuMigrationCookieShutdownCheck(qemuMigrationCookiePtr mig, + virDomainObjPtr dom) +{ + qemuDomainObjPrivatePtr priv = dom->privateData; + + if (mig->gotShutdown) { + qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Migration shutdown value already set")); + return -1; + } + + if (priv->gotShutdown) + mig->gotShutdown = priv->gotShutdown; + + return 0; +} + static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf, qemuMigrationCookieGraphicsPtr grap) @@ -404,6 +450,13 @@ qemuMigrationCookieXMLFormat(struct qemud_driver *driver, virBufferAsprintf(buf, "<qemu-migration>\n"); virBufferEscapeString(buf, " <name>%s</name>\n", mig->name); virBufferAsprintf(buf, " <uuid>%s</uuid>\n", uuidstr); + + if (mig->fakeReboot) + virBufferAsprintf(buf, " <fakereboot/>\n"); + + if (mig->gotShutdown) + virBufferAsprintf(buf, " <gotshutdown/>\n"); + virBufferEscapeString(buf, " <hostname>%s</hostname>\n", mig->localHostname); virBufferAsprintf(buf, " <hostuuid>%s</hostuuid>\n", hostuuidstr); @@ -562,6 +615,11 @@ qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, } VIR_FREE(tmp); + /* Set fakeReboot value if present */ + mig->fakeReboot = virXPathBoolean("boolean(./fakereboot)", ctxt) == 1; + /* Set shutdown value if present */ + mig->gotShutdown = virXPathBoolean("boolean(./gotshutdown)", ctxt) == 1; + /* Check & forbid "localhost" migration */ if (!(mig->remoteHostname = virXPathString("string(./hostname[1])", ctxt))) { qemuReportError(VIR_ERR_INTERNAL_ERROR, @@ -721,6 +779,14 @@ qemuMigrationBakeCookie(qemuMigrationCookiePtr mig, qemuMigrationCookieAddPersistent(mig, dom) < 0) return -1; + if (flags & QEMU_MIGRATION_COOKIE_FAKEREBOOT && + qemuMigrationCookieFakeRebootCheck(mig, dom) < 0) + return -1; + + if (flags & QEMU_MIGRATION_COOKIE_SHUTDOWN && + qemuMigrationCookieShutdownCheck(mig, dom) < 0) + return -1; + if (!(*cookieout = qemuMigrationCookieXMLFormatStr(driver, mig))) return -1; @@ -1144,7 +1210,9 @@ char *qemuMigrationBegin(struct qemud_driver *driver, if (qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, - QEMU_MIGRATION_COOKIE_LOCKSTATE) < 0) + QEMU_MIGRATION_COOKIE_LOCKSTATE | + QEMU_MIGRATION_COOKIE_FAKEREBOOT) < 0) + goto cleanup; if (xmlin) { @@ -1194,6 +1262,31 @@ qemuMigrationPrepareCleanup(struct qemud_driver *driver, qemuDomainObjDiscardAsyncJob(driver, vm); } + +static void +qemuMigrationPrepareVMPrivateSave(qemuDomainObjPrivatePtr priv, + qemuMigrationCookiePtr mig) +{ + if (!mig || !priv) + return; + + if (mig->lockState) { + VIR_DEBUG("Received lockstate %s", mig->lockState); + VIR_FREE(priv->lockState); + priv->lockState = mig->lockState; + mig->lockState = NULL; + } else { + VIR_DEBUG("Received no lockstate"); + } + + if (mig->fakeReboot) { + VIR_DEBUG("Received fakeReboot is True"); + priv->fakeReboot = mig->fakeReboot; + mig->fakeReboot = false; + } +} + + static int qemuMigrationPrepareAny(struct qemud_driver *driver, virConnectPtr dconn, @@ -1335,14 +1428,7 @@ qemuMigrationPrepareAny(struct qemud_driver *driver, dataFD[1] = -1; /* 'st' owns the FD now & will close it */ } - if (mig->lockState) { - VIR_DEBUG("Received lockstate %s", mig->lockState); - VIR_FREE(priv->lockState); - priv->lockState = mig->lockState; - mig->lockState = NULL; - } else { - VIR_DEBUG("Received no lockstate"); - } + qemuMigrationPrepareVMPrivateSave(priv, mig); if (qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, QEMU_MIGRATION_COOKIE_GRAPHICS) < 0) { @@ -1990,7 +2076,8 @@ cleanup: if (ret == 0 && qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, - QEMU_MIGRATION_COOKIE_PERSISTENT ) < 0) + QEMU_MIGRATION_COOKIE_PERSISTENT | + QEMU_MIGRATION_COOKIE_SHUTDOWN ) < 0) VIR_WARN("Unable to encode migration cookie"); qemuMigrationCookieFree(mig); @@ -2934,6 +3021,16 @@ qemuMigrationFinish(struct qemud_driver *driver, cookieinlen, cookie_flags))) goto endjob; + /* + * If fakeReboot is not true, we ignore the value of shutdown. + * Because the guest is shutting down in that case but reboot. + */ + if (priv->fakeReboot && mig->gotShutdown) { + VIR_DEBUG("Received shutdown is True"); + priv->gotShutdown = mig->gotShutdown; + } + mig->gotShutdown = false; + /* Did the migration go as planned? If yes, return the domain * object, but if no, clean up the empty qemu process. */ @@ -2944,15 +3041,8 @@ qemuMigrationFinish(struct qemud_driver *driver, goto endjob; } - if (qemuMigrationVPAssociatePortProfiles(vm->def) < 0) { - qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, - VIR_QEMU_PROCESS_STOP_MIGRATED); - virDomainAuditStop(vm, "failed"); - event = virDomainEventNewFromObj(vm, - VIR_DOMAIN_EVENT_STOPPED, - VIR_DOMAIN_EVENT_STOPPED_FAILED); - goto endjob; - } + if (qemuMigrationVPAssociatePortProfiles(vm->def) < 0) + goto stopqemu; if (flags & VIR_MIGRATE_PERSIST_DEST) { virDomainDefPtr vmdef; @@ -3002,6 +3092,23 @@ qemuMigrationFinish(struct qemud_driver *driver, } if (!(flags & VIR_MIGRATE_PAUSED)) { + /* If both fakeReboot annd shutdown are true, that means the guest + * has done with the work of shutdown on the source server. we just + * need todo a reset of virtual hardware, followed by restart of the + * CPUS. + */ + if (priv->fakeReboot && priv->gotShutdown) { + if (qemuDomainObjEnterMonitorAsync(driver, vm, + QEMU_ASYNC_JOB_MIGRATION_IN) < 0) + goto stopqemu; + + if (qemuMonitorSystemReset(priv->mon) < 0) { + qemuDomainObjExitMonitorWithDriver(driver, vm); + goto stopqemu; + } + qemuDomainObjExitMonitorWithDriver(driver, vm); + } + /* run 'cont' on the destination, which allows migration on qemu * >= 0.10.6 to work properly. This isn't strictly necessary on * older qemu's, but it also doesn't hurt anything there @@ -3025,14 +3132,9 @@ qemuMigrationFinish(struct qemud_driver *driver, * target in paused state, in case admin can fix * things up */ - if (v3proto) { - qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, - VIR_QEMU_PROCESS_STOP_MIGRATED); - virDomainAuditStop(vm, "failed"); - event = virDomainEventNewFromObj(vm, - VIR_DOMAIN_EVENT_STOPPED, - VIR_DOMAIN_EVENT_STOPPED_FAILED); - } + if (v3proto) + goto stopqemu; + goto endjob; } } @@ -3057,18 +3159,21 @@ qemuMigrationFinish(struct qemud_driver *driver, /* Guest is successfully running, so cancel previous auto destroy */ qemuProcessAutoDestroyRemove(driver, vm); - } else { - qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, - VIR_QEMU_PROCESS_STOP_MIGRATED); - virDomainAuditStop(vm, "failed"); - event = virDomainEventNewFromObj(vm, - VIR_DOMAIN_EVENT_STOPPED, - VIR_DOMAIN_EVENT_STOPPED_FAILED); } if (qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, 0) < 0) VIR_WARN("Unable to encode migration cookie"); + goto endjob; + +stopqemu: + qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, + VIR_QEMU_PROCESS_STOP_MIGRATED); + virDomainAuditStop(vm, "failed"); + event = virDomainEventNewFromObj(vm, + VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_FAILED); + endjob: if (qemuMigrationJobFinish(driver, vm) == 0) { vm = NULL; -- 1.7.7.5
participants (1)
-
Guannan Ren