[libvirt] [PATCH 6/6] Implement migration v3 protocol in QEMU driver

Implement the v3 migration protocol, which has two extra steps, 'begin' on the source host and 'confirm' on the source host. All other methods also gain both input and output cookies to allow bi-directional data passing at all stages * src/qemu/qemu_driver.c: Wire up migrate v3 APIs * src/qemu/qemu_migration.c, src/qemu/qemu_migration.h: Add begin & confirm methods --- src/qemu/qemu_driver.c | 318 +++++++++++++++++++++++++++++++++++++++++++- src/qemu/qemu_migration.c | 141 ++++++++++++++++++-- src/qemu/qemu_migration.h | 16 ++- 3 files changed, 454 insertions(+), 21 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 4f72c07..91caeea 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -866,6 +866,7 @@ qemudSupportsFeature (virConnectPtr conn ATTRIBUTE_UNUSED, int feature) { switch (feature) { case VIR_DRV_FEATURE_MIGRATION_V2: + case VIR_DRV_FEATURE_MIGRATION_V3: case VIR_DRV_FEATURE_MIGRATION_P2P: return 1; default: @@ -5343,7 +5344,9 @@ qemuDomainEventDeregisterAny(virConnectPtr conn, } -/* Migration support. */ +/******************************************************************* + * Migration Protocol Version 2 + *******************************************************************/ /* Prepare is the first step, and it runs on the destination host. * @@ -5361,6 +5364,15 @@ qemudDomainMigratePrepareTunnel(virConnectPtr dconn, struct qemud_driver *driver = dconn->privateData; int ret = -1; + virCheckFlags(VIR_MIGRATE_LIVE | + VIR_MIGRATE_PEER2PEER | + VIR_MIGRATE_TUNNELLED | + VIR_MIGRATE_PERSIST_DEST | + VIR_MIGRATE_UNDEFINE_SOURCE | + VIR_MIGRATE_PAUSED | + VIR_MIGRATE_NON_SHARED_DISK | + VIR_MIGRATE_NON_SHARED_INC, -1); + if (!dom_xml) { qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("no domain XML passed")); @@ -5480,7 +5492,7 @@ qemudDomainMigratePerform (virDomainPtr dom, ret = qemuMigrationPerform(driver, dom->conn, vm, uri, cookie, cookielen, NULL, NULL, /* No output cookies in v2 */ - flags, dname, resource); + flags, dname, resource, true); cleanup: qemuDriverUnlock(driver); @@ -5537,6 +5549,296 @@ cleanup: } +/******************************************************************* + * Migration Protocol Version 3 + *******************************************************************/ + +static char * +qemuDomainMigrateBegin3(virDomainPtr domain, + char **cookieout, + int *cookieoutlen, + unsigned long flags, + const char *dname ATTRIBUTE_UNUSED, + unsigned long resource ATTRIBUTE_UNUSED) +{ + struct qemud_driver *driver = domain->conn->privateData; + virDomainObjPtr vm; + char *xml = NULL; + + virCheckFlags(VIR_MIGRATE_LIVE | + VIR_MIGRATE_PEER2PEER | + VIR_MIGRATE_TUNNELLED | + VIR_MIGRATE_PERSIST_DEST | + VIR_MIGRATE_UNDEFINE_SOURCE | + VIR_MIGRATE_PAUSED | + VIR_MIGRATE_NON_SHARED_DISK | + VIR_MIGRATE_NON_SHARED_INC, NULL); + + qemuDriverLock(driver); + vm = virDomainFindByUUID(&driver->domains, domain->uuid); + if (!vm) { + char uuidstr[VIR_UUID_STRING_BUFLEN]; + virUUIDFormat(domain->uuid, uuidstr); + qemuReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s'"), uuidstr); + goto cleanup; + } + + xml = qemuMigrationBegin(driver, vm, + cookieout, cookieoutlen); + +cleanup: + qemuDriverUnlock(driver); + return xml; +} + +static int +qemuDomainMigratePrepare3(virConnectPtr dconn, + const char *cookiein, + int cookieinlen, + char **cookieout, + int *cookieoutlen, + const char *uri_in, + char **uri_out, + unsigned long flags, + const char *dname, + unsigned long resource ATTRIBUTE_UNUSED, + const char *dom_xml) +{ + struct qemud_driver *driver = dconn->privateData; + int ret = -1; + + virCheckFlags(VIR_MIGRATE_LIVE | + VIR_MIGRATE_PEER2PEER | + VIR_MIGRATE_TUNNELLED | + VIR_MIGRATE_PERSIST_DEST | + VIR_MIGRATE_UNDEFINE_SOURCE | + VIR_MIGRATE_PAUSED | + VIR_MIGRATE_NON_SHARED_DISK | + VIR_MIGRATE_NON_SHARED_INC, -1); + + *uri_out = NULL; + + qemuDriverLock(driver); + if (flags & VIR_MIGRATE_TUNNELLED) { + /* this is a logical error; we never should have gotten here with + * VIR_MIGRATE_TUNNELLED set + */ + qemuReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("Tunnelled migration requested but invalid RPC method called")); + goto cleanup; + } + + if (!dom_xml) { + qemuReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("no domain XML passed")); + goto cleanup; + } + + ret = qemuMigrationPrepareDirect(driver, dconn, + cookiein, cookieinlen, + cookieout, cookieoutlen, + uri_in, uri_out, + dname, dom_xml); + +cleanup: + qemuDriverUnlock(driver); + return ret; +} + + +static int +qemuDomainMigratePrepareTunnel3(virConnectPtr dconn, + virStreamPtr st, + const char *cookiein, + int cookieinlen, + char **cookieout, + int *cookieoutlen, + unsigned long flags, + const char *dname, + unsigned long resource ATTRIBUTE_UNUSED, + const char *dom_xml) +{ + struct qemud_driver *driver = dconn->privateData; + int ret = -1; + + virCheckFlags(VIR_MIGRATE_LIVE | + VIR_MIGRATE_PEER2PEER | + VIR_MIGRATE_TUNNELLED | + VIR_MIGRATE_PERSIST_DEST | + VIR_MIGRATE_UNDEFINE_SOURCE | + VIR_MIGRATE_PAUSED | + VIR_MIGRATE_NON_SHARED_DISK | + VIR_MIGRATE_NON_SHARED_INC, -1); + + if (!dom_xml) { + qemuReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("no domain XML passed")); + goto cleanup; + } + if (!(flags & VIR_MIGRATE_TUNNELLED)) { + qemuReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("PrepareTunnel called but no TUNNELLED flag set")); + goto cleanup; + } + if (st == NULL) { + qemuReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("tunnelled migration requested but NULL stream passed")); + goto cleanup; + } + + qemuDriverLock(driver); + ret = qemuMigrationPrepareTunnel(driver, dconn, + cookiein, cookieinlen, + cookieout, cookieoutlen, + st, dname, dom_xml); + qemuDriverUnlock(driver); + +cleanup: + return ret; +} + + +static int +qemuDomainMigratePerform3(virDomainPtr dom, + const char *cookiein, + int cookieinlen, + char **cookieout, + int *cookieoutlen, + const char *uri, + unsigned long flags, + const char *dname, + unsigned long resource) +{ + struct qemud_driver *driver = dom->conn->privateData; + virDomainObjPtr vm; + int ret = -1; + + virCheckFlags(VIR_MIGRATE_LIVE | + VIR_MIGRATE_PEER2PEER | + VIR_MIGRATE_TUNNELLED | + VIR_MIGRATE_PERSIST_DEST | + VIR_MIGRATE_UNDEFINE_SOURCE | + VIR_MIGRATE_PAUSED | + VIR_MIGRATE_NON_SHARED_DISK | + VIR_MIGRATE_NON_SHARED_INC, -1); + + qemuDriverLock(driver); + vm = virDomainFindByUUID(&driver->domains, dom->uuid); + if (!vm) { + char uuidstr[VIR_UUID_STRING_BUFLEN]; + virUUIDFormat(dom->uuid, uuidstr); + qemuReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s'"), uuidstr); + goto cleanup; + } + + ret = qemuMigrationPerform(driver, dom->conn, vm, + uri, cookiein, cookieinlen, + cookieout, cookieoutlen, + flags, dname, resource, false); + +cleanup: + qemuDriverUnlock(driver); + return ret; +} + + +static int +qemuDomainMigrateFinish3(virConnectPtr dconn, + const char *dname, + const char *cookiein, + int cookieinlen, + char **cookieout, + int *cookieoutlen, + const char *uri ATTRIBUTE_UNUSED, + unsigned long flags, + int cancelled, + virDomainPtr *newdom) +{ + struct qemud_driver *driver = dconn->privateData; + virDomainObjPtr vm; + virErrorPtr orig_err; + int ret = -1; + + virCheckFlags(VIR_MIGRATE_LIVE | + VIR_MIGRATE_PEER2PEER | + VIR_MIGRATE_TUNNELLED | + VIR_MIGRATE_PERSIST_DEST | + VIR_MIGRATE_UNDEFINE_SOURCE | + VIR_MIGRATE_PAUSED | + VIR_MIGRATE_NON_SHARED_DISK | + VIR_MIGRATE_NON_SHARED_INC, -1); + + /* Migration failed. Save the current error so nothing squashes it */ + orig_err = virSaveLastError(); + + qemuDriverLock(driver); + vm = virDomainFindByName(&driver->domains, dname); + if (!vm) { + qemuReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching name '%s'"), dname); + goto cleanup; + } + + *newdom = qemuMigrationFinish(driver, dconn, vm, + cookiein, cookieinlen, + cookieout, cookieoutlen, + flags, cancelled); + + ret = 0; + +cleanup: + if (orig_err) { + virSetError(orig_err); + virFreeError(orig_err); + } + qemuDriverUnlock(driver); + return ret; +} + +static int +qemuDomainMigrateConfirm3(virDomainPtr domain, + const char *cookiein, + int cookieinlen, + unsigned long flags, + int cancelled) +{ + struct qemud_driver *driver = domain->conn->privateData; + virDomainObjPtr vm; + int ret = -1; + + virCheckFlags(VIR_MIGRATE_LIVE | + VIR_MIGRATE_PEER2PEER | + VIR_MIGRATE_TUNNELLED | + VIR_MIGRATE_PERSIST_DEST | + VIR_MIGRATE_UNDEFINE_SOURCE | + VIR_MIGRATE_PAUSED | + VIR_MIGRATE_NON_SHARED_DISK | + VIR_MIGRATE_NON_SHARED_INC, -1); + + /* Migration failed. Save the current error so nothing squashes it */ + + qemuDriverLock(driver); + vm = virDomainFindByUUID(&driver->domains, domain->uuid); + if (!vm) { + char uuidstr[VIR_UUID_STRING_BUFLEN]; + virUUIDFormat(domain->uuid, uuidstr); + qemuReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s'"), uuidstr); + goto cleanup; + } + + ret = qemuMigrationConfirm(driver, domain->conn, vm, + cookiein, cookieinlen, + flags, cancelled); + +cleanup: + qemuDriverUnlock(driver); + return ret; +} + + static int qemudNodeDeviceGetPciInfo (virNodeDevicePtr dev, unsigned *domain, @@ -6860,12 +7162,12 @@ static virDriver qemuDriver = { qemuDomainSetMemoryParameters, /* domainSetMemoryParameters */ qemuDomainGetMemoryParameters, /* domainGetMemoryParameters */ qemuDomainOpenConsole, /* domainOpenConsole */ - NULL, /* domainMigrateBegin3 */ - NULL, /* domainMigratePrepare3 */ - NULL, /* domainMigratePrepareTunnel3 */ - NULL, /* domainMigratePerform3 */ - NULL, /* domainMigrateFinish3 */ - NULL, /* domainMigrateConfirm3 */ + qemuDomainMigrateBegin3, /* domainMigrateBegin3 */ + qemuDomainMigratePrepare3, /* domainMigratePrepare3 */ + qemuDomainMigratePrepareTunnel3, /* domainMigratePrepareTunnel3 */ + qemuDomainMigratePerform3, /* domainMigratePerform3 */ + qemuDomainMigrateFinish3, /* domainMigrateFinish3 */ + qemuDomainMigrateConfirm3, /* domainMigrateConfirm3 */ }; diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 097acaf..4901918 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -731,6 +731,42 @@ cleanup: } +char *qemuMigrationBegin(struct qemud_driver *driver, + virDomainObjPtr vm, + char **cookieout, + int *cookieoutlen) +{ + char *rv = NULL; + qemuMigrationCookiePtr mig = NULL; + + if (!virDomainObjIsActive(vm)) { + qemuReportError(VIR_ERR_OPERATION_INVALID, + "%s", _("domain is not running")); + goto cleanup; + } + + if (!qemuMigrationIsAllowed(vm->def)) + goto cleanup; + + if (!(mig = qemuMigrationEatCookie(vm, NULL, 0, 0))) + goto cleanup; + + if (qemuMigrationBakeCookie(mig, driver, vm, + cookieout, cookieoutlen, + 0) < 0) + goto cleanup; + + rv = qemuDomainFormatXML(driver, vm, + VIR_DOMAIN_XML_SECURE | + VIR_DOMAIN_XML_UPDATE_CPU); + +cleanup: + virDomainObjUnlock(vm); + qemuMigrationCookieFree(mig); + return rv; +} + + /* Prepare is the first step, and it runs on the destination host. * * This version starts an empty VM listening on a localhost TCP port, and @@ -1621,7 +1657,8 @@ int qemuMigrationPerform(struct qemud_driver *driver, int *cookieoutlen, unsigned long flags, const char *dname, - unsigned long resource) + unsigned long resource, + bool killOnFinish) { virDomainEventPtr event = NULL; int ret = -1; @@ -1665,18 +1702,20 @@ int qemuMigrationPerform(struct qemud_driver *driver, } /* Clean up the source domain. */ - qemuProcessStop(driver, vm, 1); - qemuDomainStopAudit(vm, "migrated"); - resume = 0; + if (killOnFinish) { + qemuProcessStop(driver, vm, 1); + qemuDomainStopAudit(vm, "migrated"); + resume = 0; - event = virDomainEventNewFromObj(vm, - VIR_DOMAIN_EVENT_STOPPED, - VIR_DOMAIN_EVENT_STOPPED_MIGRATED); - if (!vm->persistent || (flags & VIR_MIGRATE_UNDEFINE_SOURCE)) { - virDomainDeleteConfig(driver->configDir, driver->autostartDir, vm); - if (qemuDomainObjEndJob(vm) > 0) - virDomainRemoveInactive(&driver->domains, vm); - vm = NULL; + event = virDomainEventNewFromObj(vm, + VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_MIGRATED); + if (!vm->persistent || (flags & VIR_MIGRATE_UNDEFINE_SOURCE)) { + virDomainDeleteConfig(driver->configDir, driver->autostartDir, vm); + if (qemuDomainObjEndJob(vm) > 0) + virDomainRemoveInactive(&driver->domains, vm); + vm = NULL; + } } ret = 0; @@ -1881,3 +1920,81 @@ cleanup: qemuMigrationCookieFree(mig); return dom; } + +int qemuMigrationConfirm(struct qemud_driver *driver, + virConnectPtr conn, + virDomainObjPtr vm, + const char *cookiein, + int cookieinlen, + unsigned int flags, + int retcode) +{ + qemuMigrationCookiePtr mig; + virDomainEventPtr event = NULL; + int rv = -1; + + if (!(mig = qemuMigrationEatCookie(vm, cookiein, cookieinlen, 0))) + return -1; + + if (qemuDomainObjBeginJobWithDriver(driver, vm) < 0) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("guest unexpectedly quit")); + goto endjob; + } + + /* Did the migration go as planned? If yes, kill off the + * domain object, but if no, resume CPUs + */ + if (retcode == 0) { + qemuProcessStop(driver, vm, 1); + qemuDomainStopAudit(vm, "migrated"); + + event = virDomainEventNewFromObj(vm, + VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_MIGRATED); + if (!vm->persistent || (flags & VIR_MIGRATE_UNDEFINE_SOURCE)) { + virDomainDeleteConfig(driver->configDir, driver->autostartDir, vm); + if (qemuDomainObjEndJob(vm) > 0) + virDomainRemoveInactive(&driver->domains, vm); + vm = NULL; + } + } else { + + /* run 'cont' on the destination, which allows migration on qemu + * >= 0.10.6 to work properly. This isn't strictly necessary on + * older qemu's, but it also doesn't hurt anything there + */ + if (qemuProcessStartCPUs(driver, vm, conn) < 0) { + if (virGetLastError() == NULL) + qemuReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("resume operation failed")); + goto endjob; + } + + event = virDomainEventNewFromObj(vm, + VIR_DOMAIN_EVENT_RESUMED, + VIR_DOMAIN_EVENT_RESUMED_MIGRATED); + if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0) { + VIR_WARN("Failed to save status on vm %s", vm->def->name); + goto endjob; + } + } + + qemuMigrationCookieFree(mig); + rv = 0; + +endjob: + if (vm && + qemuDomainObjEndJob(vm) == 0) + vm = NULL; + +cleanup: + if (vm) + virDomainObjUnlock(vm); + if (event) + qemuDomainEventQueue(driver, event); + return rv; +} diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index e4e68dc..3c7bf62 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -32,6 +32,11 @@ int qemuMigrationSetOffline(struct qemud_driver *driver, int qemuMigrationWaitForCompletion(struct qemud_driver *driver, virDomainObjPtr vm); +char *qemuMigrationBegin(struct qemud_driver *driver, + virDomainObjPtr vm, + char **cookieout, + int *cookieoutlen); + int qemuMigrationPrepareTunnel(struct qemud_driver *driver, virConnectPtr dconn, const char *cookiein, @@ -63,7 +68,8 @@ int qemuMigrationPerform(struct qemud_driver *driver, int *cookieoutlen, unsigned long flags, const char *dname, - unsigned long resource); + unsigned long resource, + bool killOnFinish); virDomainPtr qemuMigrationFinish(struct qemud_driver *driver, virConnectPtr dconn, @@ -75,5 +81,13 @@ virDomainPtr qemuMigrationFinish(struct qemud_driver *driver, unsigned long flags, int retcode); +int qemuMigrationConfirm(struct qemud_driver *driver, + virConnectPtr conn, + virDomainObjPtr vm, + const char *cookiein, + int cookieinlen, + unsigned int flags, + int retcode); + #endif /* __QEMU_MIGRATION_H__ */ -- 1.7.4