Migration just seems togo from bad to worse. We already had to
introduce a second migration protocol when adding the QEMU driver,
since the one from Xen was insufficiently flexible to cope with
passing the data the QEMU driver required.
It turns out that this protocol still has some flaws that we
need to address. The current sequence is
* Src: DumpXML
- Generate XML to pass to dst
* Dst: Prepare
- Get ready to accept incoming VM
- Generate optional cookie to pass to src
* Src: Perform
- Start migration and wait for send completion
- Kill off VM if successful, resume if failed
* Dst: Finish
- Wait for recv completion and check status
- Kill off VM if unsuccessful
The problems with this are:
- Since the first step is a generic 'DumpXML' call, we can't
add in other migration specific data. eg, we can't include
any VM lease data from lock manager plugins
- Since the first step is a generic 'DumpXML' call, we can't
emit any 'migration begin' event on the source, or have
any hook that runs right at the start of the process
- Since there is no final step on the source, if the Finish
method fails to receive all migration data & has to kill
the VM, then there's no way to resume the original VM
on the source
This patch attempts to introduce a version 3 that uses the
improved 5 step sequence
* Src: Begin
- Generate XML to pass to dst
- Generate optional cookie to pass to dst
* Dst: Prepare
- Get ready to accept incoming VM
- Generate optional cookie to pass to src
* Src: Perform
- Start migration and wait for send completion
- Generate optional cookie to pass to dst
* Dst: Finish
- Wait for recv completion and check status
- Kill off VM if failed, resume if success
- Generate optional cookie to pass to src
* Src: Confirm
- Kill off VM if success, resume if failed
The API is designed to allow both input and output cookies
in all methods where applicable. This lets us pass around
arbitrary extra driver specific data between src & dst during
migration. Combined with the extra 'Begin' method this lets
us pass lease information from source to dst at the start of
migration
Moving the killing of the source VM out of Perform and
into Confirm, means we can now recover if the dst host
can't successfully Finish receiving migration data.
---
src/driver.h | 64 +++++++++++++++++++
src/libvirt.c | 164 ++++++++++++++++++++++++++++++++++++++++++++++++
src/libvirt_internal.h | 7 ++
3 files changed, 235 insertions(+), 0 deletions(-)
diff --git a/src/driver.h b/src/driver.h
index e3a232d..b2514ed 100644
--- a/src/driver.h
+++ b/src/driver.h
@@ -486,6 +486,65 @@ typedef int
virStreamPtr st,
unsigned int flags);
+typedef char *
+ (*virDrvDomainMigrateBegin3)
+ (virConnectPtr dconn,
+ char **cookiesrc,
+ int *cookiesrclen,
+ const char *uri,
+ unsigned long flags,
+ const char *dname,
+ unsigned long resource);
+
+typedef int
+ (*virDrvDomainMigratePrepare3)
+ (virConnectPtr dconn,
+ const char *cookiesrc,
+ int cookiesrclen,
+ char **cookiedst,
+ int *cookiedstlen,
+ const char *uri_in,
+ char **uri_out,
+ unsigned long flags,
+ const char *dname,
+ unsigned long resource,
+ const char *dom_xml);
+
+typedef int
+ (*virDrvDomainMigratePerform3)
+ (virDomainPtr domain,
+ char **cookiesrc,
+ int *cookiesrclen,
+ const char *cookiedst,
+ int cookiedstlen,
+ const char *uri,
+ unsigned long flags,
+ const char *dname,
+ unsigned long resource);
+
+typedef virDomainPtr
+ (*virDrvDomainMigrateFinish3)
+ (virConnectPtr dconn,
+ const char *dname,
+ const char *cookiesrc,
+ int cookiesrclen,
+ char **cookiedst,
+ int *cookiedstlen,
+ const char *uri,
+ unsigned long flags,
+ int retcode);
+
+typedef int
+ (*virDrvDomainMigrateConfirm3)
+ (virConnectPtr dconn,
+ const char *dname,
+ const char *cookiesrc,
+ int cookiesrclen,
+ const char *cookiedst,
+ int cookiedstlen,
+ const char *uri,
+ unsigned long flags,
+ int retcode);
/**
* _virDriver:
@@ -604,6 +663,11 @@ struct _virDriver {
virDrvDomainSetMemoryParameters domainSetMemoryParameters;
virDrvDomainGetMemoryParameters domainGetMemoryParameters;
virDrvDomainOpenConsole domainOpenConsole;
+ virDrvDomainMigrateBegin3 domainMigrateBegin3;
+ virDrvDomainMigratePrepare3 domainMigratePrepare3;
+ virDrvDomainMigratePerform3 domainMigratePerform3;
+ virDrvDomainMigrateFinish3 domainMigrateFinish3;
+ virDrvDomainMigrateConfirm3 domainMigrateConfirm3;
};
typedef int
diff --git a/src/libvirt.c b/src/libvirt.c
index e24fb9e..04a7cd0 100644
--- a/src/libvirt.c
+++ b/src/libvirt.c
@@ -3342,6 +3342,22 @@ error:
}
+/*
+ * Sequence v1:
+ *
+ * Dst: Prepare
+ * - Get ready to accept incoming VM
+ * - Generate optional cookie to pass to src
+ *
+ * Src: Perform
+ * - Start migration and wait for send completion
+ * - Kill off VM if successful, resume if failed
+ *
+ * Dst: Finish
+ * - Wait for recv completion and check status
+ * - Kill off VM if unsuccessful
+ *
+ */
static virDomainPtr
virDomainMigrateVersion1 (virDomainPtr domain,
virConnectPtr dconn,
@@ -3411,6 +3427,25 @@ virDomainMigrateVersion1 (virDomainPtr domain,
return ddomain;
}
+/*
+ * Sequence v2:
+ *
+ * Src: DumpXML
+ * - Generate XML to pass to dst
+ *
+ * Dst: Prepare
+ * - Get ready to accept incoming VM
+ * - Generate optional cookie to pass to src
+ *
+ * Src: Perform
+ * - Start migration and wait for send completion
+ * - Kill off VM if successful, resume if failed
+ *
+ * Dst: Finish
+ * - Wait for recv completion and check status
+ * - Kill off VM if unsuccessful
+ *
+ */
static virDomainPtr
virDomainMigrateVersion2 (virDomainPtr domain,
virConnectPtr dconn,
@@ -3505,6 +3540,130 @@ virDomainMigrateVersion2 (virDomainPtr domain,
}
+/*
+ * Sequence v3:
+ *
+ * Src: Begin
+ * - Generate XML to pass to dst
+ * - Generate optional cookie to pass to dst
+ *
+ * Dst: Prepare
+ * - Get ready to accept incoming VM
+ * - Generate optional cookie to pass to src
+ *
+ * Src: Perform
+ * - Start migration and wait for send completion
+ * - Generate optional cookie to pass to dst
+ *
+ * Dst: Finish
+ * - Wait for recv completion and check status
+ * - Kill off VM if failed, resume if success
+ * - Generate optional cookie to pass to src
+ *
+ * Src: Confirm
+ * - Kill off VM if success, resume if failed
+ *
+ */
+static virDomainPtr
+virDomainMigrateVersion3 (virDomainPtr domain,
+ virConnectPtr dconn,
+ unsigned long flags,
+ const char *dname,
+ const char *uri,
+ unsigned long bandwidth)
+{
+ virDomainPtr ddomain = NULL;
+ char *uri_out = NULL;
+ char *cookiesrc = NULL;
+ char *cookiedst = NULL;
+ char *dom_xml = NULL;
+ int cookiesrclen = 0;
+ int cookiedstlen = 0;
+ int ret;
+ virDomainInfo info;
+ virErrorPtr orig_err = NULL;
+
+ if (!domain->conn->driver->domainMigrateBegin3) {
+ virLibConnError (domain->conn, VIR_ERR_INTERNAL_ERROR, __FUNCTION__);
+ virDispatchError(domain->conn);
+ return NULL;
+ }
+ dom_xml = domain->conn->driver->domainMigrateBegin3
+ (domain->conn, &cookiesrc, &cookiesrclen, uri, flags, dname,
+ bandwidth);
+ if (!dom_xml)
+ goto done;
+
+ ret = virDomainGetInfo (domain, &info);
+ if (ret == 0 && info.state == VIR_DOMAIN_PAUSED) {
+ flags |= VIR_MIGRATE_PAUSED;
+ }
+
+ ret = dconn->driver->domainMigratePrepare3
+ (dconn, cookiesrc, cookiesrclen, &cookiedst, &cookiedstlen,
+ uri, &uri_out, flags, dname, bandwidth, dom_xml);
+ VIR_FREE (dom_xml);
+ if (ret == -1)
+ goto done;
+
+ if (uri == NULL && uri_out == NULL) {
+ virLibConnError (domain->conn, VIR_ERR_INTERNAL_ERROR,
+ _("domainMigratePrepare2 did not set uri"));
+ virDispatchError(domain->conn);
+ goto done;
+ }
+ if (uri_out)
+ uri = uri_out; /* Did domainMigratePrepare2 change URI? */
+ assert (uri != NULL);
+
+ /* Perform the migration. The driver isn't supposed to return
+ * until the migration is complete. The src VM should remain
+ * running, but in paused state until the destination can
+ * confirm migration completion.
+ */
+ ret = domain->conn->driver->domainMigratePerform3
+ (domain, &cookiesrc, &cookiesrclen, cookiedst, cookiedstlen,
+ uri, flags, dname, bandwidth);
+
+ /* Perform failed. Make sure Finish doesn't overwrite the error */
+ if (ret < 0)
+ orig_err = virSaveLastError();
+
+ /*
+ * The status code from the source is passed to the destination.
+ * The dest can cleanup in the source indicated it failed to
+ * send all migration data. Returns NULL for ddomain if
+ * the dest was unable to complete migration.
+ */
+ dname = dname ? dname : domain->name;
+ ddomain = dconn->driver->domainMigrateFinish3
+ (dconn, dname, cookiesrc, cookiesrclen, &cookiedst, &cookiedstlen,
+ uri, flags, ret);
+
+ /*
+ * If Perform3 indicated an error, or if NULL was returned
+ * from Finish3, then the status code tells the source
+ * to resume CPUs on the original VM.
+ */
+ ret = domain->conn->driver->domainMigrateConfirm3
+ (domain->conn, dname, cookiesrc, cookiesrclen, cookiedst, cookiedstlen,
+ uri, flags, ret >= 0 && domain ? 0 : -1);
+ /* XXX is there anything we can do if Confirm3 returns -1.
+ * Probably nothing beyond praying
+ */
+
+ done:
+ if (orig_err) {
+ virSetError(orig_err);
+ virFreeError(orig_err);
+ }
+ VIR_FREE(uri_out);
+ VIR_FREE(cookiesrc);
+ VIR_FREE(cookiedst);
+ return ddomain;
+}
+
+
/*
* This is sort of a migration v3
*
@@ -3716,6 +3875,11 @@ virDomainMigrate (virDomainPtr domain,
VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
VIR_DRV_FEATURE_MIGRATION_V2))
ddomain = virDomainMigrateVersion2(domain, dconn, flags, dname, uri, bandwidth);
+ else if (VIR_DRV_SUPPORTS_FEATURE(domain->conn->driver, domain->conn,
+ VIR_DRV_FEATURE_MIGRATION_V3) &&
+ VIR_DRV_SUPPORTS_FEATURE(dconn->driver, dconn,
+ VIR_DRV_FEATURE_MIGRATION_V3))
+ ddomain = virDomainMigrateVersion3(domain, dconn, flags, dname, uri, bandwidth);
else {
/* This driver does not support any migration method */
virLibConnError(domain->conn, VIR_ERR_NO_SUPPORT, __FUNCTION__);
diff --git a/src/libvirt_internal.h b/src/libvirt_internal.h
index 1c4fa4f..d781d6b 100644
--- a/src/libvirt_internal.h
+++ b/src/libvirt_internal.h
@@ -66,6 +66,13 @@ enum {
* perform step is used.
*/
VIR_DRV_FEATURE_MIGRATION_DIRECT = 5,
+
+ /*
+ * Driver supports V3-style virDomainMigrate, ie domainMigrateBegin3/
+ * domainMigratePrepare3/domainMigratePerform3/domainMigrateFinish3/
+ * domainMigrateConfirm3.
+ */
+ VIR_DRV_FEATURE_MIGRATION_V3 = 6,
};
--
1.7.2.3
