[libvirt] [PATCH 0/3] Transport Open vSwitch per-port data during live migration
This series of commits has the end goal of allowing per-port data stored in the Open vSwitch DB to be transported during live migration. This is done by first providing a generic infrastructure for transporting network data, adding some utility functions specific to Open vSwitch, and hooking the two together. The framework provided is generic in that other networking data could be transferred as well by simply adding in additional hooks as needed. Kyle Mestery (3): Add the ability for the Qemu V3 migration protocol to include transporting network configuration. A generic framework is proposed with this patch to allow for the transfer of opaque data. Add utility functions for Open vSwitch to both save per-port data before a live migration, and restore the per-port data after a live migration. Transport Open vSwitch per-port data during live migration by using the utility functions virNetDevOpenvswitchGetMigrateData() and virNetDevOpenvswitchSetMigrateData(). cfg.mk | 9 + configure.ac | 36 +- docs/formatdomain.html.in | 45 +- docs/schemas/domaincommon.rng | 75 ++- src/conf/domain_conf.c | 394 +++++++++++- src/conf/domain_conf.h | 32 + src/esx/esx_util.c | 7 +- src/libvirt.c | 20 +- src/libvirt_private.syms | 5 + src/parallels/parallels_driver.c | 133 +++- src/qemu/qemu_capabilities.c | 679 +++++++++++++++------ src/qemu/qemu_capabilities.h | 49 +- src/qemu/qemu_command.c | 660 +++++++++++--------- src/qemu/qemu_command.h | 56 +- src/qemu/qemu_domain.c | 16 +- src/qemu/qemu_domain.h | 4 +- src/qemu/qemu_driver.c | 50 +- src/qemu/qemu_hotplug.c | 141 ++--- src/qemu/qemu_migration.c | 295 ++++++++- src/qemu/qemu_monitor.c | 114 ++-- src/qemu/qemu_monitor.h | 10 +- src/qemu/qemu_monitor_json.c | 77 ++- src/qemu/qemu_monitor_json.h | 5 +- src/qemu/qemu_monitor_text.c | 335 +++------- src/qemu/qemu_process.c | 62 +- src/rpc/virnetserverprogram.c | 11 +- src/rpc/virnettlscontext.c | 6 +- src/util/bitmap.c | 19 + src/util/bitmap.h | 6 + src/util/virnetdevopenvswitch.c | 71 +++ src/util/virnetdevopenvswitch.h | 6 + src/vbox/vbox_tmpl.c | 14 +- src/vmware/vmware_driver.c | 4 +- tests/Makefile.am | 10 +- .../domain-parallels-ct-simple.xml | 27 + tests/qemuhelptest.c | 16 +- tests/qemumonitortestutils.c | 12 +- .../qemuxml2argv-cpu-eoi-disabled.args | 4 + .../qemuxml2argv-cpu-eoi-disabled.xml | 28 + .../qemuxml2argv-cpu-eoi-enabled.args | 4 + .../qemuxml2argv-cpu-eoi-enabled.xml | 28 + .../qemuxml2argv-eoi-disabled.args | 4 + .../qemuxml2argvdata/qemuxml2argv-eoi-disabled.xml | 25 + .../qemuxml2argvdata/qemuxml2argv-eoi-enabled.args | 4 + .../qemuxml2argvdata/qemuxml2argv-eoi-enabled.xml | 25 + .../qemuxml2argv-usb-redir-filter.args | 10 + .../qemuxml2argv-usb-redir-filter.xml | 45 ++ tests/qemuxml2argvtest.c | 23 +- tests/qemuxml2xmltest.c | 6 + tests/qemuxmlnstest.c | 6 +- tests/undefine | 72 --- tests/virsh-undefine | 72 +++ tools/virsh-domain.c | 4 +- tools/virsh-network.c | 8 +- tools/virsh-pool.c | 8 +- tools/virsh-volume.c | 8 +- 56 files changed, 2678 insertions(+), 1217 deletions(-) create mode 100644 tests/domainschemadata/domain-parallels-ct-simple.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-eoi-disabled.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-eoi-disabled.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-eoi-enabled.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-eoi-enabled.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-eoi-disabled.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-eoi-disabled.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-eoi-enabled.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-eoi-enabled.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-usb-redir-filter.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-usb-redir-filter.xml delete mode 100755 tests/undefine create mode 100755 tests/virsh-undefine -- 1.7.11.4
Add the ability for the Qemu V3 migration protocol to include transporting network configuration. A generic framework is proposed with this patch to allow for the transfer of opaque data. Signed-off-by: Kyle Mestery <kmestery@cisco.com> --- src/qemu/qemu_migration.c | 260 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 258 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 1b21ef6..539b361 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -70,6 +70,7 @@ enum qemuMigrationCookieFlags { QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS, QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE, QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT, + QEMU_MIGRATION_COOKIE_FLAG_NETWORK, QEMU_MIGRATION_COOKIE_FLAG_LAST }; @@ -77,12 +78,13 @@ enum qemuMigrationCookieFlags { VIR_ENUM_DECL(qemuMigrationCookieFlag); VIR_ENUM_IMPL(qemuMigrationCookieFlag, QEMU_MIGRATION_COOKIE_FLAG_LAST, - "graphics", "lockstate", "persistent"); + "graphics", "lockstate", "persistent", "network"); enum qemuMigrationCookieFeatures { QEMU_MIGRATION_COOKIE_GRAPHICS = (1 << QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS), QEMU_MIGRATION_COOKIE_LOCKSTATE = (1 << QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE), QEMU_MIGRATION_COOKIE_PERSISTENT = (1 << QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT), + QEMU_MIGRATION_COOKIE_NETWORK = (1 << QEMU_MIGRATION_COOKIE_FLAG_NETWORK), }; typedef struct _qemuMigrationCookieGraphics qemuMigrationCookieGraphics; @@ -95,6 +97,22 @@ struct _qemuMigrationCookieGraphics { char *tlsSubject; }; +typedef struct _qemuMigrationCookieNetwork qemuMigrationCookieNetwork; +typedef qemuMigrationCookieNetwork *qemuMigrationCookieNetworkPtr; +struct _qemuMigrationCookieNetwork { + /* How many virtual NICs are we saving data for? */ + int nnets; + + /* + * Array of pointers to saved data. Each VIF will have it's own + * data to transfer. + */ + char **netdata; + + /* What type is each VIF? */ + int **viftype; +}; + typedef struct _qemuMigrationCookie qemuMigrationCookie; typedef qemuMigrationCookie *qemuMigrationCookiePtr; struct _qemuMigrationCookie { @@ -120,6 +138,9 @@ struct _qemuMigrationCookie { /* If (flags & QEMU_MIGRATION_COOKIE_PERSISTENT) */ virDomainDefPtr persistent; + + /* If (flags & QEMU_MIGRATION_COOKIE_NETWORK) */ + qemuMigrationCookieNetworkPtr network; }; static void qemuMigrationCookieGraphicsFree(qemuMigrationCookieGraphicsPtr grap) @@ -132,6 +153,27 @@ static void qemuMigrationCookieGraphicsFree(qemuMigrationCookieGraphicsPtr grap) } +static void qemuMigrationCookieNetworkFree(qemuMigrationCookieNetworkPtr + network) +{ + int i; + + if (!network) + return; + + for (i = 0; i < network->nnets; i++) { + VIR_FREE(network->viftype[i]); + VIR_FREE(network->netdata[i]); + } + + VIR_FREE(network->viftype); + + VIR_FREE(network->netdata); + + VIR_FREE(network); +} + + static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig) { if (!mig) @@ -140,6 +182,10 @@ static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig) if (mig->flags & QEMU_MIGRATION_COOKIE_GRAPHICS) qemuMigrationCookieGraphicsFree(mig->graphics); + if (mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) { + qemuMigrationCookieNetworkFree(mig->network); + } + VIR_FREE(mig->localHostname); VIR_FREE(mig->remoteHostname); VIR_FREE(mig->name); @@ -256,6 +302,59 @@ error: } +static qemuMigrationCookieNetworkPtr +qemuMigrationCookieNetworkAlloc(struct qemud_driver *driver ATTRIBUTE_UNUSED, + virDomainDefPtr def) +{ + qemuMigrationCookieNetworkPtr mig; + int i; + virDomainNetDefPtr netptr ATTRIBUTE_UNUSED; + + if (VIR_ALLOC(mig) < 0) + goto no_memory; + + mig->nnets = def->nnets; + + if (VIR_ALLOC_N(mig->netdata, def->nnets) < 0) + goto no_memory; + + if (VIR_ALLOC_N(mig->viftype, def->nnets) < 0) + goto no_memory; + + for (i = 0; i < def->nnets; i++) { + virNetDevVPortProfilePtr vport = virDomainNetGetActualVirtPortProfile(def->nets[i]); + netptr = def->nets[i]; + + if (vport) { + if (VIR_ALLOC(mig->viftype[i]) < 0) + goto no_memory; + + *mig->viftype[i] = vport->virtPortType; + + switch (vport->virtPortType) { + case VIR_NETDEV_VPORT_PROFILE_NONE: + break; + case VIR_NETDEV_VPORT_PROFILE_8021QBG: + break; + case VIR_NETDEV_VPORT_PROFILE_8021QBH: + break; + case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + break; + default: + break; + } + } + } + + return mig; + +no_memory: + virReportOOMError(); + qemuMigrationCookieNetworkFree(mig); + return NULL; +} + + static qemuMigrationCookiePtr qemuMigrationCookieNew(virDomainObjPtr dom) { @@ -370,6 +469,27 @@ qemuMigrationCookieAddPersistent(qemuMigrationCookiePtr mig, } +static int +qemuMigrationCookieAddNetwork(qemuMigrationCookiePtr mig, + struct qemud_driver *driver, + virDomainObjPtr dom) +{ + if (mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Network migration data already present")); + return -1; + } + + if (dom->def->nnets >= 1) { + if (!(mig->network = qemuMigrationCookieNetworkAlloc( + driver, dom->def))) + return -1; + mig->flags |= QEMU_MIGRATION_COOKIE_NETWORK; + } + + return 0; +} + static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf, qemuMigrationCookieGraphicsPtr grap) @@ -389,6 +509,25 @@ static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf, } +static void qemuMigrationCookieNetworkXMLFormat(virBufferPtr buf, + qemuMigrationCookieNetworkPtr optr) +{ + int i; + + virBufferAsprintf(buf, " <network nnets='%d'>\n", optr->nnets); + if (optr->nnets > 0) + virBufferAsprintf(buf, " <vifs>\n"); + for (i = 0; i < optr->nnets; i++) { + virBufferAsprintf(buf, " <vif num='%d' viftype='%d' netdata='%s'/>\n", + i, *optr->viftype[i], optr->netdata[i]); + } + if (optr->nnets > 0) + virBufferAsprintf(buf, " </vifs>\n"); + + virBufferAddLit(buf, " </network>\n"); +} + + static int qemuMigrationCookieXMLFormat(struct qemud_driver *driver, virBufferPtr buf, @@ -439,6 +578,10 @@ qemuMigrationCookieXMLFormat(struct qemud_driver *driver, virBufferAdjustIndent(buf, -2); } + if ((mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) && + mig->network) + qemuMigrationCookieNetworkXMLFormat(buf, mig->network); + virBufferAddLit(buf, "</qemu-migration>\n"); return 0; } @@ -516,6 +659,74 @@ error: } +static qemuMigrationCookieNetworkPtr +qemuMigrationCookieNetworkXMLParse(xmlXPathContextPtr ctxt) +{ + qemuMigrationCookieNetworkPtr optr; + int i; + int n; + xmlNodePtr *vifs = NULL; + char *viftype; + + if (VIR_ALLOC(optr) < 0) + goto no_memory; + + if (virXPathInt("string(./network/@nnets)", ctxt, &optr->nnets) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing nnets attribute in migration data")); + goto error; + } + + if (VIR_ALLOC_N(optr->netdata, optr->nnets) < 0) + goto no_memory; + + if (VIR_ALLOC_N(optr->viftype, optr->nnets) < 0) + goto no_memory; + + if ((n = virXPathNodeSet("./network/vifs/vif", ctxt, &vifs)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing vif information")); + goto error; + } + + for (i = 0; i < n; i++) { + if (VIR_ALLOC(optr->netdata[i]) < 0) + goto no_memory; + + if (VIR_ALLOC(optr->viftype[i]) < 0) + goto no_memory; + + if (!(optr->netdata[i] = virXMLPropString(vifs[i], "netdata"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing netdata attribute in migration data")); + goto error; + } + + if (!(viftype = virXMLPropString(vifs[i], "viftype"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing viftype attribute in migration data")); + goto error; + } + + *optr->viftype[i] = atoi(viftype); + + VIR_FREE(viftype); + } + + VIR_FREE(vifs); + + return optr; + +no_memory: + virReportOOMError(); +error: + if (vifs) + VIR_FREE(vifs); + qemuMigrationCookieNetworkFree(optr); + return NULL; +} + + static int qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, struct qemud_driver *driver, @@ -662,6 +873,11 @@ qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, VIR_FREE(nodes); } + if ((flags & QEMU_MIGRATION_COOKIE_NETWORK) && + virXPathBoolean("count(./network) > 0", ctxt) && + (!(mig->network = qemuMigrationCookieNetworkXMLParse(ctxt)))) + goto error; + return 0; error: @@ -721,6 +937,10 @@ qemuMigrationBakeCookie(qemuMigrationCookiePtr mig, qemuMigrationCookieAddPersistent(mig, dom) < 0) return -1; + if (flags & QEMU_MIGRATION_COOKIE_NETWORK && + qemuMigrationCookieAddNetwork(mig, driver, dom) < 0) + return -1; + if (!(*cookieout = qemuMigrationCookieXMLFormatStr(driver, mig))) return -1; @@ -1050,6 +1270,36 @@ qemuDomainMigrateGraphicsRelocate(struct qemud_driver *driver, } +static int +qemuDomainMigrateOPDRelocate(struct qemud_driver *driver ATTRIBUTE_UNUSED, + virDomainObjPtr vm, + qemuMigrationCookiePtr cookie) +{ + virDomainNetDefPtr netptr ATTRIBUTE_UNUSED; + int ret = 0; + int i; + + for (i = 0; i < cookie->network->nnets; i++) { + netptr = vm->def->nets[i]; + + switch (*cookie->network->viftype[i]) { + case VIR_NETDEV_VPORT_PROFILE_NONE: + break; + case VIR_NETDEV_VPORT_PROFILE_8021QBG: + break; + case VIR_NETDEV_VPORT_PROFILE_8021QBH: + break; + case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + break; + default: + break; + } + } + + return ret; +} + + /* This is called for outgoing non-p2p migrations when a connection to the * client which initiated the migration was closed but we were waiting for it * to follow up with the next phase, that is, in between @@ -1994,7 +2244,8 @@ cleanup: if (ret == 0 && qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, - QEMU_MIGRATION_COOKIE_PERSISTENT ) < 0) + QEMU_MIGRATION_COOKIE_PERSISTENT | + QEMU_MIGRATION_COOKIE_NETWORK) < 0) VIR_WARN("Unable to encode migration cookie"); qemuMigrationCookieFree(mig); @@ -2929,6 +3180,7 @@ qemuMigrationFinish(struct qemud_driver *driver, qemuDomainCleanupRemove(vm, qemuMigrationPrepareCleanup); + cookie_flags = QEMU_MIGRATION_COOKIE_NETWORK; if (flags & VIR_MIGRATE_PERSIST_DEST) cookie_flags |= QEMU_MIGRATION_COOKIE_PERSISTENT; @@ -2956,6 +3208,10 @@ qemuMigrationFinish(struct qemud_driver *driver, goto endjob; } + if (mig->network) + if (qemuDomainMigrateOPDRelocate(driver, vm, mig) < 0) + VIR_WARN("unable to provide network data for relocation"); + if (flags & VIR_MIGRATE_PERSIST_DEST) { virDomainDefPtr vmdef; if (vm->persistent) -- 1.7.11.4
On 09/14/2012 12:38 PM, Kyle Mestery wrote:
Add the ability for the Qemu V3 migration protocol to include transporting network configuration. A generic framework is proposed with this patch to allow for the transfer of opaque data.
Signed-off-by: Kyle Mestery <kmestery@cisco.com> --- src/qemu/qemu_migration.c | 260 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 258 insertions(+), 2 deletions(-)
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 1b21ef6..539b361 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -70,6 +70,7 @@ enum qemuMigrationCookieFlags { QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS, QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE, QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT, + QEMU_MIGRATION_COOKIE_FLAG_NETWORK,
QEMU_MIGRATION_COOKIE_FLAG_LAST }; @@ -77,12 +78,13 @@ enum qemuMigrationCookieFlags { VIR_ENUM_DECL(qemuMigrationCookieFlag); VIR_ENUM_IMPL(qemuMigrationCookieFlag, QEMU_MIGRATION_COOKIE_FLAG_LAST, - "graphics", "lockstate", "persistent"); + "graphics", "lockstate", "persistent", "network");
enum qemuMigrationCookieFeatures { QEMU_MIGRATION_COOKIE_GRAPHICS = (1 << QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS), QEMU_MIGRATION_COOKIE_LOCKSTATE = (1 << QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE), QEMU_MIGRATION_COOKIE_PERSISTENT = (1 << QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT), + QEMU_MIGRATION_COOKIE_NETWORK = (1 << QEMU_MIGRATION_COOKIE_FLAG_NETWORK), };
typedef struct _qemuMigrationCookieGraphics qemuMigrationCookieGraphics; @@ -95,6 +97,22 @@ struct _qemuMigrationCookieGraphics { char *tlsSubject; };
+typedef struct _qemuMigrationCookieNetwork qemuMigrationCookieNetwork; +typedef qemuMigrationCookieNetwork *qemuMigrationCookieNetworkPtr; +struct _qemuMigrationCookieNetwork { + /* How many virtual NICs are we saving data for? */ + int nnets; + + /* + * Array of pointers to saved data. Each VIF will have it's own + * data to transfer. + */ + char **netdata; + + /* What type is each VIF? */ + int **viftype;
Since each netdata has a matching viftype, please put the two of them together in a struct (e.g. qemuMigrationCookieNetdata), then replace the "char **netdata; int **viftype;" with "qemuMigrationCookieNetdata **net" Also, what exactly is viftype? If it's describing what type of interface, use a string rather than a cryptic integer value. ENUM_DECL() and ENUM_IMPL() (and the XXXToString and XXXFromString functions they define) are a big help with that. Also, why "vif"? why not "interface"? Are you concerned the types will be confused with the types of a domain's <interface> device? If so, I wouldn't worry about that, especially if you use strings rather than integers for the type.
+}; + typedef struct _qemuMigrationCookie qemuMigrationCookie; typedef qemuMigrationCookie *qemuMigrationCookiePtr; struct _qemuMigrationCookie { @@ -120,6 +138,9 @@ struct _qemuMigrationCookie {
/* If (flags & QEMU_MIGRATION_COOKIE_PERSISTENT) */ virDomainDefPtr persistent; + + /* If (flags & QEMU_MIGRATION_COOKIE_NETWORK) */ + qemuMigrationCookieNetworkPtr network; };
static void qemuMigrationCookieGraphicsFree(qemuMigrationCookieGraphicsPtr grap) @@ -132,6 +153,27 @@ static void qemuMigrationCookieGraphicsFree(qemuMigrationCookieGraphicsPtr grap) }
+static void qemuMigrationCookieNetworkFree(qemuMigrationCookieNetworkPtr + network) +{ + int i; + + if (!network) + return; + + for (i = 0; i < network->nnets; i++) { + VIR_FREE(network->viftype[i]); + VIR_FREE(network->netdata[i]); + } + + VIR_FREE(network->viftype); + + VIR_FREE(network->netdata); + + VIR_FREE(network);
There's a lot of extra blank lines in there :-) (Of course if you put viftype and netdata into a single struct, this code will be redone anyway).
+} + + static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig) { if (!mig) @@ -140,6 +182,10 @@ static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig) if (mig->flags & QEMU_MIGRATION_COOKIE_GRAPHICS) qemuMigrationCookieGraphicsFree(mig->graphics);
+ if (mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) { + qemuMigrationCookieNetworkFree(mig->network); + } + VIR_FREE(mig->localHostname); VIR_FREE(mig->remoteHostname); VIR_FREE(mig->name); @@ -256,6 +302,59 @@ error: }
+static qemuMigrationCookieNetworkPtr +qemuMigrationCookieNetworkAlloc(struct qemud_driver *driver ATTRIBUTE_UNUSED, + virDomainDefPtr def) +{ + qemuMigrationCookieNetworkPtr mig; + int i; + virDomainNetDefPtr netptr ATTRIBUTE_UNUSED; + + if (VIR_ALLOC(mig) < 0) + goto no_memory; + + mig->nnets = def->nnets; + + if (VIR_ALLOC_N(mig->netdata, def->nnets) < 0) + goto no_memory; + + if (VIR_ALLOC_N(mig->viftype, def->nnets) < 0) + goto no_memory;
Same here -> this will be a single VIR_ALLOC_N(mig->net, def->nnets)
+ + for (i = 0; i < def->nnets; i++) { + virNetDevVPortProfilePtr vport = virDomainNetGetActualVirtPortProfile(def->nets[i]); + netptr = def->nets[i]; + + if (vport) { + if (VIR_ALLOC(mig->viftype[i]) < 0) + goto no_memory; + + *mig->viftype[i] = vport->virtPortType;
Ah, I see - you're directly putting the integer equivalent of the virtPortType in here. That's fine for storing it in memory, but when you get to the Format function, you should use virNetDevVPortTypeToString(type) to put it in the XML as a mnemonic. That way it won't be broken if someone messes with the enum values.
+ + switch (vport->virtPortType) { + case VIR_NETDEV_VPORT_PROFILE_NONE: + break; + case VIR_NETDEV_VPORT_PROFILE_8021QBG: + break; + case VIR_NETDEV_VPORT_PROFILE_8021QBH: + break; + case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + break; + default: + break; + } + } + } + + return mig; + +no_memory: + virReportOOMError(); + qemuMigrationCookieNetworkFree(mig); + return NULL; +} + + static qemuMigrationCookiePtr qemuMigrationCookieNew(virDomainObjPtr dom) { @@ -370,6 +469,27 @@ qemuMigrationCookieAddPersistent(qemuMigrationCookiePtr mig, }
+static int +qemuMigrationCookieAddNetwork(qemuMigrationCookiePtr mig, + struct qemud_driver *driver, + virDomainObjPtr dom) +{ + if (mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Network migration data already present")); + return -1; + } + + if (dom->def->nnets >= 1) { + if (!(mig->network = qemuMigrationCookieNetworkAlloc( + driver, dom->def))) + return -1; + mig->flags |= QEMU_MIGRATION_COOKIE_NETWORK; + } + + return 0; +} +
static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf, qemuMigrationCookieGraphicsPtr grap) @@ -389,6 +509,25 @@ static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf, }
+static void qemuMigrationCookieNetworkXMLFormat(virBufferPtr buf, + qemuMigrationCookieNetworkPtr optr) +{ + int i; + + virBufferAsprintf(buf, " <network nnets='%d'>\n", optr->nnets);
Did you say last time why you need to put nnets in the XML? Seems like it could be deduced from the number/index of the <vif> elements.
+ if (optr->nnets > 0) + virBufferAsprintf(buf, " <vifs>\n");
Same question as last time - why not just put the <vif> elements directly inside <network> rather than having an extra level? (and, as I asked above, is there a reason to call it <vif> instead of <interface>?
+ for (i = 0; i < optr->nnets; i++) { + virBufferAsprintf(buf, " <vif num='%d' viftype='%d' netdata='%s'/>\n", + i, *optr->viftype[i], optr->netdata[i]);
Instead of num='n', maybe index='n'? num might be misinterpreted as "there are 'n' of [something]" rather than "this is for the 'n'th interface in the domain's list" (actually, this points out the usefulness of calling the element <interface> rather than <vif> - makes it clearer that this information is associated with the <interface> element of the domain. Also, the "viftype='%d'" should use '%s' and virNetDevVPortTypeToString(*optr->viftype) instead. At the end, this example seems to me to have the same information you have, but is more compact, just as easy to parse, and makes it more clear where the info came from / what it's for: <network> <interface index='1' vporttype='openvswitch' netdata='blah blah blah'/> <interface index='7' vporttype='openvswitch' netdata='blah blah blah'/> </network> (or maybe "portdata" or "openvswitchdata" instead of "netdata"?).
+ } + if (optr->nnets > 0) + virBufferAsprintf(buf, " </vifs>\n"); + + virBufferAddLit(buf, " </network>\n"); +} + + static int qemuMigrationCookieXMLFormat(struct qemud_driver *driver, virBufferPtr buf, @@ -439,6 +578,10 @@ qemuMigrationCookieXMLFormat(struct qemud_driver *driver, virBufferAdjustIndent(buf, -2); }
+ if ((mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) && + mig->network) + qemuMigrationCookieNetworkXMLFormat(buf, mig->network); + virBufferAddLit(buf, "</qemu-migration>\n"); return 0; } @@ -516,6 +659,74 @@ error: }
+static qemuMigrationCookieNetworkPtr +qemuMigrationCookieNetworkXMLParse(xmlXPathContextPtr ctxt) +{ + qemuMigrationCookieNetworkPtr optr; + int i; + int n; + xmlNodePtr *vifs = NULL; + char *viftype; + + if (VIR_ALLOC(optr) < 0) + goto no_memory; + + if (virXPathInt("string(./network/@nnets)", ctxt, &optr->nnets) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing nnets attribute in migration data")); + goto error; + } + + if (VIR_ALLOC_N(optr->netdata, optr->nnets) < 0) + goto no_memory; + + if (VIR_ALLOC_N(optr->viftype, optr->nnets) < 0) + goto no_memory; + + if ((n = virXPathNodeSet("./network/vifs/vif", ctxt, &vifs)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing vif information")); + goto error; + } + + for (i = 0; i < n; i++) { + if (VIR_ALLOC(optr->netdata[i]) < 0) + goto no_memory; + + if (VIR_ALLOC(optr->viftype[i]) < 0) + goto no_memory; + + if (!(optr->netdata[i] = virXMLPropString(vifs[i], "netdata"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing netdata attribute in migration data")); + goto error; + } + + if (!(viftype = virXMLPropString(vifs[i], "viftype"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing viftype attribute in migration data")); + goto error; + } + + *optr->viftype[i] = atoi(viftype); + + VIR_FREE(viftype); + } + + VIR_FREE(vifs); + + return optr; + +no_memory: + virReportOOMError(); +error: + if (vifs) + VIR_FREE(vifs); + qemuMigrationCookieNetworkFree(optr); + return NULL; +} + + static int qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, struct qemud_driver *driver, @@ -662,6 +873,11 @@ qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, VIR_FREE(nodes); }
+ if ((flags & QEMU_MIGRATION_COOKIE_NETWORK) && + virXPathBoolean("count(./network) > 0", ctxt) && + (!(mig->network = qemuMigrationCookieNetworkXMLParse(ctxt)))) + goto error; + return 0;
error: @@ -721,6 +937,10 @@ qemuMigrationBakeCookie(qemuMigrationCookiePtr mig, qemuMigrationCookieAddPersistent(mig, dom) < 0) return -1;
+ if (flags & QEMU_MIGRATION_COOKIE_NETWORK && + qemuMigrationCookieAddNetwork(mig, driver, dom) < 0) + return -1; + if (!(*cookieout = qemuMigrationCookieXMLFormatStr(driver, mig))) return -1;
@@ -1050,6 +1270,36 @@ qemuDomainMigrateGraphicsRelocate(struct qemud_driver *driver, }
+static int +qemuDomainMigrateOPDRelocate(struct qemud_driver *driver ATTRIBUTE_UNUSED, + virDomainObjPtr vm, + qemuMigrationCookiePtr cookie) +{ + virDomainNetDefPtr netptr ATTRIBUTE_UNUSED; + int ret = 0; + int i; + + for (i = 0; i < cookie->network->nnets; i++) { + netptr = vm->def->nets[i]; + + switch (*cookie->network->viftype[i]) { + case VIR_NETDEV_VPORT_PROFILE_NONE: + break; + case VIR_NETDEV_VPORT_PROFILE_8021QBG: + break; + case VIR_NETDEV_VPORT_PROFILE_8021QBH: + break; + case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + break; + default: + break;
You can just reduce these down to a single break;
+ } + } + + return ret; +} + + /* This is called for outgoing non-p2p migrations when a connection to the * client which initiated the migration was closed but we were waiting for it * to follow up with the next phase, that is, in between @@ -1994,7 +2244,8 @@ cleanup:
if (ret == 0 && qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, - QEMU_MIGRATION_COOKIE_PERSISTENT ) < 0) + QEMU_MIGRATION_COOKIE_PERSISTENT | + QEMU_MIGRATION_COOKIE_NETWORK) < 0) VIR_WARN("Unable to encode migration cookie");
qemuMigrationCookieFree(mig); @@ -2929,6 +3180,7 @@ qemuMigrationFinish(struct qemud_driver *driver,
qemuDomainCleanupRemove(vm, qemuMigrationPrepareCleanup);
+ cookie_flags = QEMU_MIGRATION_COOKIE_NETWORK; if (flags & VIR_MIGRATE_PERSIST_DEST) cookie_flags |= QEMU_MIGRATION_COOKIE_PERSISTENT;
@@ -2956,6 +3208,10 @@ qemuMigrationFinish(struct qemud_driver *driver, goto endjob; }
+ if (mig->network) + if (qemuDomainMigrateOPDRelocate(driver, vm, mig) < 0) + VIR_WARN("unable to provide network data for relocation"); + if (flags & VIR_MIGRATE_PERSIST_DEST) { virDomainDefPtr vmdef; if (vm->persistent)
Add utility functions for Open vSwitch to both save per-port data before a live migration, and restore the per-port data after a live migration. Signed-off-by: Kyle Mestery <kmestery@cisco.com> --- src/libvirt_private.syms | 2 ++ src/util/virnetdevopenvswitch.c | 71 +++++++++++++++++++++++++++++++++++++++++ src/util/virnetdevopenvswitch.h | 6 ++++ 3 files changed, 79 insertions(+) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 0494e1f..64ea290 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1449,7 +1449,9 @@ virNetDevMacVLanVPortProfileRegisterCallback; # virnetdevopenvswitch.h virNetDevOpenvswitchAddPort; +virNetDevOpenvswitchGetMigrateData; virNetDevOpenvswitchRemovePort; +virNetDevOpenvswitchSetMigrateData; # virnetdevtap.h diff --git a/src/util/virnetdevopenvswitch.c b/src/util/virnetdevopenvswitch.c index 97b9d52..1763a25 100644 --- a/src/util/virnetdevopenvswitch.c +++ b/src/util/virnetdevopenvswitch.c @@ -179,3 +179,74 @@ int virNetDevOpenvswitchRemovePort(const char *brname ATTRIBUTE_UNUSED, const ch virCommandFree(cmd); return ret; } + +/** + * virNetDevOpenvswitchGetMigrateData: + * @migrate: a pointer to store the data into, allocated by this function + * @ifname: name of the interface for which data is being migrated + * + * Allocates data to be migrated specific to Open vSwitch + * + * Returns 0 in case of success or -1 in case of failure + */ +int virNetDevOpenvswitchGetMigrateData(char **migrate, const char *ifname) +{ + virCommandPtr cmd = NULL; + int ret = 0; + + cmd = virCommandNewArgList(OVSVSCTL, "get", "Interface", + ifname, "external_ids:PortData", NULL); + + virCommandSetOutputBuffer(cmd, migrate); + + /* Run the command */ + if (virCommandRun(cmd, NULL) < 0) { + virReportSystemError(VIR_ERR_INTERNAL_ERROR, + _("Unable to run command to get OVS port data for " + "interface %s"), ifname); + ret = -1; + goto error; + } + + /* Wipeout the newline */ + (*migrate)[strlen(*migrate) - 1] = '\0'; + +error: + return ret; +} + +/** + * virNetDevOpenvswitchSetMigrateData: + * @migrate: the data which was transferred during migration + * @ifname: the name of the interface the data is associated with + * + * Repopulates OVS per-port data on destination host + * + * Returns 0 in case of success or -1 in case of failure + */ +int virNetDevOpenvswitchSetMigrateData(char *migrate, const char *ifname) +{ + virCommandPtr cmd = NULL; + int ret = 0; + virBufferPtr buf; + + if (VIR_ALLOC(buf) < 0) { + ret = -1; + goto error; + } + + virBufferAsprintf(buf, "external_ids:PortData=%s", migrate); + + cmd = virCommandNewArgList(OVSVSCTL, "set", "Interface", ifname, + virBufferCurrentContent(buf), NULL); + /* Run the command */ + if ((ret = virCommandRun(cmd, NULL)) < 0) { + virReportSystemError(VIR_ERR_INTERNAL_ERROR, + _("Unable to run command to set OVS port data for " + "interface %s"), ifname); + } + +error: + return ret; +} + diff --git a/src/util/virnetdevopenvswitch.h b/src/util/virnetdevopenvswitch.h index 58b4dda..a80fbef 100644 --- a/src/util/virnetdevopenvswitch.h +++ b/src/util/virnetdevopenvswitch.h @@ -42,4 +42,10 @@ int virNetDevOpenvswitchAddPort(const char *brname, int virNetDevOpenvswitchRemovePort(const char *brname, const char *ifname) ATTRIBUTE_NONNULL(2) ATTRIBUTE_RETURN_CHECK; +int virNetDevOpenvswitchGetMigrateData(char **migrate, const char *ifname) + ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_RETURN_CHECK; + +int virNetDevOpenvswitchSetMigrateData(char *migrate, const char *ifname) + ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_RETURN_CHECK; + #endif /* __VIR_NETDEV_OPENVSWITCH_H__ */ -- 1.7.11.4
Transport Open vSwitch per-port data during live migration by using the utility functions virNetDevOpenvswitchGetMigrateData() and virNetDevOpenvswitchSetMigrateData(). Signed-off-by: Kyle Mestery <kmestery@cisco.com> --- src/qemu/qemu_migration.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 539b361..50884c8 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -308,7 +308,7 @@ qemuMigrationCookieNetworkAlloc(struct qemud_driver *driver ATTRIBUTE_UNUSED, { qemuMigrationCookieNetworkPtr mig; int i; - virDomainNetDefPtr netptr ATTRIBUTE_UNUSED; + virDomainNetDefPtr netptr; if (VIR_ALLOC(mig) < 0) goto no_memory; @@ -339,6 +339,13 @@ qemuMigrationCookieNetworkAlloc(struct qemud_driver *driver ATTRIBUTE_UNUSED, case VIR_NETDEV_VPORT_PROFILE_8021QBH: break; case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + if (virNetDevOpenvswitchGetMigrateData(&mig->netdata[i], + netptr->ifname) != 0) { + virReportSystemError(VIR_ERR_INTERNAL_ERROR, + _("Unable to run command to get OVS port data for " + "interface %s"), netptr->ifname); + goto error; + } break; default: break; @@ -346,6 +353,7 @@ qemuMigrationCookieNetworkAlloc(struct qemud_driver *driver ATTRIBUTE_UNUSED, } } +error: return mig; no_memory: @@ -1275,7 +1283,7 @@ qemuDomainMigrateOPDRelocate(struct qemud_driver *driver ATTRIBUTE_UNUSED, virDomainObjPtr vm, qemuMigrationCookiePtr cookie) { - virDomainNetDefPtr netptr ATTRIBUTE_UNUSED; + virDomainNetDefPtr netptr; int ret = 0; int i; @@ -1290,12 +1298,23 @@ qemuDomainMigrateOPDRelocate(struct qemud_driver *driver ATTRIBUTE_UNUSED, case VIR_NETDEV_VPORT_PROFILE_8021QBH: break; case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + VIR_ERROR(_("KM-> Setting OVS data (%s)"), cookie->network->netdata[i]); + if (virNetDevOpenvswitchSetMigrateData(cookie->network->netdata[i], + netptr->ifname) != 0) { + virReportSystemError(VIR_ERR_INTERNAL_ERROR, + _("Unable to run command to set OVS port data for " + "interface %s"), netptr->ifname); + ret = -1; + goto error; + } + break; default: break; } } +error: return ret; } -- 1.7.11.4
I found an issue with this patch when migrating VMs without OVS interfaces. I'm fixing that issue now, and will be posting V2 of this series soon. On Sep 14, 2012, at 11:38 AM, Kyle Mestery wrote:
This series of commits has the end goal of allowing per-port data stored in the Open vSwitch DB to be transported during live migration. This is done by first providing a generic infrastructure for transporting network data, adding some utility functions specific to Open vSwitch, and hooking the two together.
The framework provided is generic in that other networking data could be transferred as well by simply adding in additional hooks as needed.
Kyle Mestery (3): Add the ability for the Qemu V3 migration protocol to include transporting network configuration. A generic framework is proposed with this patch to allow for the transfer of opaque data. Add utility functions for Open vSwitch to both save per-port data before a live migration, and restore the per-port data after a live migration. Transport Open vSwitch per-port data during live migration by using the utility functions virNetDevOpenvswitchGetMigrateData() and virNetDevOpenvswitchSetMigrateData().
cfg.mk | 9 + configure.ac | 36 +- docs/formatdomain.html.in | 45 +- docs/schemas/domaincommon.rng | 75 ++- src/conf/domain_conf.c | 394 +++++++++++- src/conf/domain_conf.h | 32 + src/esx/esx_util.c | 7 +- src/libvirt.c | 20 +- src/libvirt_private.syms | 5 + src/parallels/parallels_driver.c | 133 +++- src/qemu/qemu_capabilities.c | 679 +++++++++++++++------ src/qemu/qemu_capabilities.h | 49 +- src/qemu/qemu_command.c | 660 +++++++++++--------- src/qemu/qemu_command.h | 56 +- src/qemu/qemu_domain.c | 16 +- src/qemu/qemu_domain.h | 4 +- src/qemu/qemu_driver.c | 50 +- src/qemu/qemu_hotplug.c | 141 ++--- src/qemu/qemu_migration.c | 295 ++++++++- src/qemu/qemu_monitor.c | 114 ++-- src/qemu/qemu_monitor.h | 10 +- src/qemu/qemu_monitor_json.c | 77 ++- src/qemu/qemu_monitor_json.h | 5 +- src/qemu/qemu_monitor_text.c | 335 +++------- src/qemu/qemu_process.c | 62 +- src/rpc/virnetserverprogram.c | 11 +- src/rpc/virnettlscontext.c | 6 +- src/util/bitmap.c | 19 + src/util/bitmap.h | 6 + src/util/virnetdevopenvswitch.c | 71 +++ src/util/virnetdevopenvswitch.h | 6 + src/vbox/vbox_tmpl.c | 14 +- src/vmware/vmware_driver.c | 4 +- tests/Makefile.am | 10 +- .../domain-parallels-ct-simple.xml | 27 + tests/qemuhelptest.c | 16 +- tests/qemumonitortestutils.c | 12 +- .../qemuxml2argv-cpu-eoi-disabled.args | 4 + .../qemuxml2argv-cpu-eoi-disabled.xml | 28 + .../qemuxml2argv-cpu-eoi-enabled.args | 4 + .../qemuxml2argv-cpu-eoi-enabled.xml | 28 + .../qemuxml2argv-eoi-disabled.args | 4 + .../qemuxml2argvdata/qemuxml2argv-eoi-disabled.xml | 25 + .../qemuxml2argvdata/qemuxml2argv-eoi-enabled.args | 4 + .../qemuxml2argvdata/qemuxml2argv-eoi-enabled.xml | 25 + .../qemuxml2argv-usb-redir-filter.args | 10 + .../qemuxml2argv-usb-redir-filter.xml | 45 ++ tests/qemuxml2argvtest.c | 23 +- tests/qemuxml2xmltest.c | 6 + tests/qemuxmlnstest.c | 6 +- tests/undefine | 72 --- tests/virsh-undefine | 72 +++ tools/virsh-domain.c | 4 +- tools/virsh-network.c | 8 +- tools/virsh-pool.c | 8 +- tools/virsh-volume.c | 8 +- 56 files changed, 2678 insertions(+), 1217 deletions(-) create mode 100644 tests/domainschemadata/domain-parallels-ct-simple.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-eoi-disabled.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-eoi-disabled.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-eoi-enabled.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-eoi-enabled.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-eoi-disabled.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-eoi-disabled.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-eoi-enabled.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-eoi-enabled.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-usb-redir-filter.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-usb-redir-filter.xml delete mode 100755 tests/undefine create mode 100755 tests/virsh-undefine
-- 1.7.11.4
participants (3)
-
Kyle Mestery -
Kyle Mestery (kmestery)
-
Laine Stump