From: Kyle Mestery <kmestery(a)cisco.com&gt; Add the ability for the Qemu V3 migration protocol to include transporting network configuration. A generic framework is proposed with this patch to allow for the transfer of opaque data. Signed-off-by: Kyle Mestery <kmestery(a)cisco.com&gt; --- src/qemu/qemu_migration.c | 238 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 236 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 487182e..67276f0 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -1,3 +1,4 @@ + /* * qemu_migration.c: QEMU migration handling * @@ -70,6 +71,7 @@ enum qemuMigrationCookieFlags { QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS, QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE, QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT, + QEMU_MIGRATION_COOKIE_FLAG_NETWORK, QEMU_MIGRATION_COOKIE_FLAG_LAST }; @@ -77,12 +79,13 @@ enum qemuMigrationCookieFlags { VIR_ENUM_DECL(qemuMigrationCookieFlag); VIR_ENUM_IMPL(qemuMigrationCookieFlag, QEMU_MIGRATION_COOKIE_FLAG_LAST, - "graphics", "lockstate", "persistent"); + "graphics", "lockstate", "persistent", "network"); enum qemuMigrationCookieFeatures { QEMU_MIGRATION_COOKIE_GRAPHICS = (1 << QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS), QEMU_MIGRATION_COOKIE_LOCKSTATE = (1 << QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE), QEMU_MIGRATION_COOKIE_PERSISTENT = (1 << QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT), + QEMU_MIGRATION_COOKIE_NETWORK = (1 << QEMU_MIGRATION_COOKIE_FLAG_NETWORK), }; typedef struct _qemuMigrationCookieGraphics qemuMigrationCookieGraphics; @@ -95,6 +98,27 @@ struct _qemuMigrationCookieGraphics { char *tlsSubject; }; +typedef struct _qemuMigrationCookieNetdata qemuMigrationCookieNetdata; +typedef qemuMigrationCookieNetdata *qemuMigrationCookieNetdataPtr; +struct _qemuMigrationCookieNetdata { + int vporttype; /* enum virNetDevVPortProfile */ + + /* + * Array of pointers to saved data. Each VIF will have it's own + * data to transfer. + */ + char *portdata; +}; + +typedef struct _qemuMigrationCookieNetwork qemuMigrationCookieNetwork; +typedef qemuMigrationCookieNetwork *qemuMigrationCookieNetworkPtr; +struct _qemuMigrationCookieNetwork { + /* How many virtual NICs are we saving data for? */ + int nnets; + + qemuMigrationCookieNetdataPtr net; +}; + typedef struct _qemuMigrationCookie qemuMigrationCookie; typedef qemuMigrationCookie *qemuMigrationCookiePtr; struct _qemuMigrationCookie { @@ -120,6 +144,9 @@ struct _qemuMigrationCookie { /* If (flags & QEMU_MIGRATION_COOKIE_PERSISTENT) */ virDomainDefPtr persistent; + + /* If (flags & QEMU_MIGRATION_COOKIE_NETWORK) */ + qemuMigrationCookieNetworkPtr network; }; static void qemuMigrationCookieGraphicsFree(qemuMigrationCookieGraphicsPtr grap) @@ -132,6 +159,23 @@ static void qemuMigrationCookieGraphicsFree(qemuMigrationCookieGraphicsPtr grap) } +static void +qemuMigrationCookieNetworkFree(qemuMigrationCookieNetworkPtr network) +{ + int i; + + if (!network) + return; + + if (network->net) { + for (i = 0; i < network->nnets; i++) + VIR_FREE(network->net[i].portdata); + } + VIR_FREE(network->net); + VIR_FREE(network); +} + + static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig) { if (!mig) @@ -140,6 +184,10 @@ static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig) if (mig->flags & QEMU_MIGRATION_COOKIE_GRAPHICS) qemuMigrationCookieGraphicsFree(mig->graphics); + if (mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) { + qemuMigrationCookieNetworkFree(mig->network); + } + VIR_FREE(mig->localHostname); VIR_FREE(mig->remoteHostname); VIR_FREE(mig->name); @@ -256,6 +304,49 @@ error: } +static qemuMigrationCookieNetworkPtr +qemuMigrationCookieNetworkAlloc(struct qemud_driver *driver ATTRIBUTE_UNUSED, + virDomainDefPtr def) +{ + qemuMigrationCookieNetworkPtr mig; + int i; + + if (VIR_ALLOC(mig) < 0) + goto no_memory; + + mig->nnets = def->nnets; + + if (VIR_ALLOC_N(mig->net, def->nnets) <0) + goto no_memory; + + for (i = 0; i < def->nnets; i++) { + virDomainNetDefPtr netptr; + virNetDevVPortProfilePtr vport; + + netptr = def->nets[i]; + vport = virDomainNetGetActualVirtPortProfile(netptr); + + if (vport) { + mig->net[i].vporttype = vport->virtPortType; + + switch (vport->virtPortType) { + case VIR_NETDEV_VPORT_PROFILE_NONE: + case VIR_NETDEV_VPORT_PROFILE_8021QBG: + case VIR_NETDEV_VPORT_PROFILE_8021QBH: + case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + default: + break; + } + } + } + return mig; + +no_memory: + virReportOOMError(); + qemuMigrationCookieNetworkFree(mig); + return NULL; +} + static qemuMigrationCookiePtr qemuMigrationCookieNew(virDomainObjPtr dom) { @@ -370,6 +461,27 @@ qemuMigrationCookieAddPersistent(qemuMigrationCookiePtr mig, } +static int +qemuMigrationCookieAddNetwork(qemuMigrationCookiePtr mig, + struct qemud_driver *driver, + virDomainObjPtr dom) +{ + if (mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Network migration data already present")); + return -1; + } + + if (dom->def->nnets > 0) { + mig->network = qemuMigrationCookieNetworkAlloc(driver, dom->def); + if (!mig->network) + return -1; + mig->flags |= QEMU_MIGRATION_COOKIE_NETWORK; + } + + return 0; +} + static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf, qemuMigrationCookieGraphicsPtr grap) @@ -389,6 +501,32 @@ static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf, } +static void +qemuMigrationCookieNetworkXMLFormat(virBufferPtr buf, + qemuMigrationCookieNetworkPtr optr) +{ + int i; + + virBufferAsprintf(buf, " <network>\n"); + for (i = 0; i < optr->nnets; i++) { + /* If optr->net.vporttype[i] is not set, there is nothing to transfer */ + if (optr->net[i].vporttype != VIR_NETDEV_VPORT_PROFILE_NONE) { + virBufferAsprintf(buf, " <interface index='%d' vporttype='%s'", + i, virNetDevVPortTypeToString(optr->net[i].vporttype)); + if (optr->net[i].portdata) { + virBufferAddLit(buf, ">\n"); + virBufferEscapeString(buf, " <portdata>%s</portdata>\n", + optr->net[i].portdata); + virBufferAddLit(buf, " </interface>\n"); + } else { + virBufferAddLit(buf, "/>\n"); + } + } + } + virBufferAddLit(buf, " </network>\n"); +} + + static int qemuMigrationCookieXMLFormat(struct qemud_driver *driver, virBufferPtr buf, @@ -439,6 +577,9 @@ qemuMigrationCookieXMLFormat(struct qemud_driver *driver, virBufferAdjustIndent(buf, -2); } + if ((mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) && mig->network) + qemuMigrationCookieNetworkXMLFormat(buf, mig->network); + virBufferAddLit(buf, "</qemu-migration>\n"); return 0; } @@ -516,6 +657,58 @@ error: } +static qemuMigrationCookieNetworkPtr +qemuMigrationCookieNetworkXMLParse(xmlXPathContextPtr ctxt) +{ + qemuMigrationCookieNetworkPtr optr; + int i; + int n; + xmlNodePtr *interfaces = NULL; + char *vporttype; + xmlNodePtr save_ctxt = ctxt->node; + + if (VIR_ALLOC(optr) < 0) + goto no_memory; + + if ((n = virXPathNodeSet("./network/interface", ctxt, &interfaces)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing interface information")); + goto error; + } + + optr->nnets = n; + if (VIR_ALLOC_N(optr->net, optr->nnets) <0) + goto no_memory; + + for (i = 0; i < n; i++) { + /* portdata is optional, and may not exist */ + ctxt->node = interfaces[i]; + optr->net[i].portdata = virXPathString("string(./portdata[1])", ctxt); + + if (!(vporttype = virXMLPropString(interfaces[i], "vporttype"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing vporttype attribute in migration data")); + goto error; + } + optr->net[i].vporttype = virNetDevVPortTypeFromString(vporttype); + } + + VIR_FREE(interfaces); + +cleanup: + ctxt->node = save_ctxt; + return optr; + +no_memory: + virReportOOMError(); +error: + VIR_FREE(interfaces); + qemuMigrationCookieNetworkFree(optr); + optr = NULL; + goto cleanup; +} + + static int qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, struct qemud_driver *driver, @@ -663,6 +856,11 @@ qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, VIR_FREE(nodes); } + if ((flags & QEMU_MIGRATION_COOKIE_NETWORK) && + virXPathBoolean("count(./network) > 0", ctxt) && + (!(mig->network = qemuMigrationCookieNetworkXMLParse(ctxt)))) + goto error; + return 0; error: @@ -722,6 +920,10 @@ qemuMigrationBakeCookie(qemuMigrationCookiePtr mig, qemuMigrationCookieAddPersistent(mig, dom) < 0) return -1; + if (flags & QEMU_MIGRATION_COOKIE_NETWORK && + qemuMigrationCookieAddNetwork(mig, driver, dom) < 0) + return -1; + if (!(*cookieout = qemuMigrationCookieXMLFormatStr(driver, mig))) return -1; @@ -1084,6 +1286,32 @@ qemuDomainMigrateGraphicsRelocate(struct qemud_driver *driver, } +static int +qemuDomainMigrateOPDRelocate(struct qemud_driver *driver ATTRIBUTE_UNUSED, + virDomainObjPtr vm, + qemuMigrationCookiePtr cookie) +{ + virDomainNetDefPtr netptr ATTRIBUTE_UNUSED; + int ret = 0; + int i; + + for (i = 0; i < cookie->network->nnets; i++) { + netptr = vm->def->nets[i]; + + switch (cookie->network->net[i].vporttype) { + case VIR_NETDEV_VPORT_PROFILE_NONE: + case VIR_NETDEV_VPORT_PROFILE_8021QBG: + case VIR_NETDEV_VPORT_PROFILE_8021QBH: + case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + default: + break; + } + } + + return ret; +} + + /* This is called for outgoing non-p2p migrations when a connection to the * client which initiated the migration was closed but we were waiting for it * to follow up with the next phase, that is, in between @@ -2029,7 +2257,8 @@ cleanup: if (ret == 0 && qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, - QEMU_MIGRATION_COOKIE_PERSISTENT ) < 0) + QEMU_MIGRATION_COOKIE_PERSISTENT | + QEMU_MIGRATION_COOKIE_NETWORK) < 0) VIR_WARN("Unable to encode migration cookie"); qemuMigrationCookieFree(mig); @@ -2963,6 +3192,7 @@ qemuMigrationFinish(struct qemud_driver *driver, qemuDomainCleanupRemove(vm, qemuMigrationPrepareCleanup); + cookie_flags = QEMU_MIGRATION_COOKIE_NETWORK; if (flags & VIR_MIGRATE_PERSIST_DEST) cookie_flags |= QEMU_MIGRATION_COOKIE_PERSISTENT; @@ -2990,6 +3220,10 @@ qemuMigrationFinish(struct qemud_driver *driver, goto endjob; } + if (mig->network) + if (qemuDomainMigrateOPDRelocate(driver, vm, mig) < 0) + VIR_WARN("unable to provide network data for relocation"); + if (flags & VIR_MIGRATE_PERSIST_DEST) { virDomainDefPtr vmdef; if (vm->persistent) -- 1.7.11.7

On 10/23/2012 08:07 AM, Michal Privoznik wrote: Well, what would have saved another level of nesting would be if nnets was part of qemuMigrationCookie instead of qemuMigrationCookieNetwork. I had already removed one extra layer of nesting when I updated Kyle's patches, and thought of making this change as well, but kind of liked the consistency of all the "sub-cookies" being a single pointer that could be checked for NULL: struct _qemuMigrationCookieNetdata { int vporttype; /* enum virNetDevVPortProfile */ char *portdata; }; struct _qemuMigrationCookieNetwork { int nnets; qemuMigrationCookieNetdataPtr net; }; struct _qemuMigrationCookie { ... /* If (flags & QEMU_MIGRATION_COOKIE_GRAPHICS) */ qemuMigrationCookieGraphicsPtr graphics; /* If (flags & QEMU_MIGRATION_COOKIE_PERSISTENT) */ virDomainDefPtr persistent; /* If (flags & QEMU_MIGRATION_COOKIE_NETWORK) */ qemuMigrationCookieNetworkPtr network; }; On the other hand, I noticed at the last minute that there is a LOCKSTATE cookie that *doesn't* follow this pattern: /* If (flags & QEMU_MIGRATION_COOKIE_LOCKSTATE) */ char *lockState; char *lockDriver; so I'd be just as happy with this: struct _qemuMigrationCookie { ... /* If (flags & QEMU_MIGRATION_COOKIE_GRAPHICS) */ qemuMigrationCookieGraphicsPtr graphics; /* If (flags & QEMU_MIGRATION_COOKIE_PERSISTENT) */ virDomainDefPtr persistent; /* If (flags & QEMU_MIGRATION_COOKIE_NETWORK) */ size_t nnets; qemuMigrationCookieNetdataPtr net; }; (i.e. eliminating the qemuMigrationCookieNet object completely) Not sure how that by itself solves the problem. I was a bit bothered by that extra object in there anyway - I'm going to respin to remove it - the array will be directly off the main cookie object.

From: Kyle Mestery <kmestery(a)cisco.com&gt; Add utility functions for Open vSwitch to both save per-port data before a live migration, and restore the per-port data after a live migration. Signed-off-by: Kyle Mestery <kmestery(a)cisco.com&gt; --- src/libvirt_private.syms | 2 ++ src/util/virnetdevopenvswitch.c | 70 +++++++++++++++++++++++++++++++++++++++++ src/util/virnetdevopenvswitch.h | 6 ++++ 3 files changed, 78 insertions(+) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 60f9c7f..699c9a3 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1500,7 +1500,9 @@ virNetDevMacVLanVPortProfileRegisterCallback; # virnetdevopenvswitch.h virNetDevOpenvswitchAddPort; +virNetDevOpenvswitchGetMigrateData; virNetDevOpenvswitchRemovePort; +virNetDevOpenvswitchSetMigrateData; # virnetdevtap.h diff --git a/src/util/virnetdevopenvswitch.c b/src/util/virnetdevopenvswitch.c index a6993b6..841f693 100644 --- a/src/util/virnetdevopenvswitch.c +++ b/src/util/virnetdevopenvswitch.c @@ -179,3 +179,73 @@ int virNetDevOpenvswitchRemovePort(const char *brname ATTRIBUTE_UNUSED, const ch virCommandFree(cmd); return ret; } + +/** + * virNetDevOpenvswitchGetMigrateData: + * @migrate: a pointer to store the data into, allocated by this function + * @ifname: name of the interface for which data is being migrated + * + * Allocates data to be migrated specific to Open vSwitch + * + * Returns 0 in case of success or -1 in case of failure + */ +int virNetDevOpenvswitchGetMigrateData(char **migrate, const char *ifname) +{ + virCommandPtr cmd = NULL; + int ret = 0; + + cmd = virCommandNewArgList(OVSVSCTL, "--timeout=5", "get", "Interface", + ifname, "external_ids:PortData", NULL); + + virCommandSetOutputBuffer(cmd, migrate); + + /* Run the command */ + if (virCommandRun(cmd, NULL) < 0) { + virReportSystemError(VIR_ERR_INTERNAL_ERROR, + _("Unable to run command to get OVS port data for " + "interface %s"), ifname); + ret = -1; + goto error; + } + + /* Wipeout the newline */ + (*migrate)[strlen(*migrate) - 1] = '\0'; + +error: + return ret; +} + +/** + * virNetDevOpenvswitchSetMigrateData: + * @migrate: the data which was transferred during migration + * @ifname: the name of the interface the data is associated with + * + * Repopulates OVS per-port data on destination host + * + * Returns 0 in case of success or -1 in case of failure + */ +int virNetDevOpenvswitchSetMigrateData(char *migrate, const char *ifname) +{ + virCommandPtr cmd = NULL; + int ret = 0; + virBufferPtr buf; + + if (VIR_ALLOC(buf) < 0) { + ret = -1; + goto error; + } + + virBufferAsprintf(buf, "external_ids:PortData=%s", migrate); + + cmd = virCommandNewArgList(OVSVSCTL, "--timeout=5", "set", "Interface", ifname, + virBufferCurrentContent(buf), NULL); + /* Run the command */ + if ((ret = virCommandRun(cmd, NULL)) < 0) { + virReportSystemError(VIR_ERR_INTERNAL_ERROR, + _("Unable to run command to set OVS port data for " + "interface %s"), ifname); + } + +error: + return ret; +} diff --git a/src/util/virnetdevopenvswitch.h b/src/util/virnetdevopenvswitch.h index 7e5b618..147cd6f 100644 --- a/src/util/virnetdevopenvswitch.h +++ b/src/util/virnetdevopenvswitch.h @@ -42,4 +42,10 @@ int virNetDevOpenvswitchAddPort(const char *brname, int virNetDevOpenvswitchRemovePort(const char *brname, const char *ifname) ATTRIBUTE_NONNULL(2) ATTRIBUTE_RETURN_CHECK; +int virNetDevOpenvswitchGetMigrateData(char **migrate, const char *ifname) + ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_RETURN_CHECK; + +int virNetDevOpenvswitchSetMigrateData(char *migrate, const char *ifname) + ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_RETURN_CHECK; + #endif /* __VIR_NETDEV_OPENVSWITCH_H__ */ -- 1.7.11.7

From: Kyle Mestery <kmestery(a)cisco.com&gt; Transport Open vSwitch per-port data during live migration by using the utility functions virNetDevOpenvswitchGetMigrateData() and virNetDevOpenvswitchSetMigrateData(). Signed-off-by: Kyle Mestery <kmestery(a)cisco.com&gt; --- src/qemu/qemu_migration.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 67276f0..abe8ffb 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -333,7 +333,16 @@ qemuMigrationCookieNetworkAlloc(struct qemud_driver *driver ATTRIBUTE_UNUSED, case VIR_NETDEV_VPORT_PROFILE_NONE: case VIR_NETDEV_VPORT_PROFILE_8021QBG: case VIR_NETDEV_VPORT_PROFILE_8021QBH: + break; case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + if (virNetDevOpenvswitchGetMigrateData(&mig->net[i].portdata, + netptr->ifname) != 0) { + virReportSystemError(VIR_ERR_INTERNAL_ERROR, + _("Unable to run command to get OVS port data for " + "interface %s"), netptr->ifname); + goto error; + } + break; default: break; } @@ -343,6 +352,7 @@ qemuMigrationCookieNetworkAlloc(struct qemud_driver *driver ATTRIBUTE_UNUSED, no_memory: virReportOOMError(); +error: qemuMigrationCookieNetworkFree(mig); return NULL; } @@ -1291,8 +1301,8 @@ qemuDomainMigrateOPDRelocate(struct qemud_driver *driver ATTRIBUTE_UNUSED, virDomainObjPtr vm, qemuMigrationCookiePtr cookie) { - virDomainNetDefPtr netptr ATTRIBUTE_UNUSED; - int ret = 0; + virDomainNetDefPtr netptr; + int ret = -1; int i; for (i = 0; i < cookie->network->nnets; i++) { @@ -1302,12 +1312,23 @@ qemuDomainMigrateOPDRelocate(struct qemud_driver *driver ATTRIBUTE_UNUSED, case VIR_NETDEV_VPORT_PROFILE_NONE: case VIR_NETDEV_VPORT_PROFILE_8021QBG: case VIR_NETDEV_VPORT_PROFILE_8021QBH: + break; case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + if (virNetDevOpenvswitchSetMigrateData(cookie->network->net[i].portdata, + netptr->ifname) != 0) { + virReportSystemError(VIR_ERR_INTERNAL_ERROR, + _("Unable to run command to set OVS port data for " + "interface %s"), netptr->ifname); + goto cleanup; + } + break; default: break; } } + ret = 0; +cleanup: return ret; } -- 1.7.11.7