This patch introduces the usage of the pre-associate state of the IEEE 802.1Qbg standard on incoming VM migration on the target host. It is in response to bugzilla entry 632750. https://bugzilla.redhat.com/show_bug.cgi?id=632750 For being able to differentiate the exact reason as to why a macvtap device is being created, either due to a VM creation or an incoming VM migration, I needed to pass that reason as a parameter from wherever qemudStartVMDaemon is being called in order to determine whether to send an ASSOCIATE (VM creation) or a PRE-ASSOCIATE (incoming VM migration) towards lldpad. I am also fixing problem with the virsh domainxml-to-native call on the way. Gerhard successfully tested the patch with a recent blade network 802.1Qbg-compliant switch. The patch should not have any side-effects on the 802.1Qbh support in libvirt, but Scott (cc'ed) may want to verify this. Signed-off-by: Stefan Berger <stefanb@us.ibm.com> Signed-off-by: Gerhard Stenzel <gerhard.stenzel@de.ibm.com> --- src/libvirt_macvtap.syms | 2 + src/qemu/qemu_conf.c | 12 +++++-- src/qemu/qemu_conf.h | 7 +++- src/qemu/qemu_driver.c | 72 +++++++++++++++++++++++++++++++++++++---------- src/util/macvtap.c | 54 ++++++++++++++++++++++++++++++----- src/util/macvtap.h | 20 +++++++++++-- 6 files changed, 137 insertions(+), 30 deletions(-) Index: libvirt-acl/src/qemu/qemu_conf.c =================================================================== --- libvirt-acl.orig/src/qemu/qemu_conf.c +++ libvirt-acl/src/qemu/qemu_conf.c @@ -1640,7 +1640,8 @@ qemudPhysIfaceConnect(virConnectPtr conn struct qemud_driver *driver, virDomainNetDefPtr net, unsigned long long qemuCmdFlags, - const unsigned char *vmuuid) + const unsigned char *vmuuid, + enum vmOperation vmop) { int rc; #if WITH_MACVTAP @@ -1654,7 +1655,8 @@ qemudPhysIfaceConnect(virConnectPtr conn rc = openMacvtapTap(net->ifname, net->mac, net->data.direct.linkdev, net->data.direct.mode, vnet_hdr, vmuuid, -&net->data.direct.virtPortProfile,&res_ifname); +&net->data.direct.virtPortProfile,&res_ifname, + vmop); if (rc>= 0) { VIR_FREE(net->ifname); net->ifname = res_ifname; @@ -3951,7 +3953,8 @@ int qemudBuildCommandLine(virConnectPtr int **vmfds, int *nvmfds, const char *migrateFrom, - virDomainSnapshotObjPtr current_snapshot) + virDomainSnapshotObjPtr current_snapshot, + enum vmOperation vmop) { int i; char memory[50]; @@ -4794,7 +4797,8 @@ int qemudBuildCommandLine(virConnectPtr } else if (net->type == VIR_DOMAIN_NET_TYPE_DIRECT) { int tapfd = qemudPhysIfaceConnect(conn, driver, net, qemuCmdFlags, - def->uuid); + def->uuid, + vmop); if (tapfd< 0) goto error; Index: libvirt-acl/src/qemu/qemu_conf.h =================================================================== --- libvirt-acl.orig/src/qemu/qemu_conf.h +++ libvirt-acl/src/qemu/qemu_conf.h @@ -40,6 +40,7 @@ # include "cpu_conf.h" # include "driver.h" # include "bitmap.h" +# include "macvtap.h" # define qemudDebug(fmt, ...) do {} while(0) @@ -238,7 +239,8 @@ int qemudBuildCommandLine int **vmfds, int *nvmfds, const char *migrateFrom, - virDomainSnapshotObjPtr current_snapshot) + virDomainSnapshotObjPtr current_snapshot, + enum vmOperation vmop) ATTRIBUTE_NONNULL(1); /* With vlan == -1, use netdev syntax, else old hostnet */ @@ -317,7 +319,8 @@ int qemudPhysIfaceConnect(virConnectPtr struct qemud_driver *driver, virDomainNetDefPtr net, unsigned long long qemuCmdFlags, - const unsigned char *vmuuid); + const unsigned char *vmuuid, + enum vmOperation vmop); int qemudProbeMachineTypes (const char *binary, virCapsGuestMachinePtr **machines, Index: libvirt-acl/src/qemu/qemu_driver.c =================================================================== --- libvirt-acl.orig/src/qemu/qemu_driver.c +++ libvirt-acl/src/qemu/qemu_driver.c @@ -163,7 +163,8 @@ static int qemudStartVMDaemon(virConnect const char *migrateFrom, bool start_paused, int stdin_fd, - const char *stdin_path); + const char *stdin_path, + enum vmOperation vmop); static void qemudShutdownVMDaemon(struct qemud_driver *driver, virDomainObjPtr vm, @@ -3864,7 +3865,8 @@ static int qemudStartVMDaemon(virConnect const char *migrateFrom, bool start_paused, int stdin_fd, - const char *stdin_path) { + const char *stdin_path, + enum vmOperation vmop) { const char **argv = NULL, **tmp; const char **progenv = NULL; int i, ret, runflags; @@ -4065,7 +4067,7 @@ static int qemudStartVMDaemon(virConnect if (qemudBuildCommandLine(conn, driver, vm->def, priv->monConfig, priv->monJSON, qemuCmdFlags,&argv,&progenv, &vmfds,&nvmfds, migrateFrom, - vm->current_snapshot)< 0) + vm->current_snapshot, vmop)< 0) goto cleanup; if (qemuDomainSnapshotSetInactive(vm, driver->snapshotDir)< 0) @@ -4854,7 +4856,7 @@ static virDomainPtr qemudDomainCreate(vi if (qemudStartVMDaemon(conn, driver, vm, NULL, (flags& VIR_DOMAIN_START_PAUSED) != 0, - -1, NULL)< 0) { + -1, NULL, VM_OP_CREATE)< 0) { qemuDomainStartAudit(vm, "booted", false); if (qemuDomainObjEndJob(vm)> 0) virDomainRemoveInactive(&driver->domains, @@ -6997,7 +6999,8 @@ qemudDomainSaveImageStartVM(virConnectPt } /* Set the migration source and start it up. */ - ret = qemudStartVMDaemon(conn, driver, vm, "stdio", true, fd, path); + ret = qemudStartVMDaemon(conn, driver, vm, "stdio", true, fd, path, + VM_OP_RESTORE); if (intermediate_pid != -1) { /* Wait for intermediate process to exit */ @@ -7316,14 +7319,15 @@ static char *qemuDomainXMLToNative(virCo if (!def) goto cleanup; - /* Since we're just exporting args, we can't do bridge/network - * setups, since libvirt will normally create TAP devices + /* Since we're just exporting args, we can't do bridge/network/direct + * setups, since libvirt will normally create TAP/macvtap devices * directly. We convert those configs into generic 'ethernet' * config and assume the user has suitable 'ifup-qemu' scripts */ for (i = 0 ; i< def->nnets ; i++) { virDomainNetDefPtr net = def->nets[i]; - if (net->type == VIR_DOMAIN_NET_TYPE_NETWORK) { + if (net->type == VIR_DOMAIN_NET_TYPE_NETWORK || + net->type == VIR_DOMAIN_NET_TYPE_DIRECT) { VIR_FREE(net->data.network.name); memset(net, 0, sizeof *net); @@ -7379,7 +7383,8 @@ static char *qemuDomainXMLToNative(virCo &monConfig, 0, qemuCmdFlags, &retargv,&retenv, NULL, NULL, /* Don't want it to create TAP devices */ - NULL, NULL)< 0) { + NULL, NULL, + VM_OP_NO_OP)< 0) { goto cleanup; } @@ -7466,7 +7471,8 @@ static int qemudDomainObjStart(virConnec goto cleanup; } - ret = qemudStartVMDaemon(conn, driver, vm, NULL, start_paused, -1, NULL); + ret = qemudStartVMDaemon(conn, driver, vm, NULL, start_paused, -1, NULL, + VM_OP_CREATE); qemuDomainStartAudit(vm, "booted", ret>= 0); if (ret>= 0) { virDomainEventPtr event = @@ -8309,7 +8315,8 @@ static int qemudDomainAttachNetDevice(vi if ((tapfd = qemudPhysIfaceConnect(conn, driver, net, qemuCmdFlags, - vm->def->uuid))< 0) + vm->def->uuid, + VM_OP_CREATE))< 0) return -1; } @@ -10983,7 +10990,7 @@ qemudDomainMigratePrepareTunnel(virConne * -incoming unix:/path/to/file or exec:nc -U /path/to/file */ internalret = qemudStartVMDaemon(dconn, driver, vm, migrateFrom, true, - -1, NULL); + -1, NULL, VM_OP_MIGRATE_IN_START); VIR_FREE(migrateFrom); if (internalret< 0) { qemuDomainStartAudit(vm, "migrated", false); @@ -11229,7 +11236,7 @@ qemudDomainMigratePrepare2 (virConnectPt */ snprintf (migrateFrom, sizeof (migrateFrom), "tcp:0.0.0.0:%d", this_port); if (qemudStartVMDaemon (dconn, driver, vm, migrateFrom, true, - -1, NULL)< 0) { + -1, NULL, VM_OP_MIGRATE_IN_START)< 0) { qemuDomainStartAudit(vm, "migrated", false); /* Note that we don't set an error here because qemudStartVMDaemon * should have already done that. @@ -11844,6 +11851,41 @@ cleanup: return ret; } +static void +qemudVPAssociatePortProfiles(virDomainDefPtr def) { + int i; + int last_good_net = -1; + virDomainNetDefPtr net; + + for (i = 0; i< def->nnets; i++) { + net = def->nets[i]; + if (net->type == VIR_DOMAIN_NET_TYPE_DIRECT) { + if (vpAssociatePortProfileId(net->ifname, + net->mac, + net->data.direct.linkdev, +&net->data.direct.virtPortProfile, + def->uuid, + VM_OP_MIGRATE_IN_FINISH) != 0) + goto err_exit; + } + last_good_net = i; + } + + return; + +err_exit: + for (i = 0; i< last_good_net; i++) { + net = def->nets[i]; + if (net->type == VIR_DOMAIN_NET_TYPE_DIRECT) { + vpDisassociatePortProfileId(net->ifname, + net->mac, + net->data.direct.linkdev, +&net->data.direct.virtPortProfile, + VM_OP_MIGRATE_IN_FINISH); + } + } +} + /* Finish is the third and final step, and it runs on the destination host. */ static virDomainPtr qemudDomainMigrateFinish2 (virConnectPtr dconn, @@ -11904,6 +11946,8 @@ qemudDomainMigrateFinish2 (virConnectPtr goto cleanup; } + qemudVPAssociatePortProfiles(vm->def); + if (flags& VIR_MIGRATE_PERSIST_DEST) { if (vm->persistent) newVM = 0; @@ -12796,7 +12840,7 @@ static int qemuDomainRevertToSnapshot(vi goto endjob; rc = qemudStartVMDaemon(snapshot->domain->conn, driver, vm, NULL, - false, -1, NULL); + false, -1, NULL, VM_OP_CREATE); qemuDomainStartAudit(vm, "from-snapshot", rc>= 0); if (qemuDomainSnapshotSetInactive(vm, driver->snapshotDir)< 0) goto endjob; Index: libvirt-acl/src/util/macvtap.c =================================================================== --- libvirt-acl.orig/src/util/macvtap.c +++ libvirt-acl/src/util/macvtap.c @@ -77,9 +77,22 @@ # define LLDPAD_PID_FILE "/var/run/lldpad.pid" +static const char *vmOpStr[] = { + [VM_OP_CREATE] = "create", + [VM_OP_SAVE] = "save", + [VM_OP_RESTORE] = "restore", + [VM_OP_DESTROY] = "destroy", + [VM_OP_MIGRATE_OUT] = "migrate out", + [VM_OP_MIGRATE_IN_START] = "migrate in start", + [VM_OP_MIGRATE_IN_FINISH] = "migrate in finish", + [VM_OP_NO_OP] = "no-op" +}; + + enum virVirtualPortOp { ASSOCIATE = 0x1, DISASSOCIATE = 0x2, + PREASSOCIATE = 0x3, }; @@ -551,7 +564,8 @@ openMacvtapTap(const char *tgifname, int vnet_hdr, const unsigned char *vmuuid, virVirtualPortProfileParamsPtr virtPortProfile, - char **res_ifname) + char **res_ifname, + enum vmOperation vmOp) { const char *type = "macvtap"; int c, rc; @@ -563,6 +577,8 @@ openMacvtapTap(const char *tgifname, *res_ifname = NULL; + VIR_DEBUG("%s: VM OPERATION: %s", __FUNCTION__, vmOpStr[vmOp]); + if (tgifname) { if(ifaceGetIndex(false, tgifname,&ifindex) == 0) { if (STRPREFIX(tgifname, @@ -601,7 +617,7 @@ create_name: macaddress, linkdev, virtPortProfile, - vmuuid) != 0) { + vmuuid, vmOp) != 0) { rc = -1; goto link_del_exit; } @@ -634,7 +650,8 @@ disassociate_exit: vpDisassociatePortProfileId(cr_ifname, macaddress, linkdev, - virtPortProfile); + virtPortProfile, + vmOp); link_del_exit: link_del(cr_ifname); @@ -662,7 +679,8 @@ delMacvtap(const char *ifname, if (ifname) { vpDisassociatePortProfileId(ifname, macaddr, linkdev, - virtPortProfile); + virtPortProfile, + VM_OP_DESTROY); link_del(ifname); } } @@ -1320,6 +1338,9 @@ doPortProfileOp8021Qbg(const char *ifnam portVsi.vsi_type_id[0] = virtPort->u.virtPort8021Qbg.typeID; switch (virtPortOp) { + case PREASSOCIATE: + op = PORT_REQUEST_PREASSOCIATE; + break; case ASSOCIATE: op = PORT_REQUEST_ASSOCIATE; break; @@ -1484,6 +1505,7 @@ err_exit: * @macvtap_ifname: The name of the macvtap device * @virtPort: pointer to the object holding port profile parameters * @vmuuid : the UUID of the virtual machine + * @vmOp : The VM operation (i.e., create, no-op) * * Associate a port on a swtich with a profile. This function * may notify a kernel driver or an external daemon to run @@ -1499,13 +1521,19 @@ vpAssociatePortProfileId(const char *mac const unsigned char *macvtap_macaddr, const char *linkdev, const virVirtualPortProfileParamsPtr virtPort, - const unsigned char *vmuuid) + const unsigned char *vmuuid, + enum vmOperation vmOp) { int rc = 0; VIR_DEBUG("Associating port profile '%p' on link device '%s'", virtPort, macvtap_ifname); + VIR_DEBUG("%s: VM OPERATION: %s", __FUNCTION__, vmOpStr[vmOp]); + + if (vmOp == VM_OP_NO_OP) + return 0; + switch (virtPort->virtPortType) { case VIR_VIRTUALPORT_NONE: case VIR_VIRTUALPORT_TYPE_LAST: @@ -1513,10 +1541,16 @@ vpAssociatePortProfileId(const char *mac case VIR_VIRTUALPORT_8021QBG: rc = doPortProfileOp8021Qbg(macvtap_ifname, macvtap_macaddr, - virtPort, ASSOCIATE); + virtPort, + (vmOp == VM_OP_MIGRATE_IN_START) + ? PREASSOCIATE + : ASSOCIATE); break; case VIR_VIRTUALPORT_8021QBH: + /* avoid associating twice */ + if (vmOp == VM_OP_MIGRATE_IN_FINISH) + break; rc = doPortProfileOp8021Qbh(linkdev, virtPort, vmuuid, ASSOCIATE); @@ -1542,13 +1576,16 @@ int vpDisassociatePortProfileId(const char *macvtap_ifname, const unsigned char *macvtap_macaddr, const char *linkdev, - const virVirtualPortProfileParamsPtr virtPort) + const virVirtualPortProfileParamsPtr virtPort, + enum vmOperation vmOp) { int rc = 0; VIR_DEBUG("Disassociating port profile id '%p' on link device '%s' ", virtPort, macvtap_ifname); + VIR_DEBUG("%s: VM OPERATION: %s", __FUNCTION__, vmOpStr[vmOp]); + switch (virtPort->virtPortType) { case VIR_VIRTUALPORT_NONE: case VIR_VIRTUALPORT_TYPE_LAST: @@ -1560,6 +1597,9 @@ vpDisassociatePortProfileId(const char * break; case VIR_VIRTUALPORT_8021QBH: + /* avoid disassociating twice */ + if (vmOp == VM_OP_MIGRATE_IN_FINISH) + break; rc = doPortProfileOp8021Qbh(linkdev, virtPort, NULL, DISASSOCIATE); Index: libvirt-acl/src/util/macvtap.h =================================================================== --- libvirt-acl.orig/src/util/macvtap.h +++ libvirt-acl/src/util/macvtap.h @@ -62,6 +62,17 @@ struct _virVirtualPortProfileParams { # include "internal.h" +enum vmOperation { + VM_OP_CREATE, + VM_OP_SAVE, + VM_OP_RESTORE, + VM_OP_DESTROY, + VM_OP_MIGRATE_OUT, + VM_OP_MIGRATE_IN_START, + VM_OP_MIGRATE_IN_FINISH, + VM_OP_NO_OP, +}; + int openMacvtapTap(const char *ifname, const unsigned char *macaddress, const char *linkdev, @@ -69,7 +80,8 @@ int openMacvtapTap(const char *ifname, int vnet_hdr, const unsigned char *vmuuid, virVirtualPortProfileParamsPtr virtPortProfile, - char **res_ifname); + char **res_ifname, + enum vmOperation vmop); void delMacvtap(const char *ifname, const unsigned char *macaddress, @@ -86,12 +98,14 @@ int vpAssociatePortProfileId(const char const unsigned char *macvtap_macaddr, const char *linkdev, const virVirtualPortProfileParamsPtr virtPort, - const unsigned char *vmuuid); + const unsigned char *vmuuid, + enum vmOperation vmOp); int vpDisassociatePortProfileId(const char *macvtap_ifname, const unsigned char *macvtap_macaddr, const char *linkdev, - const virVirtualPortProfileParamsPtr virtPort); + const virVirtualPortProfileParamsPtr virtPort, + enum vmOperation vmOp); VIR_ENUM_DECL(virVirtualPort) Index: libvirt-acl/src/libvirt_macvtap.syms =================================================================== --- libvirt-acl.orig/src/libvirt_macvtap.syms +++ libvirt-acl/src/libvirt_macvtap.syms @@ -3,3 +3,5 @@ # macvtap.h openMacvtapTap; delMacvtap; +vpAssociatePortProfileId; +vpDisassociatePortProfileId;