Re: [libvirt] [PATCH v3] network: Bring netdevs online later

Currently, MAC registration occurs during device creation, which is early enough that, during live migration, you end up with duplicate MAC addresses on still-running source and target devices, even though the target device isn't actually being used yet. This patch proposes to defer MAC registration until right before the guest can actually use the device -- In other words, right before starting guest CPUs. Signed-off-by: Matthew Rosato <mjrosato(a)linux.vnet.ibm.com&gt; ---

Associated BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1081461 Changes for v3: * Some minor formatting fixes. * in qemuNetworkIfaceConnect, set VIR_NETDEV_TAP_CREATE_IFUP unconditionally. * in qemuDomainAttachNetDevice, call qemuInterfaceStartDevice only for VIR_DOMAIN_NET_TYPE_DIRECT, _BRIDGE and _NETWORK. * in qemuProcessStartCPUs, use 'reason' to determine whether or not qemuInterfaceStartDevices needs to be called. Basically, it needs to be called for any reason that the system would be initializing (or re-initializing). src/Makefile.am | 3 +- src/conf/domain_conf.h | 2 ++ src/lxc/lxc_process.c | 4 ++- src/qemu/qemu_command.c | 3 ++ src/qemu/qemu_hotplug.c | 8 +++++ src/qemu/qemu_interface.c | 78 +++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_interface.h | 32 ++++++++++++++++++ src/qemu/qemu_process.c | 7 ++++ src/util/virnetdevmacvlan.c | 8 +++-- src/util/virnetdevmacvlan.h | 2 ++ 10 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 src/qemu/qemu_interface.c create mode 100644 src/qemu/qemu_interface.h diff --git a/src/Makefile.am b/src/Makefile.am index fa741a8..035120e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -703,7 +703,8 @@ QEMU_DRIVER_SOURCES = \ qemu/qemu_monitor_text.h \ qemu/qemu_monitor_json.c \ qemu/qemu_monitor_json.h \ - qemu/qemu_driver.c qemu/qemu_driver.h + qemu/qemu_driver.c qemu/qemu_driver.h \ + qemu/qemu_interface.c qemu/qemu_interface.h XENAPI_DRIVER_SOURCES = \ xenapi/xenapi_driver.c xenapi/xenapi_driver.h \ diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 0862bd7..5f328cf 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -951,6 +951,8 @@ struct _virDomainNetDef { virNetDevBandwidthPtr bandwidth; virNetDevVlan vlan; int linkstate; + /* vmOp value saved if deferring interface start */ + virNetDevVPortProfileOp vmOp; }; /* Used for prefix of ifname of any network name generated dynamically diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c index ed30c37..b2256c0 100644 --- a/src/lxc/lxc_process.c +++ b/src/lxc/lxc_process.c @@ -300,6 +300,7 @@ char *virLXCProcessSetupInterfaceDirect(virConnectPtr conn, virNetDevBandwidthPtr bw; virNetDevVPortProfilePtr prof; virLXCDriverConfigPtr cfg = virLXCDriverGetConfig(driver); + unsigned int macvlan_create_flags = VIR_NETDEV_MACVLAN_CREATE_IFUP; /* XXX how todo bandwidth controls ? * Since the 'net-ifname' is about to be moved to a different @@ -336,7 +337,8 @@ char *virLXCProcessSetupInterfaceDirect(virConnectPtr conn, &res_ifname, VIR_NETDEV_VPORT_PROFILE_OP_CREATE, cfg->stateDir, - virDomainNetGetActualBandwidth(net), 0) < 0) + virDomainNetGetActualBandwidth(net), + macvlan_create_flags) < 0) goto cleanup; ret = res_ifname; diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index e5270bd..229dff4 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -199,6 +199,9 @@ qemuPhysIfaceConnect(virDomainDefPtr def, net->ifname = res_ifname; } + /* Save vport profile op for later */ + net->vmOp = vmop; + virObjectUnref(cfg); return rc; } diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 7bc19cd..530e6da 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -30,6 +30,7 @@ #include "qemu_domain.h" #include "qemu_command.h" #include "qemu_hostdev.h" +#include "qemu_interface.h" #include "domain_audit.h" #include "domain_nwfilter.h" #include "virlog.h" @@ -922,6 +923,8 @@ int qemuDomainAttachNetDevice(virConnectPtr conn, priv->qemuCaps, tapfd, &tapfdSize) < 0) goto cleanup; iface_connected = true; + /* Set device online immediately */ + qemuInterfaceStartDevice(net); if (qemuOpenVhostNet(vm->def, net, priv->qemuCaps, vhostfd, &vhostfdSize) < 0) goto cleanup; } else if (actualType == VIR_DOMAIN_NET_TYPE_DIRECT) { @@ -937,6 +940,8 @@ int qemuDomainAttachNetDevice(virConnectPtr conn, VIR_NETDEV_VPORT_PROFILE_OP_CREATE)) < 0) goto cleanup; iface_connected = true; + /* Set device online immediately */ + qemuInterfaceStartDevice(net); if (qemuOpenVhostNet(vm->def, net, priv->qemuCaps, vhostfd, &vhostfdSize) < 0) goto cleanup; } else if (actualType == VIR_DOMAIN_NET_TYPE_ETHERNET) { @@ -2070,6 +2075,9 @@ qemuDomainChangeNet(virQEMUDriverPtr driver, goto cleanup; } + /* Set device online immediately */ + qemuInterfaceStartDevice(newdev); + newType = virDomainNetGetActualType(newdev); if (newType == VIR_DOMAIN_NET_TYPE_HOSTDEV) { diff --git a/src/qemu/qemu_interface.c b/src/qemu/qemu_interface.c new file mode 100644 index 0000000..dccfcc4 --- /dev/null +++ b/src/qemu/qemu_interface.c @@ -0,0 +1,78 @@ +/* + * qemu_interface.c: QEMU interface management + * + * Copyright IBM Corp. 2014 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Matthew J. Rosato <mjrosato(a)linux.vnet.ibm.com&gt; + */ + +#include <config.h> + +#include "qemu_interface.h" +#include "virnetdev.h" +#include "virnetdevtap.h" +#include "virnetdevmacvlan.h" +#include "virnetdevvportprofile.h" + +/** + * qemuInterfaceStartDevice: + * @net: net device to start + * + * Based upon the type of device provided, perform the appropriate + * work to set the device online. + */ +void +qemuInterfaceStartDevice(virDomainNetDefPtr net) +{ + switch (virDomainNetGetActualType(net)) { + case VIR_DOMAIN_NET_TYPE_BRIDGE: + case VIR_DOMAIN_NET_TYPE_NETWORK: + if (virNetDevSetOnline(net->ifname, true) < 0) { + ignore_value(virNetDevTapDelete(net->ifname)); + } + break; + case VIR_DOMAIN_NET_TYPE_DIRECT: + if (virNetDevSetOnline(net->ifname, true) < 0) { + ignore_value(virNetDevVPortProfileDisassociate(net->ifname, + virDomainNetGetActualVirtPortProfile(net), + &net->mac, + virDomainNetGetActualDirectDev(net), + -1, + net->vmOp)); + } + break; + } +} + +/** + * qemuInterfaceStartDevices: + * @def: domain definition + * + * Set all ifaces associated with this domain to the online state. + */ +void +qemuInterfaceStartDevices(virDomainDefPtr def) +{ + size_t i; + + for (i = 0; i < def->nnets; i++) { + qemuInterfaceStartDevice(def->nets[i]); + } + + return; +} diff --git a/src/qemu/qemu_interface.h b/src/qemu/qemu_interface.h new file mode 100644 index 0000000..5810cda --- /dev/null +++ b/src/qemu/qemu_interface.h @@ -0,0 +1,32 @@ +/* + * qemu_interface.h: QEMU interface management + * + * Copyright IBM Corp. 2014 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Matthew J. Rosato <mjrosato(a)linux.vnet.ibm.com&gt; + */ + +#ifndef __QEMU_INTERFACE_H__ +# define __QEMU_INTERFACE_H__ + +# include "domain_conf.h" + +void qemuInterfaceStartDevice(virDomainNetDefPtr net); +void qemuInterfaceStartDevices(virDomainDefPtr def); + +#endif /* __QEMU_INTERFACE_H__ */ diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index f391743..3fc50a0 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -42,6 +42,7 @@ #include "qemu_hostdev.h" #include "qemu_hotplug.h" #include "qemu_migration.h" +#include "qemu_interface.h" #include "cpu/cpu.h" #include "datatypes.h" @@ -2947,6 +2948,12 @@ qemuProcessStartCPUs(virQEMUDriverPtr driver, virDomainObjPtr vm, qemuDomainObjPrivatePtr priv = vm->privateData; virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver); + /* Bring up netdevs before starting CPUs */ + if (reason != VIR_DOMAIN_RUNNING_UNPAUSED && + reason != VIR_DOMAIN_RUNNING_SAVE_CANCELED) { + qemuInterfaceStartDevices(vm->def); + } + VIR_DEBUG("Using lock state '%s'", NULLSTR(priv->lockState)); if (virDomainLockProcessResume(driver->lockManager, cfg->uri, vm, priv->lockState) < 0) { diff --git a/src/util/virnetdevmacvlan.c b/src/util/virnetdevmacvlan.c index c83341c..1edf3ae 100644 --- a/src/util/virnetdevmacvlan.c +++ b/src/util/virnetdevmacvlan.c @@ -902,9 +902,11 @@ int virNetDevMacVLanCreateWithVPortProfile(const char *tgifname, goto link_del_exit; } - if (virNetDevSetOnline(cr_ifname, true) < 0) { - rc = -1; - goto disassociate_exit; + if (flags & VIR_NETDEV_MACVLAN_CREATE_IFUP) { + if (virNetDevSetOnline(cr_ifname, true) < 0) { + rc = -1; + goto disassociate_exit; + } } if (flags & VIR_NETDEV_MACVLAN_CREATE_WITH_TAP) { diff --git a/src/util/virnetdevmacvlan.h b/src/util/virnetdevmacvlan.h index 41aa4e2..41b4014 100644 --- a/src/util/virnetdevmacvlan.h +++ b/src/util/virnetdevmacvlan.h @@ -44,6 +44,8 @@ typedef enum { VIR_NETDEV_MACVLAN_CREATE_NONE = 0, /* Create with a tap device */ VIR_NETDEV_MACVLAN_CREATE_WITH_TAP = 1 << 0, + /* Bring the interface up */ + VIR_NETDEV_MACVLAN_CREATE_IFUP = 1 << 1, } virNetDevMacVLanCreateFlags; int virNetDevMacVLanCreate(const char *ifname,