Handle preparing and releasing host pci devices for PCI passthrough. Preparing host pci devices will be used before vm is started, while releasing (or, reattaching) will be used after vm is shut off. Code refers to qemu driver, so add Red Hat copyright. Signed-off-by: Chunyan Liu <cyliu@suse.com> --- po/POTFILES.in | 1 + src/Makefile.am | 1 + src/libxl/libxl_hostdev.c | 600 +++++++++++++++++++++++++++++++++++++++++++++ src/libxl/libxl_hostdev.h | 44 ++++ 4 files changed, 646 insertions(+), 0 deletions(-) create mode 100644 src/libxl/libxl_hostdev.c create mode 100644 src/libxl/libxl_hostdev.h diff --git a/po/POTFILES.in b/po/POTFILES.in index 95619f9..a2b8d6b 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -66,6 +66,7 @@ src/lxc/lxc_monitor.c src/lxc/lxc_process.c src/libxl/libxl_driver.c src/libxl/libxl_conf.c +src/libxl/libxl_hostdev.c src/network/bridge_driver.c src/node_device/node_device_driver.c src/node_device/node_device_hal.c diff --git a/src/Makefile.am b/src/Makefile.am index da571c7..d7149b5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -531,6 +531,7 @@ XENAPI_DRIVER_SOURCES = \ LIBXL_DRIVER_SOURCES = \ libxl/libxl_conf.c libxl/libxl_conf.h \ + libxl/libxl_hostdev.c libxl/libxl_hostdev.h \ libxl/libxl_driver.c libxl/libxl_driver.h UML_DRIVER_SOURCES = \ diff --git a/src/libxl/libxl_hostdev.c b/src/libxl/libxl_hostdev.c new file mode 100644 index 0000000..19c6d82 --- /dev/null +++ b/src/libxl/libxl_hostdev.c @@ -0,0 +1,600 @@ +/*---------------------------------------------------------------------------*/ +/* Copyright (C) 2006-2012 Red Hat, Inc. + * Copyright (c) 2013 SUSE LINUX Products GmbH, Nuernberg, Germany. + * Copyright (C) 2011 Univention GmbH. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + * + * Authors: + * Daniel P. Berrange <berrange@redhat.com> + * Chunyan Liu <cyliu@suse.com> + */ +/*---------------------------------------------------------------------------*/ + +#include <config.h> + +#include "libxl_hostdev.h" +#include "logging.h" +#include "virterror_internal.h" +#include "memory.h" +#include "pci.h" +#include "virnetdev.h" + +#define VIR_FROM_THIS VIR_FROM_LIBXL + +static pciDeviceList * +libxlGetActivePciHostDeviceList(libxlDriverPrivatePtr driver, + virDomainHostdevDefPtr *hostdevs, + int nhostdevs) +{ + pciDeviceList *list; + int i; + + if (!(list = pciDeviceListNew())) + return NULL; + + for (i = 0 ; i < nhostdevs ; i++) { + virDomainHostdevDefPtr hostdev = hostdevs[i]; + pciDevice *dev; + pciDevice *activeDev; + + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + continue; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + continue; + + dev = pciGetDevice(hostdev->source.subsys.u.pci.domain, + hostdev->source.subsys.u.pci.bus, + hostdev->source.subsys.u.pci.slot, + hostdev->source.subsys.u.pci.function); + if (!dev) { + pciDeviceListFree(list); + return NULL; + } + + if ((activeDev = pciDeviceListFind(driver->activePciHostdevs, dev))) { + if (pciDeviceListAdd(list, activeDev) < 0) { + pciFreeDevice(dev); + pciDeviceListFree(list); + return NULL; + } + } + + pciFreeDevice(dev); + } + + return list; +} + +static pciDeviceList * +libxlGetPciHostDeviceList(virDomainHostdevDefPtr *hostdevs, int nhostdevs) +{ + pciDeviceList *list; + int i; + + if (!(list = pciDeviceListNew())) + return NULL; + + for (i = 0 ; i < nhostdevs ; i++) { + virDomainHostdevDefPtr hostdev = hostdevs[i]; + pciDevice *dev; + + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + continue; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + continue; + + dev = pciGetDevice(hostdev->source.subsys.u.pci.domain, + hostdev->source.subsys.u.pci.bus, + hostdev->source.subsys.u.pci.slot, + hostdev->source.subsys.u.pci.function); + if (!dev) { + pciDeviceListFree(list); + return NULL; + } + + if (pciDeviceListAdd(list, dev) < 0) { + pciFreeDevice(dev); + pciDeviceListFree(list); + return NULL; + } + + pciDeviceSetManaged(dev, hostdev->managed); + } + + return list; +} + +void libxlReattachPciDevice(pciDevice *dev, libxlDriverPrivatePtr driver) +{ + /* If the device is not managed and was attached to guest + * successfully, it must have been inactive. + */ + if (!pciDeviceGetManaged(dev)) { + pciDeviceListAdd(driver->inactivePciHostdevs, dev); + return; + } + + if (pciReAttachDevice(dev, driver->activePciHostdevs, + driver->inactivePciHostdevs, "pciback") < 0) { + virErrorPtr err = virGetLastError(); + VIR_ERROR(_("Failed to re-attach PCI device: %s"), + err ? err->message : _("unknown error")); + virResetError(err); + } +} + +static int +libxlDomainHostdevPciSysfsPath(virDomainHostdevDefPtr hostdev, char **sysfs_path) +{ + struct pci_config_address config_address; + + config_address.domain = hostdev->source.subsys.u.pci.domain; + config_address.bus = hostdev->source.subsys.u.pci.bus; + config_address.slot = hostdev->source.subsys.u.pci.slot; + config_address.function = hostdev->source.subsys.u.pci.function; + + return pciConfigAddressToSysfsFile(&config_address, sysfs_path); +} + +static int +libxlDomainHostdevIsVirtualFunction(virDomainHostdevDefPtr hostdev) +{ + char *sysfs_path = NULL; + int ret = -1; + + if (libxlDomainHostdevPciSysfsPath(hostdev, &sysfs_path) < 0) + return ret; + + ret = pciDeviceIsVirtualFunction(sysfs_path); + + VIR_FREE(sysfs_path); + + return ret; +} + +static int +libxlDomainHostdevNetDevice(virDomainHostdevDefPtr hostdev, char **linkdev, + int *vf) +{ + int ret = -1; + char *sysfs_path = NULL; + + if (libxlDomainHostdevPciSysfsPath(hostdev, &sysfs_path) < 0) + return ret; + + if (pciDeviceIsVirtualFunction(sysfs_path) == 1) { + if (pciDeviceGetVirtualFunctionInfo(sysfs_path, linkdev, + vf) < 0) + goto cleanup; + } else { + if (pciDeviceNetName(sysfs_path, linkdev) < 0) + goto cleanup; + *vf = -1; + } + + ret = 0; + +cleanup: + VIR_FREE(sysfs_path); + + return ret; +} + +static int +libxlDomainHostdevNetConfigVirtPortProfile(const char *linkdev, int vf, + virNetDevVPortProfilePtr virtPort, + const virMacAddrPtr macaddr, + const unsigned char *uuid, + int associate) +{ + int ret = -1; + + if (!virtPort) + return ret; + + switch (virtPort->virtPortType) { + case VIR_NETDEV_VPORT_PROFILE_NONE: + case VIR_NETDEV_VPORT_PROFILE_OPENVSWITCH: + case VIR_NETDEV_VPORT_PROFILE_8021QBG: + case VIR_NETDEV_VPORT_PROFILE_LAST: + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("virtualport type %s is " + "currently not supported on interfaces of type " + "hostdev"), + virNetDevVPortTypeToString(virtPort->virtPortType)); + break; + + case VIR_NETDEV_VPORT_PROFILE_8021QBH: + if (associate) + ret = virNetDevVPortProfileAssociate(NULL, virtPort, macaddr, + linkdev, vf, uuid, + VIR_NETDEV_VPORT_PROFILE_OP_CREATE, false); + else + ret = virNetDevVPortProfileDisassociate(NULL, virtPort, + macaddr, linkdev, vf, + VIR_NETDEV_VPORT_PROFILE_OP_DESTROY); + break; + } + + return ret; +} + +static int +libxlDomainHostdevNetConfigReplace(virDomainHostdevDefPtr hostdev, + const unsigned char *uuid, + char *stateDir) +{ + char *linkdev = NULL; + virNetDevVlanPtr vlan; + virNetDevVPortProfilePtr virtPort; + int ret = -1; + int vf = -1; + int vlanid = -1; + int port_profile_associate = 1; + int isvf; + + isvf = libxlDomainHostdevIsVirtualFunction(hostdev); + if (isvf <= 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Interface type hostdev is currently supported on" + " SR-IOV Virtual Functions only")); + return ret; + } + + if (libxlDomainHostdevNetDevice(hostdev, &linkdev, &vf) < 0) + return ret; + + vlan = virDomainNetGetActualVlan(hostdev->parent.data.net); + virtPort = virDomainNetGetActualVirtPortProfile( + hostdev->parent.data.net); + if (virtPort) { + if (vlan) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("direct setting of the vlan tag is not allowed " + "for hostdev devices using %s mode"), + virNetDevVPortTypeToString(virtPort->virtPortType)); + goto cleanup; + } + ret = libxlDomainHostdevNetConfigVirtPortProfile(linkdev, vf, + virtPort, &hostdev->parent.data.net->mac, uuid, + port_profile_associate); + } else { + /* Set only mac and vlan */ + if (vlan) { + if (vlan->nTags != 1 || vlan->trunk) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("vlan trunking is not supported " + "by SR-IOV network devices")); + goto cleanup; + } + if (vf == -1) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("vlan can only be set for SR-IOV VFs, but " + "%s is not a VF"), linkdev); + goto cleanup; + } + vlanid = vlan->tag[0]; + } else if (vf >= 0) { + vlanid = 0; /* assure any current vlan tag is reset */ + } + + ret = virNetDevReplaceNetConfig(linkdev, vf, + &hostdev->parent.data.net->mac, + vlanid, stateDir); + } +cleanup: + VIR_FREE(linkdev); + return ret; +} + +static int +libxlDomainHostdevNetConfigRestore(virDomainHostdevDefPtr hostdev, + char *stateDir) +{ + char *linkdev = NULL; + virNetDevVPortProfilePtr virtPort; + int ret = -1; + int vf = -1; + int port_profile_associate = 0; + int isvf; + + isvf = libxlDomainHostdevIsVirtualFunction(hostdev); + if (isvf <= 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Interface type hostdev is currently supported on" + " SR-IOV Virtual Functions only")); + return ret; + } + + if (libxlDomainHostdevNetDevice(hostdev, &linkdev, &vf) < 0) + return ret; + + virtPort = virDomainNetGetActualVirtPortProfile( + hostdev->parent.data.net); + if (virtPort) + ret = libxlDomainHostdevNetConfigVirtPortProfile(linkdev, vf, virtPort, + &hostdev->parent.data.net->mac, NULL, + port_profile_associate); + else + ret = virNetDevRestoreNetConfig(linkdev, vf, stateDir); + + VIR_FREE(linkdev); + + return ret; +} + +void libxlDomainReAttachHostdevPciDevices(libxlDriverPrivatePtr driver, + const char *name, + virDomainHostdevDefPtr *hostdevs, + int nhostdevs) +{ + pciDeviceList *pcidevs; + int i; + + if (!(pcidevs = libxlGetActivePciHostDeviceList(driver, + hostdevs, + nhostdevs))) { + virErrorPtr err = virGetLastError(); + VIR_ERROR(_("Failed to allocate pciDeviceList: %s"), + err ? err->message : _("unknown error")); + virResetError(err); + return; + } + + /* Again 4 loops; mark all devices as inactive before reset + * them and reset all the devices before re-attach. + * Attach mac and port profile parameters to devices + */ + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + pciDevice *activeDev = NULL; + + /* Never delete the dev from list driver->activePciHostdevs + * if it's used by other domain. + */ + activeDev = pciDeviceListFind(driver->activePciHostdevs, dev); + if (activeDev && + STRNEQ_NULLABLE(name, pciDeviceGetUsedBy(activeDev))) { + pciDeviceListSteal(pcidevs, dev); + continue; + } + + /* pciDeviceListFree() will take care of freeing the dev. */ + pciDeviceListSteal(driver->activePciHostdevs, dev); + } + + /* + * For SRIOV net host devices, unset mac and port profile before + * reset and reattach device + */ + for (i = 0; i < nhostdevs; i++) { + virDomainHostdevDefPtr hostdev = hostdevs[i]; + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + continue; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + continue; + if (hostdev->parent.type == VIR_DOMAIN_DEVICE_NET && + hostdev->parent.data.net) { + libxlDomainHostdevNetConfigRestore(hostdev, driver->stateDir); + } + } + + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + if (pciResetDevice(dev, driver->activePciHostdevs, + driver->inactivePciHostdevs) < 0) { + virErrorPtr err = virGetLastError(); + VIR_ERROR(_("Failed to reset PCI device: %s"), + err ? err->message : _("unknown error")); + virResetError(err); + } + } + + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + libxlReattachPciDevice(dev, driver); + } + + pciDeviceListFree(pcidevs); +} + +void libxlDomainReAttachHostDevices(libxlDriverPrivatePtr driver, + virDomainDefPtr def) +{ + if (!def->nhostdevs) + return; + + libxlDomainReAttachHostdevPciDevices(driver, def->name, def->hostdevs, + def->nhostdevs); +} + +int libxlPrepareHostdevPCIDevices(libxlDriverPrivatePtr driver, + const char *name, + const unsigned char *uuid, + virDomainHostdevDefPtr *hostdevs, + int nhostdevs) +{ + pciDeviceList *pcidevs; + int last_processed_hostdev_vf = -1; + int i; + int ret = -1; + + if (!(pcidevs = libxlGetPciHostDeviceList(hostdevs, nhostdevs))) + return -1; + + /* Loop 1: validate that non-managed device isn't in use */ + + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + pciDevice *other; + + /* The device is in use by other active domain if + * the dev is in list driver->activePciHostdevs. + */ + if ((other = pciDeviceListFind(driver->activePciHostdevs, dev))) { + const char *other_name = pciDeviceGetUsedBy(other); + + if (other_name) + virReportError(VIR_ERR_OPERATION_INVALID, + _("PCI device %s is in use by domain %s"), + pciDeviceGetName(dev), other_name); + else + virReportError(VIR_ERR_OPERATION_INVALID, + _("PCI device %s is already in use"), + pciDeviceGetName(dev)); + goto cleanup; + } + } + + /* Loop 2: detach managed devices */ + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + if (pciDeviceGetManaged(dev) && + pciDettachDevice(dev, driver->activePciHostdevs, NULL, "pciback") < 0) + goto reattachdevs; + } + + /* Loop 3: Now that all the PCI hostdevs have been detached, we + * can safely reset them */ + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + if (pciResetDevice(dev, driver->activePciHostdevs, + driver->inactivePciHostdevs) < 0) + goto reattachdevs; + } + + /* Loop 4: For SRIOV network devices, Now that we have detached the + * the network device, set the netdev config */ + for (i = 0; i < nhostdevs; i++) { + virDomainHostdevDefPtr hostdev = hostdevs[i]; + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + continue; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + continue; + if (hostdev->parent.type == VIR_DOMAIN_DEVICE_NET && + hostdev->parent.data.net) { + if (libxlDomainHostdevNetConfigReplace(hostdev, uuid, + driver->stateDir) < 0) { + goto resetvfnetconfig; + } + } + last_processed_hostdev_vf = i; + } + + /* Loop 5: Now mark all the devices as active */ + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + if (pciDeviceListAdd(driver->activePciHostdevs, dev) < 0) { + pciFreeDevice(dev); + goto inactivedevs; + } + } + + /* Loop 6: Now remove the devices from inactive list. */ + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + pciDeviceListDel(driver->inactivePciHostdevs, dev); + } + + /* Loop 7: Now set the used_by_domain of the device in + * driver->activePciHostdevs as domain name. + */ + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev, *activeDev; + + dev = pciDeviceListGet(pcidevs, i); + activeDev = pciDeviceListFind(driver->activePciHostdevs, dev); + + pciDeviceSetUsedBy(activeDev, name); + } + + /* Loop 8: Now set the original states for hostdev def */ + for (i = 0; i < nhostdevs; i++) { + pciDevice *dev; + pciDevice *pcidev; + virDomainHostdevDefPtr hostdev = hostdevs[i]; + + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + continue; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + continue; + + dev = pciGetDevice(hostdev->source.subsys.u.pci.domain, + hostdev->source.subsys.u.pci.bus, + hostdev->source.subsys.u.pci.slot, + hostdev->source.subsys.u.pci.function); + + /* original states "unbind_from_stub", "remove_slot", + * "reprobe" were already set by pciDettachDevice in + * loop 2. + */ + if ((pcidev = pciDeviceListFind(pcidevs, dev))) { + hostdev->origstates.states.pci.unbind_from_stub = + pciDeviceGetUnbindFromStub(pcidev); + hostdev->origstates.states.pci.remove_slot = + pciDeviceGetRemoveSlot(pcidev); + hostdev->origstates.states.pci.reprobe = + pciDeviceGetReprobe(pcidev); + } + + pciFreeDevice(dev); + } + + /* Loop 9: Now steal all the devices from pcidevs */ + while (pciDeviceListCount(pcidevs) > 0) { + pciDevice *dev = pciDeviceListGet(pcidevs, 0); + pciDeviceListSteal(pcidevs, dev); + } + + ret = 0; + goto cleanup; + +inactivedevs: + /* Only steal all the devices from driver->activePciHostdevs. We will + * free them in pciDeviceListFree(). + */ + while (pciDeviceListCount(pcidevs) > 0) { + pciDevice *dev = pciDeviceListGet(pcidevs, 0); + pciDeviceListSteal(driver->activePciHostdevs, dev); + } + +resetvfnetconfig: + for (i = 0; i < last_processed_hostdev_vf; i++) { + virDomainHostdevDefPtr hostdev = hostdevs[i]; + if (hostdev->parent.type == VIR_DOMAIN_DEVICE_NET && + hostdev->parent.data.net) { + libxlDomainHostdevNetConfigRestore(hostdev, driver->stateDir); + } + } + +reattachdevs: + for (i = 0; i < pciDeviceListCount(pcidevs); i++) { + pciDevice *dev = pciDeviceListGet(pcidevs, i); + pciReAttachDevice(dev, driver->activePciHostdevs, NULL, "pciback"); + } + +cleanup: + pciDeviceListFree(pcidevs); + return ret; +} + +int +libxlPrepareHostDevices(libxlDriverPrivatePtr driver, + virDomainDefPtr def) +{ + return libxlPrepareHostdevPCIDevices(driver, def->name, def->uuid, + def->hostdevs, def->nhostdevs); +} diff --git a/src/libxl/libxl_hostdev.h b/src/libxl/libxl_hostdev.h new file mode 100644 index 0000000..e5a5058 --- /dev/null +++ b/src/libxl/libxl_hostdev.h @@ -0,0 +1,44 @@ +/*---------------------------------------------------------------------------*/ +/* Copyright (c) 2013 SUSE LINUX Products GmbH, Nuernberg, Germany. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + * + * Authors: + * Chunyan Liu <cyliu@suse.com> + */ +/*---------------------------------------------------------------------------*/ + +#ifndef __LIBXL_HOSTDEV_H__ +# define __LIBXL_HOSTDEV_H__ + +# include "libxl_conf.h" +# include "domain_conf.h" + +int libxlPrepareHostdevPCIDevices(libxlDriverPrivatePtr driver, + const char *name, + const unsigned char *uuid, + virDomainHostdevDefPtr *hostdevs, + int nhostdevs); +int libxlPrepareHostDevices(libxlDriverPrivatePtr driver, + virDomainDefPtr def); +void libxlReattachPciDevice(pciDevice *dev, libxlDriverPrivatePtr driver); +void libxlDomainReAttachHostdevPciDevices(libxlDriverPrivatePtr driver, + const char *name, + virDomainHostdevDefPtr *hostdevs, + int nhostdevs); +void libxlDomainReAttachHostDevices(libxlDriverPrivatePtr driver, + virDomainDefPtr def); + +#endif /* __LIBXL_HOSTDEV_H__ */ -- 1.7.3.4