[libvirt] [RFC PATCH 12/28] qemu: address: Enable auto addressing multifunction cards

For existing domains using the primary function alone of a multifunction card, the card is still treated as a multifunction card. This is done to prevent hotplug of other functions when the primary function is already hotplugged. If the secondary functions are part of the xml without the primary function being part of the xml, this has never been supported. So, the libvirt doesn't consider this either as a multifunction card. Signed-off-by: Shivaprasad G Bhat <sbhat(a)linux.vnet.ibm.com&gt; --- src/qemu/qemu_domain.h | 11 + src/qemu/qemu_domain_address.c | 169 +++++++++++++++++++- tests/qemuhotplugtest.c | 1 .../hostdev-pci-multifunction.args | 18 +- tests/qemuxml2argvdata/pseries-hostdevs-1.args | 5 - tests/qemuxml2argvdata/pseries-hostdevs-3.args | 5 - tests/qemuxml2argvtest.c | 5 - .../hostdev-pci-multifunction.xml | 16 +- tests/qemuxml2xmloutdata/pseries-hostdevs-1.xml | 4 tests/qemuxml2xmloutdata/pseries-hostdevs-3.xml | 4 10 files changed, 207 insertions(+), 31 deletions(-) diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 6d3e6eb5e3..fbfc994652 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -473,6 +473,17 @@ struct _qemuDomainSaveCookie { qemuDomainSaveCookiePtr qemuDomainSaveCookieNew(virDomainObjPtr vm); +typedef struct _qemuDomainPCIHostdevdata qemuDomainPCIHostdevdata; +typedef qemuDomainPCIHostdevdata *qemuDomainPCIHostdevDataPtr; +struct _qemuDomainPCIHostdevdata { + const virDomainDef *def; + virDomainPCIAddressSetPtr addrs; + virDomainHostdevDefPtr device; +}; + +typedef int (*virDomainPCIHostdevCallback)(qemuDomainPCIHostdevDataPtr data, + virDomainHostdevDefPtr hostdev); + const char *qemuDomainAsyncJobPhaseToString(qemuDomainAsyncJob job, int phase); int qemuDomainAsyncJobPhaseFromString(qemuDomainAsyncJob job, diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c index 424b56dac9..bc72b6e94c 100644 --- a/src/qemu/qemu_domain_address.c +++ b/src/qemu/qemu_domain_address.c @@ -1186,11 +1186,155 @@ qemuDomainSetupIsolationGroups(virDomainDefPtr def) } +#define PCI_MAX_BRIDGE_NUMBER 0xff +#define PCI_MAX_DEVICES 32 + + +/** + * qemuDomainPCIHostDevicesIter: + * @data - The data->device is the one which is called-back with for + * each hostdev + * cb() - callback to be called for each hostdev + * Return : + * If the callback for any of the hostdev fails, the Iter returns + * with the return value for that callback. + * Zero on success. + */ +static +int qemuDomainPCIHostDevicesIter(qemuDomainPCIHostdevDataPtr data, + virDomainPCIHostdevCallback cb) +{ + size_t i; + int ret = -1; + + /* Iterate through the PCI Hostdevices, the Mdev source is of type + * UUID, so skip that. */ + for (i = 0; i < data->def->nhostdevs; i++) { + virDomainHostdevSubsysPtr subsys = &data->def->hostdevs[i]->source.subsys; + virDomainHostdevDefPtr hostdev = data->def->hostdevs[i]; + if (data->device == hostdev) + continue; + if (data->def->hostdevs[i]->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + continue; + if (subsys->type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI && + subsys->type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI_HOST && + (subsys->type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_MDEV && + subsys->u.mdev.model == VIR_MDEV_MODEL_TYPE_VFIO_PCI)) { + continue; + } + + if ((ret = cb(data, hostdev)) != 0) + return ret; + } + + return 0; +} + + + +static int +qemuDomainFindNextAggregationSlotIdxIter(virDomainDefPtr def ATTRIBUTE_UNUSED, + virDomainDeviceDefPtr dev ATTRIBUTE_UNUSED, + virDomainDeviceInfoPtr info, + void *opaque) +{ + int *aggregationSlotIdx = opaque; + + if (info && info->aggregateSlotIdx == *aggregationSlotIdx) + return -1; + + return 0; +} + + +static unsigned int +qemuDomainFindNextSlotAggregationIdx(virDomainDefPtr def) +{ + int aggregateSlotIdx = 2; + + while (aggregateSlotIdx < PCI_MAX_BRIDGE_NUMBER * PCI_MAX_DEVICES && + virDomainDeviceInfoIterate(def, + qemuDomainFindNextAggregationSlotIdxIter, + &aggregateSlotIdx) < 0) { + aggregateSlotIdx++; + } + + return aggregateSlotIdx; +} + + +static int +qemuDomainDefHostdevGetSlotAggregateIdx(qemuDomainPCIHostdevDataPtr data, + virDomainHostdevDefPtr hostdev) +{ + if (data->device && + virHostdevPCIDevicesBelongToSameSlot(data->device, hostdev)) { + if (hostdev->info->aggregateSlotIdx > 0) + return hostdev->info->aggregateSlotIdx; + } + + return 0; +} + +/** + * qemuDomainDefDeviceFindSlotAggregateIdx: + * @def : domain def + * @dev : Find the slot aggregate for the device if other + * functions are already part of the def and have + * a slot aggreate idx assigned. + * Return: + * -1: if not assigned. + * 0: If the device is not a hostdev or not a + * multifunction device. + * >0: If assinged a value; + **/ +int +qemuDomainDefDeviceFindSlotAggregateIdx(virDomainDefPtr def, + virDomainDeviceDefPtr dev) +{ + int aggregateSlotIdx = 0; + virPCIDevicePtr pciDev; + virDomainHostdevDefPtr hostdev = dev->data.hostdev; + qemuDomainPCIHostdevdata temp = {def, NULL, hostdev}; + virPCIDeviceAddressPtr hostAddr = &hostdev->source.subsys.u.pci.addr; + + /* Only PCI host devices are subject to isolation */ + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS || + hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) { + return 0; + } + + if (!(pciDev = virPCIDeviceNew(hostAddr->domain, + hostAddr->bus, + hostAddr->slot, + hostAddr->function))) { + /* libvirt should be able to perform all the + * operations in virPCIDeviceNew() even if it's + * running unprivileged, so if this fails, the device + * apparently doesn't currently exist on the host. + * Since majority of host are non-multifunction, + * assume this one is too. + */ + return 0; + } + + if (!virPCIDeviceIsMultifunction(pciDev)) + return 0; + + aggregateSlotIdx = qemuDomainPCIHostDevicesIter(&temp, qemuDomainDefHostdevGetSlotAggregateIdx); + if (aggregateSlotIdx > 0) + return aggregateSlotIdx; + + return -1; +} + + void -qemuDomainSetDeviceSlotAggregateIdx(virDomainDefPtr def ATTRIBUTE_UNUSED, +qemuDomainSetDeviceSlotAggregateIdx(virDomainDefPtr def, virDomainDeviceDefPtr dev) { virDomainDeviceInfoPtr info = virDomainDeviceGetInfo(dev); + int aggregateSlotIdx = 0; if (!info) return; @@ -1203,6 +1347,12 @@ qemuDomainSetDeviceSlotAggregateIdx(virDomainDefPtr def ATTRIBUTE_UNUSED, cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT_PORT) { info->aggregateSlotIdx = 1; } + } else if (dev->type == VIR_DOMAIN_DEVICE_HOSTDEV) { + aggregateSlotIdx = qemuDomainDefDeviceFindSlotAggregateIdx(def, dev); + if (aggregateSlotIdx > 0) + info->aggregateSlotIdx = aggregateSlotIdx; + else if (aggregateSlotIdx < 0) + info->aggregateSlotIdx = qemuDomainFindNextSlotAggregationIdx(def); } return; @@ -2073,10 +2223,12 @@ qemuDomainAssignDevicePCISlots(virDomainDefPtr def, /* Host PCI devices */ for (i = 0; i < def->nhostdevs; i++) { - virDomainHostdevSubsysPtr subsys = &def->hostdevs[i]->source.subsys; - if (!virDeviceInfoPCIAddressWanted(def->hostdevs[i]->info)) + int function = 0; + virDomainHostdevDefPtr hostdev = def->hostdevs[i]; + virDomainHostdevSubsysPtr subsys = &hostdev->source.subsys; + if (!virDeviceInfoPCIAddressWanted(hostdev->info)) continue; - if (def->hostdevs[i]->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) continue; if (subsys->type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI && subsys->type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI_HOST && @@ -2085,9 +2237,14 @@ qemuDomainAssignDevicePCISlots(virDomainDefPtr def, continue; } - if (qemuDomainPCIAddressReserveNextAddr(addrs, - def->hostdevs[i]->info) < 0) + if (hostdev->info->aggregateSlotIdx > 1) + function = hostdev->source.subsys.u.pci.addr.function; + + if (virDomainPCIAddressReserveNextAddr(addrs, hostdev->info, + hostdev->info->pciConnectFlags, + function) < 0) { goto error; + } } /* VirtIO balloon */ diff --git a/tests/qemuhotplugtest.c b/tests/qemuhotplugtest.c index 31ce8d43b9..b5dca5e5c9 100644 --- a/tests/qemuhotplugtest.c +++ b/tests/qemuhotplugtest.c @@ -81,6 +81,7 @@ qemuHotplugCreateObjects(virDomainXMLOptionPtr xmlopt, virQEMUCapsSet(priv->qemuCaps, QEMU_CAPS_SCSI_DISK_WWN); virQEMUCapsSet(priv->qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI); virQEMUCapsSet(priv->qemuCaps, QEMU_CAPS_DEVICE_SPAPR_PCI_HOST_BRIDGE); + virQEMUCapsSet(priv->qemuCaps, QEMU_CAPS_VIRTIO_PCI_DISABLE_LEGACY); if (event) virQEMUCapsSet(priv->qemuCaps, QEMU_CAPS_DEVICE_DEL_EVENT); diff --git a/tests/qemuxml2argvdata/hostdev-pci-multifunction.args b/tests/qemuxml2argvdata/hostdev-pci-multifunction.args index 6b57f5713f..243ca8f2aa 100644 --- a/tests/qemuxml2argvdata/hostdev-pci-multifunction.args +++ b/tests/qemuxml2argvdata/hostdev-pci-multifunction.args @@ -19,11 +19,13 @@ server,nowait \ -no-acpi \ -boot c \ -usb \ --device vfio-pci,host=0005:90:01.0,id=hostdev0,bus=pci.0,addr=0x3 \ --device vfio-pci,host=0001:01:00.1,id=hostdev1,bus=pci.0,addr=0x4 \ --device vfio-pci,host=0001:01:00.0,id=hostdev2,bus=pci.0,addr=0x5 \ --device vfio-pci,host=0005:90:01.2,id=hostdev3,bus=pci.0,addr=0x6 \ --device vfio-pci,host=0005:90:01.3,id=hostdev4,bus=pci.0,addr=0x7 \ --device vfio-pci,host=06:12.1,id=hostdev5,bus=pci.0,addr=0x8 \ --device vfio-pci,host=06:12.2,id=hostdev6,bus=pci.0,addr=0x9 \ --device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xa +-device vfio-pci,host=0005:90:01.0,id=hostdev0,bus=pci.0,multifunction=on,\ +addr=0x3 \ +-device vfio-pci,host=0001:01:00.1,id=hostdev1,bus=pci.0,addr=0x4.0x1 \ +-device vfio-pci,host=0001:01:00.0,id=hostdev2,bus=pci.0,multifunction=on,\ +addr=0x4 \ +-device vfio-pci,host=0005:90:01.2,id=hostdev3,bus=pci.0,addr=0x3.0x2 \ +-device vfio-pci,host=0005:90:01.3,id=hostdev4,bus=pci.0,addr=0x3.0x3 \ +-device vfio-pci,host=06:12.1,id=hostdev5,bus=pci.0,addr=0x5 \ +-device vfio-pci,host=06:12.2,id=hostdev6,bus=pci.0,addr=0x6 \ +-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x7 diff --git a/tests/qemuxml2argvdata/pseries-hostdevs-1.args b/tests/qemuxml2argvdata/pseries-hostdevs-1.args index 8a4a4c5a63..6195350291 100644 --- a/tests/qemuxml2argvdata/pseries-hostdevs-1.args +++ b/tests/qemuxml2argvdata/pseries-hostdevs-1.args @@ -21,5 +21,6 @@ server,nowait \ -device spapr-pci-host-bridge,index=1,id=pci.1 \ -device spapr-pci-host-bridge,index=2,id=pci.2 \ -device vfio-pci,host=0005:90:01.0,id=hostdev0,bus=pci.1.0,addr=0x1 \ --device vfio-pci,host=0001:01:00.0,id=hostdev1,bus=pci.2.0,addr=0x1 \ --device vfio-pci,host=0001:01:00.1,id=hostdev2,bus=pci.2.0,addr=0x2 +-device vfio-pci,host=0001:01:00.0,id=hostdev1,bus=pci.2.0,multifunction=on,\ +addr=0x1 \ +-device vfio-pci,host=0001:01:00.1,id=hostdev2,bus=pci.2.0,addr=0x1.0x1 diff --git a/tests/qemuxml2argvdata/pseries-hostdevs-3.args b/tests/qemuxml2argvdata/pseries-hostdevs-3.args index 66a31ba1a8..2e800574de 100644 --- a/tests/qemuxml2argvdata/pseries-hostdevs-3.args +++ b/tests/qemuxml2argvdata/pseries-hostdevs-3.args @@ -20,5 +20,6 @@ server,nowait \ -boot c \ -device spapr-pci-host-bridge,index=1,id=pci.1 \ -device spapr-pci-host-bridge,index=2,id=pci.2 \ --device vfio-pci,host=0001:01:00.0,id=hostdev0,bus=pci.2.0,addr=0x1 \ --device vfio-pci,host=0001:01:00.1,id=hostdev1,bus=pci.2.0,addr=0x2 +-device vfio-pci,host=0001:01:00.0,id=hostdev0,bus=pci.2.0,multifunction=on,\ +addr=0x1 \ +-device vfio-pci,host=0001:01:00.1,id=hostdev1,bus=pci.2.0,addr=0x1.0x1 diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 1e00eb167a..583598dfec 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -1607,7 +1607,8 @@ mymain(void) DO_TEST("hostdev-pci-multifunction", QEMU_CAPS_KVM, QEMU_CAPS_DEVICE_VFIO_PCI, - QEMU_CAPS_HOST_PCI_MULTIDOMAIN, + QEMU_CAPS_VIRTIO_PCI_DISABLE_LEGACY, + QEMU_CAPS_HOST_PCI_MULTIDOMAIN, QEMU_CAPS_PCI_MULTIFUNCTION); DO_TEST("hostdev-mdev-precreated", QEMU_CAPS_NODEFCONFIG, @@ -1905,6 +1906,7 @@ mymain(void) QEMU_CAPS_NODEFCONFIG, QEMU_CAPS_DEVICE_SPAPR_PCI_HOST_BRIDGE, QEMU_CAPS_HOST_PCI_MULTIDOMAIN, + QEMU_CAPS_PCI_MULTIFUNCTION, QEMU_CAPS_VIRTIO_SCSI, QEMU_CAPS_DEVICE_VFIO_PCI); DO_TEST("pseries-hostdevs-2", @@ -1918,6 +1920,7 @@ mymain(void) QEMU_CAPS_DEVICE_SPAPR_PCI_HOST_BRIDGE, QEMU_CAPS_HOST_PCI_MULTIDOMAIN, QEMU_CAPS_VIRTIO_SCSI, + QEMU_CAPS_PCI_MULTIFUNCTION, QEMU_CAPS_DEVICE_VFIO_PCI); DO_TEST("pseries-features-hpt", diff --git a/tests/qemuxml2xmloutdata/hostdev-pci-multifunction.xml b/tests/qemuxml2xmloutdata/hostdev-pci-multifunction.xml index 52ed86e305..3396a73cf8 100644 --- a/tests/qemuxml2xmloutdata/hostdev-pci-multifunction.xml +++ b/tests/qemuxml2xmloutdata/hostdev-pci-multifunction.xml @@ -28,52 +28,52 @@ <source> <address domain='0x0005' bus='0x90' slot='0x01' function='0x0'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/> </hostdev> <hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0001' bus='0x01' slot='0x00' function='0x1'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x1'/> </hostdev> <hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0001' bus='0x01' slot='0x00' function='0x0'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0' multifunction='on'/> </hostdev> <hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0005' bus='0x90' slot='0x01' function='0x2'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/> </hostdev> <hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0005' bus='0x90' slot='0x01' function='0x3'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x3'/> </hostdev> <hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0000' bus='0x06' slot='0x12' function='0x1'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/> </hostdev> <hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0000' bus='0x06' slot='0x12' function='0x2'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/> </hostdev> <memballoon model='virtio'> - <address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/> </memballoon> </devices> </domain> diff --git a/tests/qemuxml2xmloutdata/pseries-hostdevs-1.xml b/tests/qemuxml2xmloutdata/pseries-hostdevs-1.xml index e77a060a38..c09588de9d 100644 --- a/tests/qemuxml2xmloutdata/pseries-hostdevs-1.xml +++ b/tests/qemuxml2xmloutdata/pseries-hostdevs-1.xml @@ -40,14 +40,14 @@ <source> <address domain='0x0001' bus='0x01' slot='0x00' function='0x0'/> </source> - <address type='pci' domain='0x0000' bus='0x02' slot='0x01' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x02' slot='0x01' function='0x0' multifunction='on'/> </hostdev> <hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0001' bus='0x01' slot='0x00' function='0x1'/> </source> - <address type='pci' domain='0x0000' bus='0x02' slot='0x02' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x02' slot='0x01' function='0x1'/> </hostdev> <memballoon model='none'/> <panic model='pseries'/> diff --git a/tests/qemuxml2xmloutdata/pseries-hostdevs-3.xml b/tests/qemuxml2xmloutdata/pseries-hostdevs-3.xml index f91959b805..f01adf6d25 100644 --- a/tests/qemuxml2xmloutdata/pseries-hostdevs-3.xml +++ b/tests/qemuxml2xmloutdata/pseries-hostdevs-3.xml @@ -32,14 +32,14 @@ <source> <address domain='0x0001' bus='0x01' slot='0x00' function='0x0'/> </source> - <address type='pci' domain='0x0000' bus='0x02' slot='0x01' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x02' slot='0x01' function='0x0' multifunction='on'/> </hostdev> <hostdev mode='subsystem' type='pci' managed='yes'> <driver name='vfio'/> <source> <address domain='0x0001' bus='0x01' slot='0x00' function='0x1'/> </source> - <address type='pci' domain='0x0000' bus='0x02' slot='0x02' function='0x0'/> + <address type='pci' domain='0x0000' bus='0x02' slot='0x01' function='0x1'/> </hostdev> <memballoon model='none'/> <panic model='pseries'/>