[libvirt] [PATCH 23/26] conf: Introduce isolation groups

Isolation groups will eventually allow us to make sure certain devices, eg. PCI hostdevs, are assigned to guest PCI buses in a way that guarantees improved isolation, error detection and recovery for machine types and hypervisors that support it, eg. pSeries guest on QEMU. --- src/bhyve/bhyve_device.c | 4 ++-- src/conf/device_conf.h | 4 ++++ src/conf/domain_addr.c | 33 +++++++++++++++++++++++++-------- src/conf/domain_addr.h | 7 ++++++- src/conf/domain_conf.h | 4 ++++ src/qemu/qemu_domain_address.c | 34 +++++++++++++++++----------------- 6 files changed, 58 insertions(+), 28 deletions(-) diff --git a/src/bhyve/bhyve_device.c b/src/bhyve/bhyve_device.c index fdfd512..03aa6c9 100644 --- a/src/bhyve/bhyve_device.c +++ b/src/bhyve/bhyve_device.c @@ -57,7 +57,7 @@ bhyveCollectPCIAddress(virDomainDefPtr def ATTRIBUTE_UNUSED, } if (virDomainPCIAddressReserveAddr(addrs, addr, - VIR_PCI_CONNECT_TYPE_PCI_DEVICE) < 0) { + VIR_PCI_CONNECT_TYPE_PCI_DEVICE, 0) < 0) { goto cleanup; } @@ -100,7 +100,7 @@ bhyveAssignDevicePCISlots(virDomainDefPtr def, lpc_addr.slot = 0x1; if (virDomainPCIAddressReserveAddr(addrs, &lpc_addr, - VIR_PCI_CONNECT_TYPE_PCI_DEVICE) < 0) { + VIR_PCI_CONNECT_TYPE_PCI_DEVICE, 0) < 0) { goto error; } diff --git a/src/conf/device_conf.h b/src/conf/device_conf.h index 58b2c9c..4b1377c 100644 --- a/src/conf/device_conf.h +++ b/src/conf/device_conf.h @@ -163,6 +163,10 @@ struct _virDomainDeviceInfo { * assignment, never saved and never reported. */ int pciConnectFlags; /* enum virDomainPCIConnectFlags */ + + /* PCI devices will only be automatically placed on a PCI bus + * that shares the same isolation group */ + int isolationGroup; }; diff --git a/src/conf/domain_addr.c b/src/conf/domain_addr.c index e6a6399..cd26b21 100644 --- a/src/conf/domain_addr.c +++ b/src/conf/domain_addr.c @@ -365,7 +365,8 @@ virDomainPCIAddressBusSetModel(virDomainPCIAddressBusPtr bus, static int virDomainPCIAddressSetGrow(virDomainPCIAddressSetPtr addrs, virPCIDeviceAddressPtr addr, - virDomainPCIConnectFlags flags) + virDomainPCIConnectFlags flags, + int isolationGroup) { int add; size_t i; @@ -492,6 +493,9 @@ virDomainPCIAddressSetGrow(virDomainPCIAddressSetPtr addrs, for (; i < addrs->nbuses; i++) { if (virDomainPCIAddressBusSetModel(&addrs->buses[i], model) < 0) return -1; + + /* Set isolation group for the new bus */ + addrs->buses[i].isolationGroup = isolationGroup; } return add; @@ -534,6 +538,7 @@ static int ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs, virPCIDeviceAddressPtr addr, virDomainPCIConnectFlags flags, + int isolationGroup, bool fromConfig) { int ret = -1; @@ -546,8 +551,10 @@ virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs, goto cleanup; /* Add an extra bus if necessary */ - if (addrs->dryRun && virDomainPCIAddressSetGrow(addrs, addr, flags) < 0) + if (addrs->dryRun && + virDomainPCIAddressSetGrow(addrs, addr, flags, isolationGroup) < 0) { goto cleanup; + } /* Check that the requested bus exists, is the correct type, and we * are asking for a valid slot */ @@ -586,9 +593,11 @@ virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs, int virDomainPCIAddressReserveAddr(virDomainPCIAddressSetPtr addrs, virPCIDeviceAddressPtr addr, - virDomainPCIConnectFlags flags) + virDomainPCIConnectFlags flags, + int isolationGroup) { - return virDomainPCIAddressReserveAddrInternal(addrs, addr, flags, true); + return virDomainPCIAddressReserveAddrInternal(addrs, addr, flags, + isolationGroup, true); } int @@ -624,7 +633,8 @@ virDomainPCIAddressEnsureAddr(virDomainPCIAddressSetPtr addrs, goto cleanup; ret = virDomainPCIAddressReserveAddrInternal(addrs, &dev->addr.pci, - flags, true); + flags, dev->isolationGroup, + true); } else { ret = virDomainPCIAddressReserveNextAddr(addrs, dev, flags, -1); } @@ -745,6 +755,7 @@ static int ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs, virPCIDeviceAddressPtr next_addr, virDomainPCIConnectFlags flags, + int isolationGroup, int function) { virPCIDeviceAddress a = { 0 }; @@ -765,6 +776,10 @@ virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs, for (a.bus = 0; a.bus < addrs->nbuses; a.bus++) { bool found = false; + /* Isolation groups for bus and device must match */ + if (addrs->buses[a.bus].isolationGroup != isolationGroup) + continue; + a.slot = addrs->buses[a.bus].minSlot; if (virDomainPCIAddressFindUnusedFunctionOnBus(&addrs->buses[a.bus], @@ -780,7 +795,7 @@ virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs, /* There were no free slots after the last used one */ if (addrs->dryRun) { /* a is already set to the first new bus */ - if (virDomainPCIAddressSetGrow(addrs, &a, flags) < 0) + if (virDomainPCIAddressSetGrow(addrs, &a, flags, isolationGroup) < 0) goto error; /* this device will use the first slot of the new bus */ a.slot = addrs->buses[a.bus].minSlot; @@ -825,10 +840,12 @@ virDomainPCIAddressReserveNextAddr(virDomainPCIAddressSetPtr addrs, { virPCIDeviceAddress addr; - if (virDomainPCIAddressGetNextAddr(addrs, &addr, flags, function) < 0) + if (virDomainPCIAddressGetNextAddr(addrs, &addr, flags, + dev->isolationGroup, function) < 0) return -1; - if (virDomainPCIAddressReserveAddrInternal(addrs, &addr, flags, false) < 0) + if (virDomainPCIAddressReserveAddrInternal(addrs, &addr, flags, + dev->isolationGroup, false) < 0) return -1; if (!addrs->dryRun) { diff --git a/src/conf/domain_addr.h b/src/conf/domain_addr.h index 1849d2b..23c7900 100644 --- a/src/conf/domain_addr.h +++ b/src/conf/domain_addr.h @@ -100,6 +100,10 @@ typedef struct { * bit is set, that function is in use by a device. */ virDomainPCIAddressSlot slot[VIR_PCI_ADDRESS_SLOT_LAST + 1]; + + /* PCI devices will only be automatically placed on a PCI bus + * that shares the same isolation group */ + int isolationGroup; } virDomainPCIAddressBus; typedef virDomainPCIAddressBus *virDomainPCIAddressBusPtr; @@ -147,7 +151,8 @@ bool virDomainPCIAddressSlotInUse(virDomainPCIAddressSetPtr addrs, int virDomainPCIAddressReserveAddr(virDomainPCIAddressSetPtr addrs, virPCIDeviceAddressPtr addr, - virDomainPCIConnectFlags flags) + virDomainPCIConnectFlags flags, + int isolationGroup) ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2); int virDomainPCIAddressReserveNextAddr(virDomainPCIAddressSetPtr addrs, diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 244977a..507891a 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -786,6 +786,10 @@ struct _virDomainPCIControllerOpts { * item in memory target config) -1 == unspecified */ int numaNode; + + /* PCI devices will only be automatically placed on a PCI bus + * that shares the same isolation group */ + int isolationGroup; }; typedef struct _virDomainUSBControllerOpts virDomainUSBControllerOpts; diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c index 08773c0..384d6fd 100644 --- a/src/qemu/qemu_domain_address.c +++ b/src/qemu/qemu_domain_address.c @@ -1032,7 +1032,8 @@ qemuDomainCollectPCIAddress(virDomainDefPtr def ATTRIBUTE_UNUSED, } if (virDomainPCIAddressReserveAddr(addrs, addr, - info->pciConnectFlags) < 0) { + info->pciConnectFlags, + info->isolationGroup) < 0) { goto cleanup; } @@ -1077,6 +1078,8 @@ qemuDomainPCIAddressSetCreate(virDomainDefPtr def, if (virDomainPCIAddressBusSetModel(&addrs->buses[idx], cont->model) < 0) goto error; + addrs->buses[idx].isolationGroup = cont->opts.pciopts.isolationGroup; + if (cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT) hasPCIeRoot = true; } @@ -1193,7 +1196,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def, if (addrs->nbuses) { memset(&tmp_addr, 0, sizeof(tmp_addr)); tmp_addr.slot = 1; - if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags) < 0) + if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) goto cleanup; } @@ -1228,7 +1231,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def, goto cleanup; } } else { - if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags) < 0) + if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) goto cleanup; primaryVideo->info.addr.pci = tmp_addr; primaryVideo->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; @@ -1253,7 +1256,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def, VIR_DEBUG("PCI address 0:0:2.0 in use, future addition of a video" " device will not be possible without manual" " intervention"); - } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags) < 0) { + } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) { goto cleanup; } } @@ -1329,10 +1332,8 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def, assign = true; } if (assign) { - if (virDomainPCIAddressReserveAddr(addrs, - &tmp_addr, flags) < 0) { + if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) goto cleanup; - } cont->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; cont->info.addr.pci.domain = 0; @@ -1354,10 +1355,8 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def, memset(&tmp_addr, 0, sizeof(tmp_addr)); tmp_addr.slot = 0x1E; if (!virDomainPCIAddressSlotInUse(addrs, &tmp_addr)) { - if (virDomainPCIAddressReserveAddr(addrs, - &tmp_addr, flags) < 0) { + if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) goto cleanup; - } cont->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; cont->info.addr.pci.domain = 0; @@ -1380,12 +1379,12 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def, tmp_addr.slot = 0x1F; tmp_addr.function = 0; tmp_addr.multi = VIR_TRISTATE_SWITCH_ON; - if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags) < 0) + if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) goto cleanup; tmp_addr.function = 3; tmp_addr.multi = VIR_TRISTATE_SWITCH_ABSENT; - if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags) < 0) + if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) goto cleanup; } @@ -1419,7 +1418,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def, goto cleanup; } } else { - if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags) < 0) + if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) goto cleanup; primaryVideo->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; primaryVideo->info.addr.pci = tmp_addr; @@ -1445,8 +1444,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def, " device will not be possible without manual" " intervention"); virResetLastError(); - } else if (virDomainPCIAddressReserveAddr(addrs, - &tmp_addr, flags) < 0) { + } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) { goto cleanup; } } @@ -1467,7 +1465,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def, !virDeviceInfoPCIAddressWanted(&sound->info)) { continue; } - if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags) < 0) + if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) goto cleanup; sound->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; @@ -1684,7 +1682,8 @@ qemuDomainAssignDevicePCISlots(virDomainDefPtr def, if (foundAddr) { /* Reserve this function on the slot we found */ if (virDomainPCIAddressReserveAddr(addrs, &addr, - cont->info.pciConnectFlags) < 0) { + cont->info.pciConnectFlags, + cont->info.isolationGroup) < 0) { goto error; } @@ -2150,6 +2149,7 @@ qemuDomainAssignPCIAddresses(virDomainDefPtr def, } dev.type = VIR_DOMAIN_DEVICE_CONTROLLER; dev.data.controller = def->controllers[contIndex]; + dev.data.controller->opts.pciopts.isolationGroup = bus->isolationGroup; /* set connect flags so it will be properly addressed */ qemuDomainFillDevicePCIConnectFlags(def, &dev, qemuCaps, driver); -- 2.7.5