Add support for parsing "nestedSmmuv3" devices from the VM definition. When a "nestedSmmuv3" IOMMU model is parsed from the VM definition, add a "nestedSmmuv3" device to the VM definition for each host SMMU node detected from sysfs. Specify the associated host SMMU sysfs filename as the "name" attribute for each "nestedSmmuv3" device in the VM definition. Signed-off-by: Nathan Chen <nathanc(a)nvidia.com&gt; --- docs/formatdomain.rst | 20 ++++ src/ch/ch_domain.c | 1 + src/conf/domain_addr.h | 1 + src/conf/domain_conf.c | 186 ++++++++++++++++++++++++++++++ src/conf/domain_conf.h | 12 ++ src/conf/domain_postparse.c | 1 + src/conf/domain_validate.c | 22 ++++ src/conf/schemas/domaincommon.rng | 16 +++ src/conf/virconftypes.h | 2 + src/lxc/lxc_driver.c | 6 + src/qemu/qemu_command.c | 58 ++++++++++ src/qemu/qemu_command.h | 4 + src/qemu/qemu_domain.c | 2 + src/qemu/qemu_domain_address.c | 5 + src/qemu/qemu_driver.c | 3 + src/qemu/qemu_hotplug.c | 5 + src/qemu/qemu_postparse.c | 1 + src/qemu/qemu_validate.c | 1 + src/test/test_driver.c | 4 + tests/schemas/device.rng.in | 1 + 20 files changed, 351 insertions(+) diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index 63bb565991..739b5b1b89 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -8800,6 +8800,26 @@ The optional ``driver`` element allows to specify virtio options, see ... +NestedSmmuv3 +~~~~~~~~~~~~ +A representation of a host SMMU node. This is used to keep track of which +controller to assign a VFIO device to in the VM if it is associated with a +host SMMU node. This supports having multiple vSMMU nodes in the VM by +attaching devices with different SMMU nodes to different pcie-expander-bus +controllers in the VM. The ``name`` attribute denotes the host SMMU node's +identifier parsed from the host's sysfs. + +:: + + ... + <devices> + <nestedSmmuv3> + <name>smmu3.0x0000000005000000</name> + </nestedSmmuv3> + </devices> + ... + + Crypto ~~~~~~ diff --git a/src/ch/ch_domain.c b/src/ch/ch_domain.c index bfccabed49..7e7ca1c6ac 100644 --- a/src/ch/ch_domain.c +++ b/src/ch/ch_domain.c @@ -180,6 +180,7 @@ chValidateDomainDeviceDef(const virDomainDeviceDef *dev, case VIR_DOMAIN_DEVICE_IOMMU: case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_PSTORE: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Cloud-Hypervisor doesn't support '%1$s' device"), diff --git a/src/conf/domain_addr.h b/src/conf/domain_addr.h index e72fb48847..9781685903 100644 --- a/src/conf/domain_addr.h +++ b/src/conf/domain_addr.h @@ -57,6 +57,7 @@ typedef enum { VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS = 1 << 11, VIR_PCI_CONNECT_TYPE_PCI_BRIDGE = 1 << 12, VIR_PCI_CONNECT_TYPE_PCIE_TO_PCI_BRIDGE = 1 << 13, + VIR_PCI_CONNECT_TYPE_NESTED_SMMUV3 = 1 << 14, } virDomainPCIConnectFlags; /* a combination of all bits that describe the type of connections diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index c1092551e0..24aff1cfbe 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -341,6 +341,7 @@ VIR_ENUM_IMPL(virDomainDevice, "audio", "crypto", "pstore", + "nestedSmmuv3", ); VIR_ENUM_IMPL(virDomainDiskDevice, @@ -3449,6 +3450,19 @@ virDomainHostdevDefNew(void) } +virDomainNestedSmmuv3Def * +virDomainNestedSmmuv3DefNew(void) +{ + virDomainNestedSmmuv3Def *def; + + def = g_new0(virDomainNestedSmmuv3Def, 1); + + def->info = g_new0(virDomainDeviceInfo, 1); + + return def; +} + + static virDomainTPMDef * virDomainTPMDefNew(virDomainXMLOption *xmlopt) { @@ -3509,6 +3523,18 @@ void virDomainHostdevDefFree(virDomainHostdevDef *def) g_free(def); } +void virDomainNestedSmmuv3DefFree(virDomainNestedSmmuv3Def *def) +{ + if (!def) + return; + + g_free(def->name); + + virDomainDeviceInfoFree(def->info); + + g_free(def); +} + void virDomainHubDefFree(virDomainHubDef *def) { if (!def) @@ -3671,6 +3697,9 @@ void virDomainDeviceDefFree(virDomainDeviceDef *def) case VIR_DOMAIN_DEVICE_PSTORE: virDomainPstoreDefFree(def->data.pstore); break; + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: + virDomainNestedSmmuv3DefFree(def->data.nestedsmmuv3); + break; case VIR_DOMAIN_DEVICE_LAST: case VIR_DOMAIN_DEVICE_NONE: break; @@ -4597,6 +4626,8 @@ virDomainDeviceGetInfo(const virDomainDeviceDef *device) return &device->data.crypto->info; case VIR_DOMAIN_DEVICE_PSTORE: return &device->data.pstore->info; + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: + return device->data.nestedsmmuv3->info; /* The following devices do not contain virDomainDeviceInfo */ case VIR_DOMAIN_DEVICE_LEASE: @@ -4705,6 +4736,9 @@ virDomainDeviceSetData(virDomainDeviceDef *device, case VIR_DOMAIN_DEVICE_PSTORE: device->data.pstore = devicedata; break; + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: + device->data.nestedsmmuv3 = devicedata; + break; case VIR_DOMAIN_DEVICE_NONE: case VIR_DOMAIN_DEVICE_LAST: break; @@ -4930,6 +4964,13 @@ virDomainDeviceInfoIterateFlags(virDomainDef *def, return rc; } + device.type = VIR_DOMAIN_DEVICE_NESTED_SMMUV3; + for (i = 0; i < def->nnestedsmmus; i++) { + device.data.nestedsmmuv3 = def->nestedsmmus[i]; + if ((rc = cb(def, &device, def->nestedsmmus[i]->info, opaque)) != 0) + return rc; + } + /* If the flag below is set, make sure @cb can handle @info being NULL */ if (iteratorFlags & DOMAIN_DEVICE_ITERATE_MISSING_INFO) { device.type = VIR_DOMAIN_DEVICE_GRAPHICS; @@ -4990,6 +5031,7 @@ virDomainDeviceInfoIterateFlags(virDomainDef *def, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: break; } #endif @@ -13365,6 +13407,40 @@ virDomainHostdevDefParseXML(virDomainXMLOption *xmlopt, } +static virDomainNestedSmmuv3Def * +virDomainNestedSmmuv3DefParseXML(virDomainXMLOption *xmlopt, + xmlNodePtr node, + xmlXPathContextPtr ctxt, + unsigned int flags) +{ + virDomainNestedSmmuv3Def *def; + size_t nameLength; + VIR_XPATH_NODE_AUTORESTORE(ctxt) + + ctxt->node = node; + + if (!(def = virDomainNestedSmmuv3DefNew())) + goto error; + + nameLength = strlen(virXPathString("string(./name)", ctxt)) + 1; + VIR_REALLOC_N(def->name, nameLength); + if (!def->name) + goto error; + virStrcpy(def->name, virXPathString("string(./name)", ctxt), nameLength); + + if (def->info->type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE) { + if (virDomainDeviceInfoParseXML(xmlopt, node, ctxt, def->info, + flags) < 0) + goto error; + } + return def; + + error: + virDomainNestedSmmuv3DefFree(def); + return NULL; +} + + static virDomainRedirdevDef * virDomainRedirdevDefParseXML(virDomainXMLOption *xmlopt, xmlNodePtr node, @@ -14366,6 +14442,12 @@ virDomainDeviceDefParse(const char *xmlStr, return NULL; } break; + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: + if (!(dev->data.nestedsmmuv3 = virDomainNestedSmmuv3DefParseXML(xmlopt, node, + ctxt, flags))) { + return NULL; + } + break; case VIR_DOMAIN_DEVICE_NONE: case VIR_DOMAIN_DEVICE_LAST: break; @@ -19445,6 +19527,21 @@ virDomainDefParseXML(xmlXPathContextPtr ctxt, VIR_FREE(nodes); + /* analysis of the nested SMMUs */ + if ((n = virXPathNodeSet("./devices/nestedSmmuv3", ctxt, &nodes)) < 0) + return NULL; + if (n > 0) + VIR_REALLOC_N(def->nestedsmmus, def->nnestedsmmus + n); + for (i = 0; i < n; i++) { + virDomainNestedSmmuv3Def *nestedsmmuv3; + nestedsmmuv3 = virDomainNestedSmmuv3DefParseXML(xmlopt, nodes[i], ctxt, + flags); + if (!nestedsmmuv3) + return NULL; + def->nestedsmmus[def->nnestedsmmus++] = nestedsmmuv3; + } + VIR_FREE(nodes); + /* analysis of the host devices */ if ((n = virXPathNodeSet("./devices/hostdev", ctxt, &nodes)) < 0) return NULL; @@ -20664,6 +20761,20 @@ virDomainHostdevDefCheckABIStability(virDomainHostdevDef *src, } +static bool +virDomainNestedSmmuv3DefCheckABIStability(virDomainNestedSmmuv3Def *src, + virDomainNestedSmmuv3Def *dst) +{ + if (STRNEQ(src->name, dst->name)) + return false; + + if (!virDomainDeviceInfoCheckABIStability(src->info, dst->info)) + return false; + + return true; +} + + static bool virDomainSmartcardDefCheckABIStability(virDomainSmartcardDef *src, virDomainSmartcardDef *dst) @@ -22001,6 +22112,18 @@ virDomainDefCheckABIStabilityFlags(virDomainDef *src, goto error; } + if (src->nnestedsmmus != dst->nnestedsmmus) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain nested SMMUs count %1$zu does not match source %2$zu"), + dst->nnestedsmmus, src->nnestedsmmus); + goto error; + } + + for (i = 0; i < src->nnestedsmmus; i++) + if (!virDomainNestedSmmuv3DefCheckABIStability(src->nestedsmmus[i], + dst->nestedsmmus[i])) + goto error; + if ((!src->redirfilter && dst->redirfilter) || (src->redirfilter && !dst->redirfilter)) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, @@ -22171,6 +22294,7 @@ virDomainDefCheckABIStabilityFlags(virDomainDef *src, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: break; } #endif @@ -22366,6 +22490,36 @@ virDomainDefAddImplicitVideo(virDomainDef *def, virDomainXMLOption *xmlopt) return 0; } +static int +virDomainDefAddImplicitNestedSmmuv3(virDomainDef *def) +{ + // Get the number of host-level SMMUv3 instances + g_autoptr(DIR) dir = NULL; + struct dirent *dent; + int num = 0; + + virDomainNestedSmmuv3Def* nestedsmmuv3 = NULL; + + if (virDirOpen(&dir, "/sys/class/iommu") < 0) + return -1; + + while (virDirRead(dir, &dent, "/sys/class/iommu") > 0) { + if (!(nestedsmmuv3 = virDomainNestedSmmuv3DefNew())) + return -1; + if (STRPREFIX(dent->d_name, "smmu3.0x")) { + VIR_REALLOC_N(nestedsmmuv3->name, strlen(dent->d_name) + 1); + virStrcpy(nestedsmmuv3->name, dent->d_name, strlen(dent->d_name) + 1); + VIR_REALLOC_N(def->nestedsmmus, def->nnestedsmmus + 1); + def->nestedsmmus[def->nnestedsmmus++] = nestedsmmuv3; + num++; + } + } + if (num == 0) + return -1; + + return 0; +} + int virDomainDefAddImplicitDevices(virDomainDef *def, virDomainXMLOption *xmlopt) { @@ -22379,6 +22533,13 @@ virDomainDefAddImplicitDevices(virDomainDef *def, virDomainXMLOption *xmlopt) if (virDomainDefAddImplicitVideo(def, xmlopt) < 0) return -1; + if (def->iommu != NULL && + def->iommu->model == VIR_DOMAIN_IOMMU_MODEL_NESTED_SMMUV3 && + def->nnestedsmmus < 1) { + if (virDomainDefAddImplicitNestedSmmuv3(def) < 0) + return -1; + } + return 0; } @@ -26814,6 +26975,24 @@ virDomainHostdevDefFormat(virBuffer *buf, return 0; } +static int +virDomainNestedSmmuv3DefFormat(virBuffer *buf, + virDomainNestedSmmuv3Def *def, + unsigned int flags) +{ + virBufferAddLit(buf, "<nestedSmmuv3>\n"); + virBufferAdjustIndent(buf, 2); + + virBufferAsprintf(buf, "<name>%s</name>\n", def->name); + + virDomainDeviceInfoFormat(buf, def->info, flags); + + virBufferAdjustIndent(buf, -2); + virBufferAddLit(buf, "</nestedSmmuv3>\n"); + + return 0; +} + static int virDomainRedirdevDefFormat(virBuffer *buf, virDomainRedirdevDef *def, @@ -28672,6 +28851,12 @@ virDomainDefFormatInternalSetRootName(virDomainDef *def, for (n = 0; n < def->ncryptos; n++) { virDomainCryptoDefFormat(buf, def->cryptos[n], flags); } + + for (n = 0; n < def->nnestedsmmus; n++) { + if (virDomainNestedSmmuv3DefFormat(buf, def->nestedsmmus[n], flags) < 0) + return -1; + } + if (def->iommu) virDomainIOMMUDefFormat(buf, def->iommu); @@ -28841,6 +29026,7 @@ virDomainDeviceIsUSB(virDomainDeviceDef *dev, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: break; } diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index f8ab1b7d2f..edde9b63d6 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -88,6 +88,7 @@ typedef enum { VIR_DOMAIN_DEVICE_AUDIO, VIR_DOMAIN_DEVICE_CRYPTO, VIR_DOMAIN_DEVICE_PSTORE, + VIR_DOMAIN_DEVICE_NESTED_SMMUV3, VIR_DOMAIN_DEVICE_LAST } virDomainDeviceType; @@ -122,6 +123,7 @@ struct _virDomainDeviceDef { virDomainAudioDef *audio; virDomainCryptoDef *crypto; virDomainPstoreDef *pstore; + virDomainNestedSmmuv3Def *nestedsmmuv3; } data; }; @@ -353,6 +355,11 @@ typedef enum { VIR_DOMAIN_STARTUP_POLICY_LAST } virDomainStartupPolicy; +struct _virDomainNestedSmmuv3Def { + char *name; + virDomainDeviceInfo *info; /* Guest address */ +}; + /* basic device for direct passthrough */ struct _virDomainHostdevDef { /* If 'parentnet' is non-NULL it means this host dev was @@ -3197,6 +3204,9 @@ struct _virDomainDef { size_t ntpms; virDomainTPMDef **tpms; + size_t nnestedsmmus; + virDomainNestedSmmuv3Def **nestedsmmus; + /* Only 1 */ virDomainMemballoonDef *memballoon; virDomainNVRAMDef *nvram; @@ -3677,6 +3687,8 @@ virDomainVideoDef *virDomainVideoDefNew(virDomainXMLOption *xmlopt); void virDomainVideoDefFree(virDomainVideoDef *def); G_DEFINE_AUTOPTR_CLEANUP_FUNC(virDomainVideoDef, virDomainVideoDefFree); void virDomainVideoDefClear(virDomainVideoDef *def); +virDomainNestedSmmuv3Def *virDomainNestedSmmuv3DefNew(void); +void virDomainNestedSmmuv3DefFree(virDomainNestedSmmuv3Def *def); virDomainHostdevDef *virDomainHostdevDefNew(void); void virDomainHostdevDefFree(virDomainHostdevDef *def); void virDomainHubDefFree(virDomainHubDef *def); diff --git a/src/conf/domain_postparse.c b/src/conf/domain_postparse.c index bf33f29638..b838a9688e 100644 --- a/src/conf/domain_postparse.c +++ b/src/conf/domain_postparse.c @@ -757,6 +757,7 @@ virDomainDeviceDefPostParseCommon(virDomainDeviceDef *dev, case VIR_DOMAIN_DEVICE_IOMMU: case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_PSTORE: ret = 0; break; diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index c8b7b701bf..20d16c8b84 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -2310,6 +2310,25 @@ virDomainHostdevDefValidate(const virDomainHostdevDef *hostdev) } +static int +virDomainNestedSmmuv3DefValidate(const virDomainNestedSmmuv3Def *nestedsmmuv3) +{ + if (nestedsmmuv3->info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE && + nestedsmmuv3->info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("nestedsmmuv3 must use 'pci' address type")); + return -1; + } + + if (nestedsmmuv3->name[0] == '\0') { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("nestedsmmuv3 must have an associated sysfs node name")); + return -1; + } + return 0; +} + + /** * virDomainMemoryGetMappedSize: * @mem: memory device definition @@ -3230,6 +3249,9 @@ virDomainDeviceDefValidateInternal(const virDomainDeviceDef *dev, case VIR_DOMAIN_DEVICE_PSTORE: return virDomainPstoreDefValidate(dev->data.pstore); + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: + return virDomainNestedSmmuv3DefValidate(dev->data.nestedsmmuv3); + case VIR_DOMAIN_DEVICE_LEASE: case VIR_DOMAIN_DEVICE_WATCHDOG: case VIR_DOMAIN_DEVICE_HUB: diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index de6a1e5c7e..e39559952e 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -6359,6 +6359,21 @@ </element> </define> + <define name="nestedSmmuv3"> + <element name="nestedSmmuv3"> + <interleave> + <optional> + <element name="name"> + <text/> + </element> + </optional> + <optional> + <ref name="address"/> + </optional> + </interleave> + </element> + </define> + <define name="hostdev"> <element name="hostdev"> <interleave> @@ -6808,6 +6823,7 @@ <ref name="shmem"/> <ref name="memorydev"/> <ref name="crypto"/> + <ref name="nestedSmmuv3"/> </choice> </zeroOrMore> <zeroOrMore> diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h index 59be61cea4..b36020d9a5 100644 --- a/src/conf/virconftypes.h +++ b/src/conf/virconftypes.h @@ -118,6 +118,8 @@ typedef struct _virDomainHostdevCaps virDomainHostdevCaps; typedef struct _virDomainHostdevDef virDomainHostdevDef; +typedef struct _virDomainNestedSmmuv3Def virDomainNestedSmmuv3Def; + typedef struct _virDomainHostdevSubsys virDomainHostdevSubsys; typedef struct _virDomainHostdevSubsysMediatedDev virDomainHostdevSubsysMediatedDev; diff --git a/src/lxc/lxc_driver.c b/src/lxc/lxc_driver.c index 2488940feb..5a7fb9b62d 100644 --- a/src/lxc/lxc_driver.c +++ b/src/lxc/lxc_driver.c @@ -3021,6 +3021,7 @@ lxcDomainAttachDeviceConfig(virDomainDef *vmdef, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("persistent attach of device is not supported")); break; @@ -3087,6 +3088,7 @@ lxcDomainUpdateDeviceConfig(virDomainDef *vmdef, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("persistent update of device is not supported")); break; @@ -3169,6 +3171,7 @@ lxcDomainDetachDeviceConfig(virDomainDef *vmdef, case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("persistent detach of device is not supported")); break; @@ -3271,6 +3274,7 @@ lxcDomainAttachDeviceMknodHelper(pid_t pid G_GNUC_UNUSED, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: virReportError(VIR_ERR_INTERNAL_ERROR, _("Unexpected device type %1$d"), data->def->type); @@ -3947,6 +3951,7 @@ lxcDomainAttachDeviceLive(virLXCDriver *driver, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("device type '%1$s' cannot be attached"), virDomainDeviceTypeToString(dev->type)); @@ -4365,6 +4370,7 @@ lxcDomainDetachDeviceLive(virLXCDriver *driver, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("device type '%1$s' cannot be detached"), virDomainDeviceTypeToString(dev->type)); diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 6629addace..3186e60a0e 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -975,6 +975,7 @@ qemuBuildVirtioDevGetConfigDev(const virDomainDeviceDef *device, case VIR_DOMAIN_DEVICE_IOMMU: case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: default: break; @@ -4564,6 +4565,33 @@ qemuBuildVideoCommandLine(virCommand *cmd, } +virJSONValue * +qemuBuildPCINestedSmmuv3DevProps(const virDomainDef *def, + virDomainNestedSmmuv3Def *nestedsmmuv3) +{ + g_autoptr(virJSONValue) props = NULL; + g_autofree char *pciaddr = NULL; + g_autofree char *bus = qemuBuildDeviceAddressPCIGetBus(def, nestedsmmuv3->info); + + if (!bus) + return NULL; + + if (nestedsmmuv3->info->addr.pci.function != 0) + pciaddr = g_strdup_printf("0x%x.0x%x", nestedsmmuv3->info->addr.pci.slot, + nestedsmmuv3->info->addr.pci.function); + else + pciaddr = g_strdup_printf("0x%x", nestedsmmuv3->info->addr.pci.slot); + + if (virJSONValueObjectAdd(&props, + "s:driver", "arm-smmuv3-nested", + "s:bus", bus, + NULL) < 0) + return NULL; + + return g_steal_pointer(&props); +} + + virJSONValue * qemuBuildPCIHostdevDevProps(const virDomainDef *def, virDomainHostdevDef *dev) @@ -4988,6 +5016,33 @@ qemuBuildHostdevSCSICommandLine(virCommand *cmd, } +static int +qemuBuildNestedSmmuv3CommandLine(virCommand *cmd, + const virDomainDef *def, + virQEMUCaps *qemuCaps) +{ + size_t i; + + for (i = 0; i < def->nnestedsmmus; i++) { + g_autoptr(virJSONValue) devprops = NULL; + virDomainNestedSmmuv3Def *nestedsmmuv3 = def->nestedsmmus[i]; + + if (nestedsmmuv3->info->type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_UNASSIGNED) + continue; + + if (qemuCommandAddExtDevice(cmd, nestedsmmuv3->info, def, qemuCaps) < 0) + return -1; + + if (!(devprops = qemuBuildPCINestedSmmuv3DevProps(def, nestedsmmuv3))) + return -1; + + if (qemuBuildDeviceCommandlineFromJSON(cmd, devprops, def, qemuCaps) < 0) + return -1; + } + return 0; +} + + static int qemuBuildHostdevCommandLine(virCommand *cmd, const virDomainDef *def, @@ -10558,6 +10613,9 @@ qemuBuildCommandLine(virDomainObj *vm, if (qemuBuildHostdevCommandLine(cmd, def, qemuCaps) < 0) return NULL; + if (qemuBuildNestedSmmuv3CommandLine(cmd, def, qemuCaps) < 0) + return NULL; + if (migrateURI) virCommandAddArgList(cmd, "-incoming", migrateURI, NULL); diff --git a/src/qemu/qemu_command.h b/src/qemu/qemu_command.h index 76c514b5f7..f09dcc514c 100644 --- a/src/qemu/qemu_command.h +++ b/src/qemu/qemu_command.h @@ -157,6 +157,10 @@ qemuBuildThreadContextProps(virJSONValue **tcProps, qemuDomainObjPrivate *priv, virBitmap *nodemask); +virJSONValue * +qemuBuildPCINestedSmmuv3DevProps(const virDomainDef *def, + virDomainNestedSmmuv3Def *nestedsmmuv3); + /* Current, best practice */ virJSONValue * qemuBuildPCIHostdevDevProps(const virDomainDef *def, diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 3366346624..2567d620d9 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -8718,6 +8718,7 @@ qemuDomainPrepareChardevSourceOne(virDomainDeviceDef *dev, case VIR_DOMAIN_DEVICE_VSOCK: case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_PSTORE: break; } @@ -10670,6 +10671,7 @@ qemuDomainDeviceBackendChardevForeachOne(virDomainDeviceDef *dev, case VIR_DOMAIN_DEVICE_VSOCK: case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_PSTORE: /* no chardev backend */ break; diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c index 51f4bbd6eb..31004bfc7e 100644 --- a/src/qemu/qemu_domain_address.c +++ b/src/qemu/qemu_domain_address.c @@ -470,6 +470,7 @@ qemuDomainDeviceSupportZPCI(virDomainDeviceDef *device) case VIR_DOMAIN_DEVICE_VSOCK: case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_PSTORE: break; @@ -819,6 +820,10 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDef *dev, return pciFlags; } + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: { + return VIR_PCI_CONNECT_TYPE_NESTED_SMMUV3; + } + case VIR_DOMAIN_DEVICE_MEMBALLOON: switch (dev->data.memballoon->model) { case VIR_DOMAIN_MEMBALLOON_MODEL_VIRTIO_TRANSITIONAL: diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 09f7edda7d..d9ade7000b 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -6880,6 +6880,7 @@ qemuDomainAttachDeviceConfig(virDomainDef *vmdef, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("persistent attach of device '%1$s' is not supported"), @@ -7099,6 +7100,7 @@ qemuDomainDetachDeviceConfig(virDomainDef *vmdef, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("persistent detach of device '%1$s' is not supported"), @@ -7225,6 +7227,7 @@ qemuDomainUpdateDeviceConfig(virDomainDef *vmdef, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("persistent update of device '%1$s' is not supported"), diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 3c18af6b0c..0a67df8605 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -3521,6 +3521,7 @@ qemuDomainAttachDeviceLive(virDomainObj *vm, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("live attach of device '%1$s' is not supported"), @@ -5420,6 +5421,7 @@ qemuDomainRemoveAuditDevice(virDomainObj *vm, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: /* libvirt doesn't yet support detaching these devices */ break; @@ -5525,6 +5527,7 @@ qemuDomainRemoveDevice(virQEMUDriver *driver, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("don't know how to remove a %1$s device"), @@ -6420,6 +6423,7 @@ qemuDomainDetachDeviceLive(virDomainObj *vm, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("live detach of device '%1$s' is not supported"), @@ -7412,6 +7416,7 @@ qemuDomainUpdateDeviceLive(virDomainObj *vm, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("live update of device '%1$s' is not supported"), diff --git a/src/qemu/qemu_postparse.c b/src/qemu/qemu_postparse.c index 03b5ef825a..4e8892b838 100644 --- a/src/qemu/qemu_postparse.c +++ b/src/qemu/qemu_postparse.c @@ -897,6 +897,7 @@ qemuDomainDeviceDefPostParse(virDomainDeviceDef *dev, case VIR_DOMAIN_DEVICE_NVRAM: case VIR_DOMAIN_DEVICE_RNG: case VIR_DOMAIN_DEVICE_AUDIO: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_CRYPTO: ret = 0; break; diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c index 9f07ffe4a3..d84f04903f 100644 --- a/src/qemu/qemu_validate.c +++ b/src/qemu/qemu_validate.c @@ -5470,6 +5470,7 @@ qemuValidateDomainDeviceDef(const virDomainDeviceDef *dev, case VIR_DOMAIN_DEVICE_PSTORE: return qemuValidateDomainDeviceDefPstore(dev->data.pstore, def, qemuCaps); + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LEASE: case VIR_DOMAIN_DEVICE_PANIC: case VIR_DOMAIN_DEVICE_NONE: diff --git a/src/test/test_driver.c b/src/test/test_driver.c index f1cefb5c50..c63e76baa5 100644 --- a/src/test/test_driver.c +++ b/src/test/test_driver.c @@ -10451,6 +10451,7 @@ testDomainAttachDeviceLive(virDomainObj *vm, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("live attach of device '%1$s' is not supported"), @@ -10594,6 +10595,7 @@ testDomainUpdateDevice(virDomainDef *vmdef, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("persistent update of device '%1$s' is not supported"), @@ -10965,6 +10967,7 @@ testDomainRemoveDevice(testDriver *driver, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("live detach of device '%1$s' is not supported"), @@ -11036,6 +11039,7 @@ testDomainDetachDeviceLive(testDriver *driver, case VIR_DOMAIN_DEVICE_AUDIO: case VIR_DOMAIN_DEVICE_CRYPTO: case VIR_DOMAIN_DEVICE_PSTORE: + case VIR_DOMAIN_DEVICE_NESTED_SMMUV3: case VIR_DOMAIN_DEVICE_LAST: virReportError(VIR_ERR_OPERATION_UNSUPPORTED, _("live detach of device '%1$s' is not supported"), diff --git a/tests/schemas/device.rng.in b/tests/schemas/device.rng.in index b322b5275e..fc59700574 100644 --- a/tests/schemas/device.rng.in +++ b/tests/schemas/device.rng.in @@ -45,6 +45,7 @@ <ref name="panic"/> <ref name="iommu"/> <ref name="vsock"/> + <ref name="nestedSmmuv3"/> </choice> </start> -- 2.34.1

Update the PXB controller busNrs to account for devices we attached in the previous commit, ensuring there are enough VM bus numbers to be assigned for each device attached downstream from each PXB controller. Signed-off-by: Nathan Chen <nathanc(a)nvidia.com&gt; --- src/qemu/qemu_domain_address.c | 52 ++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c index dee198a7d2..001d1ec0b9 100644 --- a/src/qemu/qemu_domain_address.c +++ b/src/qemu/qemu_domain_address.c @@ -2678,6 +2678,23 @@ qemuDomainAddressFindNewTargetIndex(virDomainDef *def) } +static int +qemuDomainFindAttachedDevicesBusNr(virDomainDef *def, + int lowestBusNr, + unsigned int contIdx) +{ + size_t i; + for (i = 0; i < def->ncontrollers; i++) { + virDomainControllerDef *cont = def->controllers[i]; + if (cont->info.addr.pci.bus == contIdx && cont->idx != 0) + lowestBusNr = qemuDomainFindAttachedDevicesBusNr(def, lowestBusNr - 1, cont->idx); + if (lowestBusNr <= 2) + return -1; + } + return lowestBusNr; +} + + static int qemuDomainAddressFindNewBusNr(virDomainDef *def) { @@ -2725,8 +2742,43 @@ qemuDomainAddressFindNewBusNr(virDomainDef *def) */ size_t i; + size_t lowestBusNrContIdx = 0; int lowestBusNr = 256; + if (def->iommu && def->iommu->model && + def->iommu->model == VIR_DOMAIN_IOMMU_MODEL_NESTED_SMMUV3) { + for (i = 0; i < def->ncontrollers; i++) { + virDomainControllerDef *cont = def->controllers[i]; + if (cont->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI) { + int thisBusNr = cont->opts.pciopts.busNr; + if (thisBusNr >= 0 && thisBusNr < lowestBusNr) { + lowestBusNr = thisBusNr; + lowestBusNrContIdx = i; + } + } + } + if (lowestBusNr <= 2) + return -1; + if (lowestBusNrContIdx == 0) { + for (i = 0; i < def->ncontrollers; i++) { + if (def->controllers[i]->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_EXPANDER_BUS) { + lowestBusNrContIdx = i; + break; + } + } + } else { + for (i = lowestBusNrContIdx + 1; i < def->ncontrollers; i++) { + if (def->controllers[i]->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_EXPANDER_BUS) { + lowestBusNrContIdx = i; + break; + } + } + } + lowestBusNr = qemuDomainFindAttachedDevicesBusNr(def, lowestBusNr, + def->controllers[lowestBusNrContIdx]->idx); + return lowestBusNr - 2; + } + for (i = 0; i < def->ncontrollers; i++) { virDomainControllerDef *cont = def->controllers[i]; -- 2.34.1