Introduce support for "pciBus" driver attribute for "smmuv3" IOMMU model. The "pciBus" attribute indicates the index of the controller that a smmuv3 IOMMU device is attached to, and differentiates the device-pluggable arm-smmuv3 model from the virt-machine-associated smmuv3 model. Signed-off-by: Nathan Chen <nathanc@nvidia.com> --- docs/formatdomain.rst | 4 ++ src/conf/domain_conf.c | 16 +++++++ src/conf/domain_conf.h | 1 + src/conf/domain_validate.c | 28 ++++++++++-- src/conf/schemas/domaincommon.rng | 5 ++ src/qemu/qemu_command.c | 76 ++++++++++++++++++++++++++++--- 6 files changed, 121 insertions(+), 9 deletions(-) diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index 39bdecf9a1..4fa9e1dd2f 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -9218,6 +9218,10 @@ Example: Enable x2APIC mode. Useful for higher number of guest CPUs. :since:`Since 11.5.0` (QEMU/KVM and ``amd`` model only) + ``pciBus`` + The ``pciBus`` attribute notes the index of the controller that an + IOMMU device is attached to. (QEMU/KVM and ``smmuv3`` model only) + The ``virtio`` IOMMU devices can further have ``address`` element as described in `Device addresses`_ (address has to by type of ``pci``). diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 6e320713bb..a91989a277 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -2814,6 +2814,8 @@ virDomainIOMMUDefNew(void) iommu = g_new0(virDomainIOMMUDef, 1); + iommu->pci_bus = -1; + return g_steal_pointer(&iommu); } @@ -14449,6 +14451,10 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt, if (virXMLPropTristateSwitch(driver, "passthrough", VIR_XML_PROP_NONE, &iommu->pt) < 0) return NULL; + + if (virXMLPropInt(driver, "pciBus", 10, VIR_XML_PROP_NONE, + &iommu->pci_bus, -1) < 0) + return NULL; } if (virDomainDeviceInfoParseXML(xmlopt, node, ctxt, @@ -22142,6 +22148,12 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDef *src, dst->aw_bits, src->aw_bits); return false; } + if (src->pci_bus != dst->pci_bus) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device pci_bus value '%1$d' does not match source '%2$d'"), + dst->pci_bus, src->pci_bus); + return false; + } if (src->dma_translation != dst->dma_translation) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Target domain IOMMU device dma translation '%1$s' does not match source '%2$s'"), @@ -28461,6 +28473,10 @@ virDomainIOMMUDefFormat(virBuffer *buf, virBufferAsprintf(&driverAttrBuf, " xtsup='%s'", virTristateSwitchTypeToString(iommu->xtsup)); } + if (iommu->pci_bus >= 0) { + virBufferAsprintf(&driverAttrBuf, " pciBus='%d'", + iommu->pci_bus); + } virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, NULL); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 139bcdc1af..1463dc83c4 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -3051,6 +3051,7 @@ struct _virDomainIOMMUDef { virTristateSwitch eim; virTristateSwitch iotlb; unsigned int aw_bits; + int pci_bus; virDomainDeviceInfo info; virTristateSwitch dma_translation; virTristateSwitch xtsup; diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index dd1b491dfe..99b6ac7f7b 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -1860,6 +1860,11 @@ virDomainDefIOMMUValidate(const virDomainDef *def) _("IOMMU model smmuv3 must be specified for multiple IOMMU definitions")); } + if (def->niommus > 1 && iommu->pci_bus < 0) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("device-pluggable IOMMU with pciBus attribute must be specified for multiple IOMMU definitions")); + } + if (iommu->intremap == VIR_TRISTATE_SWITCH_ON && def->features[VIR_DOMAIN_FEATURE_IOAPIC] != VIR_DOMAIN_IOAPIC_QEMU) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", @@ -3109,13 +3114,28 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) { switch (iommu->model) { case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: + if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT || + iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT || + iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || + iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT || + iommu->aw_bits != 0 || + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pt != VIR_TRISTATE_SWITCH_ABSENT) { + virReportError(VIR_ERR_XML_ERROR, + _("iommu model '%1$s' doesn't support some additional attributes"), + virDomainIOMMUModelTypeToString(iommu->model)); + return -1; + } + break; case VIR_DOMAIN_IOMMU_MODEL_VIRTIO: if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT || iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT || iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT || iommu->aw_bits != 0 || - iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pci_bus >= 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3127,7 +3147,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) if (iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT || iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || iommu->aw_bits != 0 || - iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pci_bus >= 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3137,7 +3158,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) case VIR_DOMAIN_IOMMU_MODEL_INTEL: if (iommu->pt != VIR_TRISTATE_SWITCH_ABSENT || - iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pci_bus >= 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index 98596d4cea..69cc9ab13c 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -6311,6 +6311,11 @@ <ref name="virOnOff"/> </attribute> </optional> + <optional> + <attribute name="pciBus"> + <data type="unsignedInt"/> + </attribute> + </optional> </element> </optional> <optional> diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 61457c4f7a..014c27e695 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -6222,6 +6222,62 @@ qemuBuildBootCommandLine(virCommand *cmd, } +static virJSONValue * +qemuBuildPCINestedSmmuv3DevProps(const virDomainDef *def, + const virDomainIOMMUDef *iommu, + size_t id) +{ + g_autoptr(virJSONValue) props = NULL; + g_autofree char *bus = NULL; + g_autofree char *smmuv3_id = NULL; + size_t i; + + for (i = 0; i < def->ncontrollers; i++) { + virDomainControllerDef *cont = def->controllers[i]; + if (cont->idx == iommu->pci_bus) { + if (cont->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI) { + const char *alias = cont->info.alias; + + if (!alias) + return NULL; + + if (virDomainDeviceAliasIsUserAlias(alias)) { + if (cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT && + iommu->pci_bus == 0) { + if (qemuDomainSupportsPCIMultibus(def)) + bus = g_strdup("pci.0"); + else + bus = g_strdup("pci"); + } else if (cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT) { + bus = g_strdup("pcie.0"); + } + } else { + bus = g_strdup(alias); + } + break; + } + } + } + + if (!bus) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Could not find a suitable controller for smmuv3.")); + return NULL; + } + + smmuv3_id = g_strdup_printf("smmuv3.%zu", id); + + if (virJSONValueObjectAdd(&props, + "s:driver", "arm-smmuv3", + "s:primary-bus", bus, + "s:id", smmuv3_id, + NULL) < 0) + return NULL; + + return g_steal_pointer(&props); +} + + static int qemuBuildIOMMUCommandLine(virCommand *cmd, const virDomainDef *def, @@ -6269,9 +6325,6 @@ qemuBuildIOMMUCommandLine(virCommand *cmd, return -1; return 0; - case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: - /* There is no -device for SMMUv3, so nothing to be done here */ - return 0; case VIR_DOMAIN_IOMMU_MODEL_AMD: if (virJSONValueObjectAdd(&wrapperProps, @@ -6301,6 +6354,17 @@ qemuBuildIOMMUCommandLine(virCommand *cmd, return 0; + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: + if (iommu->pci_bus >= 0) { + if (!(props = qemuBuildPCINestedSmmuv3DevProps(def, iommu, i))) + return -1; + if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0) + return -1; + break; + } else { + return 0; + } + case VIR_DOMAIN_IOMMU_MODEL_LAST: default: virReportEnumRangeError(virDomainIOMMUModel, iommu->model); @@ -10805,15 +10869,15 @@ qemuBuildCommandLine(virDomainObj *vm, if (qemuBuildBootCommandLine(cmd, def) < 0) return NULL; - if (qemuBuildIOMMUCommandLine(cmd, def, qemuCaps) < 0) - return NULL; - if (qemuBuildGlobalControllerCommandLine(cmd, def) < 0) return NULL; if (qemuBuildControllersCommandLine(cmd, def, qemuCaps) < 0) return NULL; + if (qemuBuildIOMMUCommandLine(cmd, def, qemuCaps) < 0) + return NULL; + if (qemuBuildMemoryDeviceCommandLine(cmd, cfg, def, priv) < 0) return NULL; -- 2.43.0