Introduce support for "pciBus" driver attribute for "smmuv3" IOMMU model. The "pciBus" attribute indicates the index of the controller that a smmuv3 IOMMU device is attached to, and differentiates the device-pluggable arm-smmuv3 model from the virt-machine-associated smmuv3 model. Signed-off-by: Nathan Chen <nathanc@nvidia.com> --- docs/formatdomain.rst | 4 +++ src/conf/domain_conf.c | 16 +++++++++ src/conf/domain_conf.h | 1 + src/conf/domain_validate.c | 28 +++++++++++++-- src/conf/schemas/domaincommon.rng | 5 +++ src/qemu/qemu_command.c | 58 +++++++++++++++++++++++++++++-- 6 files changed, 106 insertions(+), 6 deletions(-) diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index c6d0b183d0..160e7ad9c7 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -9238,6 +9238,10 @@ Example: Enable x2APIC mode. Useful for higher number of guest CPUs. :since:`Since 11.5.0` (QEMU/KVM and ``amd`` model only) + ``pciBus`` + The ``pciBus`` attribute notes the index of the controller that an + IOMMU device is attached to. (QEMU/KVM and ``smmuv3`` model only) + The ``virtio`` IOMMU devices can further have ``address`` element as described in `Device addresses`_ (address has to by type of ``pci``). diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 0c7a32b03c..af9c4cc39b 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -2815,6 +2815,8 @@ virDomainIOMMUDefNew(void) iommu = g_new0(virDomainIOMMUDef, 1); + iommu->pci_bus = -1; + return g_steal_pointer(&iommu); } @@ -14497,6 +14499,10 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt, if (virXMLPropTristateSwitch(driver, "passthrough", VIR_XML_PROP_NONE, &iommu->pt) < 0) return NULL; + + if (virXMLPropInt(driver, "pciBus", 10, VIR_XML_PROP_NONE, + &iommu->pci_bus, -1) < 0) + return NULL; } if (virDomainDeviceInfoParseXML(xmlopt, node, ctxt, @@ -22223,6 +22229,12 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDef *src, dst->aw_bits, src->aw_bits); return false; } + if (src->pci_bus != dst->pci_bus) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device pci_bus value '%1$d' does not match source '%2$d'"), + dst->pci_bus, src->pci_bus); + return false; + } if (src->dma_translation != dst->dma_translation) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("Target domain IOMMU device dma translation '%1$s' does not match source '%2$s'"), @@ -28554,6 +28566,10 @@ virDomainIOMMUDefFormat(virBuffer *buf, virBufferAsprintf(&driverAttrBuf, " xtsup='%s'", virTristateSwitchTypeToString(iommu->xtsup)); } + if (iommu->pci_bus >= 0) { + virBufferAsprintf(&driverAttrBuf, " pciBus='%d'", + iommu->pci_bus); + } virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, NULL); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 6aa716b907..4c78589c9e 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -3055,6 +3055,7 @@ struct _virDomainIOMMUDef { virTristateSwitch eim; virTristateSwitch iotlb; unsigned int aw_bits; + int pci_bus; virDomainDeviceInfo info; virTristateSwitch dma_translation; virTristateSwitch xtsup; diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index 3339fc83e0..8bbea5f000 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -1853,6 +1853,11 @@ virDomainDefIOMMUValidate(const virDomainDef *def) _("IOMMU model smmuv3 must be specified for multiple IOMMU definitions")); } + if (def->niommus > 1 && iommu->pci_bus < 0) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("device-pluggable IOMMU with pciBus attribute must be specified for multiple IOMMU definitions")); + } + if (iommu->intremap == VIR_TRISTATE_SWITCH_ON && def->features[VIR_DOMAIN_FEATURE_IOAPIC] != VIR_DOMAIN_IOAPIC_QEMU) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", @@ -3107,13 +3112,28 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) { switch (iommu->model) { case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: + if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT || + iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT || + iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || + iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT || + iommu->aw_bits != 0 || + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pt != VIR_TRISTATE_SWITCH_ABSENT) { + virReportError(VIR_ERR_XML_ERROR, + _("iommu model '%1$s' doesn't support some additional attributes"), + virDomainIOMMUModelTypeToString(iommu->model)); + return -1; + } + break; case VIR_DOMAIN_IOMMU_MODEL_VIRTIO: if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT || iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT || iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT || iommu->aw_bits != 0 || - iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pci_bus >= 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3125,7 +3145,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) if (iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT || iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || iommu->aw_bits != 0 || - iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pci_bus >= 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3135,7 +3156,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) case VIR_DOMAIN_IOMMU_MODEL_INTEL: if (iommu->pt != VIR_TRISTATE_SWITCH_ABSENT || - iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pci_bus >= 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index ae3fa95904..9afc679258 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -6322,6 +6322,11 @@ <ref name="virOnOff"/> </attribute> </optional> + <optional> + <attribute name="pciBus"> + <data type="unsignedInt"/> + </attribute> + </optional> </element> </optional> <optional> diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 9c8330645b..16521bb4e5 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -6239,6 +6239,51 @@ qemuBuildBootCommandLine(virCommand *cmd, } +static virJSONValue * +qemuBuildPCINestedSmmuv3DevProps(const virDomainDef *def, + const virDomainIOMMUDef *iommu, + size_t id) +{ + g_autoptr(virJSONValue) props = NULL; + g_autofree char *bus = NULL; + g_autofree char *smmuv3_id = NULL; + size_t i; + + for (i = 0; i < def->ncontrollers; i++) { + virDomainControllerDef *cont = def->controllers[i]; + if (cont->idx == iommu->pci_bus) { + if (cont->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI) { + const char *alias = cont->info.alias; + + if (!alias) { + return NULL; + } else { + bus = g_strdup(alias); + } + break; + } + } + } + + if (!bus) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Could not find a suitable controller for smmuv3.")); + return NULL; + } + + smmuv3_id = g_strdup_printf("smmuv3.%zu", id); + + if (virJSONValueObjectAdd(&props, + "s:driver", "arm-smmuv3", + "s:primary-bus", bus, + "s:id", smmuv3_id, + NULL) < 0) + return NULL; + + return g_steal_pointer(&props); +} + + static int qemuBuildIOMMUCommandLine(virCommand *cmd, const virDomainDef *def, @@ -6268,6 +6313,7 @@ qemuBuildIOMMUCommandLine(virCommand *cmd, return -1; break; + case VIR_DOMAIN_IOMMU_MODEL_VIRTIO: if (virJSONValueObjectAdd(&props, "s:driver", "virtio-iommu", @@ -6283,9 +6329,6 @@ qemuBuildIOMMUCommandLine(virCommand *cmd, return -1; break; - case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: - /* There is no -device for SMMUv3, so nothing to be done here */ - break; case VIR_DOMAIN_IOMMU_MODEL_AMD: if (virJSONValueObjectAdd(&wrapperProps, @@ -6315,6 +6358,15 @@ qemuBuildIOMMUCommandLine(virCommand *cmd, break; + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: + if (iommu->pci_bus >= 0) { + if (!(props = qemuBuildPCINestedSmmuv3DevProps(def, iommu, i))) + return -1; + if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0) + return -1; + } + break; + case VIR_DOMAIN_IOMMU_MODEL_LAST: default: virReportEnumRangeError(virDomainIOMMUModel, iommu->model); -- 2.43.0