
Introduce support for "smmuv3Dev" IOMMU model and its driver attributes "parentIdx", "accel", "ats", "ril", "pasid", and "oas". Signed-off-by: Nathan Chen <nathanc@nvidia.com> --- docs/formatdomain.rst | 36 ++++++++++++- src/conf/domain_conf.c | 88 +++++++++++++++++++++++++++++++ src/conf/domain_conf.h | 7 +++ src/conf/domain_validate.c | 41 ++++++++++++-- src/conf/schemas/domaincommon.rng | 31 +++++++++++ src/qemu/qemu_command.c | 83 +++++++++++++++++++++++++++-- src/qemu/qemu_domain_address.c | 2 + src/qemu/qemu_validate.c | 16 ++++++ 8 files changed, 295 insertions(+), 9 deletions(-) diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index f50dce477f..972493e62c 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -9161,8 +9161,9 @@ Example: ``model`` Supported values are ``intel`` (for Q35 guests) ``smmuv3`` (:since:`since 5.5.0`, for ARM virt guests), ``virtio`` - (:since:`since 8.3.0`, for Q35 and ARM virt guests) and - ``amd`` (:since:`since 11.5.0`). + (:since:`since 8.3.0`, for Q35 and ARM virt guests), + ``amd`` (:since:`since 11.5.0`), and ``smmuv3Dev`` (for + ARM virt guests). ``driver`` The ``driver`` subelement can be used to configure additional options, some @@ -9212,6 +9213,37 @@ Example: Enable x2APIC mode. Useful for higher number of guest CPUs. :since:`Since 11.5.0` (QEMU/KVM and ``amd`` model only) + ``parentIdx`` + The ``parentIdx`` attribute notes the index of the controller that an + IOMMU device is attached to. (QEMU/KVM and ``smmuv3Dev`` model only) + + ``accel`` + The ``accel`` attribute with possible values ``on`` and ``off`` can be used + to enable hardware acceleration support for smmuv3Dev IOMMU devices. + (QEMU/KVM and ``smmuv3Dev`` model only) + + ``ats`` + The ``ats`` attribute with possible values ``on`` and ``off`` can be used + to enable reporting Address Translation Services capability to the guest + for smmuv3Dev IOMMU devices with ``accel`` set to ``on``, if the host + SMMUv3 supports ATS and the associated passthrough device supports ATS. + (QEMU/KVM and ``smmuv3Dev`` model only) + + ``ril`` + The ``ril`` attribute with possible values ``on`` and ``off`` can be used + to report whether Range Invalidation for smmuv3Dev IOMMU devices with + ``accel`` set to ``on`` is compatible with host SMMUv3 support. + (QEMU/KVM and ``smmuv3Dev`` model only) + + ``pasid`` + The ``pasid`` attribute with possible values ``on`` and ``off`` can be + used to enable Process Address Space ID support for smmuv3Dev IOMMU devices + with ``accel`` set to ``on``. (QEMU/KVM and ``smmuv3Dev`` model only) + + ``oas`` + The ``oas`` attribute sets the output address size in units of bits. + (QEMU/KVM and ``smmuv3Dev`` model only) + The ``virtio`` IOMMU devices can further have ``address`` element as described in `Device addresses`_ (address has to by type of ``pci``). diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 281846dfbe..b9a1c29b73 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -1353,6 +1353,7 @@ VIR_ENUM_IMPL(virDomainIOMMUModel, "smmuv3", "virtio", "amd", + "smmuv3Dev", ); VIR_ENUM_IMPL(virDomainVsockModel, @@ -2813,6 +2814,8 @@ virDomainIOMMUDefNew(void) iommu = g_new0(virDomainIOMMUDef, 1); + iommu->parent_idx = -1; + return g_steal_pointer(&iommu); } @@ -14407,6 +14410,7 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt, VIR_XML_PROP_REQUIRED, &iommu->model) < 0) return NULL; + if ((driver = virXPathNode("./driver", ctxt))) { if (virXMLPropTristateSwitch(driver, "intremap", VIR_XML_PROP_NONE, &iommu->intremap) < 0) @@ -14439,6 +14443,30 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt, if (virXMLPropTristateSwitch(driver, "passthrough", VIR_XML_PROP_NONE, &iommu->pt) < 0) return NULL; + + if (virXMLPropInt(driver, "parentIdx", 10, VIR_XML_PROP_NONE, + &iommu->parent_idx, -1) < 0) + return NULL; + + if (virXMLPropTristateSwitch(driver, "accel", VIR_XML_PROP_NONE, + &iommu->accel) < 0) + return NULL; + + if (virXMLPropTristateSwitch(driver, "ats", VIR_XML_PROP_NONE, + &iommu->ats) < 0) + return NULL; + + if (virXMLPropTristateSwitch(driver, "ril", VIR_XML_PROP_NONE, + &iommu->ril) < 0) + return NULL; + + if (virXMLPropTristateSwitch(driver, "pasid", VIR_XML_PROP_NONE, + &iommu->pasid) < 0) + return NULL; + + if (virXMLPropUInt(driver, "oas", 10, VIR_XML_PROP_NONE, + &iommu->oas) < 0) + return NULL; } if (virDomainDeviceInfoParseXML(xmlopt, node, ctxt, @@ -22113,6 +22141,42 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDef *src, virTristateSwitchTypeToString(src->xtsup)); return false; } + if (src->parent_idx != dst->parent_idx) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device parent_idx value '%1$d' does not match source '%2$d'"), + dst->parent_idx, src->parent_idx); + return false; + } + if (src->accel != dst->accel) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device accel value '%1$d' does not match source '%2$d'"), + dst->accel, src->accel); + return false; + } + if (src->ats != dst->ats) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device ATS value '%1$d' does not match source '%2$d'"), + dst->ats, src->ats); + return false; + } + if (src->ril != dst->ril) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device ril value '%1$d' does not match source '%2$d'"), + dst->ril, src->ril); + return false; + } + if (src->pasid != dst->pasid) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device pasid value '%1$d' does not match source '%2$d'"), + dst->pasid, src->pasid); + return false; + } + if (src->oas != dst->oas) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device oas value '%1$d' does not match source '%2$d'"), + dst->oas, src->oas); + return false; + } return virDomainDeviceInfoCheckABIStability(&src->info, &dst->info); } @@ -28409,6 +28473,30 @@ virDomainIOMMUDefFormat(virBuffer *buf, virBufferAsprintf(&driverAttrBuf, " xtsup='%s'", virTristateSwitchTypeToString(iommu->xtsup)); } + if (iommu->parent_idx >= 0) { + virBufferAsprintf(&driverAttrBuf, " parentIdx='%d'", + iommu->parent_idx); + } + if (iommu->accel != VIR_TRISTATE_SWITCH_ABSENT) { + virBufferAsprintf(&driverAttrBuf, " accel='%s'", + virTristateSwitchTypeToString(iommu->accel)); + } + if (iommu->ats != VIR_TRISTATE_SWITCH_ABSENT) { + virBufferAsprintf(&driverAttrBuf, " ats='%s'", + virTristateSwitchTypeToString(iommu->ats)); + } + if (iommu->ril != VIR_TRISTATE_SWITCH_ABSENT) { + virBufferAsprintf(&driverAttrBuf, " ril='%s'", + virTristateSwitchTypeToString(iommu->ril)); + } + if (iommu->pasid != VIR_TRISTATE_SWITCH_ABSENT) { + virBufferAsprintf(&driverAttrBuf, " pasid='%s'", + virTristateSwitchTypeToString(iommu->pasid)); + } + if (iommu->oas > 0) { + virBufferAsprintf(&driverAttrBuf, " oas='%d'", + iommu->oas); + } virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, NULL); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 39807b5fe3..1bc52e1a63 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -3039,6 +3039,7 @@ typedef enum { VIR_DOMAIN_IOMMU_MODEL_SMMUV3, VIR_DOMAIN_IOMMU_MODEL_VIRTIO, VIR_DOMAIN_IOMMU_MODEL_AMD, + VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV, VIR_DOMAIN_IOMMU_MODEL_LAST } virDomainIOMMUModel; @@ -3054,6 +3055,12 @@ struct _virDomainIOMMUDef { virTristateSwitch dma_translation; virTristateSwitch xtsup; virTristateSwitch pt; + int parent_idx; + virTristateSwitch accel; + virTristateSwitch ats; + virTristateSwitch ril; + virTristateSwitch pasid; + unsigned int oas; }; typedef enum { diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index 93a2bc9b01..117338b4da 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -3108,7 +3108,13 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT || iommu->aw_bits != 0 || - iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->parent_idx != -1 || + iommu->accel != VIR_TRISTATE_SWITCH_ABSENT || + iommu->ats != VIR_TRISTATE_SWITCH_ABSENT || + iommu->ril != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pasid != VIR_TRISTATE_SWITCH_ABSENT || + iommu->oas != 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3120,7 +3126,13 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) if (iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT || iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || iommu->aw_bits != 0 || - iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->parent_idx != -1 || + iommu->accel != VIR_TRISTATE_SWITCH_ABSENT || + iommu->ats != VIR_TRISTATE_SWITCH_ABSENT || + iommu->ril != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pasid != VIR_TRISTATE_SWITCH_ABSENT || + iommu->oas != 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3130,7 +3142,29 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) case VIR_DOMAIN_IOMMU_MODEL_INTEL: if (iommu->pt != VIR_TRISTATE_SWITCH_ABSENT || - iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT || + iommu->parent_idx != -1 || + iommu->accel != VIR_TRISTATE_SWITCH_ABSENT || + iommu->ats != VIR_TRISTATE_SWITCH_ABSENT || + iommu->ril != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pasid != VIR_TRISTATE_SWITCH_ABSENT || + iommu->oas != 0) { + virReportError(VIR_ERR_XML_ERROR, + _("iommu model '%1$s' doesn't support some additional attributes"), + virDomainIOMMUModelTypeToString(iommu->model)); + return -1; + } + break; + + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV: + if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT || + iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT || + iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || + iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT || + iommu->aw_bits != 0 || + iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || + iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT || + iommu->pt != VIR_TRISTATE_SWITCH_ABSENT) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3155,6 +3189,7 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) case VIR_DOMAIN_IOMMU_MODEL_VIRTIO: case VIR_DOMAIN_IOMMU_MODEL_AMD: + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV: case VIR_DOMAIN_IOMMU_MODEL_LAST: break; } diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index b9230a35b4..64fb320ca9 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -6266,6 +6266,7 @@ <value>smmuv3</value> <value>virtio</value> <value>amd</value> + <value>smmuv3Dev</value> </choice> </attribute> <interleave> @@ -6311,6 +6312,36 @@ <ref name="virOnOff"/> </attribute> </optional> + <optional> + <attribute name="parentIdx"> + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="accel"> + <ref name="virOnOff"/> + </attribute> + </optional> + <optional> + <attribute name="ats"> + <ref name="virOnOff"/> + </attribute> + </optional> + <optional> + <attribute name="ril"> + <ref name="virOnOff"/> + </attribute> + </optional> + <optional> + <attribute name="pasid"> + <ref name="virOnOff"/> + </attribute> + </optional> + <optional> + <attribute name="oas"> + <data type="unsignedInt"/> + </attribute> + </optional> </element> </optional> <optional> diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 031f09b7a5..6ec4801a7e 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -6294,6 +6294,73 @@ qemuBuildBootCommandLine(virCommand *cmd, } +static virJSONValue * +qemuBuildPCISmmuv3DevDevProps(const virDomainDef *def, + const virDomainIOMMUDef *iommu) +{ + g_autoptr(virJSONValue) props = NULL; + g_autofree char *bus = NULL; + size_t i; + bool contIsPHB = false; + + for (i = 0; i < def->ncontrollers; i++) { + virDomainControllerDef *cont = def->controllers[i]; + if (cont->idx == iommu->parent_idx) { + if (cont->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI) { + const char *alias = cont->info.alias; + contIsPHB = virDomainControllerIsPSeriesPHB(cont); + + if (!alias) + return NULL; + + if (virDomainDeviceAliasIsUserAlias(alias)) { + if (cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT && + iommu->parent_idx <= 0) { + if (qemuDomainSupportsPCIMultibus(def)) + bus = g_strdup("pci.0"); + else + bus = g_strdup("pci"); + } else if (cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT) { + bus = g_strdup("pcie.0"); + } + } else { + bus = g_strdup(alias); + } + break; + } + } + } + + if (!bus) + return NULL; + + if (contIsPHB && iommu->parent_idx > 0) { + char *temp_bus = g_strdup_printf("%s.0", bus); + g_free(bus); + bus = temp_bus; + } + + if (virJSONValueObjectAdd(&props, + "s:driver", "arm-smmuv3", + "S:primary-bus", bus, + "b:accel", (iommu->accel == VIR_TRISTATE_SWITCH_ON), + "b:ats", (iommu->ats == VIR_TRISTATE_SWITCH_ON), + "b:ril", (iommu->ril == VIR_TRISTATE_SWITCH_ON), + "b:pasid", (iommu->pasid == VIR_TRISTATE_SWITCH_ON), + NULL) < 0) + return NULL; + + if (iommu->oas > 0) { + if (virJSONValueObjectAdd(&props, + "U:oas", (unsigned long long)iommu->oas, + NULL) < 0) + return NULL; + } + + return g_steal_pointer(&props); +} + + static int qemuBuildIOMMUCommandLine(virCommand *cmd, const virDomainDef *def, @@ -6342,7 +6409,6 @@ qemuBuildIOMMUCommandLine(virCommand *cmd, return 0; case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: - /* There is no -device for SMMUv3, so nothing to be done here */ return 0; case VIR_DOMAIN_IOMMU_MODEL_AMD: @@ -6373,6 +6439,14 @@ qemuBuildIOMMUCommandLine(virCommand *cmd, return 0; + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV: + if (!(props = qemuBuildPCISmmuv3DevDevProps(def, iommu))) + return -1; + if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0) + return -1; + + return 0; + case VIR_DOMAIN_IOMMU_MODEL_LAST: default: virReportEnumRangeError(virDomainIOMMUModel, iommu->model); @@ -7206,6 +7280,7 @@ qemuBuildMachineCommandLine(virCommand *cmd, case VIR_DOMAIN_IOMMU_MODEL_INTEL: case VIR_DOMAIN_IOMMU_MODEL_VIRTIO: case VIR_DOMAIN_IOMMU_MODEL_AMD: + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV: /* These IOMMUs are formatted in qemuBuildIOMMUCommandLine */ break; @@ -10860,15 +10935,15 @@ qemuBuildCommandLine(virDomainObj *vm, if (qemuBuildBootCommandLine(cmd, def) < 0) return NULL; - if (qemuBuildIOMMUCommandLine(cmd, def, qemuCaps) < 0) - return NULL; - if (qemuBuildGlobalControllerCommandLine(cmd, def) < 0) return NULL; if (qemuBuildControllersCommandLine(cmd, def, qemuCaps) < 0) return NULL; + if (qemuBuildIOMMUCommandLine(cmd, def, qemuCaps) < 0) + return NULL; + if (qemuBuildMemoryDeviceCommandLine(cmd, cfg, def, priv) < 0) return NULL; diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c index 96a9ca9b14..06bf4fab32 100644 --- a/src/qemu/qemu_domain_address.c +++ b/src/qemu/qemu_domain_address.c @@ -952,6 +952,7 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDef *dev, case VIR_DOMAIN_IOMMU_MODEL_INTEL: case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV: case VIR_DOMAIN_IOMMU_MODEL_LAST: /* These are not PCI devices */ return 0; @@ -2378,6 +2379,7 @@ qemuDomainAssignDevicePCISlots(virDomainDef *def, case VIR_DOMAIN_IOMMU_MODEL_INTEL: case VIR_DOMAIN_IOMMU_MODEL_SMMUV3: + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV: case VIR_DOMAIN_IOMMU_MODEL_LAST: /* These are not PCI devices */ break; diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c index c7ecb467a3..aac004c544 100644 --- a/src/qemu/qemu_validate.c +++ b/src/qemu/qemu_validate.c @@ -5414,6 +5414,22 @@ qemuValidateDomainDeviceDefIOMMU(const virDomainIOMMUDef *iommu, } break; + case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV: + if (!qemuDomainIsARMVirt(def)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("IOMMU device: '%1$s' is only supported with ARM Virt machines"), + virDomainIOMMUModelTypeToString(iommu->model)); + return -1; + } + // TODO: Check for pluggable device SMMUv3 qemu capability + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MACHINE_VIRT_IOMMU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("IOMMU device: '%1$s' is not supported with this QEMU binary"), + virDomainIOMMUModelTypeToString(iommu->model)); + return -1; + } + break; + case VIR_DOMAIN_IOMMU_MODEL_LAST: default: virReportEnumRangeError(virDomainIOMMUModel, iommu->model); -- 2.43.0