From: Michal Privoznik <mprivozn@redhat.com> In PCI assignment scenario the virtio-iommu needs to know the guest page size also known as granule. Expose it as an attribute to the <driver/> element of a virtio-iommu. This is possibly interesting only for aarch64 since it supports virtio-iommu and also supports running guests with different page size than the host. Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- docs/formatdomain.rst | 38 ++++++++- src/conf/domain_conf.c | 77 ++++++++++++++++++- src/conf/domain_conf.h | 1 + src/conf/domain_validate.c | 9 ++- src/conf/schemas/domaincommon.rng | 22 ++++++ src/qemu/qemu_validate.c | 11 +++ .../virtio-iommu-aarch64.aarch64-latest.xml | 4 +- .../qemuxmlconfdata/virtio-iommu-aarch64.xml | 4 +- .../virtio-iommu-x86_64.x86_64-latest.xml | 3 + tests/qemuxmlconfdata/virtio-iommu-x86_64.xml | 6 +- 10 files changed, 166 insertions(+), 9 deletions(-) diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index 4b34a8a963..50563741a0 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -9194,7 +9194,7 @@ IOMMU devices The ``iommu`` element can be used to add an IOMMU device. :since:`Since 2.1.0` -Example: +Examples: :: @@ -9206,6 +9206,17 @@ Example: </devices> ... + + ... + <devices> + <iommu model='virtio'> + <driver aw_bits='48'> + <granule size='64' unit='KiB'/> + </driver> + </iommu> + </devices> + ... + ``model`` Supported values are ``intel`` (for Q35 guests) ``smmuv3`` (:since:`since 5.5.0`, for ARM virt guests), ``virtio`` @@ -9264,6 +9275,31 @@ Example: The ``pciBus`` attribute notes the index of the controller that an IOMMU device is attached to. (QEMU/KVM and ``smmuv3`` model only) +In case of ``virtio`` IOMMU device, the ``driver`` element can optionally +contain ``granule`` subelement that allows to choose which granule will be +used by default. It is useful when running guests with different page size +than the host. :since:`Since 12.1.0` (QEMU/KVM and ``virtio`` model only). +There are two possible options: + +:: + + <iommu model='virtio'> + <driver> + <granule mode='host'/> + </driver> + </iommu> + + <iommu model='virtio'> + <driver> + <granule size='64' unit='KiB'/> + </driver> + </iommu> + +The ``mode='host'`` case matches the host page size, the other sets desired +granule size. Please note that hypervisor might support only some selected +values. For instance, QEMU supports only 4KiB, 8KiB, 16KiB and 64KiB large +granules. + The ``virtio`` IOMMU devices can further have ``address`` element as described in `Device addresses`_ (address has to by type of ``pci``). diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 1b2a439ca4..4503fc0095 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -1592,6 +1592,21 @@ VIR_ENUM_IMPL(virDomainChrSourceMode, ); +/*virDomainIOMMUGranuleModeTypeToString: + * @val: value to format + * + * Reuturns: an allocated string. Caller must free it. + */ +static char * +virDomainIOMMUGranuleModeTypeToString(int val) +{ + if (val == -1) + return g_strdup("host"); + + return g_strdup_printf("%dKiB", val); +} + + static virClass *virDomainObjClass; static virClass *virDomainXMLOptionClass; static void virDomainObjDispose(void *obj); @@ -14487,6 +14502,8 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt, return NULL; if ((driver = virXPathNode("./driver", ctxt))) { + xmlNodePtr granule; + if (virXMLPropTristateSwitch(driver, "intremap", VIR_XML_PROP_NONE, &iommu->intremap) < 0) return NULL; @@ -14522,6 +14539,41 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt, if (virXMLPropInt(driver, "pciBus", 10, VIR_XML_PROP_NONE, &iommu->pci_bus, -1) < 0) return NULL; + + if ((granule = virXPathNode("./driver/granule", ctxt))) { + g_autofree char *mode = virXMLPropString(granule, "mode"); + unsigned long long size; + int rc; + + rc = virDomainParseMemory("./driver/granule/@size", + "./driver/granule/@unit", + ctxt, &size, false, false); + if (rc < 0) { + return NULL; + } else if (rc > 0) { + if (mode) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("'mode' and 'size' can't be specified at the same time for 'granule'")); + return NULL; + } + + if (size != (int) size) { + virReportError(VIR_ERR_OVERFLOW, "%s", _("size value too large")); + return NULL; + } + + iommu->granule = (int)size; + } else { + if (STREQ_NULLABLE(mode, "host")) { + iommu->granule = -1; + } else if (mode) { + virReportError(VIR_ERR_XML_ERROR, + _("Invalid value for attribute '%1$s' in element '%2$s': '%3$s'."), + "mode", "granule", mode); + return NULL; + } + } + } } if (virDomainDeviceInfoParseXML(xmlopt, node, ctxt, @@ -16585,7 +16637,8 @@ virDomainIOMMUDefEquals(const virDomainIOMMUDef *a, a->eim != b->eim || a->iotlb != b->iotlb || a->aw_bits != b->aw_bits || - a->dma_translation != b->dma_translation) + a->dma_translation != b->dma_translation || + a->granule != b->granule) return false; if (a->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE && @@ -22340,6 +22393,16 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDef *src, virTristateSwitchTypeToString(src->xtsup)); return false; } + if (src->granule != dst->granule) { + g_autofree char *src_granule = virDomainIOMMUGranuleModeTypeToString(src->granule); + g_autofree char *dst_granule = virDomainIOMMUGranuleModeTypeToString(dst->granule); + + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target domain IOMMU device granule '%1$s' does not match source '%2$s'"), + dst_granule, + src_granule); + return false; + } return virDomainDeviceInfoCheckABIStability(&src->info, &dst->info); } @@ -28628,6 +28691,7 @@ virDomainIOMMUDefFormat(virBuffer *buf, g_auto(virBuffer) childBuf = VIR_BUFFER_INIT_CHILD(buf); g_auto(virBuffer) attrBuf = VIR_BUFFER_INITIALIZER; g_auto(virBuffer) driverAttrBuf = VIR_BUFFER_INITIALIZER; + g_auto(virBuffer) driverChildBuf = VIR_BUFFER_INIT_CHILD(&childBuf); if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT) { virBufferAsprintf(&driverAttrBuf, " intremap='%s'", @@ -28665,8 +28729,17 @@ virDomainIOMMUDefFormat(virBuffer *buf, virBufferAsprintf(&driverAttrBuf, " pciBus='%d'", iommu->pci_bus); } + if (iommu->granule != 0) { + if (iommu->granule == -1) { + virBufferAddLit(&driverChildBuf, "<granule mode='host'/>\n"); + } else { + virBufferAsprintf(&driverChildBuf, + "<granule size='%d' unit='KiB'/>\n", + iommu->granule); + } + } - virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, NULL); + virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, &driverChildBuf); virDomainDeviceInfoFormat(&childBuf, &iommu->info, 0); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 83d49969d3..72badb32a6 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -3062,6 +3062,7 @@ struct _virDomainIOMMUDef { virTristateSwitch dma_translation; virTristateSwitch xtsup; virTristateSwitch pt; + int granule; /* -1 means 'host', 0 unset, page size in KiB otherwise */ }; typedef enum { diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index c83fff132b..1ad614935f 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -3194,7 +3194,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) iommu->aw_bits != 0 || iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT || - iommu->pt != VIR_TRISTATE_SWITCH_ABSENT) { + iommu->pt != VIR_TRISTATE_SWITCH_ABSENT || + iommu->granule != 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3229,7 +3230,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) iommu->eim != VIR_TRISTATE_SWITCH_ABSENT || iommu->aw_bits != 0 || iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT || - iommu->pci_bus >= 0) { + iommu->pci_bus >= 0 || + iommu->granule != 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); @@ -3240,7 +3242,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu) case VIR_DOMAIN_IOMMU_MODEL_INTEL: if (iommu->pt != VIR_TRISTATE_SWITCH_ABSENT || iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT || - iommu->pci_bus >= 0) { + iommu->pci_bus >= 0 || + iommu->granule != 0) { virReportError(VIR_ERR_XML_ERROR, _("iommu model '%1$s' doesn't support some additional attributes"), virDomainIOMMUModelTypeToString(iommu->model)); diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index 114dd3f96f..3d5eb7d4f3 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -6329,6 +6329,28 @@ <data type="unsignedInt"/> </attribute> </optional> + <optional> + <element name="granule"> + <choice> + <group> + <attribute name="mode"> + <value>host</value> + </attribute> + </group> + <group> + <attribute name="size"> + <ref name="unsignedInt"/> + </attribute> + <optional> + <attribute name="unit"> + <ref name="unit"/> + </attribute> + </optional> + </group> + </choice> + <empty/> + </element> + </optional> </element> </optional> <optional> diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c index ab8a1938c1..8f85334cf9 100644 --- a/src/qemu/qemu_validate.c +++ b/src/qemu/qemu_validate.c @@ -5685,6 +5685,17 @@ qemuValidateDomainDeviceDefIOMMU(const virDomainIOMMUDef *iommu, return -1; } + /* QEMU supports only 4KiB, 8KiB, 16KiB and 64KiB granule size */ + if (iommu->granule > 0 && + !(iommu->granule == 4 || + iommu->granule == 8 || + iommu->granule == 16 || + iommu->granule == 64)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("iommu: unsupported granule size. Supported values are 4, 8, 16 and 64 KiB")); + return -1; + } + return 0; } diff --git a/tests/qemuxmlconfdata/virtio-iommu-aarch64.aarch64-latest.xml b/tests/qemuxmlconfdata/virtio-iommu-aarch64.aarch64-latest.xml index 4ae628ab5a..ad2fedaab7 100644 --- a/tests/qemuxmlconfdata/virtio-iommu-aarch64.aarch64-latest.xml +++ b/tests/qemuxmlconfdata/virtio-iommu-aarch64.aarch64-latest.xml @@ -29,7 +29,9 @@ <audio id='1' type='none'/> <memballoon model='none'/> <iommu model='virtio'> - <driver aw_bits='48'/> + <driver aw_bits='48'> + <granule size='64' unit='KiB'/> + </driver> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/> </iommu> </devices> diff --git a/tests/qemuxmlconfdata/virtio-iommu-aarch64.xml b/tests/qemuxmlconfdata/virtio-iommu-aarch64.xml index 96e5ea05ae..8d5f081b92 100644 --- a/tests/qemuxmlconfdata/virtio-iommu-aarch64.xml +++ b/tests/qemuxmlconfdata/virtio-iommu-aarch64.xml @@ -14,7 +14,9 @@ <controller type='usb' model='none'/> <memballoon model='none'/> <iommu model='virtio'> - <driver aw_bits='48'/> + <driver aw_bits='48'> + <granule size='64' unit='KiB'/> + </driver> </iommu> </devices> </domain> diff --git a/tests/qemuxmlconfdata/virtio-iommu-x86_64.x86_64-latest.xml b/tests/qemuxmlconfdata/virtio-iommu-x86_64.x86_64-latest.xml index f458f9a706..56cee393d3 100644 --- a/tests/qemuxmlconfdata/virtio-iommu-x86_64.x86_64-latest.xml +++ b/tests/qemuxmlconfdata/virtio-iommu-x86_64.x86_64-latest.xml @@ -31,6 +31,9 @@ <watchdog model='itco' action='reset'/> <memballoon model='none'/> <iommu model='virtio'> + <driver> + <granule mode='host'/> + </driver> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/> </iommu> </devices> diff --git a/tests/qemuxmlconfdata/virtio-iommu-x86_64.xml b/tests/qemuxmlconfdata/virtio-iommu-x86_64.xml index 51c13d2ef6..db0af57f69 100644 --- a/tests/qemuxmlconfdata/virtio-iommu-x86_64.xml +++ b/tests/qemuxmlconfdata/virtio-iommu-x86_64.xml @@ -13,6 +13,10 @@ <emulator>/usr/bin/qemu-system-x86_64</emulator> <controller type='usb' model='none'/> <memballoon model='none'/> - <iommu model='virtio'/> + <iommu model='virtio'> + <driver> + <granule mode='host'/> + </driver> + </iommu> </devices> </domain> -- 2.52.0