From: Nathan Chen <nathanc@nvidia.com> Support the vfio-pci ATS attribute that can be used to control exposing or masking a vfio-pci device's ATS extended capability in the PCIe config space within the guest. Place the ATS attribute under the hostdev <driver> and reject it for network and port XML definitions as it only applies to individual vfio-pci devices per VM. Signed-off-by: Nathan Chen <nathanc@nvidia.com> --- docs/formatdomain.rst | 8 +++ src/conf/device_conf.c | 8 +++ src/conf/device_conf.h | 1 + src/conf/network_conf.c | 6 +++ src/conf/schemas/basictypes.rng | 5 ++ src/conf/virnetworkportdef.c | 6 +++ src/qemu/qemu_command.c | 1 + src/qemu/qemu_validate.c | 7 +++ .../hostdev-vfio-ats.x86_64-latest.args | 36 +++++++++++++ .../hostdev-vfio-ats.x86_64-latest.xml | 52 +++++++++++++++++++ tests/qemuxmlconfdata/hostdev-vfio-ats.xml | 41 +++++++++++++++ tests/qemuxmlconftest.c | 1 + 12 files changed, 172 insertions(+) create mode 100644 tests/qemuxmlconfdata/hostdev-vfio-ats.x86_64-latest.args create mode 100644 tests/qemuxmlconfdata/hostdev-vfio-ats.x86_64-latest.xml create mode 100644 tests/qemuxmlconfdata/hostdev-vfio-ats.xml diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index 77cd6b504b..71fe6f8834 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -5105,6 +5105,14 @@ or: assigned devices. Libvirt will open the /dev/iommu and VFIO device cdev and pass associated file descriptors to QEMU. + :since:`Since 12.10.0 (QEMU and KVM only)`, the ``ats`` attribute + can be used to control exposing a device's ATS extended capability + in its PCIe config space. ``off`` masks the capability, ``on`` + requires ATS support and fails to start the guest if the host kernel + reports lack of ATS support for the device. If omitted, the + capability is exposed unless the host kernel reports ATS as + unsupported for the device. + (Note: :since:`Since 1.0.5`, the ``name`` attribute has been described to be used to select the type of PCI device assignment ("vfio", "kvm", or "xen"), but those values have been mostly diff --git a/src/conf/device_conf.c b/src/conf/device_conf.c index d68232a4f4..468f0c1058 100644 --- a/src/conf/device_conf.c +++ b/src/conf/device_conf.c @@ -72,6 +72,10 @@ virDeviceHostdevPCIDriverInfoParseXML(xmlNodePtr node, &driver->iommufd) < 0) return -1; + if (virXMLPropTristateSwitch(node, "ats", VIR_XML_PROP_NONE, + &driver->ats) < 0) + return -1; + driver->model = virXMLPropString(node, "model"); return 0; } @@ -104,6 +108,10 @@ virDeviceHostdevPCIDriverInfoFormat(virBuffer *buf, virBufferAddLit(&driverAttrBuf, " iommufd='no'"); } + if (driver->ats != VIR_TRISTATE_SWITCH_ABSENT) + virBufferAsprintf(&driverAttrBuf, " ats='%s'", + virTristateSwitchTypeToString(driver->ats)); + virXMLFormatElement(buf, "driver", &driverAttrBuf, NULL); return 0; } diff --git a/src/conf/device_conf.h b/src/conf/device_conf.h index 116b959143..84deeba1e5 100644 --- a/src/conf/device_conf.h +++ b/src/conf/device_conf.h @@ -48,6 +48,7 @@ struct _virDeviceHostdevPCIDriverInfo { virDeviceHostdevPCIDriverName name; char *model; virTristateBool iommufd; + virTristateSwitch ats; }; typedef enum { diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c index 8eba1321d6..670632b0f5 100644 --- a/src/conf/network_conf.c +++ b/src/conf/network_conf.c @@ -1379,6 +1379,12 @@ virNetworkForwardDefParseXML(const char *networkName, return -1; } + if (def->driver.ats != VIR_TRISTATE_SWITCH_ABSENT) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("'ats' is not supported for network <driver>")); + return -1; + } + /* bridge and hostdev modes can use a pool of physical interfaces */ if ((nForwardIfs = virXPathNodeSet("./interface", ctxt, &forwardIfNodes)) < 0) return -1; diff --git a/src/conf/schemas/basictypes.rng b/src/conf/schemas/basictypes.rng index a2ecfd3514..fc5cbebee8 100644 --- a/src/conf/schemas/basictypes.rng +++ b/src/conf/schemas/basictypes.rng @@ -709,6 +709,11 @@ <ref name="virYesNo"/> </attribute> </optional> + <optional> + <attribute name="ats"> + <ref name="virOnOff"/> + </attribute> + </optional> <empty/> </element> </define> diff --git a/src/conf/virnetworkportdef.c b/src/conf/virnetworkportdef.c index ad6150d5a7..bbf6887409 100644 --- a/src/conf/virnetworkportdef.c +++ b/src/conf/virnetworkportdef.c @@ -231,6 +231,12 @@ virNetworkPortDefParseXML(xmlXPathContextPtr ctxt) return NULL; } + if (def->plug.hostdevpci.driver.ats != VIR_TRISTATE_SWITCH_ABSENT) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("'ats' is not supported for network port <driver>")); + return NULL; + } + if (!(addressNode = virXPathNode("./plug/address", ctxt))) { virReportError(VIR_ERR_XML_ERROR, "%s", _("Missing network port PCI address")); diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index a27425d8da..f3d6c0f92a 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -4809,6 +4809,7 @@ qemuBuildPCIHostdevDevProps(const virDomainDef *def, "B:ramfb", ramfb, "S:iommufd", iommufdId, "S:fd", fdstr, + "S:ats", qemuOnOffAuto(pcisrc->driver.ats), NULL) < 0) return NULL; diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c index 04f33df7eb..ce97b80a15 100644 --- a/src/qemu/qemu_validate.c +++ b/src/qemu/qemu_validate.c @@ -2836,6 +2836,13 @@ qemuValidateDomainDeviceDefHostdev(const virDomainHostdevDef *hostdev, return -1; } } + + if (hostdev->source.subsys.u.pci.driver.ats != VIR_TRISTATE_SWITCH_ABSENT && + !virQEMUCapsGet(qemuCaps, QEMU_CAPS_VFIO_PCI_ATS)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("ATS control for VFIO PCI devices is not supported by this version of qemu")); + return -1; + } } if (hostdev->writeFiltering != VIR_TRISTATE_BOOL_ABSENT) { diff --git a/tests/qemuxmlconfdata/hostdev-vfio-ats.x86_64-latest.args b/tests/qemuxmlconfdata/hostdev-vfio-ats.x86_64-latest.args new file mode 100644 index 0000000000..3e6bede91c --- /dev/null +++ b/tests/qemuxmlconfdata/hostdev-vfio-ats.x86_64-latest.args @@ -0,0 +1,36 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest2 \ +USER=test \ +LOGNAME=test \ +XDG_DATA_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest2/.local/share \ +XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest2/.cache \ +XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest2/.config \ +/usr/bin/qemu-system-x86_64 \ +-name guest=QEMUGuest2,debug-threads=on \ +-S \ +-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain--1-QEMUGuest2/master-key.aes"}' \ +-machine pc,usb=off,dump-guest-core=off,memory-backend=pc.ram,acpi=off \ +-accel tcg \ +-cpu qemu64 \ +-m size=219136k \ +-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":224395264}' \ +-overcommit mem-lock=off \ +-smp 1,sockets=1,cores=1,threads=1 \ +-uuid c7a5fdbd-edaf-9466-926a-d65c16db1809 \ +-display none \ +-no-user-config \ +-nodefaults \ +-chardev socket,id=charmonitor,fd=@mon-fd@,server=on,wait=off \ +-object '{"qom-type":"monitor-qmp","id":"monitor","chardev":"charmonitor"}' \ +-rtc base=utc \ +-no-shutdown \ +-boot strict=on \ +-device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \ +-audiodev '{"id":"audio1","driver":"none"}' \ +-device '{"driver":"vfio-pci","host":"0000:06:12.1","id":"hostdev0","ats":"on","bus":"pci.0","addr":"0x2"}' \ +-device '{"driver":"vfio-pci","host":"0000:06:12.2","id":"hostdev1","ats":"off","bus":"pci.0","addr":"0x3"}' \ +-device '{"driver":"vfio-pci","host":"0000:06:12.3","id":"hostdev2","bus":"pci.0","addr":"0x4"}' \ +-device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.0","addr":"0x5"}' \ +-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ +-msg timestamp=on diff --git a/tests/qemuxmlconfdata/hostdev-vfio-ats.x86_64-latest.xml b/tests/qemuxmlconfdata/hostdev-vfio-ats.x86_64-latest.xml new file mode 100644 index 0000000000..d24ac4fcc8 --- /dev/null +++ b/tests/qemuxmlconfdata/hostdev-vfio-ats.x86_64-latest.xml @@ -0,0 +1,52 @@ +<domain type='qemu'> + <name>QEMUGuest2</name> + <uuid>c7a5fdbd-edaf-9466-926a-d65c16db1809</uuid> + <memory unit='KiB'>219100</memory> + <currentMemory unit='KiB'>219100</currentMemory> + <vcpu placement='static'>1</vcpu> + <os> + <type arch='x86_64' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <cpu mode='custom' match='exact' check='none'> + <model fallback='forbid'>qemu64</model> + </cpu> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-x86_64</emulator> + <controller type='usb' index='0' model='piix3-uhci'> + <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/> + </controller> + <controller type='pci' index='0' model='pci-root'/> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <audio id='1' type='none'/> + <hostdev mode='subsystem' type='pci' managed='yes'> + <driver name='vfio' ats='on'/> + <source> + <address domain='0x0000' bus='0x06' slot='0x12' function='0x1'/> + </source> + <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/> + </hostdev> + <hostdev mode='subsystem' type='pci' managed='yes'> + <driver name='vfio' ats='off'/> + <source> + <address domain='0x0000' bus='0x06' slot='0x12' function='0x2'/> + </source> + <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> + </hostdev> + <hostdev mode='subsystem' type='pci' managed='yes'> + <driver name='vfio'/> + <source> + <address domain='0x0000' bus='0x06' slot='0x12' function='0x3'/> + </source> + <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> + </hostdev> + <memballoon model='virtio'> + <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/> + </memballoon> + </devices> +</domain> diff --git a/tests/qemuxmlconfdata/hostdev-vfio-ats.xml b/tests/qemuxmlconfdata/hostdev-vfio-ats.xml new file mode 100644 index 0000000000..ec87b4df6f --- /dev/null +++ b/tests/qemuxmlconfdata/hostdev-vfio-ats.xml @@ -0,0 +1,41 @@ +<domain type='qemu'> + <name>QEMUGuest2</name> + <uuid>c7a5fdbd-edaf-9466-926a-d65c16db1809</uuid> + <memory unit='KiB'>219100</memory> + <currentMemory unit='KiB'>219100</currentMemory> + <vcpu placement='static'>1</vcpu> + <os> + <type arch='x86_64' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-x86_64</emulator> + <controller type='usb' index='0'/> + <controller type='pci' index='0' model='pci-root'/> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <hostdev mode='subsystem' type='pci' managed='yes'> + <driver name='vfio' ats='on'/> + <source> + <address domain='0x0000' bus='0x06' slot='0x12' function='0x1'/> + </source> + </hostdev> + <hostdev mode='subsystem' type='pci' managed='yes'> + <driver name='vfio' ats='off'/> + <source> + <address domain='0x0000' bus='0x06' slot='0x12' function='0x2'/> + </source> + </hostdev> + <hostdev mode='subsystem' type='pci' managed='yes'> + <driver name='vfio'/> + <source> + <address domain='0x0000' bus='0x06' slot='0x12' function='0x3'/> + </source> + </hostdev> + <memballoon model='virtio'/> + </devices> +</domain> diff --git a/tests/qemuxmlconftest.c b/tests/qemuxmlconftest.c index ca64f89d4c..9ea5c1e389 100644 --- a/tests/qemuxmlconftest.c +++ b/tests/qemuxmlconftest.c @@ -2275,6 +2275,7 @@ mymain(void) DO_TEST_CAPS_LATEST("hostdev-pci-display-ramfb"); DO_TEST_CAPS_LATEST("hostdev-vfio"); DO_TEST_CAPS_LATEST("hostdev-vfio-multidomain"); + DO_TEST_CAPS_LATEST("hostdev-vfio-ats"); DO_TEST_CAPS_LATEST("hostdev-mdev-precreated"); DO_TEST_CAPS_LATEST_PARSE_ERROR("hostdev-mdev-src-address-invalid"); DO_TEST_CAPS_LATEST_PARSE_ERROR("hostdev-mdev-invalid-target-address"); -- 2.43.0