[PATCH RFC 20/21] conf: Add possibility to configure multiple iothreads per disk

Introduce a new <iothreads> sub-element of disk's <driver> which will allow configuring multiple iothreads and also map them to specific virt-queues of virtio devices. Signed-off-by: Peter Krempa <pkrempa(a)redhat.com&gt; --- docs/formatdomain.rst | 23 +++++- src/conf/domain_conf.c | 76 +++++++++++++++++++ src/conf/domain_conf.h | 14 ++++ src/conf/domain_validate.c | 8 ++ src/conf/schemas/domaincommon.rng | 47 +++++++++--- .../iothreads-disk.x86_64-latest.args | 13 +++- tests/qemuxml2argvdata/iothreads-disk.xml | 25 +++++- .../iothreads-disk.x86_64-latest.xml | 26 ++++++- 8 files changed, 211 insertions(+), 21 deletions(-) diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index d831c1e527..4940f3d857 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -3233,7 +3233,28 @@ paravirtualized driver is specified via the ``disk`` element. assigned to the same IOThread and are numbered from 1 to the domain iothreads value. Available for a disk device ``target`` configured to use "virtio" ``bus`` and "pci" or "ccw" ``address`` types. :since:`Since 1.2.8 - (QEMU 2.1)` + (QEMU 2.1)` *Note:* ``iothread`` is mutually exclusive with ``iothreads``. + - The optional ``iothreads`` sub-element allows specifying multiple IOThreads + via the ``iothread`` sub-element with attribute ``id`` the disk will use + for I/O operations. Optionally the ``iothread`` element can have multiple + ``queue`` subelements specifying that given iothread should be used to + handle given queues. :since:`Since XXXXXX`. + Example:: + + <driver name='qemu' queues='2'> + <iothreads> + <iothread id='1'> + <queue id='1'/> + </iothread> + <iothread id='2'> + <queue id='1'/> + </iothread> + <iothread id='3'> + <queue id='2'/> + </iothread> + </iothreads> + </driver> + - The optional ``queues`` attribute specifies the number of virt queues for virtio-blk. ( :since:`Since 3.9.0` ) - The optional ``queue_size`` attribute specifies the size of each virt diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 9426d55f8d..6904a71c80 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -2333,6 +2333,17 @@ virDomainDefGetVcpusTopology(const virDomainDef *def, } +void +virDomainDiskIothreadDefFree(virDomainDiskIothreadDef *def) +{ + if (!def) + return; + + g_free(def->queues); + g_free(def); +} + + static virDomainDiskDef * virDomainDiskDefNewSource(virDomainXMLOption *xmlopt, virStorageSource **src) @@ -2381,6 +2392,7 @@ virDomainDiskDefFree(virDomainDiskDef *def) g_free(def->virtio); virDomainDeviceInfoClear(&def->info); virObjectUnref(def->privateData); + g_slist_free_full(def->iothreads, (GDestroyNotify) virDomainDiskIothreadDefFree); g_free(def); } @@ -7745,6 +7757,8 @@ static int virDomainDiskDefDriverParseXML(virDomainDiskDef *def, xmlNodePtr cur) { + xmlNodePtr iothreadsNode; + def->driverName = virXMLPropString(cur, "name"); if (virXMLPropEnum(cur, "cache", virDomainDiskCacheTypeFromString, @@ -7791,6 +7805,44 @@ virDomainDiskDefDriverParseXML(virDomainDiskDef *def, if (virXMLPropUInt(cur, "iothread", 10, VIR_XML_PROP_NONZERO, &def->iothread) < 0) return -1; + if ((iothreadsNode = virXMLNodeGetSubelement(cur, "iothreads"))) { + g_autoslist(virDomainDiskIothreadDef) ioth = NULL; + g_autoptr(GPtrArray) iothreadNodes = NULL; + + if ((iothreadNodes = virXMLNodeGetSubelementList(iothreadsNode, "iothread"))) { + size_t i; + + for (i = 0; i < iothreadNodes->len; i++) { + xmlNodePtr iothNode = g_ptr_array_index(iothreadNodes, i); + g_autoptr(virDomainDiskIothreadDef) iothdef = g_new0(virDomainDiskIothreadDef, 1); + g_autoptr(GPtrArray) queueNodes = NULL; + + if (virXMLPropUInt(iothNode, "id", 10, VIR_XML_PROP_REQUIRED, + &iothdef->id) < 0) + return -1; + + if ((queueNodes = virXMLNodeGetSubelementList(iothNode, "queue"))) { + size_t q; + + iothdef->queues = g_new0(unsigned int, queueNodes->len); + iothdef->nqueues = queueNodes->len; + + for (q = 0; q < queueNodes->len; q++) { + xmlNodePtr queueNode = g_ptr_array_index(queueNodes, q); + + if (virXMLPropUInt(queueNode, "id", 10, VIR_XML_PROP_REQUIRED, + &(iothdef->queues[q])) < 0) + return -1; + } + } + + ioth = g_slist_prepend(ioth, g_steal_pointer(&iothdef)); + } + + def->iothreads = g_slist_reverse(g_steal_pointer(&ioth)); + } + } + if (virXMLPropEnum(cur, "detect_zeroes", virDomainDiskDetectZeroesTypeFromString, VIR_XML_PROP_NONZERO, &def->detect_zeroes) < 0) @@ -22513,6 +22565,30 @@ virDomainDiskDefFormatDriver(virBuffer *buf, virXMLFormatElement(&childBuf, "metadata_cache", NULL, &metadataCacheChildBuf); } + if (disk->iothreads) { + g_auto(virBuffer) iothreadsChildBuf = VIR_BUFFER_INIT_CHILD(&childBuf); + GSList *n; + + for (n = disk->iothreads; n; n = n->next) { + virDomainDiskIothreadDef *iothDef = n->data; + g_auto(virBuffer) iothreadAttrBuf = VIR_BUFFER_INITIALIZER; + g_auto(virBuffer) iothreadChildBuf = VIR_BUFFER_INIT_CHILD(&iothreadsChildBuf); + + virBufferAsprintf(&iothreadAttrBuf, " id='%u'", iothDef->id); + + if (iothDef->queues) { + size_t q; + + for (q = 0; q < iothDef->nqueues; q++) + virBufferAsprintf(&iothreadChildBuf, "<queue id='%u'/>\n", iothDef->queues[q]); + } + + virXMLFormatElement(&iothreadsChildBuf, "iothread", &iothreadAttrBuf, &iothreadChildBuf); + } + + virXMLFormatElement(&childBuf, "iothreads", NULL, &iothreadsChildBuf); + } + virXMLFormatElement(buf, "driver", &attrBuf, &childBuf); } diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index f6dade62fc..708993174a 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -552,6 +552,19 @@ typedef enum { VIR_ENUM_DECL(virDomainSnapshotLocation); +struct _virDomainDiskIothreadDef { + unsigned int id; + + /* optional list of virtqueues the iothread should handle */ + unsigned int *queues; + size_t nqueues; +}; + +typedef struct _virDomainDiskIothreadDef virDomainDiskIothreadDef; +void virDomainDiskIothreadDefFree(virDomainDiskIothreadDef *def); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(virDomainDiskIothreadDef, virDomainDiskIothreadDefFree); + + /* Stores the virtual disk configuration */ struct _virDomainDiskDef { virStorageSource *src; /* non-NULL. XXX Allow NULL for empty cdrom? */ @@ -605,6 +618,7 @@ struct _virDomainDiskDef { virDomainDeviceSGIO sgio; virDomainDiskDiscard discard; unsigned int iothread; /* unused = 0, > 0 specific thread # */ + GSList *iothreads; /* List of virDomainDiskIothreadsDef */ virDomainDiskDetectZeroes detect_zeroes; char *domain_name; /* backend domain name */ unsigned int queues; diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index 5559a71e14..2aba645279 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -914,6 +914,14 @@ virDomainDiskDefValidate(const virDomainDef *def, return -1; } + /* configuring both <driver iothread='n'> and it's <iothreads> sub-element + * isn't supported */ + if (disk->iothread && disk->iothreads) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("disk driver 'iothread' attribute can't be used together with 'iothreads' subelement")); + return -1; + } + return 0; } diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index 5a1d79672f..3e54091da2 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -2467,9 +2467,26 @@ </optional> </element> </define> - <!-- - Disk may use a special driver for access. - --> + + <define name="diskDriverIothreads"> + <element name="iothreads"> + <oneOrMore> + <element name="iothread"> + <attribute name="id"> + <ref name="unsignedInt"/> + </attribute> + <zeroOrMore> + <element name="queue"> + <attribute name="id"> + <ref name="unsignedInt"/> + </attribute> + </element> + </zeroOrMore> + </element> + </oneOrMore> + </element> + </define> + <define name="diskDriver"> <element name="driver"> <optional> @@ -2516,17 +2533,23 @@ </attribute> </optional> <ref name="virtioOptions"/> - <optional> - <element name="metadata_cache"> - <optional> - <element name="max_size"> - <ref name="scaledInteger"/> - </element> - </optional> - </element> - </optional> + <interleave> + <optional> + <element name="metadata_cache"> + <optional> + <element name="max_size"> + <ref name="scaledInteger"/> + </element> + </optional> + </element> + </optional> + <optional> + <ref name="diskDriverIothreads"/> + </optional> + </interleave> </element> </define> + <define name="driverFormat"> <optional> <attribute name="name"> diff --git a/tests/qemuxml2argvdata/iothreads-disk.x86_64-latest.args b/tests/qemuxml2argvdata/iothreads-disk.x86_64-latest.args index 02fb74d945..d1953327a7 100644 --- a/tests/qemuxml2argvdata/iothreads-disk.x86_64-latest.args +++ b/tests/qemuxml2argvdata/iothreads-disk.x86_64-latest.args @@ -19,6 +19,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \ -smp 2,sockets=2,cores=1,threads=1 \ -object '{"qom-type":"iothread","id":"iothread1"}' \ -object '{"qom-type":"iothread","id":"iothread2"}' \ +-object '{"qom-type":"iothread","id":"iothread3"}' \ +-object '{"qom-type":"iothread","id":"iothread4"}' \ -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \ -display none \ -no-user-config \ @@ -30,12 +32,15 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \ -no-acpi \ -boot strict=on \ -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \ --blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest1.img","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \ +-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest1.img","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"}' \ +-blockdev '{"node-name":"libvirt-3-format","read-only":false,"driver":"raw","file":"libvirt-3-storage"}' \ +-device '{"driver":"virtio-blk-pci","iothread":"iothread1","bus":"pci.0","addr":"0x4","drive":"libvirt-3-format","id":"virtio-disk1","bootindex":1}' \ +-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest2.img","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-2-format","read-only":false,"driver":"raw","file":"libvirt-2-storage"}' \ --device '{"driver":"virtio-blk-pci","iothread":"iothread1","bus":"pci.0","addr":"0x4","drive":"libvirt-2-format","id":"virtio-disk1","bootindex":1}' \ --blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest2.img","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \ +-device '{"driver":"virtio-blk-pci","num-queues":4,"bus":"pci.0","addr":"0x2","drive":"libvirt-2-format","id":"virtio-disk2"}' \ +-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest3.img","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \ --device '{"driver":"virtio-blk-pci","iothread":"iothread2","bus":"pci.0","addr":"0x2","drive":"libvirt-1-format","id":"virtio-disk2"}' \ +-device '{"driver":"virtio-blk-pci","bus":"pci.0","addr":"0x3","drive":"libvirt-1-format","id":"virtio-disk3"}' \ -audiodev '{"id":"audio1","driver":"none"}' \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on diff --git a/tests/qemuxml2argvdata/iothreads-disk.xml b/tests/qemuxml2argvdata/iothreads-disk.xml index ad0731c79c..7ce25f559e 100644 --- a/tests/qemuxml2argvdata/iothreads-disk.xml +++ b/tests/qemuxml2argvdata/iothreads-disk.xml @@ -4,7 +4,7 @@ <memory unit='KiB'>219136</memory> <currentMemory unit='KiB'>219136</currentMemory> <vcpu placement='static'>2</vcpu> - <iothreads>2</iothreads> + <iothreads>4</iothreads> <os> <type arch='x86_64' machine='pc'>hvm</type> <boot dev='hd'/> @@ -22,10 +22,31 @@ <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> </disk> <disk type='file' device='disk'> - <driver name='qemu' type='raw' iothread='2'/> + <driver name='qemu' type='raw' queues='4'> + <iothreads> + <iothread id='2'> + <queue id='1'/> + <queue id='3'/> + </iothread> + <iothread id='3'> + <queue id='0'/> + <queue id='2'/> + </iothread> + </iothreads> + </driver> <source file='/var/lib/libvirt/images/iothrtest2.img'/> <target dev='vdc' bus='virtio'/> </disk> + <disk type='file' device='disk'> + <driver name='qemu' type='raw'> + <iothreads> + <iothread id='4'/> + <iothread id='1'/> + </iothreads> + </driver> + <source file='/var/lib/libvirt/images/iothrtest3.img'/> + <target dev='vdd' bus='virtio'/> + </disk> <controller type='usb' index='0'/> <controller type='ide' index='0'/> <controller type='pci' index='0' model='pci-root'/> diff --git a/tests/qemuxml2xmloutdata/iothreads-disk.x86_64-latest.xml b/tests/qemuxml2xmloutdata/iothreads-disk.x86_64-latest.xml index ae1da9ec2a..94864feb85 100644 --- a/tests/qemuxml2xmloutdata/iothreads-disk.x86_64-latest.xml +++ b/tests/qemuxml2xmloutdata/iothreads-disk.x86_64-latest.xml @@ -4,7 +4,7 @@ <memory unit='KiB'>219136</memory> <currentMemory unit='KiB'>219136</currentMemory> <vcpu placement='static'>2</vcpu> - <iothreads>2</iothreads> + <iothreads>4</iothreads> <os> <type arch='x86_64' machine='pc'>hvm</type> <boot dev='hd'/> @@ -25,11 +25,33 @@ <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> </disk> <disk type='file' device='disk'> - <driver name='qemu' type='raw' iothread='2'/> + <driver name='qemu' type='raw' queues='4'> + <iothreads> + <iothread id='2'> + <queue id='1'/> + <queue id='3'/> + </iothread> + <iothread id='3'> + <queue id='0'/> + <queue id='2'/> + </iothread> + </iothreads> + </driver> <source file='/var/lib/libvirt/images/iothrtest2.img'/> <target dev='vdc' bus='virtio'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/> </disk> + <disk type='file' device='disk'> + <driver name='qemu' type='raw'> + <iothreads> + <iothread id='4'/> + <iothread id='1'/> + </iothreads> + </driver> + <source file='/var/lib/libvirt/images/iothrtest3.img'/> + <target dev='vdd' bus='virtio'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> + </disk> <controller type='usb' index='0' model='piix3-uhci'> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/> </controller> -- 2.39.2