[PATCH v1 0/3] Make virtio-mem and virtio-pmem address stable
*** BLURB HERE *** Michal Prívozník (3): conf: Introduce <address/> for virtio-mem and virtio-pmem qemu: Fill virtio-mem/virtio-pmem .memaddr at runtime qemu_command: Generate .memaddr for virtio-mem and virtio-pmem docs/formatdomain.rst | 7 ++++++ src/conf/domain_conf.c | 24 ++++++++++++++++++- src/conf/domain_conf.h | 2 ++ src/conf/domain_validate.c | 6 +++++ src/conf/schemas/domaincommon.rng | 7 ++++++ src/qemu/qemu_command.c | 1 + src/qemu/qemu_domain.c | 7 ++++-- src/qemu/qemu_monitor_json.c | 12 ++++++++-- ...mory-hotplug-virtio-mem.x86_64-latest.args | 2 +- .../memory-hotplug-virtio-mem.xml | 1 + ...mory-hotplug-virtio-pmem.x86_64-5.2.0.args | 2 +- ...ory-hotplug-virtio-pmem.x86_64-latest.args | 2 +- .../memory-hotplug-virtio-pmem.xml | 1 + 13 files changed, 66 insertions(+), 8 deletions(-) -- 2.39.2
Both virtio-mem and virtio-pmem devices have '.memaddr' attribute which controls the address where they are mapped in the guest memory. Ideally, users do not need to specify this as QEMU does the right thing and computes addresses automatically on startup. But soon, we will need to record this address as it is part of guest ABI. And also, there might be some users that want to control this value. Now, we are in a bit of a pickle, because both these device types already have a PCI address, therefore we can't just use <address/> blindly. But what we can do, is introduce <address/> under the <target/> element. This is also more conceptual, as knobs under <target/> control guest visible config of memory device (and .memaddr surely falls into that category). NB, SgxEPCDeviceInfo struct in QMP definition also has .memaddr attribute, but because of the way we build cmd line there's no (easy) way to set the attribute. So ignore that for now. Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- docs/formatdomain.rst | 7 ++++++ src/conf/domain_conf.c | 24 ++++++++++++++++++- src/conf/domain_conf.h | 2 ++ src/conf/domain_validate.c | 6 +++++ src/conf/schemas/domaincommon.rng | 7 ++++++ .../memory-hotplug-virtio-mem.xml | 1 + .../memory-hotplug-virtio-pmem.xml | 1 + 7 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index 27f83e254d..092d2296d5 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -8100,6 +8100,7 @@ Example: usage of the memory devices </source> <target> <size unit='KiB'>524288</size> + <address base='0x140000000'/> </target> </memory> <memory model='virtio-mem'> @@ -8113,6 +8114,7 @@ Example: usage of the memory devices <block unit='KiB'>2048</block> <requested unit='KiB'>1048576</requested> <current unit='KiB'>524288</current> + <address base='0x150000000'/> </target> </memory> <memory model='sgx-epc'> @@ -8256,6 +8258,11 @@ Example: usage of the memory devices element is formatted into live XML and never parsed, i.e. it is output-only element. + ``address`` + For ``virtio-mem`` and ``virtio-pmem`` only. + The physical address in memory, where device is mapped. :since:`Since + 9.3.0` + IOMMU devices ~~~~~~~~~~~~~ diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 9f49c6e62d..bed6acac3a 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -13377,6 +13377,7 @@ virDomainMemoryTargetDefParseXML(xmlNodePtr node, virDomainMemoryDef *def) { VIR_XPATH_NODE_AUTORESTORE(ctxt) + xmlNodePtr addrNode = NULL; int rv; ctxt->node = node; @@ -13422,16 +13423,27 @@ virDomainMemoryTargetDefParseXML(xmlNodePtr node, if (virDomainParseMemory("./requested", "./requested/@unit", ctxt, &def->requestedsize, false, false) < 0) return -1; + + addrNode = virXPathNode("./address", ctxt); + break; + + case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM: + addrNode = virXPathNode("./address", ctxt); break; case VIR_DOMAIN_MEMORY_MODEL_NONE: case VIR_DOMAIN_MEMORY_MODEL_DIMM: - case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM: case VIR_DOMAIN_MEMORY_MODEL_SGX_EPC: case VIR_DOMAIN_MEMORY_MODEL_LAST: break; } + if (addrNode && + virXMLPropULongLong(addrNode, "base", 16, + VIR_XML_PROP_NONE, &def->address) < 0) { + return -1; + } + return 0; } @@ -20996,6 +21008,13 @@ virDomainMemoryDefCheckABIStability(virDomainMemoryDef *src, return false; } + if (src->address != dst->address) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target memory device address '0x%llx' doesn't match source memory device address '0x%llx'"), + dst->address, src->address); + return false; + } + if (src->model == VIR_DOMAIN_MEMORY_MODEL_NVDIMM) { if (src->labelsize != dst->labelsize) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, @@ -25184,6 +25203,9 @@ virDomainMemoryTargetDefFormat(virBuffer *buf, } } + if (def->address) + virBufferAsprintf(&childBuf, "<address base='0x%llx'/>\n", def->address); + virXMLFormatElement(buf, "target", NULL, &childBuf); } diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 5a2c70f012..0a8e877a63 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2646,6 +2646,8 @@ struct _virDomainMemoryDef { unsigned long long currentsize; /* kibibytes, valid for VIRTIO_MEM and active domain only, only to report never parse */ + unsigned long long address; /* address where memory is mapped, valid for + VIRTIO_PMEM and VIRTIO_MEM only. */ bool readonly; /* valid only for NVDIMM */ /* required for QEMU NVDIMM ppc64 support */ diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c index f208c0c531..ad76a21ca2 100644 --- a/src/conf/domain_validate.c +++ b/src/conf/domain_validate.c @@ -2358,6 +2358,12 @@ virDomainMemoryDefValidate(const virDomainMemoryDef *mem, _("requested size must be an integer multiple of block size")); return -1; } + + if (mem->address % mem->blocksize != 0) { + virReportError(VIR_ERR_XML_DETAIL, "%s", + _("memory device address must be aligned to blocksize")); + return -1; + } break; case VIR_DOMAIN_MEMORY_MODEL_DIMM: diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index 6158ed79ac..9af47f9688 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -7153,6 +7153,13 @@ <empty/> </element> </optional> + <optional> + <element name="address"> + <attribute name="base"> + <ref name="hexuint"/> + </attribute> + </element> + </optional> </interleave> </element> </define> diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.xml b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.xml index 73036d8602..f5cc4a35ed 100644 --- a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.xml +++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.xml @@ -65,6 +65,7 @@ <node>0</node> <block unit='KiB'>2048</block> <requested unit='KiB'>1048576</requested> + <address base='0x150000000'/> </target> <address type='pci' domain='0x0000' bus='0x01' slot='0x01' function='0x0'/> </memory> diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.xml b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.xml index 4cebd294ec..21b90e4d8a 100644 --- a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.xml +++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.xml @@ -47,6 +47,7 @@ </source> <target> <size unit='KiB'>524288</size> + <address base='0x140000000'/> </target> <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/> </memory> -- 2.39.2
On Tue, Mar 28, 2023 at 01:58:01PM +0200, Michal Privoznik wrote:
Both virtio-mem and virtio-pmem devices have '.memaddr' attribute which controls the address where they are mapped in the guest memory. Ideally, users do not need to specify this as QEMU does the right thing and computes addresses automatically on startup.
But soon, we will need to record this address as it is part of guest ABI. And also, there might be some users that want to control this value. Now, we are in a bit of a pickle, because both these device types already have a PCI address, therefore we can't just use <address/> blindly. But what we can do, is introduce <address/> under the <target/> element. This is also more conceptual, as knobs under <target/> control guest visible config of memory device (and .memaddr surely falls into that category).
NB, SgxEPCDeviceInfo struct in QMP definition also has .memaddr attribute, but because of the way we build cmd line there's no (easy) way to set the attribute. So ignore that for now.
Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- docs/formatdomain.rst | 7 ++++++ src/conf/domain_conf.c | 24 ++++++++++++++++++- src/conf/domain_conf.h | 2 ++ src/conf/domain_validate.c | 6 +++++ src/conf/schemas/domaincommon.rng | 7 ++++++ .../memory-hotplug-virtio-mem.xml | 1 + .../memory-hotplug-virtio-pmem.xml | 1 + 7 files changed, 47 insertions(+), 1 deletion(-)
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 9f49c6e62d..bed6acac3a 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -20996,6 +21008,13 @@ virDomainMemoryDefCheckABIStability(virDomainMemoryDef *src, return false; }
+ if (src->address != dst->address) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Target memory device address '0x%llx' doesn't match source memory device address '0x%llx'"),
This needs to use the new permutable format string. Other than that Reviewed-by: Martin Kletzander <mkletzan@redhat.com> and safe for freeze since this was posted long time ago and we want the back-compat to be stronger sooner rather than later.
After a QEMU domain is started, among other thing we query memory device information. And while memory address is returned by QEMU for all models, we store it only for DIMMs and NVDIMMs. Do store it for VIRTIO_MEM and VIRTIO_PMEM too. This effectively reports the address the virtio-mem/virtio-pmem is mapped to in live XML. Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- src/qemu/qemu_domain.c | 7 +++++-- src/qemu/qemu_monitor_json.c | 12 ++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 6c29c8f09f..c10e939e0c 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -8538,7 +8538,11 @@ qemuDomainUpdateMemoryDeviceInfo(virDomainObj *vm, switch (mem->model) { case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM: - mem->currentsize = VIR_DIV_UP(dimm->size, 1024); + case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM: + if (mem->model == VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM) { + mem->currentsize = VIR_DIV_UP(dimm->size, 1024); + } + mem->address = dimm->address; break; case VIR_DOMAIN_MEMORY_MODEL_DIMM: @@ -8548,7 +8552,6 @@ qemuDomainUpdateMemoryDeviceInfo(virDomainObj *vm, mem->info.addr.dimm.base = dimm->address; break; - case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM: case VIR_DOMAIN_MEMORY_MODEL_SGX_EPC: case VIR_DOMAIN_MEMORY_MODEL_NONE: case VIR_DOMAIN_MEMORY_MODEL_LAST: diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index f4416c929f..3b2caf05e2 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -7265,6 +7265,7 @@ qemuMonitorJSONGetMemoryDeviceInfo(qemuMonitor *mon, case VIR_DOMAIN_MEMORY_MODEL_DIMM: case VIR_DOMAIN_MEMORY_MODEL_NVDIMM: case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM: + case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM: /* While 'id' attribute is marked as optional in QEMU's QAPI * specification, Libvirt always sets it. Thus we can fail if not * present. */ @@ -7305,13 +7306,21 @@ qemuMonitorJSONGetMemoryDeviceInfo(qemuMonitor *mon, return -1; } - } else if (model == VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM) { + } else if (model == VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM || + model == VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM) { if (virJSONValueObjectGetNumberUlong(dimminfo, "size", &meminfo->size) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("malformed/missing size in virtio memory info")); return -1; } + + if (virJSONValueObjectGetNumberUlong(dimminfo, "memaddr", + &meminfo->address) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("malformed/missing memaddr in virtio memory info")); + return -1; + } } break; @@ -7336,7 +7345,6 @@ qemuMonitorJSONGetMemoryDeviceInfo(qemuMonitor *mon, } break; - case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM: case VIR_DOMAIN_MEMORY_MODEL_NONE: case VIR_DOMAIN_MEMORY_MODEL_LAST: /* type not handled yet */ -- 2.39.2
On Tue, Mar 28, 2023 at 01:58:02PM +0200, Michal Privoznik wrote:
After a QEMU domain is started, among other thing we query memory device information. And while memory address is returned by QEMU for all models, we store it only for DIMMs and NVDIMMs. Do store it for VIRTIO_MEM and VIRTIO_PMEM too.
This effectively reports the address the virtio-mem/virtio-pmem is mapped to in live XML.
Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Martin Kletzander <mkletzan@redhat.com>
This is fairly trivial. Just set .memaddr attribute if a value was set in the XML. Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=2180679 Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- src/qemu/qemu_command.c | 1 + .../memory-hotplug-virtio-mem.x86_64-latest.args | 2 +- .../memory-hotplug-virtio-pmem.x86_64-5.2.0.args | 2 +- .../memory-hotplug-virtio-pmem.x86_64-latest.args | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 7e75354902..717dadd492 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -3628,6 +3628,7 @@ qemuBuildMemoryDeviceProps(virQEMUDriverConfig *cfg, "T:unarmed", unarmed, "s:memdev", memdev, "B:prealloc", prealloc, + "P:memaddr", mem->address, "s:id", mem->info.alias, NULL) < 0) return NULL; diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args index ba1dd872ba..592578fb59 100644 --- a/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args +++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-mem.x86_64-latest.args @@ -32,7 +32,7 @@ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest1/.config \ -object '{"qom-type":"memory-backend-ram","id":"memvirtiomem0","reserve":false,"size":1073741824}' \ -device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":536870912,"memdev":"memvirtiomem0","id":"virtiomem0","bus":"pci.0","addr":"0x2"}' \ -object '{"qom-type":"memory-backend-file","id":"memvirtiomem1","mem-path":"/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1","reserve":false,"size":2147483648,"host-nodes":[1,2,3],"policy":"bind"}' \ --device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":1073741824,"memdev":"memvirtiomem1","prealloc":true,"id":"virtiomem1","bus":"pci.1","addr":"0x1"}' \ +-device '{"driver":"virtio-mem-pci","node":0,"block-size":2097152,"requested-size":1073741824,"memdev":"memvirtiomem1","prealloc":true,"memaddr":5637144576,"id":"virtiomem1","bus":"pci.1","addr":"0x1"}' \ -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \ -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \ diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-5.2.0.args b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-5.2.0.args index 4972285a3e..e28dd339e9 100644 --- a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-5.2.0.args +++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-5.2.0.args @@ -30,7 +30,7 @@ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest1/.config \ -boot strict=on \ -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \ -object memory-backend-file,id=memvirtiopmem0,mem-path=/tmp/virtio_pmem,share=on,size=536870912 \ --device virtio-pmem-pci,memdev=memvirtiopmem0,id=virtiopmem0,bus=pci.0,addr=0x5 \ +-device virtio-pmem-pci,memdev=memvirtiopmem0,memaddr=5368709120,id=virtiopmem0,bus=pci.0,addr=0x5 \ -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \ -device ide-hd,bus=ide.0,unit=0,drive=libvirt-1-format,id=ide0-0-0,bootindex=1 \ diff --git a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-latest.args b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-latest.args index 560be76730..bd8343c15e 100644 --- a/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-latest.args +++ b/tests/qemuxml2argvdata/memory-hotplug-virtio-pmem.x86_64-latest.args @@ -29,7 +29,7 @@ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain--1-QEMUGuest1/.config \ -boot strict=on \ -device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \ -object '{"qom-type":"memory-backend-file","id":"memvirtiopmem0","mem-path":"/tmp/virtio_pmem","share":true,"size":536870912}' \ --device '{"driver":"virtio-pmem-pci","memdev":"memvirtiopmem0","id":"virtiopmem0","bus":"pci.0","addr":"0x5"}' \ +-device '{"driver":"virtio-pmem-pci","memdev":"memvirtiopmem0","memaddr":5368709120,"id":"virtiopmem0","bus":"pci.0","addr":"0x5"}' \ -blockdev '{"driver":"host_device","filename":"/dev/HostVG/QEMUGuest1","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \ -device '{"driver":"ide-hd","bus":"ide.0","unit":0,"drive":"libvirt-1-format","id":"ide0-0-0","bootindex":1}' \ -- 2.39.2
On Tue, Mar 28, 2023 at 01:58:03PM +0200, Michal Privoznik wrote:
This is fairly trivial. Just set .memaddr attribute if a value was set in the XML.
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=2180679 Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- src/qemu/qemu_command.c | 1 + .../memory-hotplug-virtio-mem.x86_64-latest.args | 2 +- .../memory-hotplug-virtio-pmem.x86_64-5.2.0.args | 2 +- .../memory-hotplug-virtio-pmem.x86_64-latest.args | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 7e75354902..717dadd492 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -3628,6 +3628,7 @@ qemuBuildMemoryDeviceProps(virQEMUDriverConfig *cfg, "T:unarmed", unarmed, "s:memdev", memdev, "B:prealloc", prealloc, + "P:memaddr", mem->address,
Only here I figured out to think whether it might be possible to have the dimm mapped to address 0, but it's fine. Reviewed-by: Martin Kletzander <mkletzan@redhat.com> and SFF (and the second one too, I forgot to mention it there, but I think that's a given)
participants (3)
-
Martin Kletzander -
Michal Privoznik -
Michal Prívozník