[libvirt] [PATCH v2 0/2] Use huge pages on UMA guests widely
*** BLURB HERE *** Michal Privoznik (2): conf: Disallow nonexistent NUMA nodes for hugepages qemu: Honor hugepages for UMA domains src/qemu/qemu_command.c | 49 ++++++++++++++++++++-- .../qemuxml2argv-hugepages-pages4.xml | 45 ++++++++++++++++++++ .../qemuxml2argv-hugepages-pages5.args | 7 ++++ .../qemuxml2argv-hugepages-pages5.xml | 32 ++++++++++++++ tests/qemuxml2argvtest.c | 3 ++ 5 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.xml -- 1.8.5.5
As of 136ad4974 it is possible to specify different huge pages per guest NUMA node. However, there's no check if nodeset specified in ./hugepages/page contains only those guest NUMA nodes that exist. In other words with current code it is possible to define meaningless combination: <memoryBacking> <hugepages> <page size='1048576' unit='KiB' nodeset='0,2-3'/> <page size='2048' unit='KiB' nodeset='1,4'/> </hugepages> </memoryBacking> <vcpu placement='static'>4</vcpu> <cpu> <numa> <cell id='0' cpus='0' memory='1048576'/> <cell id='1' cpus='1' memory='1048576'/> <cell id='2' cpus='2' memory='1048576'/> <cell id='3' cpus='3' memory='1048576'/> </numa> </cpu> Notice the node 4 in <hugepages/>? Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- src/qemu/qemu_command.c | 24 ++++++++++++ .../qemuxml2argv-hugepages-pages4.xml | 45 ++++++++++++++++++++++ tests/qemuxml2argvtest.c | 2 + 3 files changed, 71 insertions(+) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index e5270bd..1148aec 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -6542,6 +6542,30 @@ qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg, goto cleanup; } + for (i = 0; i < def->mem.nhugepages; i++) { + ssize_t next_bit, pos = 0; + + if (!def->mem.hugepages[i].nodemask) { + /* This is the master hugepage to use. Skip it as it has no + * nodemask anyway. */ + continue; + } + + if (def->cpu && def->cpu->ncells) { + /* Fortunately, we allow only guest NUMA nodes to be continuous + * starting from zero. */ + pos = def->cpu->ncells - 1; + } + + next_bit = virBitmapNextSetBit(def->mem.hugepages[i].nodemask, pos); + if (next_bit >= 0) { + virReportError(VIR_ERR_XML_DETAIL, + _("hugepages: node %zd not found"), + next_bit); + goto cleanup; + } + } + for (i = 0; i < def->cpu->ncells; i++) { int cellmem = VIR_DIV_UP(def->cpu->cells[i].mem, 1024); def->cpu->cells[i].mem = cellmem * 1024; diff --git a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml new file mode 100644 index 0000000..a3ed29b --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml @@ -0,0 +1,45 @@ +<domain type='qemu'> + <name>QEMUGuest1</name> + <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid> + <memory unit='KiB'>4194304</memory> + <currentMemory unit='KiB'>4194304</currentMemory> + <memoryBacking> + <hugepages> + <page size='2048' unit='KiB' nodeset='1,4'/> + <page size='1048576' unit='KiB' nodeset='0,2-3'/> + </hugepages> + </memoryBacking> + <vcpu placement='static'>4</vcpu> + <numatune> + <memory mode='strict' nodeset='0-3'/> + <memnode cellid='3' mode='strict' nodeset='3'/> + </numatune> + <os> + <type arch='i686' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <cpu> + <numa> + <cell id='0' cpus='0' memory='1048576'/> + <cell id='1' cpus='1' memory='1048576'/> + <cell id='2' cpus='2' memory='1048576'/> + <cell id='3' cpus='3' memory='1048576'/> + </numa> + </cpu> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu</emulator> + <disk type='block' device='disk'> + <source dev='/dev/HostVG/QEMUGuest1'/> + <target dev='hda' bus='ide'/> + <address type='drive' controller='0' bus='0' target='0' unit='0'/> + </disk> + <controller type='usb' index='0'/> + <controller type='ide' index='0'/> + <controller type='pci' index='0' model='pci-root'/> + <memballoon model='virtio'/> + </devices> +</domain> diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 5c28253..c528499 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -684,6 +684,8 @@ mymain(void) QEMU_CAPS_OBJECT_MEMORY_FILE); DO_TEST("hugepages-pages3", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM, QEMU_CAPS_OBJECT_MEMORY_FILE); + DO_TEST_FAILURE("hugepages-pages4", QEMU_CAPS_MEM_PATH, + QEMU_CAPS_OBJECT_MEMORY_RAM, QEMU_CAPS_OBJECT_MEMORY_FILE); DO_TEST("nosharepages", QEMU_CAPS_MACHINE_OPT, QEMU_CAPS_MEM_MERGE); DO_TEST("disk-cdrom", NONE); DO_TEST("disk-cdrom-network-http", QEMU_CAPS_KVM, QEMU_CAPS_DEVICE, -- 1.8.5.5
On 09/15/2014 07:48 AM, Michal Privoznik wrote:
As of 136ad4974 it is possible to specify different huge pages per guest NUMA node. However, there's no check if nodeset specified in ./hugepages/page contains only those guest NUMA nodes that exist. In other words with current code it is possible to define meaningless combination:
<memoryBacking> <hugepages> <page size='1048576' unit='KiB' nodeset='0,2-3'/> <page size='2048' unit='KiB' nodeset='1,4'/> </hugepages> </memoryBacking> <vcpu placement='static'>4</vcpu> <cpu> <numa> <cell id='0' cpus='0' memory='1048576'/> <cell id='1' cpus='1' memory='1048576'/> <cell id='2' cpus='2' memory='1048576'/> <cell id='3' cpus='3' memory='1048576'/> </numa> </cpu>
Notice the node 4 in <hugepages/>?
Coverity detected an issue with this - it's innocuous, but I wasn't sure which way it should be handled... That is whether to add a check or remove a check...
Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- src/qemu/qemu_command.c | 24 ++++++++++++ .../qemuxml2argv-hugepages-pages4.xml | 45 ++++++++++++++++++++++ tests/qemuxml2argvtest.c | 2 + 3 files changed, 71 insertions(+) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index e5270bd..1148aec 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -6542,6 +6542,30 @@ qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg, goto cleanup; }
+ for (i = 0; i < def->mem.nhugepages; i++) { + ssize_t next_bit, pos = 0; + + if (!def->mem.hugepages[i].nodemask) { + /* This is the master hugepage to use. Skip it as it has no + * nodemask anyway. */ + continue; + } + + if (def->cpu && def->cpu->ncells) {
Because you check for def->cpu here (16) Event cond_false: Condition "def->cpu", taking false branch (18) Event var_compare_op: Comparing "def->cpu" to null implies that "def->cpu" might be null. Also see events: [var_deref_op]
+ /* Fortunately, we allow only guest NUMA nodes to be continuous + * starting from zero. */ + pos = def->cpu->ncells - 1; + } + + next_bit = virBitmapNextSetBit(def->mem.hugepages[i].nodemask, pos); + if (next_bit >= 0) { + virReportError(VIR_ERR_XML_DETAIL, + _("hugepages: node %zd not found"), + next_bit); + goto cleanup; + } + } + for (i = 0; i < def->cpu->ncells; i++) {
his makes Coverity unhappy here because it doesn't check for def->cpu first. (23) Event var_deref_op: Dereferencing null pointer "def->cpu". Also see events: [var_compare_op] John
int cellmem = VIR_DIV_UP(def->cpu->cells[i].mem, 1024); def->cpu->cells[i].mem = cellmem * 1024; diff --git a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml new file mode 100644 index 0000000..a3ed29b --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml @@ -0,0 +1,45 @@ +<domain type='qemu'> + <name>QEMUGuest1</name> + <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid> + <memory unit='KiB'>4194304</memory> + <currentMemory unit='KiB'>4194304</currentMemory> + <memoryBacking> + <hugepages> + <page size='2048' unit='KiB' nodeset='1,4'/> + <page size='1048576' unit='KiB' nodeset='0,2-3'/> + </hugepages> + </memoryBacking> + <vcpu placement='static'>4</vcpu> + <numatune> + <memory mode='strict' nodeset='0-3'/> + <memnode cellid='3' mode='strict' nodeset='3'/> + </numatune> + <os> + <type arch='i686' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <cpu> + <numa> + <cell id='0' cpus='0' memory='1048576'/> + <cell id='1' cpus='1' memory='1048576'/> + <cell id='2' cpus='2' memory='1048576'/> + <cell id='3' cpus='3' memory='1048576'/> + </numa> + </cpu> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu</emulator> + <disk type='block' device='disk'> + <source dev='/dev/HostVG/QEMUGuest1'/> + <target dev='hda' bus='ide'/> + <address type='drive' controller='0' bus='0' target='0' unit='0'/> + </disk> + <controller type='usb' index='0'/> + <controller type='ide' index='0'/> + <controller type='pci' index='0' model='pci-root'/> + <memballoon model='virtio'/> + </devices> +</domain> diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 5c28253..c528499 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -684,6 +684,8 @@ mymain(void) QEMU_CAPS_OBJECT_MEMORY_FILE); DO_TEST("hugepages-pages3", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM, QEMU_CAPS_OBJECT_MEMORY_FILE); + DO_TEST_FAILURE("hugepages-pages4", QEMU_CAPS_MEM_PATH, + QEMU_CAPS_OBJECT_MEMORY_RAM, QEMU_CAPS_OBJECT_MEMORY_FILE); DO_TEST("nosharepages", QEMU_CAPS_MACHINE_OPT, QEMU_CAPS_MEM_MERGE); DO_TEST("disk-cdrom", NONE); DO_TEST("disk-cdrom-network-http", QEMU_CAPS_KVM, QEMU_CAPS_DEVICE,
There are two ways how to tell qemu to use huge pages. The first one is suitable for domains with NUMA nodes: the path to hugetlbfs mount is appended to NUMA node definition on the command line. The second one is suitable for UMA domains: here there's this global '-mem-path' argument that accepts path to the hugetlbfs mount point. However, the latter case was not used for all the cases that it should be. For instance: <memoryBacking> <hugepages> <page size='2048' unit='KiB' nodeset='0'/> </hugepages> </memoryBacking> didn't trigger the '-mem-path' so the huge pages - despite being configured - were not used at all. Signed-off-by: Michal Privoznik <mprivozn@redhat.com> --- src/qemu/qemu_command.c | 25 ++++++++++++++--- .../qemuxml2argv-hugepages-pages5.args | 7 +++++ .../qemuxml2argv-hugepages-pages5.xml | 32 ++++++++++++++++++++++ tests/qemuxml2argvtest.c | 1 + 4 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.xml diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 1148aec..cb2fc1a 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7649,7 +7649,7 @@ qemuBuildCommandLine(virConnectPtr conn, virCommandAddArg(cmd, "-m"); def->mem.max_balloon = VIR_DIV_UP(def->mem.max_balloon, 1024) * 1024; virCommandAddArgFormat(cmd, "%llu", def->mem.max_balloon / 1024); - if (def->mem.nhugepages && !def->mem.hugepages[0].size) { + if (def->mem.nhugepages && (!def->cpu || !def->cpu->ncells)) { char *mem_path; if (!cfg->nhugetlbfs) { @@ -7665,9 +7665,26 @@ qemuBuildCommandLine(virConnectPtr conn, goto error; } - if (!(mem_path = qemuGetDefaultHugepath(cfg->hugetlbfs, - cfg->nhugetlbfs))) - goto error; + if (def->mem.hugepages[0].size) { + for (j = 0; j < cfg->nhugetlbfs; j++) { + if (cfg->hugetlbfs[j].size == def->mem.hugepages[0].size) + break; + } + + if (j == cfg->nhugetlbfs) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to find any usable hugetlbfs mount for %llu KiB"), + def->mem.hugepages[0].size); + goto error; + } + + if (!(mem_path = qemuGetHugepagePath(&cfg->hugetlbfs[j]))) + goto error; + } else { + if (!(mem_path = qemuGetDefaultHugepath(cfg->hugetlbfs, + cfg->nhugetlbfs))) + goto error; + } virCommandAddArgList(cmd, "-mem-prealloc", "-mem-path", mem_path, NULL); diff --git a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.args b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.args new file mode 100644 index 0000000..0e0f35a --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.args @@ -0,0 +1,7 @@ +LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \ +/usr/bin/qemu -S -M pc -m 1024 \ +-mem-prealloc \ +-mem-path /dev/hugepages2M/libvirt/qemu \ +-smp 2 -nographic \ +-monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -usb \ +-hda /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none diff --git a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.xml b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.xml new file mode 100644 index 0000000..50a9cd8 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.xml @@ -0,0 +1,32 @@ +<domain type='qemu'> + <name>SomeDummyHugepagesGuest</name> + <uuid>ef1bdff4-27f3-4e85-a807-5fb4d58463cc</uuid> + <memory unit='KiB'>1048576</memory> + <currentMemory unit='KiB'>1048576</currentMemory> + <memoryBacking> + <hugepages> + <page size='2048' unit='KiB'/> + </hugepages> + </memoryBacking> + <vcpu placement='static'>2</vcpu> + <os> + <type arch='i686' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu</emulator> + <disk type='block' device='disk'> + <source dev='/dev/HostVG/QEMUGuest1'/> + <target dev='hda' bus='ide'/> + <address type='drive' controller='0' bus='0' target='0' unit='0'/> + </disk> + <controller type='usb' index='0'/> + <controller type='ide' index='0'/> + <controller type='pci' index='0' model='pci-root'/> + <memballoon model='virtio'/> + </devices> +</domain> diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index c528499..5195ec5 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -686,6 +686,7 @@ mymain(void) QEMU_CAPS_OBJECT_MEMORY_FILE); DO_TEST_FAILURE("hugepages-pages4", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM, QEMU_CAPS_OBJECT_MEMORY_FILE); + DO_TEST("hugepages-pages5", QEMU_CAPS_MEM_PATH); DO_TEST("nosharepages", QEMU_CAPS_MACHINE_OPT, QEMU_CAPS_MEM_MERGE); DO_TEST("disk-cdrom", NONE); DO_TEST("disk-cdrom-network-http", QEMU_CAPS_KVM, QEMU_CAPS_DEVICE, -- 1.8.5.5
On Mon, Sep 15, 2014 at 01:48:05PM +0200, Michal Privoznik wrote:
*** BLURB HERE ***
Michal Privoznik (2): conf: Disallow nonexistent NUMA nodes for hugepages qemu: Honor hugepages for UMA domains
src/qemu/qemu_command.c | 49 ++++++++++++++++++++-- .../qemuxml2argv-hugepages-pages4.xml | 45 ++++++++++++++++++++ .../qemuxml2argv-hugepages-pages5.args | 7 ++++ .../qemuxml2argv-hugepages-pages5.xml | 32 ++++++++++++++ tests/qemuxml2argvtest.c | 3 ++ 5 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages4.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.xml
ACK series, Martin
participants (3)
-
John Ferlan -
Martin Kletzander -
Michal Privoznik