Re: [libvirt] [PATCH] Fix QEMU cpu affinity at startup to include all threads

The QEMU cpu affinity is used in NUMA scenarios to ensure that guest memory is allocated from a specific node. Normally memory is allocate on demand in vCPU threads, but when using hugepages the initial thread leader allocates memory upfront. libvirt was not setting affinity of the thread leader, or I/O threads. This patch changes the code to set the process affinity in between the fork()/exec() of QEMU. This ensures that every single QEMU thread gets the affinity * src/qemu/qemu_driver.c: Set affinity on entire QEMU process at startup --- src/qemu/qemu_driver.c | 29 ++++++++++++++++------------- 1 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 257f914..2598deb 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -1701,6 +1701,9 @@ qemuDetectVcpuPIDs(struct qemud_driver *driver, return 0; } +/* + * To be run between fork/exec of QEMU only + */ static int qemudInitCpuAffinity(virDomainObjPtr vm) { @@ -1708,7 +1711,8 @@ qemudInitCpuAffinity(virDomainObjPtr vm) virNodeInfo nodeinfo; unsigned char *cpumap; int cpumaplen; - qemuDomainObjPrivatePtr priv = vm->privateData; + + DEBUG0("Setting CPU affinity"); if (nodeGetInfo(NULL, &nodeinfo) < 0) return -1; @@ -1740,14 +1744,14 @@ qemudInitCpuAffinity(virDomainObjPtr vm) VIR_USE_CPU(cpumap, i); } - /* The XML config only gives a per-VM affinity, so we apply - * the same mapping to all vCPUs */ - for (i = 0 ; i < priv->nvcpupids ; i++) { - if (virProcessInfoSetAffinity(priv->vcpupids[i], - cpumap, cpumaplen, maxcpu) < 0) { - VIR_FREE(cpumap); - return -1; - } + /* We are assuming we are running between fork/exec of QEMU, so + * that getpid() gives the QEMU process ID and we know that + * no threads are running. + */ + if (virProcessInfoSetAffinity(getpid(), + cpumap, cpumaplen, maxcpu) < 0) { + VIR_FREE(cpumap); + return -1; } VIR_FREE(cpumap); @@ -2653,6 +2657,9 @@ struct qemudHookData { static int qemudSecurityHook(void *data) { struct qemudHookData *h = data; + if (qemudInitCpuAffinity(h->vm) < 0) + return -1; + if (qemuAddToCgroup(h->driver, h->vm->def) < 0) return -1; @@ -2943,10 +2950,6 @@ static int qemudStartVMDaemon(virConnectPtr conn, if (qemuDetectVcpuPIDs(driver, vm) < 0) goto abort; - DEBUG0("Setting CPU affinity"); - if (qemudInitCpuAffinity(vm) < 0) - goto abort; - DEBUG0("Setting any required VM passwords"); if (qemuInitPasswords(conn, driver, vm, qemuCmdFlags) < 0) goto abort;