5:22 a.m.
When either "cpuset" of <vcpu> is specified, or the "placement"
of
<vcpu> is "auto", only setting the cpuset.mems might cause the guest
starting to fail. E.g. ("placement" of both <vcpu> and <numatune>
is
"auto"):
1) Related XMLs
<vcpu placement='auto'>4</vcpu>
<numatune>
<memory mode='strict' placement='auto'/>
</numatune>
2) Host NUMA topology
% numactl --hardware
available: 8 nodes (0-7)
node 0 cpus: 0 4 8 12 16 20 24 28
node 0 size: 16374 MB
node 0 free: 11899 MB
node 1 cpus: 32 36 40 44 48 52 56 60
node 1 size: 16384 MB
node 1 free: 15318 MB
node 2 cpus: 2 6 10 14 18 22 26 30
node 2 size: 16384 MB
node 2 free: 15766 MB
node 3 cpus: 34 38 42 46 50 54 58 62
node 3 size: 16384 MB
node 3 free: 15347 MB
node 4 cpus: 3 7 11 15 19 23 27 31
node 4 size: 16384 MB
node 4 free: 15041 MB
node 5 cpus: 35 39 43 47 51 55 59 63
node 5 size: 16384 MB
node 5 free: 15202 MB
node 6 cpus: 1 5 9 13 17 21 25 29
node 6 size: 16384 MB
node 6 free: 15197 MB
node 7 cpus: 33 37 41 45 49 53 57 61
node 7 size: 16368 MB
node 7 free: 15669 MB
4) cpuset.cpus will be set as: (from debug log)
2013-05-09 16:50:17.296+0000: 417: debug : virCgroupSetValueStr:331 :
Set value '/sys/fs/cgroup/cpuset/libvirt/qemu/toy/cpuset.cpus'
to '0-63'
5) The advisory nodeset got from querying numad (from debug log)
2013-05-09 16:50:17.295+0000: 417: debug : qemuProcessStart:3614 :
Nodeset returned from numad: 1
6) cpuset.mems will be set as: (from debug log)
2013-05-09 16:50:17.296+0000: 417: debug : virCgroupSetValueStr:331 :
Set value '/sys/fs/cgroup/cpuset/libvirt/qemu/toy/cpuset.mems'
to '0-7'
I.E, the domain process's memory is restricted on the first NUMA node,
however, it can use all of the CPUs, which will very likely cause the
domain process to fail to start because of the kernel fails to allocate
memory with the possible mismatching between CPU nodes and memory nodes.
% tail -n 20 /var/log/libvirt/qemu/toy.log
...
2013-05-09 05:53:32.972+0000: 7318: debug : virCommandHandshakeChild:377 :
Handshake with parent is done
char device redirected to /dev/pts/2 (label charserial0)
kvm_init_vcpu failed: Cannot allocate memory
...
---
src/qemu/qemu_cgroup.c | 39 +++++++++++++++++++++++++++++++++------
1 file changed, 33 insertions(+), 6 deletions(-)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 0e00b47..33eebd7 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -589,7 +589,8 @@ qemuSetupCpusetCgroup(virDomainObjPtr vm,
virBitmapPtr nodemask)
{
qemuDomainObjPrivatePtr priv = vm->privateData;
- char *mask = NULL;
+ char *mem_mask = NULL;
+ char *cpu_mask = NULL;
int rc;
int ret = -1;
@@ -603,17 +604,17 @@ qemuSetupCpusetCgroup(virDomainObjPtr vm,
if (vm->def->numatune.memory.placement_mode ==
VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO)
- mask = virBitmapFormat(nodemask);
+ mem_mask = virBitmapFormat(nodemask);
else
- mask = virBitmapFormat(vm->def->numatune.memory.nodemask);
+ mem_mask = virBitmapFormat(vm->def->numatune.memory.nodemask);
- if (!mask) {
+ if (!mem_mask) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("failed to convert memory nodemask"));
goto cleanup;
}
- rc = virCgroupSetCpusetMems(priv->cgroup, mask);
+ rc = virCgroupSetCpusetMems(priv->cgroup, mem_mask);
if (rc != 0) {
virReportSystemError(-rc,
@@ -623,9 +624,35 @@ qemuSetupCpusetCgroup(virDomainObjPtr vm,
}
}
+ if (vm->def->cpumask ||
+ (vm->def->placement_mode ==
+ VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)) {
+ if (vm->def->placement_mode ==
+ VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
+ cpu_mask = virBitmapFormat(nodemask);
+ else
+ cpu_mask = virBitmapFormat(vm->def->cpumask);
+
+ if (!cpu_mask) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("failed to convert memory nodemask"));
+ goto cleanup;
+ }
+
+ rc = virCgroupSetCpusetCpus(priv->cgroup, cpu_mask);
+
+ if (rc != 0) {
+ virReportSystemError(-rc,
+ _("Unable to set cpuset.cpus for domain %s"),
+ vm->def->name);
+ goto cleanup;
+ }
+ }
+
ret = 0;
cleanup:
- VIR_FREE(mask);
+ VIR_FREE(mem_mask);
+ VIR_FREE(cpu_mask);
return ret;
}
--
1.8.1.4