On 2013年03月18日 17:04, Gao feng wrote:

qemuGetNumadAdvice will be used by LXC driver,rename it to virNumaGetAutoPlacementAdvice and move it to virnuma.c

Signed-off-by: Gao feng<gaofeng@cn.fujitsu.com> --- po/POTFILES.in | 1 + src/Makefile.am | 1 + src/libvirt_private.syms | 3 +++ src/qemu/qemu_process.c | 34 +++------------------------ src/util/virnuma.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++ src/util/virnuma.h | 28 ++++++++++++++++++++++ 6 files changed, 97 insertions(+), 31 deletions(-) create mode 100644 src/util/virnuma.c create mode 100644 src/util/virnuma.h

diff --git PATCH v3po/POTFILES.in b/po/POTFILES.in index 5b5c3c2..771e83f 100644 --- PATCH v3po/POTFILES.in +++ b/po/POTFILES.in @@ -165,6 +165,7 @@ src/util/virnetdevtap.c src/util/virnetdevvportprofile.c src/util/virnetlink.c src/util/virnodesuspend.c +src/util/virnuma.c src/util/virobject.c src/util/virpci.c src/util/virpidfile.c diff --git PATCH v3src/Makefile.am b/src/Makefile.am index 8b591d2..2db03cb 100644 --- PATCH v3src/Makefile.am +++ b/src/Makefile.am @@ -103,6 +103,7 @@ UTIL_SOURCES = \ util/virnetdevvportprofile.h util/virnetdevvportprofile.c \ util/virnetlink.c util/virnetlink.h \ util/virnodesuspend.c util/virnodesuspend.h \ + util/virnuma.c util/virnuma.h \ util/virobject.c util/virobject.h \ util/virpci.c util/virpci.h \ util/virpidfile.c util/virpidfile.h \ diff --git PATCH v3src/libvirt_private.syms b/src/libvirt_private.syms index 5cad990..1374470 100644 --- PATCH v3src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1545,6 +1545,9 @@ nodeSuspendForDuration; virNodeSuspendGetTargetMask;

+# util/virnuma.h +virNumaGetAutoPlacementAdvice; + # util/virobject.h virClassForObject; virClassForObjectLockable; diff --git PATCH v3src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 1941d4e..96ba70a 100644 --- PATCH v3src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -70,6 +70,7 @@ #include "virnetdevtap.h" #include "virbitmap.h" #include "viratomic.h" +#include "virnuma.h"

#define VIR_FROM_THIS VIR_FROM_QEMU

@@ -1903,36 +1904,6 @@ qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm, } #endif

-#if HAVE_NUMAD -static char * -qemuGetNumadAdvice(virDomainDefPtr def) -{ - virCommandPtr cmd = NULL; - char *output = NULL; - - cmd = virCommandNewArgList(NUMAD, "-w", NULL); - virCommandAddArgFormat(cmd, "%d:%llu", def->vcpus, - VIR_DIV_UP(def->mem.cur_balloon, 1024)); - - virCommandSetOutputBuffer(cmd,&output); - - if (virCommandRun(cmd, NULL)< 0) - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Failed to query numad for the " - "advisory nodeset")); - - virCommandFree(cmd); - return output; -} -#else -static char * -qemuGetNumadAdvice(virDomainDefPtr def ATTRIBUTE_UNUSED) -{ - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("numad is not available on this host")); - return NULL; -} -#endif

/* Helper to prepare cpumap for affinity setting, convert * NUMA nodeset into cpuset if @nodemask is not NULL, otherwise @@ -3637,7 +3608,8 @@ int qemuProcessStart(virConnectPtr conn, VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) || (vm->def->numatune.memory.placement_mode == VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) { - nodeset = qemuGetNumadAdvice(vm->def); + nodeset = virNumaGetAutoPlacementAdvice(vm->def->vcpus, + vm->def->mem.cur_balloon); if (!nodeset) goto cleanup;

diff --git PATCH v3src/util/virnuma.c b/src/util/virnuma.c new file mode 100644 index 0000000..f6a6eb2 --- /dev/null +++ b/src/util/virnuma.c @@ -0,0 +1,61 @@ +/* + * virnuma.c: helper APIs for managing numa + * + * Copyright (C) 2011-2013 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + *<http://www.gnu.org/licenses/>. + * + */ + +#include<config.h> + +#include "virnuma.h" +#include "vircommand.h" +#include "virerror.h" + +#define VIR_FROM_THIS VIR_FROM_NONE + +#if HAVE_NUMAD +char * +virNumaGetAutoPlacementAdvice(unsigned short vcpus, + unsigned long long balloon) +{ + virCommandPtr cmd = NULL; + char *output = NULL; + + cmd = virCommandNewArgList(NUMAD, "-w", NULL); + virCommandAddArgFormat(cmd, "%d:%llu", vcpus, + VIR_DIV_UP(balloon, 1024)); + + virCommandSetOutputBuffer(cmd,&output); + + if (virCommandRun(cmd, NULL)< 0) + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to query numad for the " + "advisory nodeset")); + + virCommandFree(cmd); + return output; +} +#else +char * +virNumaGetAutoPlacementAdvice(unsigned short vcpus ATTRIBUTE_UNUSED, + unsigned long long balloon ATTRIBUTE_UNUSED) +{ + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("numad is not available on this host")); + return NULL; +} +#endif diff --git PATCH v3src/util/virnuma.h b/src/util/virnuma.h new file mode 100644 index 0000000..d3d7d3e --- /dev/null +++ b/src/util/virnuma.h @@ -0,0 +1,28 @@ +/* + * virnuma.h: helper APIs for managing numa + * + * Copyright (C) 2011-2013 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + *<http://www.gnu.org/licenses/>. + * + */ + +#ifndef __VIR_NUMA_H__ +# define __VIR_NUMA_H__ + +char *virNumaGetAutoPlacementAdvice(unsigned short vcups, + unsigned long long balloon); + +#endif /* __VIR_NUMA_H__ */

ACK.

On 2013/03/18 17:34, Osier Yang wrote:

On 2013年03月18日 17:04, Gao feng wrote:

...

Intend to reduce the redundant code,use virNumaSetupMemoryPolicy to replace virLXCControllerSetupNUMAPolicy and qemuProcessInitNumaMemoryPolicy.

This patch also moves the numa related codes to the file virnuma.c and virnuma.h

Signed-off-by: Gao feng<gaofeng@cn.fujitsu.com> --- include/libvirt/libvirt.h.in | 15 ------ src/conf/domain_conf.c | 25 +++------ src/conf/domain_conf.h | 17 +----- src/libvirt_private.syms | 9 ++-- src/lxc/lxc_controller.c | 114 +-------------------------------------- src/qemu/qemu_cgroup.c | 6 +-- src/qemu/qemu_driver.c | 2 +- src/qemu/qemu_process.c | 121 +---------------------------------------- src/util/virnuma.c | 125 +++++++++++++++++++++++++++++++++++++++++++ src/util/virnuma.h | 45 ++++++++++++++++ tools/virsh-domain.c | 4 +- 11 files changed, 192 insertions(+), 291 deletions(-)

diff --git PATCH v3include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in index f6a7aff..b3bfd1d 100644 --- PATCH v3include/libvirt/libvirt.h.in +++ b/include/libvirt/libvirt.h.in @@ -1762,21 +1762,6 @@ typedef enum { /* Manage numa parameters */

/** - * virDomainNumatuneMemMode: - * Representation of the various modes in the<numatune> element of - * a domain. - */ -typedef enum { - VIR_DOMAIN_NUMATUNE_MEM_STRICT = 0, - VIR_DOMAIN_NUMATUNE_MEM_PREFERRED = 1, - VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE = 2, - -#ifdef VIR_ENUM_SENTINELS - VIR_DOMAIN_NUMATUNE_MEM_LAST /* This constant is subject to change */ -#endif -} virDomainNumatuneMemMode; -

NACK, this will introduce backward compatibility problems. They should *not* be changed or removed.

So what we should do is just move them from domain_conf to virnuma? The rename will cause compatibility problems. I am right? Thanks!

On 2013/03/18 19:33, Osier Yang wrote:

On 2013年03月18日 18:06, Gao feng wrote:

...

On 2013/03/18 17:34, Osier Yang wrote:

...

On 2013年03月18日 17:04, Gao feng wrote:

...

Intend to reduce the redundant code,use virNumaSetupMemoryPolicy to replace virLXCControllerSetupNUMAPolicy and qemuProcessInitNumaMemoryPolicy.

This patch also moves the numa related codes to the file virnuma.c and virnuma.h

Signed-off-by: Gao feng<gaofeng@cn.fujitsu.com> --- include/libvirt/libvirt.h.in | 15 ------ src/conf/domain_conf.c | 25 +++------ src/conf/domain_conf.h | 17 +----- src/libvirt_private.syms | 9 ++-- src/lxc/lxc_controller.c | 114 +-------------------------------------- src/qemu/qemu_cgroup.c | 6 +-- src/qemu/qemu_driver.c | 2 +- src/qemu/qemu_process.c | 121 +---------------------------------------- src/util/virnuma.c | 125 +++++++++++++++++++++++++++++++++++++++++++ src/util/virnuma.h | 45 ++++++++++++++++ tools/virsh-domain.c | 4 +- 11 files changed, 192 insertions(+), 291 deletions(-)

diff --git PATCH v3include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in index f6a7aff..b3bfd1d 100644 --- PATCH v3include/libvirt/libvirt.h.in +++ b/include/libvirt/libvirt.h.in @@ -1762,21 +1762,6 @@ typedef enum { /* Manage numa parameters */

/** - * virDomainNumatuneMemMode: - * Representation of the various modes in the<numatune> element of - * a domain. - */ -typedef enum { - VIR_DOMAIN_NUMATUNE_MEM_STRICT = 0, - VIR_DOMAIN_NUMATUNE_MEM_PREFERRED = 1, - VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE = 2, - -#ifdef VIR_ENUM_SENTINELS - VIR_DOMAIN_NUMATUNE_MEM_LAST /* This constant is subject to change */ -#endif -} virDomainNumatuneMemMode; -

NACK, this will introduce backward compatibility problems. They should *not* be changed or removed.

So what we should do is just move them from domain_conf to virnuma? The rename will cause compatibility problems. I am right?

The point is not to change/remove the stuffs which is exported to public, removing/changing on the internal files (e.g. domain_conf.{c,h}) is fine.

Get it,thanks you guys. I will rework this one and rebase 4/4 on this one. Thanks!

Intend to reduce the redundant code,use virNumaSetupMemoryPolicy to replace virLXCControllerSetupNUMAPolicy and qemuProcessInitNumaMemoryPolicy. This patch also moves the numa related codes to the file virnuma.c and virnuma.h Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- src/conf/domain_conf.c | 17 ++----- src/conf/domain_conf.h | 17 +------ src/libvirt_private.syms | 5 +- src/lxc/lxc_controller.c | 114 +----------------------------------------- src/qemu/qemu_cgroup.c | 4 +- src/qemu/qemu_process.c | 121 +-------------------------------------------- src/util/virnuma.c | 126 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virnuma.h | 30 +++++++++++ 8 files changed, 168 insertions(+), 266 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 3278e9c..72214b1 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -690,11 +690,6 @@ VIR_ENUM_IMPL(virDomainTimerMode, VIR_DOMAIN_TIMER_MODE_LAST, "paravirt", "smpsafe"); -VIR_ENUM_IMPL(virDomainNumatuneMemMode, VIR_DOMAIN_NUMATUNE_MEM_LAST, - "strict", - "preferred", - "interleave"); - VIR_ENUM_IMPL(virDomainStartupPolicy, VIR_DOMAIN_STARTUP_POLICY_LAST, "default", "mandatory", @@ -709,12 +704,6 @@ VIR_ENUM_IMPL(virDomainDiskTray, VIR_DOMAIN_DISK_TRAY_LAST, "closed", "open"); -VIR_ENUM_IMPL(virDomainNumatuneMemPlacementMode, - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_LAST, - "default", - "static", - "auto"); - VIR_ENUM_IMPL(virDomainRNGModel, VIR_DOMAIN_RNG_MODEL_LAST, "virtio"); @@ -9851,7 +9840,7 @@ virDomainDefParseXML(virCapsPtr caps, int placement_mode = 0; if (placement) { if ((placement_mode = - virDomainNumatuneMemPlacementModeTypeFromString(placement)) < 0) { + virNumatuneMemPlacementModeTypeFromString(placement)) < 0) { virReportError(VIR_ERR_XML_ERROR, _("Unsupported memory placement " "mode '%s'"), placement); @@ -9906,7 +9895,7 @@ virDomainDefParseXML(virCapsPtr caps, * and 'placement' of <vcpu> is 'auto'. */ if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) { - def->numatune.memory.placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO; + def->numatune.memory.placement_mode = VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO; def->numatune.memory.mode = VIR_DOMAIN_NUMATUNE_MEM_STRICT; } } @@ -14827,7 +14816,7 @@ virDomainDefFormatInternal(virDomainDefPtr def, virBufferAsprintf(buf, "nodeset='%s'/>\n", nodemask); VIR_FREE(nodemask); } else if (def->numatune.memory.placement_mode) { - placement = virDomainNumatuneMemPlacementModeTypeToString(def->numatune.memory.placement_mode); + placement = virNumatuneMemPlacementModeTypeToString(def->numatune.memory.placement_mode); virBufferAsprintf(buf, "placement='%s'/>\n", placement); } virBufferAddLit(buf, " </numatune>\n"); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 96f11ba..98c4745 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -47,6 +47,7 @@ # include "device_conf.h" # include "virbitmap.h" # include "virstoragefile.h" +# include "virnuma.h" /* forward declarations of all device types, required by * virDomainDeviceDef @@ -1700,18 +1701,6 @@ virDomainVcpuPinDefPtr virDomainVcpuPinFindByVcpu(virDomainVcpuPinDefPtr *def, int nvcpupin, int vcpu); -typedef struct _virDomainNumatuneDef virDomainNumatuneDef; -typedef virDomainNumatuneDef *virDomainNumatuneDefPtr; -struct _virDomainNumatuneDef { - struct { - virBitmapPtr nodemask; - int mode; - int placement_mode; /* enum virDomainNumatuneMemPlacementMode */ - } memory; - - /* Future NUMA tuning related stuff should go here. */ -}; - typedef struct _virBlkioDeviceWeight virBlkioDeviceWeight; typedef virBlkioDeviceWeight *virBlkioDeviceWeightPtr; struct _virBlkioDeviceWeight { @@ -1801,7 +1790,7 @@ struct _virDomainDef { virDomainVcpuPinDefPtr emulatorpin; } cputune; - virDomainNumatuneDef numatune; + virNumatuneDef numatune; /* These 3 are based on virDomainLifeCycleAction enum flags */ int onReboot; @@ -2396,8 +2385,6 @@ VIR_ENUM_DECL(virDomainGraphicsSpicePlaybackCompression) VIR_ENUM_DECL(virDomainGraphicsSpiceStreamingMode) VIR_ENUM_DECL(virDomainGraphicsSpiceClipboardCopypaste) VIR_ENUM_DECL(virDomainGraphicsSpiceMouseMode) -VIR_ENUM_DECL(virDomainNumatuneMemMode) -VIR_ENUM_DECL(virDomainNumatuneMemPlacementMode) VIR_ENUM_DECL(virDomainHyperv) VIR_ENUM_DECL(virDomainRNGModel) VIR_ENUM_DECL(virDomainRNGBackend) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 1374470..a4e6801 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -254,8 +254,8 @@ virDomainNostateReasonTypeFromString; virDomainNostateReasonTypeToString; virDomainNumatuneMemModeTypeFromString; virDomainNumatuneMemModeTypeToString; -virDomainNumatuneMemPlacementModeTypeFromString; -virDomainNumatuneMemPlacementModeTypeToString; +virNumatuneMemPlacementModeTypeFromString; +virNumatuneMemPlacementModeTypeToString; virDomainObjAssignDef; virDomainObjCopyPersistentDef; virDomainObjGetPersistentDef; @@ -1547,6 +1547,7 @@ virNodeSuspendGetTargetMask; # util/virnuma.h virNumaGetAutoPlacementAdvice; +virNumaSetupMemoryPolicy; # util/virobject.h virClassForObject; diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c index bb4cd16..6c5b8c8 100644 --- a/src/lxc/lxc_controller.c +++ b/src/lxc/lxc_controller.c @@ -46,11 +46,6 @@ # include <cap-ng.h> #endif -#if WITH_NUMACTL -# define NUMA_VERSION1_COMPATIBILITY 1 -# include <numa.h> -#endif - #include "virerror.h" #include "virlog.h" #include "virutil.h" @@ -479,113 +474,6 @@ cleanup: return ret; } -#if WITH_NUMACTL -static int virLXCControllerSetupNUMAPolicy(virLXCControllerPtr ctrl, - virBitmapPtr nodemask) -{ - nodemask_t mask; - int mode = -1; - int node = -1; - int ret = -1; - int i = 0; - int maxnode = 0; - bool warned = false; - virDomainNumatuneDef numatune = ctrl->def->numatune; - virBitmapPtr tmp_nodemask = NULL; - - if (numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) { - if (!numatune.memory.nodemask) - return 0; - VIR_DEBUG("Set NUMA memory policy with specified nodeset"); - tmp_nodemask = numatune.memory.nodemask; - } else if (numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) { - VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad"); - tmp_nodemask = nodemask; - } else { - return 0; - } - - VIR_DEBUG("Setting NUMA memory policy"); - - if (numa_available() < 0) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - "%s", _("Host kernel is not aware of NUMA.")); - return -1; - } - - maxnode = numa_max_node() + 1; - - /* Convert nodemask to NUMA bitmask. */ - nodemask_zero(&mask); - i = -1; - while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) { - if (i > NUMA_NUM_NODES) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("Host cannot support NUMA node %d"), i); - return -1; - } - if (i > maxnode && !warned) { - VIR_WARN("nodeset is out of range, there is only %d NUMA " - "nodes on host", maxnode); - warned = true; - } - nodemask_set(&mask, i); - } - - mode = ctrl->def->numatune.memory.mode; - - if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { - numa_set_bind_policy(1); - numa_set_membind(&mask); - numa_set_bind_policy(0); - } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) { - int nnodes = 0; - for (i = 0; i < NUMA_NUM_NODES; i++) { - if (nodemask_isset(&mask, i)) { - node = i; - nnodes++; - } - } - - if (nnodes != 1) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - "%s", _("NUMA memory tuning in 'preferred' mode " - "only supports single node")); - goto cleanup; - } - - numa_set_bind_policy(0); - numa_set_preferred(node); - } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) { - numa_set_interleave_mask(&mask); - } else { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("Unable to set NUMA policy %s"), - virDomainNumatuneMemModeTypeToString(mode)); - goto cleanup; - } - - ret = 0; - -cleanup: - return ret; -} -#else -static int virLXCControllerSetupNUMAPolicy(virLXCControllerPtr ctrl, - virBitmapPtr nodemask ATTRIBUTE_UNUSED) -{ - if (ctrl->def->numatune.memory.nodemask) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("NUMA policy is not available on this platform")); - return -1; - } - - return 0; -} -#endif - /* * To be run while still single threaded @@ -684,7 +572,7 @@ static int virLXCControllerSetupResourceLimits(virLXCControllerPtr ctrl) int ret = -1; if (virLXCControllerGetNumadAdvice(ctrl, &nodemask) < 0 || - virLXCControllerSetupNUMAPolicy(ctrl, nodemask) < 0) + virNumaSetupMemoryPolicy(ctrl->def->numatune, nodemask) < 0) goto cleanup; if (virLXCControllerSetupCpuAffinity(ctrl) < 0) diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 9d6e88b..f2c5df8 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -423,12 +423,12 @@ int qemuSetupCgroup(virQEMUDriverPtr driver, if ((vm->def->numatune.memory.nodemask || (vm->def->numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) && + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) && vm->def->numatune.memory.mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) { char *mask = NULL; if (vm->def->numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) mask = virBitmapFormat(nodemask); else mask = virBitmapFormat(vm->def->numatune.memory.nodemask); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 96ba70a..f12e54a 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -45,11 +45,6 @@ #include "qemu_bridge_filter.h" #include "qemu_migration.h" -#if WITH_NUMACTL -# define NUMA_VERSION1_COMPATIBILITY 1 -# include <numa.h> -#endif - #include "datatypes.h" #include "virlog.h" #include "virerror.h" @@ -1791,120 +1786,6 @@ qemuProcessDetectVcpuPIDs(virQEMUDriverPtr driver, } -/* - * Set NUMA memory policy for qemu process, to be run between - * fork/exec of QEMU only. - */ -#if WITH_NUMACTL -static int -qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm, - virBitmapPtr nodemask) -{ - nodemask_t mask; - int mode = -1; - int node = -1; - int ret = -1; - int i = 0; - int maxnode = 0; - bool warned = false; - virDomainNumatuneDef numatune = vm->def->numatune; - virBitmapPtr tmp_nodemask = NULL; - - if (numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) { - if (!numatune.memory.nodemask) - return 0; - VIR_DEBUG("Set NUMA memory policy with specified nodeset"); - tmp_nodemask = numatune.memory.nodemask; - } else if (numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) { - VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad"); - tmp_nodemask = nodemask; - } else { - return 0; - } - - if (numa_available() < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - "%s", _("Host kernel is not aware of NUMA.")); - return -1; - } - - maxnode = numa_max_node() + 1; - /* Convert nodemask to NUMA bitmask. */ - nodemask_zero(&mask); - i = -1; - while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) { - if (i > NUMA_NUM_NODES) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Host cannot support NUMA node %d"), i); - return -1; - } - if (i > maxnode && !warned) { - VIR_WARN("nodeset is out of range, there is only %d NUMA " - "nodes on host", maxnode); - warned = true; - } - nodemask_set(&mask, i); - } - - mode = numatune.memory.mode; - - if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { - numa_set_bind_policy(1); - numa_set_membind(&mask); - numa_set_bind_policy(0); - } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) { - int nnodes = 0; - for (i = 0; i < NUMA_NUM_NODES; i++) { - if (nodemask_isset(&mask, i)) { - node = i; - nnodes++; - } - } - - if (nnodes != 1) { - virReportError(VIR_ERR_INTERNAL_ERROR, - "%s", _("NUMA memory tuning in 'preferred' mode " - "only supports single node")); - goto cleanup; - } - - numa_set_bind_policy(0); - numa_set_preferred(node); - } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) { - numa_set_interleave_mask(&mask); - } else { - /* XXX: Shouldn't go here, as we already do checking when - * parsing domain XML. - */ - virReportError(VIR_ERR_XML_ERROR, - "%s", _("Invalid mode for memory NUMA tuning.")); - goto cleanup; - } - - ret = 0; - -cleanup: - return ret; -} -#else -static int -qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm, - virBitmapPtr nodemask ATTRIBUTE_UNUSED) -{ - if (vm->def->numatune.memory.nodemask) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("libvirt is compiled without NUMA tuning support")); - - return -1; - } - - return 0; -} -#endif - - /* Helper to prepare cpumap for affinity setting, convert * NUMA nodeset into cpuset if @nodemask is not NULL, otherwise * just return a new allocated bitmap. @@ -2654,7 +2535,7 @@ static int qemuProcessHook(void *data) qemuProcessInitCpuAffinity(h->driver, h->vm, h->nodemask) < 0) goto cleanup; - if (qemuProcessInitNumaMemoryPolicy(h->vm, h->nodemask) < 0) + if (virNumaSetupMemoryPolicy(h->vm->def->numatune, h->nodemask) < 0) goto cleanup; ret = 0; diff --git a/src/util/virnuma.c b/src/util/virnuma.c index f6a6eb2..e60df42 100644 --- a/src/util/virnuma.c +++ b/src/util/virnuma.c @@ -21,12 +21,30 @@ #include <config.h> +#if WITH_NUMACTL +# define NUMA_VERSION1_COMPATIBILITY 1 +# include <numa.h> +#endif + #include "virnuma.h" #include "vircommand.h" #include "virerror.h" +#include "virlog.h" #define VIR_FROM_THIS VIR_FROM_NONE +VIR_ENUM_IMPL(virDomainNumatuneMemMode, + VIR_DOMAIN_NUMATUNE_MEM_LAST, + "strict", + "preferred", + "interleave"); + +VIR_ENUM_IMPL(virNumatuneMemPlacementMode, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_LAST, + "default", + "static", + "auto"); + #if HAVE_NUMAD char * virNumaGetAutoPlacementAdvice(unsigned short vcpus, @@ -59,3 +77,111 @@ virNumaGetAutoPlacementAdvice(unsigned short vcpus ATTRIBUTE_UNUSED, return NULL; } #endif + +#if WITH_NUMACTL +int +virNumaSetupMemoryPolicy(virNumatuneDef numatune, + virBitmapPtr nodemask) +{ + nodemask_t mask; + int mode = -1; + int node = -1; + int ret = -1; + int i = 0; + int maxnode = 0; + bool warned = false; + virBitmapPtr tmp_nodemask = NULL; + + if (numatune.memory.placement_mode == + VIR_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) { + if (!numatune.memory.nodemask) + return 0; + VIR_DEBUG("Set NUMA memory policy with specified nodeset"); + tmp_nodemask = numatune.memory.nodemask; + } else if (numatune.memory.placement_mode == + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) { + VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad"); + tmp_nodemask = nodemask; + } else { + return 0; + } + + if (numa_available() < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("Host kernel is not aware of NUMA.")); + return -1; + } + + maxnode = numa_max_node() + 1; + /* Convert nodemask to NUMA bitmask. */ + nodemask_zero(&mask); + i = -1; + while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) { + if (i > NUMA_NUM_NODES) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Host cannot support NUMA node %d"), i); + return -1; + } + if (i > maxnode && !warned) { + VIR_WARN("nodeset is out of range, there is only %d NUMA " + "nodes on host", maxnode); + warned = true; + } + nodemask_set(&mask, i); + } + + mode = numatune.memory.mode; + + if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { + numa_set_bind_policy(1); + numa_set_membind(&mask); + numa_set_bind_policy(0); + } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) { + int nnodes = 0; + for (i = 0; i < NUMA_NUM_NODES; i++) { + if (nodemask_isset(&mask, i)) { + node = i; + nnodes++; + } + } + + if (nnodes != 1) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("NUMA memory tuning in 'preferred' mode " + "only supports single node")); + goto cleanup; + } + + numa_set_bind_policy(0); + numa_set_preferred(node); + } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) { + numa_set_interleave_mask(&mask); + } else { + /* XXX: Shouldn't go here, as we already do checking when + * parsing domain XML. + */ + virReportError(VIR_ERR_XML_ERROR, + "%s", _("Invalid mode for memory NUMA tuning.")); + goto cleanup; + } + + ret = 0; + +cleanup: + return ret; +} +#else +int +virNumaSetupMemoryPolicy(virNumatuneDef numatune, + virBitmapPtr nodemask ATTRIBUTE_UNUSED) +{ + if (numatune.memory.nodemask) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("libvirt is compiled without NUMA tuning support")); + + return -1; + } + + return 0; +} +#endif diff --git a/src/util/virnuma.h b/src/util/virnuma.h index d3d7d3e..ef201b9 100644 --- a/src/util/virnuma.h +++ b/src/util/virnuma.h @@ -22,7 +22,37 @@ #ifndef __VIR_NUMA_H__ # define __VIR_NUMA_H__ +# include "internal.h" +# include "virbitmap.h" +# include "virutil.h" + +enum virNumatuneMemPlacementMode { + VIR_NUMATUNE_MEM_PLACEMENT_MODE_DEFAULT = 0, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_STATIC, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO, + + VIR_NUMATUNE_MEM_PLACEMENT_MODE_LAST +}; + +VIR_ENUM_DECL(virNumatuneMemPlacementMode) + +VIR_ENUM_DECL(virDomainNumatuneMemMode) + +typedef struct _virNumatuneDef virNumatuneDef; +typedef virNumatuneDef *virNumatuneDefPtr; +struct _virNumatuneDef { + struct { + virBitmapPtr nodemask; + int mode; + int placement_mode; /* enum virNumatuneMemPlacementMode */ + } memory; + + /* Future NUMA tuning related stuff should go here. */ +}; + char *virNumaGetAutoPlacementAdvice(unsigned short vcups, unsigned long long balloon); +int virNumaSetupMemoryPolicy(virNumatuneDef numatune, + virBitmapPtr nodemask); #endif /* __VIR_NUMA_H__ */ -- 1.7.11.7

于 2013/03/19/ 17:59, Daniel P. Berrange wrote:

On Tue, Mar 19, 2013 at 09:59:37AM +0800, Gao feng wrote:

...

Intend to reduce the redundant code,use virNumaSetupMemoryPolicy to replace virLXCControllerSetupNUMAPolicy and qemuProcessInitNumaMemoryPolicy.

This patch also moves the numa related codes to the file virnuma.c and virnuma.h

Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- src/conf/domain_conf.c | 17 ++----- src/conf/domain_conf.h | 17 +------ src/libvirt_private.syms | 5 +- src/lxc/lxc_controller.c | 114 +----------------------------------------- src/qemu/qemu_cgroup.c | 4 +- src/qemu/qemu_process.c | 121 +-------------------------------------------- src/util/virnuma.c | 126 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virnuma.h | 30 +++++++++++ 8 files changed, 168 insertions(+), 266 deletions(-)

...

diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 96f11ba..98c4745 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -47,6 +47,7 @@ # include "device_conf.h" # include "virbitmap.h" # include "virstoragefile.h" +# include "virnuma.h"

/* forward declarations of all device types, required by * virDomainDeviceDef @@ -1700,18 +1701,6 @@ virDomainVcpuPinDefPtr virDomainVcpuPinFindByVcpu(virDomainVcpuPinDefPtr *def, int nvcpupin, int vcpu);

-typedef struct _virDomainNumatuneDef virDomainNumatuneDef; -typedef virDomainNumatuneDef *virDomainNumatuneDefPtr; -struct _virDomainNumatuneDef { - struct { - virBitmapPtr nodemask; - int mode; - int placement_mode; /* enum virDomainNumatuneMemPlacementMode */ - } memory; - - /* Future NUMA tuning related stuff should go here. */ -}; - typedef struct _virBlkioDeviceWeight virBlkioDeviceWeight; typedef virBlkioDeviceWeight *virBlkioDeviceWeightPtr; struct _virBlkioDeviceWeight { @@ -1801,7 +1790,7 @@ struct _virDomainDef { virDomainVcpuPinDefPtr emulatorpin; } cputune;

- virDomainNumatuneDef numatune; + virNumatuneDef numatune;

/* These 3 are based on virDomainLifeCycleAction enum flags */ int onReboot; @@ -2396,8 +2385,6 @@ VIR_ENUM_DECL(virDomainGraphicsSpicePlaybackCompression) VIR_ENUM_DECL(virDomainGraphicsSpiceStreamingMode) VIR_ENUM_DECL(virDomainGraphicsSpiceClipboardCopypaste) VIR_ENUM_DECL(virDomainGraphicsSpiceMouseMode) -VIR_ENUM_DECL(virDomainNumatuneMemMode) -VIR_ENUM_DECL(virDomainNumatuneMemPlacementMode) VIR_ENUM_DECL(virDomainHyperv) VIR_ENUM_DECL(virDomainRNGModel) VIR_ENUM_DECL(virDomainRNGBackend) diff --git a/src/util/virnuma.h b/src/util/virnuma.h index d3d7d3e..ef201b9 100644 --- a/src/util/virnuma.h +++ b/src/util/virnuma.h @@ -22,7 +22,37 @@ #ifndef __VIR_NUMA_H__ # define __VIR_NUMA_H__

+# include "internal.h" +# include "virbitmap.h" +# include "virutil.h" + +enum virNumatuneMemPlacementMode { + VIR_NUMATUNE_MEM_PLACEMENT_MODE_DEFAULT = 0, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_STATIC, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO, + + VIR_NUMATUNE_MEM_PLACEMENT_MODE_LAST +};

This enum is previously defined in domain_conf.h, but you've not removed it from there, so now we have 2 enums for the same thing.

Oops,I forgot to remove it.

Also you've ignore my request to use virNumaTune as the capatilization and VIR_NUMA_TUNE for enums

Get it,if I use virNumatune as the capatilization,I need to change virnuma.[c|h] to virnumatune.[c|h]? Thanks for your comments.

...

+ +VIR_ENUM_DECL(virNumatuneMemPlacementMode) + +VIR_ENUM_DECL(virDomainNumatuneMemMode) + +typedef struct _virNumatuneDef virNumatuneDef; +typedef virNumatuneDef *virNumatuneDefPtr; +struct _virNumatuneDef { + struct { + virBitmapPtr nodemask; + int mode; + int placement_mode; /* enum virNumatuneMemPlacementMode */ + } memory; + + /* Future NUMA tuning related stuff should go here. */ +};

Again virNumaTune should be the name prefix

NACK,

Daniel

On Wed, Mar 20, 2013 at 09:26:36AM +0800, Gao feng wrote:

于 2013/03/19/ 17:59, Daniel P. Berrange wrote:

...

On Tue, Mar 19, 2013 at 09:59:37AM +0800, Gao feng wrote:

...

Intend to reduce the redundant code,use virNumaSetupMemoryPolicy to replace virLXCControllerSetupNUMAPolicy and qemuProcessInitNumaMemoryPolicy.

This patch also moves the numa related codes to the file virnuma.c and virnuma.h

Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- src/conf/domain_conf.c | 17 ++----- src/conf/domain_conf.h | 17 +------ src/libvirt_private.syms | 5 +- src/lxc/lxc_controller.c | 114 +----------------------------------------- src/qemu/qemu_cgroup.c | 4 +- src/qemu/qemu_process.c | 121 +-------------------------------------------- src/util/virnuma.c | 126 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virnuma.h | 30 +++++++++++ 8 files changed, 168 insertions(+), 266 deletions(-)

...

diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 96f11ba..98c4745 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -47,6 +47,7 @@ # include "device_conf.h" # include "virbitmap.h" # include "virstoragefile.h" +# include "virnuma.h"

/* forward declarations of all device types, required by * virDomainDeviceDef @@ -1700,18 +1701,6 @@ virDomainVcpuPinDefPtr virDomainVcpuPinFindByVcpu(virDomainVcpuPinDefPtr *def, int nvcpupin, int vcpu);

-typedef struct _virDomainNumatuneDef virDomainNumatuneDef; -typedef virDomainNumatuneDef *virDomainNumatuneDefPtr; -struct _virDomainNumatuneDef { - struct { - virBitmapPtr nodemask; - int mode; - int placement_mode; /* enum virDomainNumatuneMemPlacementMode */ - } memory; - - /* Future NUMA tuning related stuff should go here. */ -}; - typedef struct _virBlkioDeviceWeight virBlkioDeviceWeight; typedef virBlkioDeviceWeight *virBlkioDeviceWeightPtr; struct _virBlkioDeviceWeight { @@ -1801,7 +1790,7 @@ struct _virDomainDef { virDomainVcpuPinDefPtr emulatorpin; } cputune;

- virDomainNumatuneDef numatune; + virNumatuneDef numatune;

/* These 3 are based on virDomainLifeCycleAction enum flags */ int onReboot; @@ -2396,8 +2385,6 @@ VIR_ENUM_DECL(virDomainGraphicsSpicePlaybackCompression) VIR_ENUM_DECL(virDomainGraphicsSpiceStreamingMode) VIR_ENUM_DECL(virDomainGraphicsSpiceClipboardCopypaste) VIR_ENUM_DECL(virDomainGraphicsSpiceMouseMode) -VIR_ENUM_DECL(virDomainNumatuneMemMode) -VIR_ENUM_DECL(virDomainNumatuneMemPlacementMode) VIR_ENUM_DECL(virDomainHyperv) VIR_ENUM_DECL(virDomainRNGModel) VIR_ENUM_DECL(virDomainRNGBackend) diff --git a/src/util/virnuma.h b/src/util/virnuma.h index d3d7d3e..ef201b9 100644 --- a/src/util/virnuma.h +++ b/src/util/virnuma.h @@ -22,7 +22,37 @@ #ifndef __VIR_NUMA_H__ # define __VIR_NUMA_H__

+# include "internal.h" +# include "virbitmap.h" +# include "virutil.h" + +enum virNumatuneMemPlacementMode { + VIR_NUMATUNE_MEM_PLACEMENT_MODE_DEFAULT = 0, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_STATIC, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO, + + VIR_NUMATUNE_MEM_PLACEMENT_MODE_LAST +};

This enum is previously defined in domain_conf.h, but you've not removed it from there, so now we have 2 enums for the same thing.

Oops,I forgot to remove it.

...

Also you've ignore my request to use virNumaTune as the capatilization and VIR_NUMA_TUNE for enums

Get it,if I use virNumatune as the capatilization,I need to change virnuma.[c|h] to virnumatune.[c|h]?

No, do not rename the file; do not use virNumatune. The enum values should use VIR_NUMA_TUNE and functions should use virNumaTune Regards, Daniel -- |: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :| |: http://libvirt.org -o- http://virt-manager.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|

On 2013年03月18日 17:04, Gao feng wrote:

This patch adds cpuset cgroup support for LXC.

Signed-off-by: Gao feng<gaofeng@cn.fujitsu.com> --- src/lxc/lxc_cgroup.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++- src/lxc/lxc_cgroup.h | 2 +- src/lxc/lxc_controller.c | 2 +- 3 files changed, 55 insertions(+), 3 deletions(-)

diff --git PATCH v3src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index a075335..d131932 100644 --- PATCH v3src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -68,6 +68,55 @@ cleanup: }

+static int virLXCCgroupSetupCpusetTune(virDomainDefPtr def, + virCgroupPtr cgroup, + virBitmapPtr nodemask) +{ + int rc = 0; + char *mask = NULL; + + if (def->placement_mode != VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO&& + def->cpumask) { + mask = virBitmapFormat(def->cpumask); + if (!mask) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to convert cpumask")); + return -1; + } + + rc = virCgroupSetCpusetCpus(cgroup, mask); + VIR_FREE(mask); + if (rc< 0) + virReportSystemError(-rc, "%s", + _("Unable to set cpuset.cpus")); + } + + if ((def->numatune.memory.nodemask || + (def->numatune.memory.placement_mode == + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO))&& + def->numatune.memory.mode == VIR_NUMATUNE_MEM_STRICT) { + if (def->numatune.memory.placement_mode == + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) + mask = virBitmapFormat(nodemask); + else + mask = virBitmapFormat(def->numatune.memory.nodemask); + + if (!mask) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to convert memory nodemask")); + return -1; + } + + rc = virCgroupSetCpusetMems(cgroup, mask); + VIR_FREE(mask); + if (rc< 0) + virReportSystemError(-rc, "%s", _("Unable to set cpuset.mems")); + } + + return rc; +} + + static int virLXCCgroupSetupBlkioTune(virDomainDefPtr def, virCgroupPtr cgroup) { @@ -472,7 +521,7 @@ cleanup: }

-int virLXCCgroupSetup(virDomainDefPtr def) +int virLXCCgroupSetup(virDomainDefPtr def, virBitmapPtr nodemask) { virCgroupPtr driver = NULL; virCgroupPtr cgroup = NULL; @@ -497,6 +546,9 @@ int virLXCCgroupSetup(virDomainDefPtr def) if (virLXCCgroupSetupCpuTune(def, cgroup)< 0) goto cleanup;

+ if (virLXCCgroupSetupCpusetTune(def, cgroup, nodemask)< 0) + goto cleanup; + if (virLXCCgroupSetupBlkioTune(def, cgroup)< 0) goto cleanup;

diff --git PATCH v3src/lxc/lxc_cgroup.h b/src/lxc/lxc_cgroup.h index fff554b..29f21d6 100644 --- PATCH v3src/lxc/lxc_cgroup.h +++ b/src/lxc/lxc_cgroup.h @@ -26,7 +26,7 @@ # include "lxc_fuse.h" # include "virusb.h"

-int virLXCCgroupSetup(virDomainDefPtr def); +int virLXCCgroupSetup(virDomainDefPtr def, virBitmapPtr nodemask); int virLXCCgroupGetMeminfo(virLXCMeminfoPtr meminfo);

int diff --git PATCH v3src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c index 6c5b8c8..eea2595 100644 --- PATCH v3src/lxc/lxc_controller.c +++ b/src/lxc/lxc_controller.c @@ -578,7 +578,7 @@ static int virLXCControllerSetupResourceLimits(virLXCControllerPtr ctrl) if (virLXCControllerSetupCpuAffinity(ctrl)< 0) goto cleanup;

- if (virLXCCgroupSetup(ctrl->def)< 0) + if (virLXCCgroupSetup(ctrl->def, nodemask)< 0) goto cleanup;

ret = 0;

ACK