This patch includes NUMA topology info in the QEMU driver capabilities XML output. It also implements the free memory driver APIs. This is done with the LGPL'd numactl library. The configure script probes for it and only enables this functionality if it is found. The numactl library has been around for quite a while - RHEL-3 vintage at least configure.in | 39 ++++++++++++++++++++++++++++++ libvirt.spec.in | 3 ++ src/Makefile.am | 2 + src/qemu_conf.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/qemu_driver.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 178 insertions(+) Regards, Daniel Index: src/qemu_conf.c =================================================================== RCS file: /data/cvs/libvirt/src/qemu_conf.c,v retrieving revision 1.64 diff -u -p -r1.64 qemu_conf.c --- src/qemu_conf.c 15 May 2008 20:07:34 -0000 1.64 +++ src/qemu_conf.c 15 May 2008 21:07:42 -0000 @@ -42,6 +42,10 @@ #include <libxml/xpath.h> #include <libxml/uri.h> +#if HAVE_NUMACTL +#include <numa.h> +#endif + #include "libvirt/virterror.h" #include "qemu_conf.h" @@ -49,6 +53,7 @@ #include "buf.h" #include "conf.h" #include "util.h" +#include "memory.h" #include <verify.h> #define qemudLog(level, msg...) fprintf(stderr, msg) @@ -389,6 +394,67 @@ qemudCapsInitGuest(virCapsPtr caps, return 0; } +#if HAVE_NUMACTL +#define MAX_CPUS 4096 +#define MAX_CPUS_MASK_SIZE (sizeof(unsigned long)) +#define MAX_CPUS_MASK_LEN (MAX_CPUS / MAX_CPUS_MASK_SIZE) +#define MAX_CPUS_MASK_BYTES (MAX_CPUS / 8) +static int +qemudCapsInitNUMA(virCapsPtr caps) +{ + int n, i; + unsigned long *mask = NULL; + int ncpus; + int *cpus = NULL; + int ret = -1; + + fprintf(stderr, "Add numa\n"); + + if (numa_available() < 0) + return 0; + + fprintf(stderr, "Start\n"); + if (VIR_ALLOC_N(mask, MAX_CPUS_MASK_LEN) < 0) + goto cleanup; + + for (n = 0 ; n <= numa_max_node() ; n++) { + fprintf(stderr, "Do node %d\n", n); + + if (numa_node_to_cpus(n, mask, MAX_CPUS_MASK_BYTES) < 0) + goto cleanup; + + for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) + if ((mask[(i / MAX_CPUS_MASK_SIZE)] >> (i % MAX_CPUS_MASK_SIZE)) & 1) + ncpus++; + + if (VIR_ALLOC_N(cpus, ncpus) < 0) + goto cleanup; + + for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) + if ((mask[(i / MAX_CPUS_MASK_SIZE)] >> (i % MAX_CPUS_MASK_SIZE)) & 1) + cpus[ncpus++] = i; + + fprintf(stderr, "Do node %d %d\n", n, ncpus); + if (virCapabilitiesAddHostNUMACell(caps, + n, + ncpus, + cpus) < 0) + goto cleanup; + + VIR_FREE(cpus); + } + + ret = 0; + +cleanup: + VIR_FREE(cpus); + VIR_FREE(mask); + return ret; +} +#else +static int qemudCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; } +#endif + virCapsPtr qemudCapsInit(void) { struct utsname utsname; virCapsPtr caps; @@ -401,6 +467,9 @@ virCapsPtr qemudCapsInit(void) { 0, 0)) == NULL) goto no_memory; + if (qemudCapsInitNUMA(caps) < 0) + goto no_memory; + for (i = 0 ; i < (sizeof(arch_info_hvm)/sizeof(arch_info_hvm[0])) ; i++) if (qemudCapsInitGuest(caps, utsname.machine, Index: src/qemu_driver.c =================================================================== RCS file: /data/cvs/libvirt/src/qemu_driver.c,v retrieving revision 1.74 diff -u -p -r1.74 qemu_driver.c --- src/qemu_driver.c 15 May 2008 16:11:40 -0000 1.74 +++ src/qemu_driver.c 15 May 2008 21:07:50 -0000 @@ -47,6 +47,10 @@ #include <sys/wait.h> #include <libxml/uri.h> +#if HAVE_NUMACTL +#include <numa.h> +#endif + #include "libvirt/virterror.h" #include "event.h" @@ -1605,6 +1609,62 @@ static char *qemudGetCapabilities(virCon } +#if HAVE_NUMACTL +static int +qemudNodeGetCellsFreeMemory(virConnectPtr conn, + unsigned long long *freeMems, + int startCell, + int maxCells) +{ + int n, lastCell, numCells; + fprintf(stderr, "Foo %d %d\n", startCell, maxCells); + if (numa_available() < 0) { + qemudReportError(conn, NULL, NULL, VIR_ERR_NO_SUPPORT, + "%s", _("NUMA not supported on this host")); + return -1; + } + lastCell = startCell + maxCells - 1; + if (lastCell > numa_max_node()) + lastCell = numa_max_node(); + + for (numCells = 0, n = startCell ; n <= lastCell ; n++) { + long long mem; + if (numa_node_size64(n, &mem) < 0) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + "%s", _("Failed to query NUMA free memory")); + return -1; + } + fprintf(stderr, "baro %d %llu\n", n, mem); + freeMems[numCells++] = mem; + } + return numCells; +} + +static unsigned long long +qemudNodeGetFreeMemory (virConnectPtr conn) +{ + unsigned long long freeMem = 0; + int n; + if (numa_available() < 0) { + qemudReportError(conn, NULL, NULL, VIR_ERR_NO_SUPPORT, + "%s", _("NUMA not supported on this host")); + return -1; + } + + for (n = 0 ; n <= numa_max_node() ; n++) { + long long mem; + if (numa_node_size64(n, &mem) < 0) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + "%s", _("Failed to query NUMA free memory")); + return -1; + } + freeMem += mem; + } + + return freeMem; +} + +#endif static int qemudGetProcessInfo(unsigned long long *cpuTime, int pid) { char proc[PATH_MAX]; @@ -3168,8 +3228,13 @@ static virDriver qemuDriver = { NULL, /* domainMigrateFinish */ qemudDomainBlockStats, /* domainBlockStats */ qemudDomainInterfaceStats, /* domainInterfaceStats */ +#if HAVE_NUMACTL + qemudNodeGetCellsFreeMemory, /* nodeGetCellsFreeMemory */ + qemudNodeGetFreeMemory, /* getFreeMemory */ +#else NULL, /* nodeGetCellsFreeMemory */ NULL, /* getFreeMemory */ +#endif }; static virNetworkDriver qemuNetworkDriver = { Index: src/Makefile.am =================================================================== RCS file: /data/cvs/libvirt/src/Makefile.am,v retrieving revision 1.79 diff -u -p -r1.79 Makefile.am --- src/Makefile.am 29 Apr 2008 15:38:13 -0000 1.79 +++ src/Makefile.am 15 May 2008 21:07:57 -0000 @@ -9,6 +9,7 @@ INCLUDES = \ $(GNUTLS_CFLAGS) \ $(SASL_CFLAGS) \ $(SELINUX_CFLAGS) \ + $(NUMACTL_CFLAGS) \ -DBINDIR=\""$(libexecdir)"\" \ -DSBINDIR=\""$(sbindir)"\" \ -DSYSCONF_DIR="\"$(sysconfdir)\"" \ @@ -100,6 +101,7 @@ endif libvirt_la_SOURCES = $(CLIENT_SOURCES) $(SERVER_SOURCES) libvirt_la_LIBADD = $(LIBXML_LIBS) $(GNUTLS_LIBS) $(SASL_LIBS) $(SELINUX_LIBS) \ + $(NUMACTL_LIBS) \ @CYGWIN_EXTRA_LIBADD@ ../gnulib/lib/libgnu.la libvirt_la_LDFLAGS = -Wl,--version-script=$(srcdir)/libvirt_sym.version \ -version-info @LIBVIRT_VERSION_INFO@ \ Index: configure.in =================================================================== RCS file: /data/cvs/libvirt/configure.in,v retrieving revision 1.143 diff -u -p -r1.143 configure.in --- configure.in 5 May 2008 19:58:56 -0000 1.143 +++ configure.in 15 May 2008 21:08:05 -0000 @@ -534,6 +534,40 @@ AM_CONDITIONAL(HAVE_SELINUX, [test "$wit AC_SUBST(SELINUX_CFLAGS) AC_SUBST(SELINUX_LIBS) +dnl NUMA lib +AC_ARG_WITH(numactl, + [ --with-numactl use numactl for host topology info], + [], + [with_numactl=check]) + +NUMACTL_CFLAGS= +NUMACTL_LIBS= +if test "$with_qemu" = "yes" -a "$with_numactl" != "no"; then + old_cflags="$CFLAGS" + old_libs="$LIBS" + if test "$with_numactl" = "check"; then + AC_CHECK_HEADER([numa.h],[],[with_numactl=no]) + AC_CHECK_LIB(numa, numa_available,[],[with_numactl=no]) + if test "$with_numactl" != "no"; then + with_numactl="yes" + fi + else + AC_CHECK_HEADER([numa.h],[], + [AC_MSG_ERROR([You must install the numactl development package in order to compile libvirt])]) + AC_CHECK_LIB(numa, numa_available,[], + [AC_MSG_ERROR([You must install the numactl development package in order to compile and run libvirt])]) + fi + CFLAGS="$old_cflags" + LIBS="$old_libs" +fi +if test "$with_numactl" = "yes"; then + NUMACTL_LIBS="-lnuma" + AC_DEFINE_UNQUOTED(HAVE_NUMACTL, 1, [whether Numactl is available for security]) +fi +AM_CONDITIONAL(HAVE_NUMACTL, [test "$with_numactl" != "no"]) +AC_SUBST(NUMACTL_CFLAGS) +AC_SUBST(NUMACTL_LIBS) + dnl virsh libraries AC_CHECK_HEADERS([readline/readline.h]) @@ -1001,6 +1035,11 @@ AC_MSG_NOTICE([ selinux: $SELINUX_CFLAG else AC_MSG_NOTICE([ selinux: no]) fi +if test "$with_numactl" = "yes" ; then +AC_MSG_NOTICE([ numactl: $NUMACTL_CFLAGS $NUMACTL_LIBS]) +else +AC_MSG_NOTICE([ numactl: no]) +fi AC_MSG_NOTICE([]) AC_MSG_NOTICE([Miscellaneous]) AC_MSG_NOTICE([]) Index: libvirt.spec.in =================================================================== RCS file: /data/cvs/libvirt/libvirt.spec.in,v retrieving revision 1.83 diff -u -p -r1.83 libvirt.spec.in --- libvirt.spec.in 8 Apr 2008 16:45:57 -0000 1.83 +++ libvirt.spec.in 15 May 2008 21:08:14 -0000 @@ -67,6 +67,9 @@ BuildRequires: dnsmasq BuildRequires: bridge-utils BuildRequires: qemu BuildRequires: cyrus-sasl-devel +%if %{with_qemu} +BuildRequires: numactl-devel +%endif %if %{with_polkit} BuildRequires: PolicyKit-devel >= 0.6 %endif -- |: Red Hat, Engineering, Boston -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|