[libvirt] [PATCH] Add NUMA memory and CPU thread siblings to capabilities
This patch is a follow up to https://www.redhat.com/archives/libvir-list/2012-October/msg01382.html. Since that one hasn't been reviewed yet I combined that work with new work to add memory information to virsh capabilities output and am submitting them both in this patch. With this virsh capabilities output will have the following form: <topology> <cells num='2'> <cell id='0'> <memory unit='KiB'>12572412</memory> <cpus num='12'> <cpu id='0' thread_siblings='0,12'/> . . </cpus> </cell> </cells> </topology> Hope you guys can find this useful. Dusty Dusty Mabe (1): Add NUMA memory and CPU thread siblings to capabilities docs/schemas/capability.rng | 15 +++ src/conf/capabilities.c | 65 ++++++++++--- src/conf/capabilities.h | 7 +- src/nodeinfo.c | 155 +++++++++++++++++++++++++++++- src/test/test_driver.c | 2 +- src/xen/xend_internal.c | 4 +- tests/capabilityschemadata/caps-test3.xml | 88 +++++++++++++++++ 7 files changed, 319 insertions(+), 17 deletions(-) create mode 100644 tests/capabilityschemadata/caps-test3.xml -- 1.7.11.7
--- docs/schemas/capability.rng | 15 +++ src/conf/capabilities.c | 65 ++++++++++--- src/conf/capabilities.h | 7 +- src/nodeinfo.c | 155 +++++++++++++++++++++++++++++- src/test/test_driver.c | 2 +- src/xen/xend_internal.c | 4 +- tests/capabilityschemadata/caps-test3.xml | 88 +++++++++++++++++ 7 files changed, 319 insertions(+), 17 deletions(-) create mode 100644 tests/capabilityschemadata/caps-test3.xml diff --git a/docs/schemas/capability.rng b/docs/schemas/capability.rng index 8c928bc..1ab36ed 100644 --- a/docs/schemas/capability.rng +++ b/docs/schemas/capability.rng @@ -176,6 +176,10 @@ </attribute> <optional> + <ref name='memory'/> + </optional> + + <optional> <element name='cpus'> <attribute name='num'> <ref name='unsignedInt'/> @@ -188,11 +192,22 @@ </element> </define> + <define name='memory'> + <element name='memory'> + <ref name='scaledInteger'/> + </element> + </define> + <define name='cpu'> <element name='cpu'> <attribute name='id'> <ref name='unsignedInt'/> </attribute> + <optional> + <attribute name='thread_siblings'> + <text/> + </attribute> + </optional> </element> </define> diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c index a8ee2cf..1b514ef 100644 --- a/src/conf/capabilities.c +++ b/src/conf/capabilities.c @@ -73,10 +73,16 @@ virCapabilitiesNew(const char *arch, static void virCapabilitiesFreeHostNUMACell(virCapsHostNUMACellPtr cell) { + int i; if (cell == NULL) return; + if (cell->threadSiblings) + for (i=0; i < cell->ncpus; i++) + VIR_FREE(cell->threadSiblings[i]); + VIR_FREE(cell->cpus); + VIR_FREE(cell->threadSiblings); VIR_FREE(cell); } @@ -253,7 +259,10 @@ int virCapabilitiesAddHostNUMACell(virCapsPtr caps, int num, int ncpus, - const int *cpus) + unsigned long long mem, + const int *cpus, + const virBitmapPtr *threadSiblings +) { virCapsHostNUMACellPtr cell; @@ -272,8 +281,20 @@ virCapabilitiesAddHostNUMACell(virCapsPtr caps, cpus, ncpus * sizeof(*cpus)); + if (threadSiblings) { + if (VIR_ALLOC_N(cell->threadSiblings, ncpus) < 0) { + VIR_FREE(cell->cpus); + VIR_FREE(cell); + return -1; + } + memcpy(cell->threadSiblings, + threadSiblings, + ncpus * sizeof(*threadSiblings)); + } + cell->ncpus = ncpus; cell->num = num; + cell->mem = mem; caps->host.numaCell[caps->host.nnumaCell++] = cell; @@ -695,6 +716,7 @@ virCapabilitiesFormatXML(virCapsPtr caps) virBuffer xml = VIR_BUFFER_INITIALIZER; int i, j, k; char host_uuid[VIR_UUID_STRING_BUFLEN]; + char *str; virBufferAddLit(&xml, "<capabilities>\n\n"); virBufferAddLit(&xml, " <host>\n"); @@ -754,22 +776,43 @@ virCapabilitiesFormatXML(virCapsPtr caps) } if (caps->host.nnumaCell) { - virBufferAddLit(&xml, " <topology>\n"); - virBufferAsprintf(&xml, " <cells num='%zu'>\n", + virBufferAdjustIndent(&xml, 4); + virBufferAddLit(&xml, "<topology>\n"); + virBufferAsprintf(&xml, " <cells num='%zu'>\n", caps->host.nnumaCell); for (i = 0 ; i < caps->host.nnumaCell ; i++) { - virBufferAsprintf(&xml, " <cell id='%d'>\n", + virBufferAsprintf(&xml, " <cell id='%d'>\n", caps->host.numaCell[i]->num); - virBufferAsprintf(&xml, " <cpus num='%d'>\n", + + /* Print out the numacell memory total if it is available */ + if (caps->host.numaCell[i]->mem) + virBufferAsprintf(&xml, " <memory unit='KiB'>%llu</memory>\n", + caps->host.numaCell[i]->mem); + + virBufferAsprintf(&xml, " <cpus num='%d'>\n", caps->host.numaCell[i]->ncpus); - for (j = 0 ; j < caps->host.numaCell[i]->ncpus ; j++) - virBufferAsprintf(&xml, " <cpu id='%d'/>\n", + for (j = 0 ; j < caps->host.numaCell[i]->ncpus ; j++) { + virBufferAsprintf(&xml, " <cpu id='%d'", caps->host.numaCell[i]->cpus[j]); - virBufferAddLit(&xml, " </cpus>\n"); - virBufferAddLit(&xml, " </cell>\n"); + + /* Print out thread siblings if they were populated */ + if (caps->host.numaCell[i]->threadSiblings) { + str = virBitmapFormat(caps->host.numaCell[i]->threadSiblings[j]); + if (str) { + virBufferAsprintf(&xml, " thread_siblings='%s'", str); + VIR_FREE(str); + } + } + + virBufferAsprintf(&xml, "/>\n"); + } + + virBufferAddLit(&xml, " </cpus>\n"); + virBufferAddLit(&xml, " </cell>\n"); } - virBufferAddLit(&xml, " </cells>\n"); - virBufferAddLit(&xml, " </topology>\n"); + virBufferAddLit(&xml, " </cells>\n"); + virBufferAddLit(&xml, "</topology>\n"); + virBufferAdjustIndent(&xml, -4); } for (i = 0; i < caps->host.nsecModels; i++) { diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h index 99056f8..ded0d57 100644 --- a/src/conf/capabilities.h +++ b/src/conf/capabilities.h @@ -88,7 +88,9 @@ typedef virCapsHostNUMACell *virCapsHostNUMACellPtr; struct _virCapsHostNUMACell { int num; int ncpus; + unsigned long long mem; /* in kibibytes */ int *cpus; + virBitmapPtr *threadSiblings; }; typedef struct _virCapsHostSecModel virCapsHostSecModel; @@ -200,7 +202,10 @@ extern int virCapabilitiesAddHostNUMACell(virCapsPtr caps, int num, int ncpus, - const int *cpus); + unsigned long long mem, + const int *cpus, + const virBitmapPtr *threadSiblings +); extern int diff --git a/src/nodeinfo.c b/src/nodeinfo.c index 36fbd66..cc42c43 100644 --- a/src/nodeinfo.c +++ b/src/nodeinfo.c @@ -56,6 +56,7 @@ #ifdef __linux__ # define CPUINFO_PATH "/proc/cpuinfo" # define SYSFS_SYSTEM_PATH "/sys/devices/system" +# define SYSFS_CPU_PATH "/sys/devices/system/cpu" # define PROCSTAT_PATH "/proc/stat" # define MEMINFO_PATH "/proc/meminfo" # define SYSFS_MEMORY_SHARED_PATH "/sys/kernel/mm/ksm" @@ -77,6 +78,7 @@ static int linuxNodeGetMemoryStats(FILE *meminfo, int cellNum, virNodeMemoryStatsPtr params, int *nparams); +static unsigned long long nodeGetCellMemory(int cell); /* Return the positive decimal contents of the given * DIR/cpu%u/FILE, or -1 on error. If MISSING_OK and the @@ -125,6 +127,66 @@ cleanup: return value; } +/** + * virNodeGetThreadSiblingsList + * @dir: directory where cpu0-N files are located. + * @cpu: the specific cpu to get the siblings for. + * + * Will open the "thread_siblings_list" file for the cpu and + * return a string representing the contents. The contents of the + * file is a string represnting the cpus that are siblings; like + * 1,9 or 1-4. + * + * Returns NULL on failure, char * on success + * + * Note: Responsibility of caller to free string + */ +static char *virNodeGetThreadSiblingsList(const char *dir, unsigned int cpu) +{ + char *path; + FILE *pathfp; + char *str = NULL; + int strsize; + + if (virAsprintf(&path, "%s/cpu%u/topology/thread_siblings_list", + dir, cpu) < 0) { + virReportOOMError(); + return str; + } + + pathfp = fopen(path, "r"); + if (pathfp == NULL) { + virReportSystemError(errno, _("cannot open %s"), path); + VIR_FREE(path); + return str; + } + + /* Detect how large of a string we need to make */ + fseek(pathfp, 0L, SEEK_END); + strsize = ftell(pathfp); + fseek(pathfp, 0L, SEEK_SET); + + + if (VIR_ALLOC_N(str, strsize) < 0) { + virReportOOMError(); + str = NULL; + goto cleanup; + } + + if (fgets(str, strsize, pathfp) == NULL) { + virReportSystemError(errno, _("cannot read from %s"), path); + VIR_FREE(str); + str = NULL; + goto cleanup; + } + +cleanup: + VIR_FORCE_FCLOSE(pathfp); + VIR_FREE(path); + + return str; +} + static unsigned long virNodeCountThreadSiblings(const char *dir, unsigned int cpu) { @@ -1313,9 +1375,14 @@ nodeCapsInitNUMA(virCapsPtr caps) int n; unsigned long *mask = NULL; unsigned long *allonesmask = NULL; + unsigned long long memory; int *cpus = NULL; int ret = -1; int max_n_cpus = NUMA_MAX_N_CPUS; + char *str = NULL; + virBitmapPtr *threadSiblings = NULL; + + if (numa_available() < 0) return 0; @@ -1343,29 +1410,56 @@ nodeCapsInitNUMA(virCapsPtr caps) continue; } + /* Detect the amount of memory in the numa cell */ + memory = nodeGetCellMemory(n); + if (memory == 0) + goto cleanup; + for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) if (MASK_CPU_ISSET(mask, i)) ncpus++; + /* Create some memory for the array of cpus. */ if (VIR_ALLOC_N(cpus, ncpus) < 0) goto cleanup; - for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) - if (MASK_CPU_ISSET(mask, i)) + /* Create some memory for the array of siblings. */ + if (VIR_ALLOC_N(threadSiblings, ncpus) < 0) + goto cleanup; + + for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) { + if (MASK_CPU_ISSET(mask, i)) { + + /* Get the string of thread siblings for this cpu */ + if (!(str = virNodeGetThreadSiblingsList(SYSFS_CPU_PATH, i))) + goto cleanup; + + /* Convert the string to a bitmap to be stored */ + if (virBitmapParse(str, 0, &threadSiblings[ncpus], max_n_cpus) < 0) + goto cleanup; + cpus[ncpus++] = i; + VIR_FREE(str); + } + } if (virCapabilitiesAddHostNUMACell(caps, n, ncpus, - cpus) < 0) + memory, + cpus, + threadSiblings) < 0) goto cleanup; + VIR_FREE(threadSiblings); VIR_FREE(cpus); } ret = 0; cleanup: + VIR_FREE(str); + VIR_FREE(threadSiblings); VIR_FREE(cpus); VIR_FREE(mask); VIR_FREE(allonesmask); @@ -1441,6 +1535,54 @@ cleanup: return freeMem; } +/** + * nodeGetCellMemory + * @cell: The number of the numa cell to get memory info for. + * + * Will call the numa_node_size64() function from libnuma to get + * the amount of total memory in bytes. It is then converted to + * KiB and returned. + * + * Returns 0 on failure, amount of memory in KiB on success. + */ +static unsigned long long +nodeGetCellMemory(int cell) +{ + long long mem; + unsigned long long memKiB = 0; + int maxCell; + + if (numa_available() < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("NUMA not supported on this host")); + goto cleanup; + } + + /* Make sure the provided cell number is valid. */ + maxCell = numa_max_node(); + if (cell > maxCell) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("cell %d out of range (0-%d)"), + cell, maxCell); + goto cleanup; + } + + /* Get the amount of memory(bytes) in the node */ + mem = numa_node_size64(cell, NULL); + if (mem < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to query NUMA total memory for node: %d"), + cell); + goto cleanup; + } + + /* Convert the memory from bytes to KiB */ + memKiB = mem >> 10; + +cleanup: + return memKiB; +} + #else int nodeCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; @@ -1462,4 +1604,11 @@ unsigned long long nodeGetFreeMemory(virConnectPtr conn ATTRIBUTE_UNUSED) _("NUMA memory information not available on this platform")); return 0; } + +static unsigned long long nodeGetCellMemory(int cell) +{ + virReportError(VIR_ERR_NO_SUPPORT, "%s", + _("NUMA memory information not available on this platform")); + return 0; +} #endif diff --git a/src/test/test_driver.c b/src/test/test_driver.c index c974e1a..9d715a0 100644 --- a/src/test/test_driver.c +++ b/src/test/test_driver.c @@ -174,7 +174,7 @@ testBuildCapabilities(virConnectPtr conn) { for (i = 0; i < privconn->numCells; i++) { if (virCapabilitiesAddHostNUMACell(caps, i, privconn->cells[i].numCpus, - privconn->cells[i].cpus) < 0) + 0, privconn->cells[i].cpus, NULL) < 0) goto no_memory; } diff --git a/src/xen/xend_internal.c b/src/xen/xend_internal.c index 922c571..d18eecb 100644 --- a/src/xen/xend_internal.c +++ b/src/xen/xend_internal.c @@ -1167,7 +1167,9 @@ sexpr_to_xend_topology(const struct sexpr *root, if (virCapabilitiesAddHostNUMACell(caps, cell, nb_cpus, - cpuNums) < 0) + 0, + cpuNums, + NULL) < 0) goto memory_error; } VIR_FREE(cpuNums); diff --git a/tests/capabilityschemadata/caps-test3.xml b/tests/capabilityschemadata/caps-test3.xml new file mode 100644 index 0000000..724ced4 --- /dev/null +++ b/tests/capabilityschemadata/caps-test3.xml @@ -0,0 +1,88 @@ +<capabilities> + + <host> + <uuid>35383339-3134-5553-4531-30314e394a50</uuid> + <cpu> + <arch>x86_64</arch> + <model>Westmere</model> + <vendor>Intel</vendor> + <topology sockets='1' cores='6' threads='2'/> + <feature name='rdtscp'/> + <feature name='pdpe1gb'/> + <feature name='dca'/> + <feature name='pdcm'/> + <feature name='xtpr'/> + <feature name='tm2'/> + <feature name='est'/> + <feature name='smx'/> + <feature name='vmx'/> + <feature name='ds_cpl'/> + <feature name='monitor'/> + <feature name='dtes64'/> + <feature name='pclmuldq'/> + <feature name='pbe'/> + <feature name='tm'/> + <feature name='ht'/> + <feature name='ss'/> + <feature name='acpi'/> + <feature name='ds'/> + <feature name='vme'/> + </cpu> + <power_management> + <suspend_disk/> + </power_management> + <migration_features> + <live/> + <uri_transports> + <uri_transport>tcp</uri_transport> + </uri_transports> + </migration_features> + <topology> + <cells num='2'> + <cell id='0'> + <memory unit='KiB'>12572412</memory> + <cpus num='12'> + <cpu id='0' thread_siblings='0,12'/> + <cpu id='2' thread_siblings='2,14'/> + <cpu id='4' thread_siblings='4,16'/> + <cpu id='6' thread_siblings='6,18'/> + <cpu id='8' thread_siblings='8,20'/> + <cpu id='10' thread_siblings='10,22'/> + <cpu id='12' thread_siblings='0,12'/> + <cpu id='14' thread_siblings='2,14'/> + <cpu id='16' thread_siblings='4,16'/> + <cpu id='18' thread_siblings='6,18'/> + <cpu id='20' thread_siblings='8,20'/> + <cpu id='22' thread_siblings='10,22'/> + </cpus> + </cell> + <cell id='1'> + <memory unit='KiB'>12582908</memory> + <cpus num='12'> + <cpu id='1' thread_siblings='1,13'/> + <cpu id='3' thread_siblings='3,15'/> + <cpu id='5' thread_siblings='5,17'/> + <cpu id='7' thread_siblings='7,19'/> + <cpu id='9' thread_siblings='9,21'/> + <cpu id='11' thread_siblings='11,23'/> + <cpu id='13' thread_siblings='1,13'/> + <cpu id='15' thread_siblings='3,15'/> + <cpu id='17' thread_siblings='5,17'/> + <cpu id='19' thread_siblings='7,19'/> + <cpu id='21' thread_siblings='9,21'/> + <cpu id='23' thread_siblings='11,23'/> + </cpus> + </cell> + </cells> + </topology> + <secmodel> + <model>none</model> + <doi>0</doi> + </secmodel> + <secmodel> + <model>dac</model> + <doi>0</doi> + </secmodel> + </host> + +</capabilities> -- 1.7.11.7
participants (1)
-
Dusty Mabe