--- docs/schemas/capability.rng | 15 +++ src/conf/capabilities.c | 65 ++++++++++--- src/conf/capabilities.h | 7 +- src/nodeinfo.c | 155 +++++++++++++++++++++++++++++- src/test/test_driver.c | 2 +- src/xen/xend_internal.c | 4 +- tests/capabilityschemadata/caps-test3.xml | 88 +++++++++++++++++ 7 files changed, 319 insertions(+), 17 deletions(-) create mode 100644 tests/capabilityschemadata/caps-test3.xml diff --git a/docs/schemas/capability.rng b/docs/schemas/capability.rng index 8c928bc..1ab36ed 100644 --- a/docs/schemas/capability.rng +++ b/docs/schemas/capability.rng @@ -176,6 +176,10 @@ </attribute> <optional> + <ref name='memory'/> + </optional> + + <optional> <element name='cpus'> <attribute name='num'> <ref name='unsignedInt'/> @@ -188,11 +192,22 @@ </element> </define> + <define name='memory'> + <element name='memory'> + <ref name='scaledInteger'/> + </element> + </define> + <define name='cpu'> <element name='cpu'> <attribute name='id'> <ref name='unsignedInt'/> </attribute> + <optional> + <attribute name='thread_siblings'> + <text/> + </attribute> + </optional> </element> </define> diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c index a8ee2cf..1b514ef 100644 --- a/src/conf/capabilities.c +++ b/src/conf/capabilities.c @@ -73,10 +73,16 @@ virCapabilitiesNew(const char *arch, static void virCapabilitiesFreeHostNUMACell(virCapsHostNUMACellPtr cell) { + int i; if (cell == NULL) return; + if (cell->threadSiblings) + for (i=0; i < cell->ncpus; i++) + VIR_FREE(cell->threadSiblings[i]); + VIR_FREE(cell->cpus); + VIR_FREE(cell->threadSiblings); VIR_FREE(cell); } @@ -253,7 +259,10 @@ int virCapabilitiesAddHostNUMACell(virCapsPtr caps, int num, int ncpus, - const int *cpus) + unsigned long long mem, + const int *cpus, + const virBitmapPtr *threadSiblings +) { virCapsHostNUMACellPtr cell; @@ -272,8 +281,20 @@ virCapabilitiesAddHostNUMACell(virCapsPtr caps, cpus, ncpus * sizeof(*cpus)); + if (threadSiblings) { + if (VIR_ALLOC_N(cell->threadSiblings, ncpus) < 0) { + VIR_FREE(cell->cpus); + VIR_FREE(cell); + return -1; + } + memcpy(cell->threadSiblings, + threadSiblings, + ncpus * sizeof(*threadSiblings)); + } + cell->ncpus = ncpus; cell->num = num; + cell->mem = mem; caps->host.numaCell[caps->host.nnumaCell++] = cell; @@ -695,6 +716,7 @@ virCapabilitiesFormatXML(virCapsPtr caps) virBuffer xml = VIR_BUFFER_INITIALIZER; int i, j, k; char host_uuid[VIR_UUID_STRING_BUFLEN]; + char *str; virBufferAddLit(&xml, "<capabilities>\n\n"); virBufferAddLit(&xml, " <host>\n"); @@ -754,22 +776,43 @@ virCapabilitiesFormatXML(virCapsPtr caps) } if (caps->host.nnumaCell) { - virBufferAddLit(&xml, " <topology>\n"); - virBufferAsprintf(&xml, " <cells num='%zu'>\n", + virBufferAdjustIndent(&xml, 4); + virBufferAddLit(&xml, "<topology>\n"); + virBufferAsprintf(&xml, " <cells num='%zu'>\n", caps->host.nnumaCell); for (i = 0 ; i < caps->host.nnumaCell ; i++) { - virBufferAsprintf(&xml, " <cell id='%d'>\n", + virBufferAsprintf(&xml, " <cell id='%d'>\n", caps->host.numaCell[i]->num); - virBufferAsprintf(&xml, " <cpus num='%d'>\n", + + /* Print out the numacell memory total if it is available */ + if (caps->host.numaCell[i]->mem) + virBufferAsprintf(&xml, " <memory unit='KiB'>%llu</memory>\n", + caps->host.numaCell[i]->mem); + + virBufferAsprintf(&xml, " <cpus num='%d'>\n", caps->host.numaCell[i]->ncpus); - for (j = 0 ; j < caps->host.numaCell[i]->ncpus ; j++) - virBufferAsprintf(&xml, " <cpu id='%d'/>\n", + for (j = 0 ; j < caps->host.numaCell[i]->ncpus ; j++) { + virBufferAsprintf(&xml, " <cpu id='%d'", caps->host.numaCell[i]->cpus[j]); - virBufferAddLit(&xml, " </cpus>\n"); - virBufferAddLit(&xml, " </cell>\n"); + + /* Print out thread siblings if they were populated */ + if (caps->host.numaCell[i]->threadSiblings) { + str = virBitmapFormat(caps->host.numaCell[i]->threadSiblings[j]); + if (str) { + virBufferAsprintf(&xml, " thread_siblings='%s'", str); + VIR_FREE(str); + } + } + + virBufferAsprintf(&xml, "/>\n"); + } + + virBufferAddLit(&xml, " </cpus>\n"); + virBufferAddLit(&xml, " </cell>\n"); } - virBufferAddLit(&xml, " </cells>\n"); - virBufferAddLit(&xml, " </topology>\n"); + virBufferAddLit(&xml, " </cells>\n"); + virBufferAddLit(&xml, "</topology>\n"); + virBufferAdjustIndent(&xml, -4); } for (i = 0; i < caps->host.nsecModels; i++) { diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h index 99056f8..ded0d57 100644 --- a/src/conf/capabilities.h +++ b/src/conf/capabilities.h @@ -88,7 +88,9 @@ typedef virCapsHostNUMACell *virCapsHostNUMACellPtr; struct _virCapsHostNUMACell { int num; int ncpus; + unsigned long long mem; /* in kibibytes */ int *cpus; + virBitmapPtr *threadSiblings; }; typedef struct _virCapsHostSecModel virCapsHostSecModel; @@ -200,7 +202,10 @@ extern int virCapabilitiesAddHostNUMACell(virCapsPtr caps, int num, int ncpus, - const int *cpus); + unsigned long long mem, + const int *cpus, + const virBitmapPtr *threadSiblings +); extern int diff --git a/src/nodeinfo.c b/src/nodeinfo.c index 36fbd66..cc42c43 100644 --- a/src/nodeinfo.c +++ b/src/nodeinfo.c @@ -56,6 +56,7 @@ #ifdef __linux__ # define CPUINFO_PATH "/proc/cpuinfo" # define SYSFS_SYSTEM_PATH "/sys/devices/system" +# define SYSFS_CPU_PATH "/sys/devices/system/cpu" # define PROCSTAT_PATH "/proc/stat" # define MEMINFO_PATH "/proc/meminfo" # define SYSFS_MEMORY_SHARED_PATH "/sys/kernel/mm/ksm" @@ -77,6 +78,7 @@ static int linuxNodeGetMemoryStats(FILE *meminfo, int cellNum, virNodeMemoryStatsPtr params, int *nparams); +static unsigned long long nodeGetCellMemory(int cell); /* Return the positive decimal contents of the given * DIR/cpu%u/FILE, or -1 on error. If MISSING_OK and the @@ -125,6 +127,66 @@ cleanup: return value; } +/** + * virNodeGetThreadSiblingsList + * @dir: directory where cpu0-N files are located. + * @cpu: the specific cpu to get the siblings for. + * + * Will open the "thread_siblings_list" file for the cpu and + * return a string representing the contents. The contents of the + * file is a string represnting the cpus that are siblings; like + * 1,9 or 1-4. + * + * Returns NULL on failure, char * on success + * + * Note: Responsibility of caller to free string + */ +static char *virNodeGetThreadSiblingsList(const char *dir, unsigned int cpu) +{ + char *path; + FILE *pathfp; + char *str = NULL; + int strsize; + + if (virAsprintf(&path, "%s/cpu%u/topology/thread_siblings_list", + dir, cpu) < 0) { + virReportOOMError(); + return str; + } + + pathfp = fopen(path, "r"); + if (pathfp == NULL) { + virReportSystemError(errno, _("cannot open %s"), path); + VIR_FREE(path); + return str; + } + + /* Detect how large of a string we need to make */ + fseek(pathfp, 0L, SEEK_END); + strsize = ftell(pathfp); + fseek(pathfp, 0L, SEEK_SET); + + + if (VIR_ALLOC_N(str, strsize) < 0) { + virReportOOMError(); + str = NULL; + goto cleanup; + } + + if (fgets(str, strsize, pathfp) == NULL) { + virReportSystemError(errno, _("cannot read from %s"), path); + VIR_FREE(str); + str = NULL; + goto cleanup; + } + +cleanup: + VIR_FORCE_FCLOSE(pathfp); + VIR_FREE(path); + + return str; +} + static unsigned long virNodeCountThreadSiblings(const char *dir, unsigned int cpu) { @@ -1313,9 +1375,14 @@ nodeCapsInitNUMA(virCapsPtr caps) int n; unsigned long *mask = NULL; unsigned long *allonesmask = NULL; + unsigned long long memory; int *cpus = NULL; int ret = -1; int max_n_cpus = NUMA_MAX_N_CPUS; + char *str = NULL; + virBitmapPtr *threadSiblings = NULL; + + if (numa_available() < 0) return 0; @@ -1343,29 +1410,56 @@ nodeCapsInitNUMA(virCapsPtr caps) continue; } + /* Detect the amount of memory in the numa cell */ + memory = nodeGetCellMemory(n); + if (memory == 0) + goto cleanup; + for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) if (MASK_CPU_ISSET(mask, i)) ncpus++; + /* Create some memory for the array of cpus. */ if (VIR_ALLOC_N(cpus, ncpus) < 0) goto cleanup; - for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) - if (MASK_CPU_ISSET(mask, i)) + /* Create some memory for the array of siblings. */ + if (VIR_ALLOC_N(threadSiblings, ncpus) < 0) + goto cleanup; + + for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) { + if (MASK_CPU_ISSET(mask, i)) { + + /* Get the string of thread siblings for this cpu */ + if (!(str = virNodeGetThreadSiblingsList(SYSFS_CPU_PATH, i))) + goto cleanup; + + /* Convert the string to a bitmap to be stored */ + if (virBitmapParse(str, 0, &threadSiblings[ncpus], max_n_cpus) < 0) + goto cleanup; + cpus[ncpus++] = i; + VIR_FREE(str); + } + } if (virCapabilitiesAddHostNUMACell(caps, n, ncpus, - cpus) < 0) + memory, + cpus, + threadSiblings) < 0) goto cleanup; + VIR_FREE(threadSiblings); VIR_FREE(cpus); } ret = 0; cleanup: + VIR_FREE(str); + VIR_FREE(threadSiblings); VIR_FREE(cpus); VIR_FREE(mask); VIR_FREE(allonesmask); @@ -1441,6 +1535,54 @@ cleanup: return freeMem; } +/** + * nodeGetCellMemory + * @cell: The number of the numa cell to get memory info for. + * + * Will call the numa_node_size64() function from libnuma to get + * the amount of total memory in bytes. It is then converted to + * KiB and returned. + * + * Returns 0 on failure, amount of memory in KiB on success. + */ +static unsigned long long +nodeGetCellMemory(int cell) +{ + long long mem; + unsigned long long memKiB = 0; + int maxCell; + + if (numa_available() < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("NUMA not supported on this host")); + goto cleanup; + } + + /* Make sure the provided cell number is valid. */ + maxCell = numa_max_node(); + if (cell > maxCell) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("cell %d out of range (0-%d)"), + cell, maxCell); + goto cleanup; + } + + /* Get the amount of memory(bytes) in the node */ + mem = numa_node_size64(cell, NULL); + if (mem < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to query NUMA total memory for node: %d"), + cell); + goto cleanup; + } + + /* Convert the memory from bytes to KiB */ + memKiB = mem >> 10; + +cleanup: + return memKiB; +} + #else int nodeCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; @@ -1462,4 +1604,11 @@ unsigned long long nodeGetFreeMemory(virConnectPtr conn ATTRIBUTE_UNUSED) _("NUMA memory information not available on this platform")); return 0; } + +static unsigned long long nodeGetCellMemory(int cell) +{ + virReportError(VIR_ERR_NO_SUPPORT, "%s", + _("NUMA memory information not available on this platform")); + return 0; +} #endif diff --git a/src/test/test_driver.c b/src/test/test_driver.c index c974e1a..9d715a0 100644 --- a/src/test/test_driver.c +++ b/src/test/test_driver.c @@ -174,7 +174,7 @@ testBuildCapabilities(virConnectPtr conn) { for (i = 0; i < privconn->numCells; i++) { if (virCapabilitiesAddHostNUMACell(caps, i, privconn->cells[i].numCpus, - privconn->cells[i].cpus) < 0) + 0, privconn->cells[i].cpus, NULL) < 0) goto no_memory; } diff --git a/src/xen/xend_internal.c b/src/xen/xend_internal.c index 922c571..d18eecb 100644 --- a/src/xen/xend_internal.c +++ b/src/xen/xend_internal.c @@ -1167,7 +1167,9 @@ sexpr_to_xend_topology(const struct sexpr *root, if (virCapabilitiesAddHostNUMACell(caps, cell, nb_cpus, - cpuNums) < 0) + 0, + cpuNums, + NULL) < 0) goto memory_error; } VIR_FREE(cpuNums); diff --git a/tests/capabilityschemadata/caps-test3.xml b/tests/capabilityschemadata/caps-test3.xml new file mode 100644 index 0000000..724ced4 --- /dev/null +++ b/tests/capabilityschemadata/caps-test3.xml @@ -0,0 +1,88 @@ +<capabilities> + + <host> + <uuid>35383339-3134-5553-4531-30314e394a50</uuid> + <cpu> + <arch>x86_64</arch> + <model>Westmere</model> + <vendor>Intel</vendor> + <topology sockets='1' cores='6' threads='2'/> + <feature name='rdtscp'/> + <feature name='pdpe1gb'/> + <feature name='dca'/> + <feature name='pdcm'/> + <feature name='xtpr'/> + <feature name='tm2'/> + <feature name='est'/> + <feature name='smx'/> + <feature name='vmx'/> + <feature name='ds_cpl'/> + <feature name='monitor'/> + <feature name='dtes64'/> + <feature name='pclmuldq'/> + <feature name='pbe'/> + <feature name='tm'/> + <feature name='ht'/> + <feature name='ss'/> + <feature name='acpi'/> + <feature name='ds'/> + <feature name='vme'/> + </cpu> + <power_management> + <suspend_disk/> + </power_management> + <migration_features> + <live/> + <uri_transports> + <uri_transport>tcp</uri_transport> + </uri_transports> + </migration_features> + <topology> + <cells num='2'> + <cell id='0'> + <memory unit='KiB'>12572412</memory> + <cpus num='12'> + <cpu id='0' thread_siblings='0,12'/> + <cpu id='2' thread_siblings='2,14'/> + <cpu id='4' thread_siblings='4,16'/> + <cpu id='6' thread_siblings='6,18'/> + <cpu id='8' thread_siblings='8,20'/> + <cpu id='10' thread_siblings='10,22'/> + <cpu id='12' thread_siblings='0,12'/> + <cpu id='14' thread_siblings='2,14'/> + <cpu id='16' thread_siblings='4,16'/> + <cpu id='18' thread_siblings='6,18'/> + <cpu id='20' thread_siblings='8,20'/> + <cpu id='22' thread_siblings='10,22'/> + </cpus> + </cell> + <cell id='1'> + <memory unit='KiB'>12582908</memory> + <cpus num='12'> + <cpu id='1' thread_siblings='1,13'/> + <cpu id='3' thread_siblings='3,15'/> + <cpu id='5' thread_siblings='5,17'/> + <cpu id='7' thread_siblings='7,19'/> + <cpu id='9' thread_siblings='9,21'/> + <cpu id='11' thread_siblings='11,23'/> + <cpu id='13' thread_siblings='1,13'/> + <cpu id='15' thread_siblings='3,15'/> + <cpu id='17' thread_siblings='5,17'/> + <cpu id='19' thread_siblings='7,19'/> + <cpu id='21' thread_siblings='9,21'/> + <cpu id='23' thread_siblings='11,23'/> + </cpus> + </cell> + </cells> + </topology> + <secmodel> + <model>none</model> + <doi>0</doi> + </secmodel> + <secmodel> + <model>dac</model> + <doi>0</doi> + </secmodel> + </host> + +</capabilities> -- 1.7.11.7