[libvirt] [PATH V10 01/12] Resctrl: Add some utils functions

This patch adds some utils struct and functions to expose resctrl information. virResCtrlAvailable: if resctrl interface exist on host. virResCtrlGet: get specific type resource control information. virResCtrlInit: initialize resctrl struct from the host's sys fs. resctrlall[]: an array to maintain resource control information. Some of host cpu related information methods was added in virhostcpu.c Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com&gt; --- include/libvirt/virterror.h | 1 + po/POTFILES.in | 1 + src/Makefile.am | 1 + src/libvirt_private.syms | 4 + src/util/virerror.c | 1 + src/util/virhostcpu.c | 186 ++++++++++++++++++++++++++++++++++++---- src/util/virhostcpu.h | 6 ++ src/util/virresctrl.c | 201 ++++++++++++++++++++++++++++++++++++++++++++ src/util/virresctrl.h | 78 +++++++++++++++++ 9 files changed, 462 insertions(+), 17 deletions(-) create mode 100644 src/util/virresctrl.c create mode 100644 src/util/virresctrl.h diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h index 2efee8f..3dd2d08 100644 --- a/include/libvirt/virterror.h +++ b/include/libvirt/virterror.h @@ -132,6 +132,7 @@ typedef enum { VIR_FROM_PERF = 65, /* Error from perf */ VIR_FROM_LIBSSH = 66, /* Error from libssh connection transport */ + VIR_FROM_RESCTRL = 67, /* Error from resource control */ # ifdef VIR_ENUM_SENTINELS VIR_ERR_DOMAIN_LAST diff --git a/po/POTFILES.in b/po/POTFILES.in index 7c7f530..4147bc6 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -241,6 +241,7 @@ src/util/virportallocator.c src/util/virprocess.c src/util/virqemu.c src/util/virrandom.c +src/util/virresctrl.c src/util/virrotatingfile.c src/util/virscsi.c src/util/virscsihost.c diff --git a/src/Makefile.am b/src/Makefile.am index 7d42eac..edb946a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -162,6 +162,7 @@ UTIL_SOURCES = \ util/virprocess.c util/virprocess.h \ util/virqemu.c util/virqemu.h \ util/virrandom.h util/virrandom.c \ + util/virresctrl.h util/virresctrl.c \ util/virrotatingfile.h util/virrotatingfile.c \ util/virscsi.c util/virscsi.h \ util/virscsihost.c util/virscsihost.h \ diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index aed1d3d..bb7c3ad 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -2320,6 +2320,10 @@ virRandomGenerateWWN; virRandomInt; +# util/virresctrl.h +virResCtrlAvailable; +virResCtrlInit; + # util/virrotatingfile.h virRotatingFileReaderConsume; virRotatingFileReaderFree; diff --git a/src/util/virerror.c b/src/util/virerror.c index ef17fb5..0ba15e6 100644 --- a/src/util/virerror.c +++ b/src/util/virerror.c @@ -139,6 +139,7 @@ VIR_ENUM_IMPL(virErrorDomain, VIR_ERR_DOMAIN_LAST, "Perf", /* 65 */ "Libssh transport layer", + "Resouce Control", ) diff --git a/src/util/virhostcpu.c b/src/util/virhostcpu.c index f29f312..e6d5102 100644 --- a/src/util/virhostcpu.c +++ b/src/util/virhostcpu.c @@ -206,29 +206,21 @@ void virHostCPUSetSysFSSystemPathLinux(const char *path) sysfs_system_path = SYSFS_SYSTEM_PATH; } -/* Return the positive decimal contents of the given - * DIR/cpu%u/FILE, or -1 on error. If DEFAULT_VALUE is non-negative - * and the file could not be found, return that instead of an error; - * this is useful for machines that cannot hot-unplug cpu0, or where - * hot-unplugging is disabled, or where the kernel is too old - * to support NUMA cells, etc. */ +/* Get a String value*/ static int -virHostCPUGetValue(const char *dir, unsigned int cpu, const char *file, - int default_value) +virHostCPUGetStrValue(const char *dir, unsigned int cpu, const char *file, char *value_str) { char *path; FILE *pathfp; - int value = -1; - char value_str[INT_BUFSIZE_BOUND(value)]; - char *tmp; + int ret = -1; if (virAsprintf(&path, "%s/cpu%u/%s", dir, cpu, file) < 0) return -1; pathfp = fopen(path, "r"); if (pathfp == NULL) { - if (default_value >= 0 && errno == ENOENT) - value = default_value; + if (errno == ENOENT) + return -2; else virReportSystemError(errno, _("cannot open %s"), path); goto cleanup; @@ -238,17 +230,84 @@ virHostCPUGetValue(const char *dir, unsigned int cpu, const char *file, virReportSystemError(errno, _("cannot read from %s"), path); goto cleanup; } + + ret = 0; + + cleanup: + VIR_FORCE_FCLOSE(pathfp); + VIR_FREE(path); + return ret; +} + + +/* Return the positive decimal contents of the given + * DIR/cpu%u/FILE, or -1 on error. If DEFAULT_VALUE is non-negative + * and the file could not be found, return that instead of an error; + * this is useful for machines that cannot hot-unplug cpu0, or where + * hot-unplugging is disabled, or where the kernel is too old + * to support NUMA cells, etc. */ +static int +virHostCPUGetValue(const char *dir, unsigned int cpu, const char *file, + int default_value) +{ + int value = -1; + char value_str[INT_BUFSIZE_BOUND(value)]; + char *tmp; + int ret; + + if ((ret = (virHostCPUGetStrValue(dir, cpu, file, value_str))) < 0) { + if (ret == -2 && default_value >= 0) + return default_value; + else + return -1; + } + if (virStrToLong_i(value_str, &tmp, 10, &value) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, _("could not convert '%s' to an integer"), value_str); - goto cleanup; + return -1; } + return value; +} - cleanup: - VIR_FORCE_FCLOSE(pathfp); - VIR_FREE(path); +/* Return specific type cache size in KiB of given cpu + -1 on error happened */ +static +int virHostCPUGetCache(unsigned int cpu, unsigned int type) +{ + char *cachedir = NULL; + char *cpudir; + char *unit = NULL; + char *tmp; + int value = -1; + unsigned long long size; + char value_str[INT_BUFSIZE_BOUND(value)]; + if (virAsprintf(&cpudir, "%s/cpu", sysfs_system_path) < 0) + return -1; + + if (virAsprintf(&cachedir, "cache/index%u/size", type) < 0) + goto error; + + if (virHostCPUGetStrValue(cpudir, cpu, cachedir, value_str) < 0) + goto error; + + if ((tmp = strchr(value_str, '\n'))) *tmp = '\0'; + + if (virStrToLong_i(value_str, &unit, 10, &value) < 0) + goto error; + + size = value; + + if (virScaleInteger(&size, unit, 1, ULLONG_MAX) < 0) + goto error; + + return size / 1024; + + error: + VIR_FREE(cpudir); + VIR_FREE(cachedir); return value; } @@ -301,6 +360,23 @@ virHostCPUParseSocket(const char *dir, return ret; } +/* return socket id of a given cpu*/ +static +int virHostCPUGetSocketId(virArch hostarch, unsigned int cpu) +{ + char *cpu_dir; + int ret = -1; + + if (virAsprintf(&cpu_dir, "%s/cpu", sysfs_system_path) < 0) + goto cleanup; + + ret = virHostCPUParseSocket(cpu_dir, hostarch, cpu); + + cleanup: + VIR_FREE(cpu_dir); + return ret; +} + /* parses a node entry, returning number of processors in the node and * filling arguments */ static int @@ -1346,3 +1422,79 @@ virHostCPUGetKVMMaxVCPUs(void) return -1; } #endif /* HAVE_LINUX_KVM_H */ + +/* Fill all cache bank informations + * Return a list of virResCacheBankPtr, and fill cache bank information + * by loop for all cpus on host, number of cache bank will be set in nbanks + * + * NULL if error happened, and nbanks will be set 0. */ +virResCacheBankPtr virHostCPUGetCacheBanks(virArch arch, int type, size_t *nbanks, int cbm_len) +{ + int npresent_cpus; + int idx; + size_t i; + virResCacheBankPtr bank; + + *nbanks = 0; + if ((npresent_cpus = virHostCPUGetCount()) < 0) + return NULL; + + switch (type) { + case VIR_RDT_RESOURCE_L3: + case VIR_RDT_RESOURCE_L3DATA: + case VIR_RDT_RESOURCE_L3CODE: + idx = 3; + break; + case VIR_RDT_RESOURCE_L2: + idx = 2; + break; + default: + idx = -1; + } + + if (idx == -1) + return NULL; + + if (VIR_ALLOC_N(bank, 1) < 0) + return NULL; + + *nbanks = 1; + + for (i = 0; i < npresent_cpus; i ++) { + int s_id; + int cache_size; + + if ((s_id = virHostCPUGetSocketId(arch, i)) < 0) + goto error; + + /* Expand cache bank array */ + if (s_id > (*nbanks - 1)) { + size_t cur = *nbanks; + size_t exp = s_id - (*nbanks) + 1; + if (VIR_EXPAND_N(bank, cur, exp) < 0) + goto error; + *nbanks = s_id + 1; + } + + if (bank[s_id].cpu_mask == NULL) { + if (!(bank[s_id].cpu_mask = virBitmapNew(npresent_cpus))) + goto error; + } + + ignore_value(virBitmapSetBit(bank[s_id].cpu_mask, i)); + + if (bank[s_id].cache_size == 0) { + if ((cache_size = virHostCPUGetCache(i, idx)) < 0) + goto error; + + bank[s_id].cache_size = cache_size; + bank[s_id].cache_min = cache_size / cbm_len; + } + } + return bank; + + error: + *nbanks = 0; + VIR_FREE(bank); + return NULL; +} diff --git a/src/util/virhostcpu.h b/src/util/virhostcpu.h index 39f7cf8..27f208e 100644 --- a/src/util/virhostcpu.h +++ b/src/util/virhostcpu.h @@ -27,6 +27,7 @@ # include "internal.h" # include "virarch.h" # include "virbitmap.h" +# include "virresctrl.h" # define VIR_HOST_CPU_MASK_LEN 1024 @@ -58,4 +59,9 @@ int virHostCPUStatsAssign(virNodeCPUStatsPtr param, const char *name, unsigned long long value); +virResCacheBankPtr virHostCPUGetCacheBanks(virArch arch, + int type, + size_t *nbanks, + int cbm_len); + #endif /* __VIR_HOSTCPU_H__*/ diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c new file mode 100644 index 0000000..44a47cc --- /dev/null +++ b/src/util/virresctrl.c @@ -0,0 +1,201 @@ +/* + * virresctrl.c: methods for managing resource control + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Eli Qiao <liyong.qiao(a)intel.com&gt; + */ +#include <config.h> + +#include "virresctrl.h" +#include "viralloc.h" +#include "virerror.h" +#include "virfile.h" +#include "virhostcpu.h" +#include "virlog.h" +#include "virstring.h" +#include "virarch.h" + +VIR_LOG_INIT("util.resctrl"); + +#define VIR_FROM_THIS VIR_FROM_RESCTRL + +#define RESCTRL_DIR "/sys/fs/resctrl" +#define RESCTRL_INFO_DIR "/sys/fs/resctrl/info" +#define SYSFS_SYSTEM_PATH "/sys/devices/system" + +#define MAX_CPU_SOCKET_NUM 8 +#define MAX_CBM_BIT_LEN 32 +#define MAX_SCHEMATA_LEN 1024 +#define MAX_FILE_LEN (10 * 1024 * 1024) + +static unsigned int host_id; + +static virResCtrl resctrlall[] = { + { + .name = "L3", + .cache_level = "l3", + }, + { + .name = "L3DATA", + .cache_level = "l3", + }, + { + .name = "L3CODE", + .cache_level = "l3", + }, + { + .name = "L2", + .cache_level = "l2", + }, +}; + +static int virResCtrlGetInfoStr(const int type, const char *item, char **str) +{ + int ret = 0; + char *tmp; + char *path; + + if (virAsprintf(&path, "%s/%s/%s", RESCTRL_INFO_DIR, resctrlall[type].name, item) < 0) + return -1; + if (virFileReadAll(path, 10, str) < 0) { + ret = -1; + goto cleanup; + } + + if ((tmp = strchr(*str, '\n'))) *tmp = '\0'; + + cleanup: + VIR_FREE(path); + return ret; +} + +static int virResCtrlReadConfig(virArch arch, int type) +{ + int ret; + size_t i, nbanks; + char *str; + + /* Read num_closids from resctrl. + eg: /sys/fs/resctrl/info/L3/num_closids */ + if ((ret = virResCtrlGetInfoStr(type, "num_closids", &str)) < 0) + goto error; + + if ((ret = virStrToLong_i(str, NULL, 10, &resctrlall[type].num_closid)) < 0) + goto error; + + VIR_FREE(str); + + /* Read min_cbm_bits from resctrl. + eg: /sys/fs/resctrl/info/L3/cbm_mask */ + if ((ret = virResCtrlGetInfoStr(type, "min_cbm_bits", &str)) < 0) + goto error; + + if ((ret = virStrToLong_i(str, NULL, 10, &resctrlall[type].min_cbm_bits)) < 0) + goto error; + + VIR_FREE(str); + + /* Read cbm_mask string from resctrl. + eg: /sys/fs/resctrl/info/L3/cbm_mask */ + if ((ret = virResCtrlGetInfoStr(type, "cbm_mask", &str)) < 0) + goto error; + + /* cbm_mask is in hex, eg: "fffff", calculate cbm length from the default + cbm_mask. */ + resctrlall[type].cbm_len = strlen(str) * 4; + + /* Get all cache bank informations */ + resctrlall[type].cache_banks = virHostCPUGetCacheBanks(arch, + type, + &nbanks, resctrlall[type].cbm_len); + + if (resctrlall[type].cache_banks == NULL) + goto error; + + resctrlall[type].num_banks = nbanks; + + for (i = 0; i < resctrlall[type].num_banks; i++) { + /* L3CODE and L3DATA shares same L3 resource, so they should + * have same host_id. */ + if (type == VIR_RDT_RESOURCE_L3CODE) + resctrlall[type].cache_banks[i].host_id = resctrlall[VIR_RDT_RESOURCE_L3DATA].cache_banks[i].host_id; + else + resctrlall[type].cache_banks[i].host_id = host_id++; + } + + resctrlall[type].enabled = true; + + ret = 0; + + error: + VIR_FREE(str); + return ret; +} + +int +virResCtrlInit(void) +{ + size_t i = 0; + char *tmp; + int rc = 0; + + virArch hostarch; + + hostarch = virArchFromHost(); + + for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) { + if ((rc = virAsprintf(&tmp, "%s/%s", RESCTRL_INFO_DIR, resctrlall[i].name)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to initialize resource control config")); + goto cleanup; + } + + if (virFileExists(tmp)) { + if ((rc = virResCtrlReadConfig(hostarch, i)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to get resource control config")); + goto cleanup; + } + } + VIR_FREE(tmp); + } + + cleanup: + VIR_FREE(tmp); + return rc; +} + +/* + * Test whether the host support resource control + */ +bool +virResCtrlAvailable(void) +{ + if (!virFileExists(RESCTRL_INFO_DIR)) + return false; + return true; +} + +/* + * Return an virResCtrlPtr point to virResCtrl object, + * We should not modify it out side of virresctrl.c + */ +virResCtrlPtr +virResCtrlGet(int type) +{ + return &resctrlall[type]; +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h new file mode 100644 index 0000000..5a6a344 --- /dev/null +++ b/src/util/virresctrl.h @@ -0,0 +1,78 @@ +/* + * virresctrl.h: header for managing resctrl control + * + * Copyright (C) 2016 Intel, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Eli Qiao <liyong.qiao(a)intel.com&gt; + */ + +#ifndef __VIR_RESCTRL_H__ +# define __VIR_RESCTRL_H__ + +# include "virbitmap.h" + +enum { + VIR_RDT_RESOURCE_L3, + VIR_RDT_RESOURCE_L3DATA, + VIR_RDT_RESOURCE_L3CODE, + VIR_RDT_RESOURCE_L2, + /* Must be the last */ + VIR_RDT_RESOURCE_LAST, +}; + + +typedef struct _virResCacheBank virResCacheBank; +typedef virResCacheBank *virResCacheBankPtr; +struct _virResCacheBank { + unsigned int host_id; + unsigned long long cache_size; + unsigned long long cache_left; + unsigned long long cache_min; + virBitmapPtr cpu_mask; +}; + +/** + * struct rdt_resource - attributes of an RDT resource + * @enabled: Is this feature enabled on this machine + * @name: Name to use in "schemata" file + * @num_closid: Number of CLOSIDs available + * @max_cbm: Largest Cache Bit Mask allowed + * @min_cbm_bits: Minimum number of consecutive bits to be set + * in a cache bit mask + * @cache_level: Which cache level defines scope of this domain + * @num_banks: Number of cache bank on this machine. + * @cache_banks: Array of cache bank + */ +typedef struct _virResCtrl virResCtrl; +typedef virResCtrl *virResCtrlPtr; +struct _virResCtrl { + bool enabled; + const char *name; + int num_closid; + int cbm_len; + int min_cbm_bits; + const char* cache_level; + int num_banks; + virResCacheBankPtr cache_banks; +}; + +bool virResCtrlAvailable(void); +int virResCtrlInit(void); +virResCtrlPtr virResCtrlGet(int); + +#endif -- 1.9.1