On Mon, Jun 26, 2017 at 06:33:39PM +0800, Eli Qiao wrote: Why would you come up with something like 'sizeKib'? That's so inconvenient. And this hunk breaks the whole compilation, of course. I'm not at work for few days plus there's a public holiday in my country, so I won't be here for a while. That shouldn't stop anyone else reviewing your code, but I'm giving you a heads-up that I'll probably work on a separate solution if I have some extra time on my hands during vacation. Have a nice day, Martin

Set cachetune if user define cachetune in domain xml when using qemu driver. This patch adds 3 major private interface. virResctrlGetFreeCache: return free cache, default cache substract cache allocated. virResctrlSetCachetunes: set cache banks which defined in a domain. virResctrlRemoveCachetunes: remove cache allocation group from the host. Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com&gt; --- include/libvirt/virterror.h | 1 + src/Makefile.am | 1 + src/libvirt_private.syms | 9 + src/qemu/qemu_process.c | 81 +++ src/util/virerror.c | 1 + src/util/virresctrl.c | 822 ++++++++++++++++++++++++++++++ src/util/virresctrl.h | 88 ++++ tests/Makefile.am | 8 +- tests/virresctrldata/L3-free.schemata | 1 + tests/virresctrldata/L3CODE-free.schemata | 1 + tests/virresctrldata/L3DATA-free.schemata | 1 + tests/virresctrldata/linux-resctrl | 1 + tests/virresctrldata/linux-resctrl-cdp | 1 + tests/virresctrltest.c | 119 +++++ 14 files changed, 1134 insertions(+), 1 deletion(-) create mode 100644 src/util/virresctrl.c create mode 100644 src/util/virresctrl.h create mode 100644 tests/virresctrldata/L3-free.schemata create mode 100644 tests/virresctrldata/L3CODE-free.schemata create mode 100644 tests/virresctrldata/L3DATA-free.schemata create mode 120000 tests/virresctrldata/linux-resctrl create mode 120000 tests/virresctrldata/linux-resctrl-cdp create mode 100644 tests/virresctrltest.c diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h index 2efee8f..4bc0c74 100644 --- a/include/libvirt/virterror.h +++ b/include/libvirt/virterror.h @@ -132,6 +132,7 @@ typedef enum { VIR_FROM_PERF = 65, /* Error from perf */ VIR_FROM_LIBSSH = 66, /* Error from libssh connection transport */ + VIR_FROM_RESCTRL = 67, /* Error from resctrl */ # ifdef VIR_ENUM_SENTINELS VIR_ERR_DOMAIN_LAST diff --git a/src/Makefile.am b/src/Makefile.am index eae32dc..8dbb778 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -167,6 +167,7 @@ UTIL_SOURCES = \ util/virprocess.c util/virprocess.h \ util/virqemu.c util/virqemu.h \ util/virrandom.h util/virrandom.c \ + util/virresctrl.h util/virresctrl.c \ util/virrotatingfile.h util/virrotatingfile.c \ util/virscsi.c util/virscsi.h \ util/virscsihost.c util/virscsihost.h \ diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index c1e9471..72e5096 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -2439,6 +2439,15 @@ virRandomGenerateWWN; virRandomInt; +# util/virresctrl.h +virResctrlBitmap2String; +virResctrlFreeSchemata; +virResctrlGetFreeCache; +virResctrlRemoveCachetunes; +virResctrlSetCachetunes; +virResctrlTypeToString; + + # util/virrotatingfile.h virRotatingFileReaderConsume; virRotatingFileReaderFree; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index fa9990e..3b11caa 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -70,6 +70,7 @@ #include "virbitmap.h" #include "viratomic.h" #include "virnuma.h" +#include "virresctrl.h" #include "virstring.h" #include "virhostdev.h" #include "secret_util.h" @@ -5130,6 +5131,77 @@ qemuProcessSetupVcpus(virDomainObjPtr vm) return 0; } +static int +qemuProcessSetCacheBanks(virCapsHostPtr caps, virDomainObjPtr vm) +{ + size_t i, j; + virDomainCachetunePtr cachetune; + virResctrlCachetunePtr resctrl_cachetune; + unsigned int max_vcpus = virDomainDefGetVcpusMax(vm->def); + pid_t *pids = NULL; + virDomainVcpuDefPtr vcpu; + size_t npids = 0; + size_t count = 0; + int ret = -1; + + cachetune = &(vm->def->cachetune); + + if (VIR_ALLOC_N(resctrl_cachetune, cachetune->n_banks) < 0) + goto cleanup; + + /* construct resctrl_cachetune array */ + for (i = 0; i < cachetune->n_banks; i++) { + + resctrl_cachetune[i].cache_id = cachetune->cache_banks[i].cache_id; + resctrl_cachetune[i].type = cachetune->cache_banks[i].type; + resctrl_cachetune[i].size = cachetune->cache_banks[i].size; + + /* get granularity from host's capabilities */ + for (j = 0; j < caps->ncaches; j++) { + /* even enable CDP, granularity for code and data are same */ + if (caps->caches[j]->id == resctrl_cachetune[i].cache_id && + caps->caches[j]->controls) { + resctrl_cachetune[i].granularity = caps->caches[j]->controls[0]->granularity; + break; + } + } + + /* create pids of vcpus array */ + if (cachetune->cache_banks[i].vcpus) { + for (j = 0; j < max_vcpus; j++) { + if (virBitmapIsBitSet(cachetune->cache_banks[i].vcpus, j)) { + + vcpu = virDomainDefGetVcpu(vm->def, j); + if (!vcpu->online) + continue; + + if (VIR_RESIZE_N(pids, npids, count, 1) < 0) + goto cleanup; + pids[count ++] = qemuDomainGetVcpuPid(vm, j); + } + } + } + } + + /* If not specify vcpus in cachetune, add vm->pid */ + if (pids == NULL) { + if (VIR_ALLOC_N(pids, 1) < 0) + goto cleanup; + pids[0] = vm->pid; + count = 1; + } + + ret = virResctrlSetCachetunes(vm->def->uuid, + resctrl_cachetune, + cachetune->n_banks, + pids, + count); + + cleanup: + VIR_FREE(resctrl_cachetune); + VIR_FREE(pids); + return ret; +} int qemuProcessSetupIOThread(virDomainObjPtr vm, @@ -5966,6 +6038,12 @@ qemuProcessLaunch(virConnectPtr conn, qemuProcessAutoDestroyAdd(driver, vm, conn) < 0) goto cleanup; + VIR_DEBUG("Cache allocation"); + if (vm->def->cachetune.n_banks > 0 && + qemuProcessSetCacheBanks(&driver->caps->host, + vm) < 0) + goto cleanup; + ret = 0; cleanup: @@ -6472,6 +6550,9 @@ void qemuProcessStop(virQEMUDriverPtr driver, virPerfFree(priv->perf); priv->perf = NULL; + if (&(vm->def->cachetune) != NULL) + virResctrlRemoveCachetunes(vm->def->uuid); + qemuProcessRemoveDomainStatus(driver, vm); /* Remove VNC and Spice ports from port reservation bitmap, but only if diff --git a/src/util/virerror.c b/src/util/virerror.c index ef17fb5..02fabcc 100644 --- a/src/util/virerror.c +++ b/src/util/virerror.c @@ -139,6 +139,7 @@ VIR_ENUM_IMPL(virErrorDomain, VIR_ERR_DOMAIN_LAST, "Perf", /* 65 */ "Libssh transport layer", + "Resource Control", ) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c new file mode 100644 index 0000000..541c69c --- /dev/null +++ b/src/util/virresctrl.c @@ -0,0 +1,822 @@ +/* + * virresctrl.c: methods for managing resource control + * + * Copyright (C) 2017 Intel, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Eli Qiao <liyong.qiao(a)intel.com&gt; + */ + +#include <config.h> +#include <fcntl.h> +#include <sys/file.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "viralloc.h" +#include "virerror.h" +#include "virfile.h" +#include "virlog.h" +#include "virresctrl.h" +#include "virstring.h" +#include "viruuid.h" + +VIR_LOG_INIT("util.resctrl"); + +#define VIR_FROM_THIS VIR_FROM_RESCTRL +#define SYSFS_RESCTRL_PATH "/sys/fs/resctrl" +#define MAX_CBM_LEN 20 +#define VIR_RESCTRL_LOCK(fd, op) flock(fd, op) +#define VIR_RESCTRL_UNLOCK(fd) flock(fd, LOCK_UN) +#define CONSTRUCT_RESCTRL_PATH(domain_name, item_name) \ +do { \ + if (NULL == domain_name) { \ + if (virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, item_name) < 0) \ + return -1; \ + } else { \ + if (virAsprintf(&path, "%s/%s/%s", SYSFS_RESCTRL_PATH, domain_name, \ + item_name) < 0) \ + return -1; \ + } \ +} while (0) + +VIR_ENUM_IMPL(virResctrl, VIR_RESCTRL_TYPE_LAST, + "L3", + "L3CODE", + "L3DATA") + +/** + * a virResctrlGroup represents a resource control group, it's a directory + * under /sys/fs/resctrl. + * e.g. /sys/fs/resctrl/CG1 + * |-- cpus + * |-- schemata + * `-- tasks + * # cat schemata + * L3DATA:0=fffff;1=fffff + * L3CODE:0=fffff;1=fffff + * + * Besides, it can also represent the default resource control group of the + * host. + */ + +typedef struct _virResctrlGroup virResctrlGroup; +typedef virResctrlGroup *virResctrlGroupPtr; +struct _virResctrlGroup { + char *name; /* resource group name, NULL for default host group */ + size_t n_tasks; /* number of tasks assigned to the resource group */ + char **tasks; /* task id list */ + virResctrlSchemataPtr schemata[VIR_RESCTRL_TYPE_LAST]; /* Array for schemata */ +}; + +/* All resource control groups on this host, including default resource group */ +typedef struct _virResctrlHost virResctrlHost; +typedef virResctrlHost *virResctrlHostPtr; +struct _virResctrlHost { + size_t n_groups; /* number of resource control group */ + virResctrlGroupPtr *groups; /* list of resource control group */ +}; + +void +virResctrlFreeSchemata(virResctrlSchemataPtr ptr) +{ + size_t i; + + if (!ptr) + return; + + for (i = 0; i < ptr->n_masks; i++) { + virBitmapFree(ptr->masks[i]->mask); + VIR_FREE(ptr->masks[i]); + } + + VIR_FREE(ptr); + ptr = NULL; +} + +static void +virResctrlFreeGroup(virResctrlGroupPtr ptr) +{ + size_t i; + + if (!ptr) + return; + + for (i = 0; i < ptr->n_tasks; i++) + VIR_FREE(ptr->tasks[i]); + VIR_FREE(ptr->name); + + for (i = 0; i < VIR_RESCTRL_TYPE_LAST; i++) + virResctrlFreeSchemata(ptr->schemata[i]); + + VIR_FREE(ptr); + ptr = NULL; +} + +/* Return specify type of schemata string from schematalval. + e.g., 0=f;1=f */ +static int +virResctrlGetSchemataString(virResctrlType type, + const char *schemataval, + char **schematastr) +{ + int rc = -1; + char *prefix = NULL; + char **lines = NULL; + + if (virAsprintf(&prefix, + "%s:", + virResctrlTypeToString(type)) < 0) + return -1; + + lines = virStringSplit(schemataval, "\n", 0); + + if (VIR_STRDUP(*schematastr, + virStringListGetFirstWithPrefix(lines, prefix)) < 0) + goto cleanup; + + if (*schematastr == NULL) + rc = -1; + else + rc = 0; + + cleanup: + VIR_FREE(prefix); + virStringListFree(lines); + return rc; +} + +static int +virResctrlRemoveSysGroup(const char* name) +{ + char *path = NULL; + int ret = -1; + + if ((ret = virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, name)) < 0) + return ret; + + ret = rmdir(path); + + VIR_FREE(path); + return ret; +} + +static int +virResctrlNewSysGroup(const char *name) +{ + char *path = NULL; + int ret = -1; + mode_t mode = 0755; + + if (virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, name) < 0) + return -1; + + if (virDirCreate(path, mode, 0, 0, 0) < 0) + goto cleanup; + + ret = 0; + + cleanup: + VIR_FREE(path); + return ret; +} + +static int +virResctrlWrite(const char *name, const char *item, const char *content) +{ + char *path; + int writefd; + int rc = -1; + + CONSTRUCT_RESCTRL_PATH(name, item); + + if (!virFileExists(path)) + goto cleanup; + + if ((writefd = open(path, O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR)) < 0) + goto cleanup; + + if (safewrite(writefd, content, strlen(content)) < 0) + goto cleanup; + rc = 0; + + cleanup: + VIR_FREE(path); + VIR_FORCE_CLOSE(writefd); + return rc; +} + +static +virBitmapPtr virResctrlMask2Bitmap(const char *mask) +{ + virBitmapPtr bitmap; + unsigned int tmp; + size_t i; + + if (virStrToLong_ui(mask, NULL, 16, &tmp) < 0) + return NULL; + + bitmap = virBitmapNewEmpty(); + + for (i = 0; i < MAX_CBM_LEN; i++) { + if (((tmp & 0x1) == 0x1) && + (virBitmapSetBitExpand(bitmap, i) < 0)) + goto error; + tmp = tmp >> 1; + } + return bitmap; + + error: + virBitmapFree(bitmap); + return NULL; +} + +char *virResctrlBitmap2String(virBitmapPtr bitmap) +{ + char *tmp; + char *ret = NULL; + char *p; + tmp = virBitmapString(bitmap); + /* skip "0x" */ + p = tmp + 2; + + /* first non-0 position */ + while (*++p == '0'); + + if (VIR_STRDUP(ret, p) < 0) + ret = NULL; + + VIR_FREE(tmp); + return ret; +} + +static int +virResctrlParseSchemata(const char* schemata_str, + virResctrlSchemataPtr schemata) +{ + VIR_DEBUG("schemata_str=%s, schemata=%p", schemata_str, schemata); + + int ret = -1; + size_t i; + virResctrlMaskPtr mask; + char **schemata_list; + char *mask_str; + + /* parse 0=fffff;1=f */ + schemata_list = virStringSplit(schemata_str, ";", 0); + + if (!schemata_list) + goto cleanup; + + for (i = 0; schemata_list[i] != NULL; i++) { + /* parse 0=fffff */ + mask_str = strchr(schemata_list[i], '='); + + if (!mask_str) + goto cleanup; + + if (VIR_ALLOC(mask) < 0) + goto cleanup; + + mask->cache_id = i; + mask->mask = virResctrlMask2Bitmap(mask_str + 1); + schemata->n_masks += 1; + schemata->masks[i] = mask; + + } + ret = 0; + + cleanup: + virStringListFree(schemata_list); + return ret; +} + +static int +virResctrlLoadGroup(const char *name, + virResctrlHostPtr host) +{ + VIR_DEBUG("name=%s, host=%p\n", name, host); + + int ret = -1; + char *schemataval = NULL; + char *schemata_str = NULL; + virResctrlType i; + int rv; + virResctrlGroupPtr grp; + virResctrlSchemataPtr schemata; + + rv = virFileReadValueString(&schemataval, + SYSFS_RESCTRL_PATH "/%s/schemata", + name ? name : ""); + + if (rv < 0) + return -1; + + if (VIR_ALLOC(grp) < 0) + goto cleanup; + + if (VIR_STRDUP(grp->name, name) < 0) + goto cleanup; + + for (i = 0; i < VIR_RESCTRL_TYPE_LAST; i++) { + rv = virResctrlGetSchemataString(i, schemataval, &schemata_str); + + if (rv < 0) + continue; + + if (VIR_ALLOC(schemata) < 0) + goto cleanup; + + schemata->type = i; + + if (virResctrlParseSchemata(schemata_str, schemata) < 0) { + VIR_FREE(schemata); + VIR_FREE(schemata_str); + goto cleanup; + } + + grp->schemata[i] = schemata; + VIR_FREE(schemata_str); + } + + if (VIR_APPEND_ELEMENT(host->groups, + host->n_groups, + grp) < 0) { + virResctrlFreeGroup(grp); + goto cleanup; + } + + ret = 0; + + cleanup: + VIR_FREE(schemataval); + return ret; +} + +static int +virResctrlLoadHost(virResctrlHostPtr host) +{ + int rv = -1; + DIR *dirp = NULL; + char *path = NULL; + struct dirent *ent; + + rv = virDirOpenIfExists(&dirp, SYSFS_RESCTRL_PATH); + if (rv < 0) + return -1; + + /* load default group first */ + if (virResctrlLoadGroup(NULL, host) < 0) + return -1; + + while ((rv = virDirRead(dirp, &ent, path)) > 0) { + /* resctrl is not hierarchical, only read directory under + /sys/fs/resctrl */ + if ((ent->d_type != DT_DIR) || STREQ(ent->d_name, "info")) + continue; + + if (virResctrlLoadGroup(ent->d_name, host) < 0) + return -1; + } + return 0; +} + +static void +virResctrlRefreshHost(virResctrlHostPtr host) +{ + virResctrlGroupPtr default_grp = NULL; + virResctrlSchemataPtr schemata = NULL; + size_t i, j; + virResctrlType t; + + default_grp = host->groups[0]; + + for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) { + if (default_grp->schemata[t] != NULL) { + for (i = 0; i < default_grp->schemata[t]->n_masks; i++) { + /* Reset default group's mask */ + virBitmapSetAll(default_grp->schemata[t]->masks[i]->mask); + /* Loop each other resource group except default group */ + for (j = 1; j < host->n_groups; j++) { + schemata = host->groups[j]->schemata[t]; + virBitmapSubtract(default_grp->schemata[t]->masks[i]->mask, + schemata->masks[i]->mask); + } + } + } + } +} + +static virResctrlGroupPtr +virResctrlGetFreeGroup(void) +{ + size_t i; + virResctrlHostPtr host = NULL; + virResctrlGroupPtr grp = NULL; + + if (VIR_ALLOC(host) < 0) + return NULL; + + if (virResctrlLoadHost(host) < 0) + goto error; + + virResctrlRefreshHost(host); + + for (i = 1; i < host->n_groups; i++) + virResctrlFreeGroup(host->groups[i]); + + grp = host->groups[0]; + VIR_FREE(host); + + return grp; + + error: + virResctrlFreeGroup(grp); + return NULL; +} + +virResctrlSchemataPtr +virResctrlGetFreeCache(virResctrlType type) +{ + VIR_DEBUG("type=%d", type); + + virResctrlType t; + virResctrlGroupPtr grp = NULL; + virResctrlSchemataPtr schemata = NULL; + int lockfd = -1; + + lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY); + if (lockfd < 0) + return NULL; + + VIR_RESCTRL_LOCK(lockfd, LOCK_SH); + + if ((grp = virResctrlGetFreeGroup()) == NULL) + goto cleanup; + + for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) { + if (t == type) + schemata = grp->schemata[t]; + else + virResctrlFreeSchemata(grp->schemata[t]); + } + + cleanup: + VIR_RESCTRL_UNLOCK(lockfd); + VIR_FORCE_CLOSE(lockfd); + return schemata; +} + +static int +virResctrlCalculateCbm(int cbm_len, + virBitmapPtr defaultcbm, + virBitmapPtr newcbm) +{ + VIR_DEBUG("cbm_len=%d, defaultcbm=%p, newcbm=%p", + cbm_len, defaultcbm, newcbm); + + ssize_t pos = -1; + size_t i; + + /* not enough cache way to be allocated */ + if (virBitmapCountBits(defaultcbm) < cbm_len + 1) + return -1; + + while ((pos = virBitmapNextSetBit(defaultcbm, pos)) >= 0) { + for (i = 0; i < cbm_len; i++) + ignore_value(virBitmapSetBitExpand(newcbm, i + pos)); + /* Test if newcbm is sub set of defaultcbm */ + if (virBitmapNextClearBit(defaultcbm, pos) > i + pos) { + break; + } else { + pos = pos + i - 1; + virBitmapClearAll(newcbm); + } + } + + if (virBitmapCountBits(newcbm) != cbm_len) + return -1; + + /* consume default cbm after allocation */ + virBitmapSubtract(defaultcbm, newcbm); + + return 0; +} + +/* Fill mask value for newly created resource group base on hostcachebank + * and domcachebank */ +static int +virResctrlFillMask(virResctrlGroupPtr grp, + virResctrlGroupPtr free_grp, + virResctrlCachetunePtr cachetune) +{ + VIR_DEBUG("grp=%p, free_grp=%p, cachetune=%p", + grp, free_grp, cachetune); + + unsigned int cache_id; + unsigned int cache_type; + int cbm_candidate_len; + virResctrlSchemataPtr schemata; + virResctrlMaskPtr mask; + + cache_type = cachetune->type; + cache_id = cachetune->cache_id; + schemata = grp->schemata[cache_type]; + + if ((schemata == NULL) && (VIR_ALLOC(schemata) < 0)) + return -1; + + if (VIR_ALLOC(mask) < 0) + return -1; + + mask->cache_id = cache_id; + mask->mask = virBitmapNewEmpty(); + + /* here should be control->granularity and control->min + also domcachebank size should be checked while define domain xml */ + cbm_candidate_len = cachetune->size / cachetune->granularity; + if (virResctrlCalculateCbm(cbm_candidate_len, + free_grp->schemata[cache_type]->masks[cache_id]->mask, + mask->mask) < 0) + goto error; + + schemata->type = cache_type; + schemata->n_masks += 1; + schemata->masks[cache_id] = mask; + grp->schemata[cache_type] = schemata; + + return 0; + + error: + VIR_FREE(schemata); + return -1; +} + +/* only keep the highest consecutive bits */ +static void +virResctrlTrimMask(virResctrlMaskPtr mask) +{ + size_t i; + ssize_t setbit = -1; + ssize_t clearbit = -1; + + clearbit = virBitmapNextClearBit(mask->mask, -1); + setbit = virBitmapNextSetBit(mask->mask, -1); + for (i = setbit; i < clearbit; i++) + ignore_value(virBitmapClearBit(mask->mask, i)); +} + +static int +virResctrlCompleteMask(virResctrlSchemataPtr schemata, + virResctrlSchemataPtr defaultschemata) +{ + size_t i; + virResctrlMaskPtr mask; + + if (schemata == NULL && VIR_ALLOC(schemata) < 0) + return -1; + + if (schemata->n_masks == defaultschemata->n_masks) + return 0; + + for (i = 0; i < defaultschemata->n_masks; i++) { + if (schemata->masks[i] == NULL) { + if (VIR_ALLOC(mask) < 0) + goto error; + + mask->cache_id = i; + mask->mask = virBitmapNewEmpty(); + schemata->n_masks += 1; + schemata->masks[i] = mask; + /* resctrl doesn't allow mask to be zero + use higher bits to fill up the cbm which + domaincache bank doens't provide */ + ignore_value(virBitmapSetBitExpand(mask->mask, + virBitmapLastSetBit(defaultschemata->masks[i]->mask))); + } + /* only keep the highest consecutive bits for default group */ + virResctrlTrimMask(defaultschemata->masks[i]); + } + + return 0; + + error: + VIR_FREE(schemata); + return -1; +} + +/* complete the schemata in the resrouce group before it can be write back + to resctrl */ +static int +virResctrlCompleteGroup(virResctrlGroupPtr grp, + virResctrlGroupPtr default_grp) +{ + virResctrlType t; + virResctrlSchemataPtr schemata; + virResctrlSchemataPtr defaultschemata; + + /* NOTES: resctrl system require we need provide all cache's cbm mask */ + for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) { + defaultschemata = default_grp->schemata[t]; + if (defaultschemata != NULL) { + schemata = grp->schemata[t]; + if (virResctrlCompleteMask(schemata, defaultschemata) < 0) + return -1; + } + } + return 0; +} + +static +char *virResctrlGetSchemataStr(virResctrlSchemataPtr schemata) +{ + virBuffer buf = VIR_BUFFER_INITIALIZER; + size_t i; + + virBufferAsprintf(&buf, "%s:%u=%s", + virResctrlTypeToString(schemata->type), + schemata->masks[0]->cache_id, + virResctrlBitmap2String(schemata->masks[0]->mask)); + + for (i = 1; i < schemata->n_masks; i ++) + virBufferAsprintf(&buf, ";%u=%s", + schemata->masks[i]->cache_id, + virResctrlBitmap2String(schemata->masks[i]->mask)); + + return virBufferContentAndReset(&buf); +} + +static int +virResctrlFlushGroup(virResctrlGroupPtr grp) +{ + int ret = -1; + size_t i; + char *schemata_str = NULL; + virResctrlType t; + virBuffer buf = VIR_BUFFER_INITIALIZER; + + VIR_DEBUG("grp=%p", grp); + + if (grp->name != NULL && virResctrlNewSysGroup(grp->name) < 0) + return -1; + + for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) { + if (grp->schemata[t] != NULL) { + schemata_str = virResctrlGetSchemataStr(grp->schemata[t]); + virBufferAsprintf(&buf, "%s\n", schemata_str); + VIR_FREE(schemata_str); + } + } + + schemata_str = virBufferContentAndReset(&buf); + + if (virResctrlWrite(grp->name, "schemata", schemata_str) < 0) + goto cleanup; + + for (i = 0; i < grp->n_tasks; i++) { + if (virResctrlWrite(grp->name, "tasks", grp->tasks[i]) < 0) + goto cleanup; + } + + ret = 0; + + cleanup: + VIR_FREE(schemata_str); + return ret; +} + +int virResctrlSetCachetunes(unsigned char* uuid, + virResctrlCachetunePtr cachetunes, size_t ncachetune, + pid_t *pids, size_t npid) +{ + size_t i; + int ret = -1; + char name[VIR_UUID_STRING_BUFLEN]; + char *tmp; + int lockfd = -1; + virResctrlGroupPtr grp = NULL; + virResctrlGroupPtr default_grp = NULL; + + virUUIDFormat(uuid, name); + + if (ncachetune < 1) + return 0; + + /* create new resource group */ + if (VIR_ALLOC(grp) < 0) + goto error; + + if (VIR_STRDUP(grp->name, name) < 0) + goto error; + + /* allocate file lock */ + lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY); + if (lockfd < 0) + goto error; + + VIR_RESCTRL_LOCK(lockfd, LOCK_EX); + + if ((default_grp = virResctrlGetFreeGroup()) == NULL) + goto error; + + /* Allocate cache for each cache bank defined in cache tune */ + for (i = 0; i < ncachetune; i++) { + /* fill up newly crated grp and consume from default_grp */ + if (virResctrlFillMask(grp, default_grp, &cachetunes[i]) < 0) + goto error; + } + + /* Add tasks to grp */ + for (i = 0; i < npid; i++) { + if (virAsprintf(&tmp, "%llu", (long long)pids[i]) < 0) + goto error; + + if (VIR_APPEND_ELEMENT(grp->tasks, + grp->n_tasks, + tmp) < 0) { + VIR_FREE(tmp); + goto error; + } + } + + if (virResctrlCompleteGroup(grp, default_grp) < 0) { + VIR_WARN("Failed to complete group"); + goto error; + } + + if (virResctrlFlushGroup(grp) < 0) + goto error; + + if (virResctrlFlushGroup(default_grp) < 0) { + virResctrlRemoveSysGroup(grp->name); + goto error; + } + + ret = 0; + + error: + VIR_RESCTRL_UNLOCK(lockfd); + VIR_FORCE_CLOSE(lockfd); + virResctrlFreeGroup(grp); + virResctrlFreeGroup(default_grp); + return ret; +} + +int virResctrlRemoveCachetunes(unsigned char* uuid) +{ + int ret = -1; + int lockfd = -1; + size_t i; + virResctrlType t; + virResctrlSchemataPtr schemata; + char name[VIR_UUID_STRING_BUFLEN]; + virResctrlGroupPtr default_grp = NULL; + + virUUIDFormat(uuid, name); + + VIR_DEBUG("name=%s", name); + + lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY); + if (lockfd < 0) + return -1; + + VIR_RESCTRL_LOCK(lockfd, LOCK_SH); + + if (virResctrlRemoveSysGroup(name) < 0) + goto cleanup; + + if ((default_grp = virResctrlGetFreeGroup()) == NULL) + goto cleanup; + + for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) { + schemata = default_grp->schemata[t]; + if (schemata != NULL) { + for (i = 0; i < schemata->n_masks; i++) + virResctrlTrimMask(schemata->masks[i]); + } + } + + if (virResctrlFlushGroup(default_grp) < 0) + goto cleanup; + + ret = 0; + + cleanup: + VIR_RESCTRL_UNLOCK(lockfd); + VIR_FORCE_CLOSE(lockfd); + return ret; +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h new file mode 100644 index 0000000..49781ad --- /dev/null +++ b/src/util/virresctrl.h @@ -0,0 +1,88 @@ +/* + * virresctrl.h: header for managing resctrl control + * + * Copyright (C) 2017 Intel, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Eli Qiao <liyong.qiao(a)intel.com&gt; + */ + +#ifndef __VIR_RESCTRL_H__ +# define __VIR_RESCTRL_H__ + +#include "virutil.h" +#include "virbitmap.h" + +#define MAX_CACHE_ID 16 + +typedef enum { + VIR_RESCTRL_TYPE_L3, + VIR_RESCTRL_TYPE_L3_CODE, + VIR_RESCTRL_TYPE_L3_DATA, + + VIR_RESCTRL_TYPE_LAST +} virResctrlType; + +VIR_ENUM_DECL(virResctrl); + +typedef struct _virResctrlCachetune virResctrlCachetune; +typedef virResctrlCachetune *virResctrlCachetunePtr; +struct _virResctrlCachetune { + unsigned int cache_id; + virResctrlType type; + unsigned long long size; /* in B */ + unsigned long long granularity; /* min size of the cache */ +}; + +/* + * a virResctrlMask represents one of mask object in a + * resource control group. + * e.g., 0=f + */ +typedef struct _virResctrlMask virResctrlMask; +typedef virResctrlMask *virResctrlMaskPtr; +struct _virResctrlMask { + unsigned int cache_id; /* cache resource id */ + virBitmapPtr mask; /* the cbm mask */ +}; + +/* + * a virResctrlSchemata represents schemata objects of specific type of + * resource in a resource control group. + * eg: L3:0=f,1=ff + */ +typedef struct _virResctrlSchemata virResctrlSchemata; +typedef virResctrlSchemata *virResctrlSchemataPtr; +struct _virResctrlSchemata { + virResctrlType type; /* resource control type, e.g., L3 */ + size_t n_masks; /* number of masks */ + virResctrlMaskPtr masks[MAX_CACHE_ID]; /* array of mask, use array for easy index */ +}; + +/* Get free cache of the host, result saved in schemata */ +virResctrlSchemataPtr virResctrlGetFreeCache(virResctrlType type); + +/* Get mask string from Bitmap */ +char *virResctrlBitmap2String(virBitmapPtr bitmap); + +void virResctrlFreeSchemata(virResctrlSchemataPtr ptr); + +int virResctrlSetCachetunes(unsigned char* uuid, + virResctrlCachetunePtr cachetunes, size_t ncachetune, + pid_t *pids, size_t npid); +int virResctrlRemoveCachetunes(unsigned char* uuid); +#endif diff --git a/tests/Makefile.am b/tests/Makefile.am index 19986dc..e0b9923 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -229,6 +229,7 @@ if WITH_LINUX test_programs += fchosttest test_programs += scsihosttest test_programs += vircaps2xmltest +test_programs += virresctrltest test_libraries += virusbmock.la \ virnetdevbandwidthmock.la \ virnumamock.la \ @@ -1149,8 +1150,13 @@ virnumamock_la_CFLAGS = $(AM_CFLAGS) virnumamock_la_LDFLAGS = $(MOCKLIBS_LDFLAGS) virnumamock_la_LIBADD = $(MOCKLIBS_LIBS) +virresctrltest_SOURCES = \ + virresctrltest.c testutils.h testutils.c virfilewrapper.h virfilewrapper.c +virresctrltest_LDADD = $(LDADDS) + else ! WITH_LINUX -EXTRA_DIST += vircaps2xmltest.c virnumamock.c virfilewrapper.c virfilewrapper.h +EXTRA_DIST += vircaps2xmltest.c virnumamock.c virfilewrapper.c \ + virfilewrapper.h virresctrltest.c endif ! WITH_LINUX if WITH_NSS diff --git a/tests/virresctrldata/L3-free.schemata b/tests/virresctrldata/L3-free.schemata new file mode 100644 index 0000000..9b47d25 --- /dev/null +++ b/tests/virresctrldata/L3-free.schemata @@ -0,0 +1 @@ +L3:0=1ffff;1=1ffff diff --git a/tests/virresctrldata/L3CODE-free.schemata b/tests/virresctrldata/L3CODE-free.schemata new file mode 100644 index 0000000..7039c45 --- /dev/null +++ b/tests/virresctrldata/L3CODE-free.schemata @@ -0,0 +1 @@ +L3CODE:0=cffff;1=cffff diff --git a/tests/virresctrldata/L3DATA-free.schemata b/tests/virresctrldata/L3DATA-free.schemata new file mode 100644 index 0000000..30f1cbd --- /dev/null +++ b/tests/virresctrldata/L3DATA-free.schemata @@ -0,0 +1 @@ +L3DATA:0=3ffff;1=3ffff diff --git a/tests/virresctrldata/linux-resctrl b/tests/virresctrldata/linux-resctrl new file mode 120000 index 0000000..069dfb2 --- /dev/null +++ b/tests/virresctrldata/linux-resctrl @@ -0,0 +1 @@ +../vircaps2xmldata/linux-resctrl \ No newline at end of file diff --git a/tests/virresctrldata/linux-resctrl-cdp b/tests/virresctrldata/linux-resctrl-cdp new file mode 120000 index 0000000..c5a973f --- /dev/null +++ b/tests/virresctrldata/linux-resctrl-cdp @@ -0,0 +1 @@ +../vircaps2xmldata/linux-resctrl-cdp \ No newline at end of file diff --git a/tests/virresctrltest.c b/tests/virresctrltest.c new file mode 100644 index 0000000..a9d75f1 --- /dev/null +++ b/tests/virresctrltest.c @@ -0,0 +1,119 @@ +/* + * Copyright (C) Intel, Inc. 2017 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Eli Qiao <liyong.qiao(a)intel.com&gt; + */ + +#include <config.h> +#include <stdlib.h> + +#include "testutils.h" +#include "virbitmap.h" +#include "virfilewrapper.h" +#include "virresctrl.h" + + +#define VIR_FROM_THIS VIR_FROM_NONE + +struct virResctrlData { + const char *filename; + virResctrlType type; +}; + +static void +GetSchemataStr(virResctrlSchemataPtr schemata, char **str) +{ + size_t i; + + virBuffer buf = VIR_BUFFER_INITIALIZER; + virBufferAsprintf(&buf, "%s:%u=%s", + virResctrlTypeToString(schemata->type), + schemata->masks[0]->cache_id, + virResctrlBitmap2String(schemata->masks[0]->mask)); + + for (i = 1; i < schemata->n_masks; i ++) + virBufferAsprintf(&buf, ";%u=%s", + schemata->masks[i]->cache_id, + virResctrlBitmap2String(schemata->masks[i]->mask)); + + *str = virBufferContentAndReset(&buf); +} + +static int +test_virResctrl(const void *opaque) +{ + struct virResctrlData *data = (struct virResctrlData *) opaque; + char *dir = NULL; + char *resctrl = NULL; + int ret = -1; + virResctrlSchemataPtr schemata = NULL; + char *schemata_str = NULL; + char *schemata_file; + + if (virAsprintf(&resctrl, "%s/virresctrldata/linux-%s/resctrl", + abs_srcdir, data->filename) < 0) + goto cleanup; + + if (virAsprintf(&schemata_file, "%s/virresctrldata/%s-free.schemata", + abs_srcdir, virResctrlTypeToString(data->type)) < 0) + goto cleanup; + + if (virFileWrapperAddPrefix("/sys/fs/resctrl", resctrl) < 0) + goto cleanup; + + if ((schemata = virResctrlGetFreeCache(data->type)) == NULL) + goto cleanup; + + virFileWrapperClearPrefixes(); + + GetSchemataStr(schemata, &schemata_str); + + if (virTestCompareToFile(schemata_str, schemata_file) < 0) + goto cleanup; + + ret = 0; + + cleanup: + VIR_FREE(dir); + VIR_FREE(resctrl); + VIR_FREE(schemata_str); + virResctrlFreeSchemata(schemata); + return ret; +} + +static int +mymain(void) +{ + int ret = 0; + +#define DO_TEST_FULL(filename, type) \ + do { \ + struct virResctrlData data = {filename, \ + type}; \ + if (virTestRun(filename, test_virResctrl, &data) < 0) \ + ret = -1; \ + } while (0) + + DO_TEST_FULL("resctrl", VIR_RESCTRL_TYPE_L3); + DO_TEST_FULL("resctrl-cdp", VIR_RESCTRL_TYPE_L3_CODE); + DO_TEST_FULL("resctrl-cdp", VIR_RESCTRL_TYPE_L3_DATA); + + return ret; +} + +VIR_TEST_MAIN(mymain) -- 1.9.1