The resctrl default allocation is introduced in this patch, which refers to the root directory (/sys/fs/resctrl) and immediately be created after mounting, owns all the tasks and cpus in the system and can make full use of all resources. It does not intentionally allocate any dedicated amount of resource, either cache or memory bandwidth, for default allocation. If a system task has no resource control applied but you want to know task's cache or memroy bandwidth utilization information, the default allocation is meaningful. We create resctrl monitor under the default allocation for such kind of task. Refactoring schemas docs and APIs to create a default cache allocation by allowing the appearance of an <cachetune> with no <cache> element. Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- docs/schemas/domaincommon.rng | 4 ++-- src/conf/domain_conf.c | 32 +++++++++++++++++++------------- src/util/virresctrl.c | 27 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index 099a949..5c533d6 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -956,7 +956,7 @@ <attribute name="vcpus"> <ref name='cpuset'/> </attribute> - <oneOrMore> + <zeroOrMore> <element name="cache"> <attribute name="id"> <ref name='unsignedInt'/> @@ -980,7 +980,7 @@ </attribute> </optional> </element> - </oneOrMore> + </zeroOrMore> </element> </zeroOrMore> <zeroOrMore> diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 9911d56..b77680e 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -19002,22 +19002,27 @@ virDomainCachetuneDefParse(virDomainDefPtr def, goto cleanup; } - if (virDomainResctrlVcpuMatch(def, vcpus, &alloc) < 0) - goto cleanup; - - if (!alloc) { - alloc = virResctrlAllocNew(); - if (!alloc) + /* If 'n' equals 0, then no <cache> element found in <cachetune>, + * this means it is a default alloction. For default allocation, + * @SetvirDomainResctrlDefPtr->alloc is set to NULL */ + if (n != 0) { + if (virDomainResctrlVcpuMatch(def, vcpus, &alloc) < 0) goto cleanup; - } else { - virReportError(VIR_ERR_XML_ERROR, "%s", - _("Identical vcpus in cachetunes found")); - goto cleanup; - } - for (i = 0; i < n; i++) { - if (virDomainCachetuneDefParseCache(ctxt, nodes[i], alloc) < 0) + if (!alloc) { + alloc = virResctrlAllocNew(); + if (!alloc) + goto cleanup; + } else { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("Identical vcpus in cachetunes found")); goto cleanup; + } + + for (i = 0; i < n; i++) { + if (virDomainCachetuneDefParseCache(ctxt, nodes[i], alloc) < 0) + goto cleanup; + } } if (virResctrlAllocIsEmpty(alloc)) { @@ -19027,6 +19032,7 @@ virDomainCachetuneDefParse(virDomainDefPtr def, if (virDomainResctrlAppend(def, node, alloc, vcpus, flags) < 0) goto cleanup; + vcpus = NULL; alloc = NULL; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index fb25ca8..74d9b6b 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -234,6 +234,10 @@ virResctrlInfoMonFree(virResctrlInfoMonPtr mon) * in case there is no allocation for that particular cache allocation (level, * cache, ...) or memory allocation for particular node). * + * Resctrl file system root directory, /sys/fs/sysctrl/, is called the default + * allocation, which is created, immediately after mounting, owns all the + * tasks and cpus in the system and can make full use of all resources. + * * =====Cache allocation technology (CAT)===== * * Since one allocation can be made for caches on different levels, the first @@ -1167,6 +1171,9 @@ virResctrlAllocSetCacheSize(virResctrlAllocPtr alloc, unsigned int cache, unsigned long long size) { + if (!alloc) + return 0; + if (virResctrlAllocCheckCollision(alloc, level, type, cache)) { virReportError(VIR_ERR_XML_ERROR, _("Colliding cache allocations for cache " @@ -1237,6 +1244,9 @@ virResctrlAllocSetMemoryBandwidth(virResctrlAllocPtr alloc, { virResctrlAllocMemBWPtr mem_bw = alloc->mem_bw; + if (!alloc) + return 0; + if (memory_bandwidth > 100) { virReportError(VIR_ERR_XML_ERROR, "%s", _("Memory Bandwidth value exceeding 100 is invalid.")); @@ -1306,6 +1316,11 @@ int virResctrlAllocSetID(virResctrlAllocPtr alloc, const char *id) { + /* If passed a default allocation in, @alloc will be NULL. This is + * a valid case, return normally. */ + if (!alloc) + return 0; + if (!id) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Resctrl allocation 'id' cannot be NULL")); @@ -1319,6 +1334,9 @@ virResctrlAllocSetID(virResctrlAllocPtr alloc, const char * virResctrlAllocGetID(virResctrlAllocPtr alloc) { + if (!alloc) + return NULL; + return alloc->id; } @@ -2211,6 +2229,9 @@ int virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, const char *machinename) { + if (!alloc) + return 0; + if (!alloc->id) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Resctrl Allocation ID must be set before creation")); @@ -2304,6 +2325,9 @@ virResctrlAllocAddPID(virResctrlAllocPtr alloc, char *pidstr = NULL; int ret = 0; + if (!alloc) + return 0; + if (!alloc->path) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Cannot add pid to non-existing resctrl allocation")); @@ -2336,6 +2360,9 @@ virResctrlAllocRemove(virResctrlAllocPtr alloc) { int ret = 0; + if (!alloc) + return 0; + if (!alloc->path) return 0; -- 2.7.4

The code of adding PID to the allocation could be reused, refactor it for later reusing. Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- src/util/virresctrl.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index bd36406..8c2bb21 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -2361,24 +2361,21 @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, } -int -virResctrlAllocAddPID(virResctrlAllocPtr alloc, - pid_t pid) +static int +virResctrlAddPID(const char *path, + pid_t pid) { char *tasks = NULL; char *pidstr = NULL; int ret = 0; - if (!alloc) - return 0; - - if (!alloc->path) { + if (!path) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Cannot add pid to non-existing resctrl allocation")); + _("Cannot add pid to non-existing resctrl group")); return -1; } - if (virAsprintf(&tasks, "%s/tasks", alloc->path) < 0) + if (virAsprintf(&tasks, "%s/tasks", path) < 0) return -1; if (virAsprintf(&pidstr, "%lld", (long long int) pid) < 0) @@ -2400,6 +2397,17 @@ virResctrlAllocAddPID(virResctrlAllocPtr alloc, int +virResctrlAllocAddPID(virResctrlAllocPtr alloc, + pid_t pid) +{ + if (!alloc) + return 0; + + return virResctrlAddPID(alloc->path, pid); +} + + +int virResctrlAllocRemove(virResctrlAllocPtr alloc) { int ret = 0; -- 2.7.4

The code for determining resctrl allocation path could be reused for monitor. Refactor it for reusing. Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- src/util/virresctrl.c | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index f2c94d2..5beed92 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -2269,6 +2269,26 @@ virResctrlAllocAssign(virResctrlInfoPtr resctrl, } +static char * +virResctrlDeterminePath(const char *pathparent, + const char *prefix, + const char *id) +{ + char *path = NULL; + + if (!id) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Resctrl resource ID must be set before creation")); + return NULL; + } + + if (virAsprintf(&path, "%s/%s-%s", pathparent, prefix, id) < 0) + return NULL; + + return path; +} + + int virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, const char *machinename) @@ -2276,15 +2296,16 @@ virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, if (!alloc) return 0; - if (!alloc->id) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Resctrl Allocation ID must be set before creation")); + if (alloc->path) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Resctrl group path is expected to be NULL")); return -1; } - if (!alloc->path && - virAsprintf(&alloc->path, "%s/%s-%s", - SYSFS_RESCTRL_PATH, machinename, alloc->id) < 0) + alloc->path = virResctrlDeterminePath(SYSFS_RESCTRL_PATH, + machinename, + alloc->id); + if (!alloc->path) return -1; return 0; -- 2.7.4

The code for creating resctrl allocation group could be reused for monitoring group, refactor it for reusing in the later patch. Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- src/util/virresctrl.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index de10a0c..d573713 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -2312,6 +2312,26 @@ virResctrlAllocDeterminePath(virResctrlAllocPtr alloc, } +/* This function creates a resctrl directory in resource control file system, + * and the directory path is specified by @path. */ +static int +virResctrlCreateGroupPath(const char *path) +{ + /* Directory exists, return */ + if (virFileExists(path)) + return 0; + + if (virFileMakePath(path) < 0) { + virReportSystemError(errno, + _("Cannot create resctrl directory '%s'"), + path); + return -1; + } + + return 0; +} + + /* This checks if the directory for the alloc exists. If not it tries to create * it and apply appropriate alloc settings. */ int @@ -2336,13 +2356,6 @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (virResctrlAllocDeterminePath(alloc, machinename) < 0) return -1; - if (virFileExists(alloc->path)) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Path '%s' for resctrl allocation exists"), - alloc->path); - goto cleanup; - } - lockfd = virResctrlLockWrite(); if (lockfd < 0) goto cleanup; @@ -2350,6 +2363,9 @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (virResctrlAllocAssign(resctrl, alloc) < 0) goto cleanup; + if (virResctrlCreateGroupPath(alloc->path) < 0) + goto cleanup; + alloc_str = virResctrlAllocFormat(alloc); if (!alloc_str) goto cleanup; @@ -2357,13 +2373,6 @@ virResctrlAllocCreate(virResctrlInfoPtr resctrl, if (virAsprintf(&schemata_path, "%s/schemata", alloc->path) < 0) goto cleanup; - if (virFileMakePath(alloc->path) < 0) { - virReportSystemError(errno, - _("Cannot create resctrl directory '%s'"), - alloc->path); - goto cleanup; - } - VIR_DEBUG("Writing resctrl schemata '%s' into '%s'", alloc_str, schemata_path); if (virFileWriteStr(schemata_path, alloc_str, 0) < 0) { rmdir(alloc->path); -- 2.7.4

Add interfaces monitor group to support operations such as add PID, set ID, remove group ... etc. The interface for getting cache occupancy information from the monitor is also added. Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- src/libvirt_private.syms | 6 ++ src/util/virresctrl.c | 209 ++++++++++++++++++++++++++++++++++++++++++++++- src/util/virresctrl.h | 23 ++++++ 3 files changed, 236 insertions(+), 2 deletions(-) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 1d6ed0b..a8932a7 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -2680,7 +2680,13 @@ virResctrlInfoNew; virResctrlMonitorAddPID; virResctrlMonitorCreate; virResctrlMonitorDeterminePath; +virResctrlMonitorGetCacheLevel; +virResctrlMonitorGetCacheOccupancy; +virResctrlMonitorGetID; virResctrlMonitorNew; +virResctrlMonitorRemove; +virResctrlMonitorSetCacheLevel; +virResctrlMonitorSetID; # util/virrotatingfile.h diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index 772dfaf..c91078f 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -225,11 +225,19 @@ virResctrlInfoMonFree(virResctrlInfoMonPtr mon) } + +/* + * virResctrlAlloc and virResctrlMonitor are representing a resource control + * group (in XML under cputune/cachetune and consequently a directory under + * /sys/fs/resctrl). virResctrlAlloc is the data structure for resource + * allocation, while the virResctrlMonitor represents the resource monitoring + * part. + */ + /* virResctrlAlloc */ /* - * virResctrlAlloc represents one allocation (in XML under cputune/cachetune and - * consequently a directory under /sys/fs/resctrl). Since it can have multiple + * virResctrlAlloc represents one allocation. Since it can have multiple * parts of multiple caches allocated it is represented as bunch of nested * sparse arrays (by sparse I mean array of pointers so that each might be NULL * in case there is no allocation for that particular cache allocation (level, @@ -347,6 +355,8 @@ struct _virResctrlMonitor { /* libvirt-generated path in /sys/fs/resctrl for this particular * monitor */ char *path; + /* The cache 'level', special for cache monitor */ + unsigned int cache_level; }; @@ -2512,6 +2522,27 @@ virResctrlMonitorDeterminePath(virResctrlMonitorPtr monitor, int +virResctrlMonitorSetID(virResctrlMonitorPtr monitor, + const char *id) +{ + if (!id) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Resctrl monitor 'id' cannot be NULL")); + return -1; + } + + return VIR_STRDUP(monitor->id, id); +} + + +const char * +virResctrlMonitorGetID(virResctrlMonitorPtr monitor) +{ + return monitor->id; +} + + +int virResctrlMonitorCreate(virResctrlAllocPtr alloc, virResctrlMonitorPtr monitor, const char *machinename) @@ -2536,3 +2567,177 @@ virResctrlMonitorCreate(virResctrlAllocPtr alloc, virResctrlUnlock(lockfd); return ret; } + + +int +virResctrlMonitorRemove(virResctrlMonitorPtr monitor) +{ + int ret = 0; + + if (!monitor->path) + return 0; + + VIR_DEBUG("Removing resctrl monitor%s", monitor->path); + if (rmdir(monitor->path) != 0 && errno != ENOENT) { + virReportSystemError(errno, + _("Unable to remove %s (%d)"), + monitor->path, errno); + ret = -errno; + VIR_ERROR(_("Unable to remove %s (%d)"), monitor->path, errno); + } + + return ret; +} + + +int +virResctrlMonitorSetCacheLevel(virResctrlMonitorPtr monitor, + unsigned int level) +{ + /* Only supports cache level 3 CMT */ + if (level != 3) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Invalid resctrl monitor cache level")); + return -1; + } + + monitor->cache_level = level; + + return 0; +} + +unsigned int +virResctrlMonitorGetCacheLevel(virResctrlMonitorPtr monitor) +{ + return monitor->cache_level; +} + + +/* + * virResctrlMonitorGetStatistic + * + * @monitor: The monitor that the statistic data will be retrieved from. + * @resource: The name for resource name. 'llc_occpancy' for cache resource. + * "mbm_totol_bytes" and "mbm_local_bytes" for memory bandwidth resource. + * @len: The array length for @ids, and @vals + * @ids: The id array for resource statistic information, ids[0] + * stores the first node id value, ids[1] stores the second node id value, + * ... and so on. + * @vals: The resource resource utilization information array. vals[0] + * stores the cache or memory bandwidth utilization value for first node, + * vals[1] stores the second value ... and so on. + * + * Get cache or memory bandwidth utilization information from monitor that + * specified by @id. + * + * Returns 0 for success, -1 for error. + */ +static int +virResctrlMonitorGetStatistic(virResctrlMonitorPtr monitor, + const char *resource, + size_t *len, + unsigned int **ids, + unsigned int **vals) +{ + int rv = -1; + int ret = -1; + size_t nids = 0; + size_t nvals = 0; + DIR *dirp = NULL; + char *datapath = NULL; + struct dirent *ent = NULL; + + if (!monitor) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Invalid resctrl monitor")); + return -1; + } + + if (virAsprintf(&datapath, "%s/mon_data", monitor->path) < 0) + return -1; + + if (virDirOpen(&dirp, datapath) < 0) + goto cleanup; + + *len = 0; + while (virDirRead(dirp, &ent, datapath) > 0) { + char *str_id = NULL; + unsigned int id = 0; + unsigned int val = 0; + size_t i = 0; + size_t cur_id_pos = 0; + unsigned int tmp_id = 0; + unsigned int tmp_val = 0; + + /* Looking for directory that contains resource utilization + * information file. The directory name is arranged in format + * "mon_<node_name>_<node_id>". For example, "mon_L3_00" and + * "mon_l3_01" are two target directories for a two nodes system + * with resource utilization data file for each node respectively. + */ + if (ent->d_type != DT_DIR) + continue; + + if (STRNEQLEN(ent->d_name, "mon_L", 5)) + continue; + + str_id = strchr(ent->d_name, '_'); + if (!str_id) + continue; + + str_id = strchr(++str_id, '_'); + if (!str_id) + continue; + + if (virStrToLong_uip(++str_id, NULL, 0, &id) < 0) + goto cleanup; + + rv = virFileReadValueUint(&val, "%s/%s/%s", datapath, + ent->d_name, resource); + if (rv == -2) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("File '%s/%s/%s' does not exist."), + datapath, ent->d_name, resource); + } + if (rv < 0) + goto cleanup; + + if (VIR_APPEND_ELEMENT(*ids, nids, id) < 0) + goto cleanup; + + if (VIR_APPEND_ELEMENT(*vals, nvals, val) < 0) + goto cleanup; + + /* Sort @ids and @vals arrays in the ascending order of id */ + cur_id_pos = nids - 1; + for (i = 0; i < cur_id_pos; i++) { + if ((*ids)[cur_id_pos] < (*ids)[i]) { + tmp_id = (*ids)[cur_id_pos]; + tmp_val = (*vals)[cur_id_pos]; + (*ids)[cur_id_pos] = (*ids)[i]; + (*vals)[cur_id_pos] = (*vals)[i]; + (*ids)[i] = tmp_id; + (*vals)[i] = tmp_val; + } + } + } + + *len = nids; + ret = 0; + cleanup: + VIR_FREE(datapath); + VIR_DIR_CLOSE(dirp); + return ret; +} + + +/* Get cache occupancy data from @monitor */ +int +virResctrlMonitorGetCacheOccupancy(virResctrlMonitorPtr monitor, + size_t *nbank, + unsigned int **bankids, + unsigned int **bankcaches) +{ + return virResctrlMonitorGetStatistic(monitor, "llc_occupancy", + nbank, bankids, bankcaches); +} diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index 1efe394..6137fee 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -202,7 +202,30 @@ virResctrlMonitorDeterminePath(virResctrlMonitorPtr monitor, const char *machinename); int +virResctrlMonitorSetID(virResctrlMonitorPtr monitor, + const char *id); + +const char * +virResctrlMonitorGetID(virResctrlMonitorPtr monitor); + +int virResctrlMonitorCreate(virResctrlAllocPtr alloc, virResctrlMonitorPtr monitor, const char *machinename); + +int +virResctrlMonitorRemove(virResctrlMonitorPtr monitor); + +int +virResctrlMonitorSetCacheLevel(virResctrlMonitorPtr monitor, + unsigned int level); + +unsigned int +virResctrlMonitorGetCacheLevel(virResctrlMonitorPtr monitor); + +int +virResctrlMonitorGetCacheOccupancy(virResctrlMonitorPtr monitor, + size_t *nbank, + unsigned int **bankids, + unsigned int **bankcaches); #endif /* __VIR_RESCTRL_H__ */ -- 2.7.4

Refactoring the code of matching the new resctrl with existing resctrl groups. Add the virObjectRef action into function virDomainResctrlVcpuMatch. Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- src/conf/domain_conf.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index b77680e..e2b4701 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -18833,7 +18833,7 @@ virDomainResctrlVcpuMatch(virDomainDefPtr def, * Just updating memory allocation information of that group */ if (virBitmapEqual(def->resctrls[i]->vcpus, vcpus)) { - *alloc = def->resctrls[i]->alloc; + *alloc = virObjectRef(def->resctrls[i]->alloc); break; } if (virBitmapOverlaps(def->resctrls[i]->vcpus, vcpus)) { @@ -19225,8 +19225,6 @@ virDomainMemorytuneDefParse(virDomainDefPtr def, if (!alloc) goto cleanup; new_alloc = true; - } else { - alloc = virObjectRef(alloc); } for (i = 0; i < n; i++) { -- 2.7.4

Introducing <monitor> element under <cachetune> to represent a cache monitor. Supports two kind of monitors, which are, monitor under default allocation or monitor under particular allocation. Monitor supervises the cache or memory bandwidth usage for interested vcpu thread set, if the vcpu thread set is belong to some resctrl allocation, then the monitor will be created under this allocation, that is, creating a resctrl monitoring group directory under the directory of '@alloc->path/mon_group'. Otherwise, the monitor will be created under default allocation. For default allocation monitor, it will have such kind of XML layout: <cachetune vcpus='1'> <monitor level=3 vcpus='1'/> </cachetune> For other type monitor, the XML layout will be something like: <cachetune vcpus='2-4'> <cache id='0' level='3' type='both' size='3' unit='MiB'/> <cache id='1' level='3' type='both' size='3' unit='MiB'/> <monitor level=3 vcpus='2'/> </cachetune> Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- docs/schemas/domaincommon.rng | 10 + src/conf/domain_conf.c | 217 ++++++++++++++++++++- src/conf/domain_conf.h | 11 ++ tests/genericxml2xmlindata/cachetune-cdp.xml | 3 + .../cachetune-colliding-monitor.xml | 30 +++ tests/genericxml2xmlindata/cachetune-small.xml | 7 + tests/genericxml2xmltest.c | 2 + 7 files changed, 275 insertions(+), 5 deletions(-) create mode 100644 tests/genericxml2xmlindata/cachetune-colliding-monitor.xml diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index 5c533d6..7ce49d3 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -981,6 +981,16 @@ </optional> </element> </zeroOrMore> + <zeroOrMore> + <element name="monitor"> + <attribute name="level"> + <ref name='unsignedInt'/> + </attribute> + <attribute name="vcpus"> + <ref name='cpuset'/> + </attribute> + </element> + </zeroOrMore> </element> </zeroOrMore> <zeroOrMore> diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 9a514a6..4f4604f 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -2955,13 +2955,30 @@ virDomainLoaderDefFree(virDomainLoaderDefPtr loader) static void +virDomainResctrlMonDefFree(virDomainResctrlMonDefPtr domresmon) +{ + if (!domresmon) + return; + + virBitmapFree(domresmon->vcpus); + virObjectUnref(domresmon->instance); +} + + +static void virDomainResctrlDefFree(virDomainResctrlDefPtr resctrl) { + size_t i = 0; + if (!resctrl) return; + for (i = 0; i < resctrl->nmonitors; i++) + virDomainResctrlMonDefFree(resctrl->monitors[i]); + virObjectUnref(resctrl->alloc); virBitmapFree(resctrl->vcpus); + VIR_FREE(resctrl->monitors); VIR_FREE(resctrl); } @@ -18919,6 +18936,154 @@ virDomainCachetuneDefParseCache(xmlXPathContextPtr ctxt, return ret; } +/* Checking if the monitor's vcpus is conflicted with existing allocation + * and monitors. + * + * Returns 1 if @vcpus equals to @resctrl->vcpus, means it is a default + * monitor. Returns - 1 if a conflict found. Returns 0 if no conflict and + * @vcpus is not equal to @resctrl->vcpus. + * */ +static int +virDomainResctrlMonValidateVcpu(virDomainResctrlDefPtr resctrl, + virBitmapPtr vcpus) +{ + size_t i = 0; + int vcpu = -1; + + if (virBitmapIsAllClear(vcpus)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("vcpus is empty")); + return -1; + } + + while ((vcpu = virBitmapNextSetBit(vcpus, vcpu)) >= 0) { + if (!virBitmapIsBitSet(resctrl->vcpus, vcpu)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Monitor vcpus conflicts with allocation")); + return -1; + } + } + + if (resctrl->alloc && virBitmapEqual(vcpus, resctrl->vcpus)) + return 1; + + for (i = 0; i < resctrl->nmonitors; i++) { + if (virBitmapEqual(resctrl->vcpus, resctrl->monitors[i]->vcpus)) + continue; + + if (virBitmapOverlaps(vcpus, resctrl->monitors[i]->vcpus)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Monitor vcpus conflicts with monitors")); + + return -1; + } + } + + return 0; +} + + +static int +virDomainResctrlMonDefParse(virDomainDefPtr def, + xmlXPathContextPtr ctxt, + xmlNodePtr node, + virResctrlMonitorType tag, + virDomainResctrlDefPtr resctrl) +{ + virDomainResctrlMonDefPtr domresmon = NULL; + xmlNodePtr oldnode = ctxt->node; + xmlNodePtr *nodes = NULL; + unsigned int level = 0; + char * tmp = NULL; + char * id = NULL; + size_t i = 0; + int n = 0; + int rv = -1; + int ret = -1; + + ctxt->node = node; + + if ((n = virXPathNodeSet("./monitor", ctxt, &nodes)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot extract monitor nodes")); + goto cleanup; + } + + for (i = 0; i < n; i++) { + + if (VIR_ALLOC(domresmon) < 0) + goto cleanup; + + domresmon->tag = tag; + + domresmon->instance = virResctrlMonitorNew(); + if (!domresmon->instance) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Could not create monitor")); + goto cleanup; + } + + if (tag == VIR_RESCTRL_MONITOR_TYPE_CACHE) { + tmp = virXMLPropString(nodes[i], "level"); + if (!tmp) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("Missing monitor attribute 'level'")); + goto cleanup; + } + + if (virStrToLong_uip(tmp, NULL, 10, &level) < 0) { + virReportError(VIR_ERR_XML_ERROR, + _("Invalid monitor attribute 'level' value '%s'"), + tmp); + goto cleanup; + } + + if (virResctrlMonitorSetCacheLevel(domresmon->instance, level) < 0) + goto cleanup; + + VIR_FREE(tmp); + } + + if (virDomainResctrlParseVcpus(def, nodes[i], &domresmon->vcpus) < 0) + goto cleanup; + + rv = virDomainResctrlMonValidateVcpu(resctrl, domresmon->vcpus); + + /* If monitor's vcpu list is identical to allocation's vcpu list, + * set as default monitor */ + if (rv == 1 && resctrl->alloc) + virResctrlMonitorSetDefault(domresmon->instance); + else if (rv < 0) + goto cleanup; + + if (!(tmp = virBitmapFormat(domresmon->vcpus))) + goto cleanup; + + if (virAsprintf(&id, "vcpus_%s", tmp) < 0) + goto cleanup; + + if (virResctrlMonitorSetID(domresmon->instance, id) < 0) + goto cleanup; + + if (VIR_APPEND_ELEMENT(resctrl->monitors, + resctrl->nmonitors, + domresmon) < 0) + goto cleanup; + + VIR_FREE(id); + VIR_FREE(tmp); + domresmon = NULL; + } + + ret = 0; + cleanup: + ctxt->node = oldnode; + VIR_FREE(id); + VIR_FREE(tmp); + virDomainResctrlMonDefFree(domresmon); + return ret; +} + static virDomainResctrlDefPtr virDomainResctrlNew(virResctrlAllocPtr alloc, @@ -19041,15 +19206,20 @@ virDomainCachetuneDefParse(virDomainDefPtr def, } } - if (virResctrlAllocIsEmpty(alloc)) { - ret = 0; - goto cleanup; - } - resctrl = virDomainResctrlNew(alloc, vcpus); if (!resctrl) goto cleanup; + if (virDomainResctrlMonDefParse(def, ctxt, node, + VIR_RESCTRL_MONITOR_TYPE_CACHE, + resctrl) < 0) + goto cleanup; + + if (virResctrlAllocIsEmpty(alloc) && !resctrl->nmonitors) { + ret = 0; + goto cleanup; + } + if (virDomainResctrlAppend(def, node, resctrl, flags) < 0) goto cleanup; @@ -27085,12 +27255,42 @@ virDomainCachetuneDefFormatHelper(unsigned int level, static int +virDomainResctrlMonDefFormatHelper(virDomainResctrlMonDefPtr domresmon, + virResctrlMonitorType tag, + virBufferPtr buf) +{ + char *vcpus = NULL; + unsigned int level = 0; + + if (domresmon->tag != tag) + return 0; + + virBufferAddLit(buf, "<monitor "); + + if (tag == VIR_RESCTRL_MONITOR_TYPE_CACHE) { + level = virResctrlMonitorGetCacheLevel(domresmon->instance); + virBufferAsprintf(buf, "level='%u' ", level); + } + + vcpus = virBitmapFormat(domresmon->vcpus); + if (!vcpus) + return -1; + + virBufferAsprintf(buf, "vcpus='%s'/>\n", vcpus); + + VIR_FREE(vcpus); + return 0; +} + + +static int virDomainCachetuneDefFormat(virBufferPtr buf, virDomainResctrlDefPtr resctrl, unsigned int flags) { virBuffer childrenBuf = VIR_BUFFER_INITIALIZER; char *vcpus = NULL; + size_t i = 0; int ret = -1; virBufferSetChildIndent(&childrenBuf, buf); @@ -27099,6 +27299,13 @@ virDomainCachetuneDefFormat(virBufferPtr buf, &childrenBuf) < 0) goto cleanup; + for (i = 0; i < resctrl->nmonitors; i ++) { + if (virDomainResctrlMonDefFormatHelper(resctrl->monitors[i], + VIR_RESCTRL_MONITOR_TYPE_CACHE, + &childrenBuf) < 0) + goto cleanup; + } + if (virBufferCheckError(&childrenBuf) < 0) goto cleanup; diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index e30a4b2..60f6464 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -2236,12 +2236,23 @@ struct _virDomainCputune { }; +typedef struct _virDomainResctrlMonDef virDomainResctrlMonDef; +typedef virDomainResctrlMonDef *virDomainResctrlMonDefPtr; +struct _virDomainResctrlMonDef { + virBitmapPtr vcpus; + virResctrlMonitorType tag; + virResctrlMonitorPtr instance; +}; + typedef struct _virDomainResctrlDef virDomainResctrlDef; typedef virDomainResctrlDef *virDomainResctrlDefPtr; struct _virDomainResctrlDef { virBitmapPtr vcpus; virResctrlAllocPtr alloc; + + virDomainResctrlMonDefPtr *monitors; + size_t nmonitors; }; diff --git a/tests/genericxml2xmlindata/cachetune-cdp.xml b/tests/genericxml2xmlindata/cachetune-cdp.xml index 9718f06..9f4c139 100644 --- a/tests/genericxml2xmlindata/cachetune-cdp.xml +++ b/tests/genericxml2xmlindata/cachetune-cdp.xml @@ -8,9 +8,12 @@ <cachetune vcpus='0-1'> <cache id='0' level='3' type='code' size='7680' unit='KiB'/> <cache id='1' level='3' type='data' size='3840' unit='KiB'/> + <monitor level='3' vcpus='0'/> + <monitor level='3' vcpus='1'/> </cachetune> <cachetune vcpus='2'> <cache id='1' level='3' type='code' size='6' unit='MiB'/> + <monitor level='3' vcpus='2'/> </cachetune> <cachetune vcpus='3'> <cache id='1' level='3' type='data' size='6912' unit='KiB'/> diff --git a/tests/genericxml2xmlindata/cachetune-colliding-monitor.xml b/tests/genericxml2xmlindata/cachetune-colliding-monitor.xml new file mode 100644 index 0000000..d481fb5 --- /dev/null +++ b/tests/genericxml2xmlindata/cachetune-colliding-monitor.xml @@ -0,0 +1,30 @@ +<domain type='qemu'> + <name>QEMUGuest1</name> + <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid> + <memory unit='KiB'>219136</memory> + <currentMemory unit='KiB'>219136</currentMemory> + <vcpu placement='static'>4</vcpu> + <cputune> + <cachetune vcpus='0-1'> + <cache id='0' level='3' type='both' size='768' unit='KiB'/> + <monitor level='3' vcpus='2'/> + </cachetune> + </cputune> + <os> + <type arch='i686' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-i686</emulator> + <controller type='usb' index='0'/> + <controller type='ide' index='0'/> + <controller type='pci' index='0' model='pci-root'/> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <memballoon model='virtio'/> + </devices> +</domain> diff --git a/tests/genericxml2xmlindata/cachetune-small.xml b/tests/genericxml2xmlindata/cachetune-small.xml index ab2d9cf..748be08 100644 --- a/tests/genericxml2xmlindata/cachetune-small.xml +++ b/tests/genericxml2xmlindata/cachetune-small.xml @@ -7,6 +7,13 @@ <cputune> <cachetune vcpus='0-1'> <cache id='0' level='3' type='both' size='768' unit='KiB'/> + <monitor level='3' vcpus='0'/> + <monitor level='3' vcpus='1'/> + <monitor level='3' vcpus='0-1'/> + </cachetune> + <cachetune vcpus='2-3'> + <monitor level='3' vcpus='2'/> + <monitor level='3' vcpus='3'/> </cachetune> </cputune> <os> diff --git a/tests/genericxml2xmltest.c b/tests/genericxml2xmltest.c index fa941f0..4393d44 100644 --- a/tests/genericxml2xmltest.c +++ b/tests/genericxml2xmltest.c @@ -137,6 +137,8 @@ mymain(void) TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); DO_TEST_FULL("cachetune-colliding-types", false, true, TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); + DO_TEST_FULL("cachetune-colliding-monitor", false, true, + TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); DO_TEST("memorytune"); DO_TEST_FULL("memorytune-colliding-allocs", false, true, TEST_COMPARE_DOM_XML2XML_RESULT_FAIL_PARSE); -- 2.7.4

Add functions for creating, destroying, reconnecting resctrl monitor in qemu according to the configuration in domain XML. Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- src/qemu/qemu_process.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 29b0ba1..9b103e8 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -2611,10 +2611,21 @@ qemuProcessResctrlCreate(virQEMUDriverPtr driver, return -1; for (i = 0; i < vm->def->nresctrls; i++) { + size_t j = 0; if (virResctrlAllocCreate(caps->host.resctrl, vm->def->resctrls[i]->alloc, priv->machineName) < 0) goto cleanup; + + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + virDomainResctrlMonDefPtr mon = vm->def->resctrls[i]->monitors[j]; + + if (virResctrlMonitorCreate(vm->def->resctrls[i]->alloc, + mon->instance, + priv->machineName) < 0) + goto cleanup; + + } } ret = 0; @@ -5443,11 +5454,22 @@ qemuProcessSetupVcpu(virDomainObjPtr vm, return -1; for (i = 0; i < vm->def->nresctrls; i++) { + size_t j = 0; virDomainResctrlDefPtr ct = vm->def->resctrls[i]; if (virBitmapIsBitSet(ct->vcpus, vcpuid)) { if (virResctrlAllocAddPID(ct->alloc, vcpupid) < 0) return -1; + + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + virDomainResctrlMonDefPtr mon = NULL; + + mon = vm->def->resctrls[i]->monitors[j]; + if (virBitmapIsBitSet(mon->vcpus, vcpuid)) { + if (virResctrlMonitorAddPID(mon->instance, vcpupid) < 0) + return -1; + } + } break; } } @@ -7210,8 +7232,15 @@ void qemuProcessStop(virQEMUDriverPtr driver, /* Remove resctrl allocation after cgroups are cleaned up which makes it * kind of safer (although removing the allocation should work even with * pids in tasks file */ - for (i = 0; i < vm->def->nresctrls; i++) + for (i = 0; i < vm->def->nresctrls; i++) { + size_t j = 0; + virResctrlAllocRemove(vm->def->resctrls[i]->alloc); + for (j = 0; j < vm->def->resctrls[i]->nmonitors; j++) { + virDomainResctrlMonDefPtr mon = vm->def->resctrls[i]->monitors[j]; + virResctrlMonitorRemove(mon->instance); + } + } qemuProcessRemoveDomainStatus(driver, vm); @@ -7947,9 +7976,19 @@ qemuProcessReconnect(void *opaque) goto error; for (i = 0; i < obj->def->nresctrls; i++) { + size_t j = 0; + if (virResctrlAllocDeterminePath(obj->def->resctrls[i]->alloc, priv->machineName) < 0) goto error; + + for (j = 0; j < obj->def->resctrls[i]->nmonitors; j++) { + virDomainResctrlMonDefPtr mon = obj->def->resctrls[i]->monitors[j]; + + if (virResctrlMonitorDeterminePath(mon->instance, + priv->machineName) < 0) + goto error; + } } /* update domain state XML with possibly updated state in virDomainObj */ -- 2.7.4

Refactoring qemuDomainGetStatsCpu, make it possible to add more CPU statistics. Signed-off-by: Wang Huaqiang <huaqiang.wang(a)intel.com&gt; --- src/qemu/qemu_driver.c | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index b238309..9cc6910 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -19708,30 +19708,29 @@ qemuDomainGetStatsCpu(virQEMUDriverPtr driver ATTRIBUTE_UNUSED, unsigned long long sys_time = 0; int err = 0; - if (!priv->cgroup) - return 0; - - err = virCgroupGetCpuacctUsage(priv->cgroup, &cpu_time); - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.time", - cpu_time) < 0) - return -1; + if (priv->cgroup) { + err = virCgroupGetCpuacctUsage(priv->cgroup, &cpu_time); + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.time", + cpu_time) < 0) + return -1; - err = virCgroupGetCpuacctStat(priv->cgroup, &user_time, &sys_time); - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.user", - user_time) < 0) - return -1; - if (!err && virTypedParamsAddULLong(&record->params, - &record->nparams, - maxparams, - "cpu.system", - sys_time) < 0) - return -1; + err = virCgroupGetCpuacctStat(priv->cgroup, &user_time, &sys_time); + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.user", + user_time) < 0) + return -1; + if (!err && virTypedParamsAddULLong(&record->params, + &record->nparams, + maxparams, + "cpu.system", + sys_time) < 0) + return -1; + } return 0; } -- 2.7.4