On Thu, Feb 16, 2017 at 05:35:12PM +0800, Eli Qiao wrote: One tiny nitpick, but it might actually help you as well. You can use -v7 parameter to git send-email or git format-patch to automatically add 'v7' to the subject-prefix keeping the "PATCH" in there. Not only could this be easier for you, but people like me, who filter patches and other mails on the list to different folders, would have these properly categorized. Anyway, for the review now. "get specific resource control information" ??? What do you need syscall.h for? If you need SYSFS_..._PATH for anything, it probably could be split into other src/util/ files. Example below. It would be more consistent if you reused parts of virHostCPUGetValue(), put them in a function, and use that one in both this one an the original one. It chould be also put in the virhostcpu.c since that's about the host cpu. No need for this function, just use virHostCPUParseSocket() So we have some places in the code that get info from sysfs. I understand that cache is controlled in the resctrl, but one doesn't have to have resctrl to get some cache info, so I would move this function into virhostcpu.c and keep here only the stuff strictly related to resource control. Why all this fuzz? You should instead pass pointer to virStrToLong_i to get where the number ends and then use virScaleInteger() to multiply it properly. Still could be in virhostcpu.c Indentation, "||" should be on the previous line but, most importantly, why not switch? That'd make sure you won't miss any enum value and if someone adds a new one, compilator will see it's missing here. set this before the first return so that this function guarantees n_sockets to be 0 on fail. Moreover, n_sockets is always set to 1 here. Due to the way the rest of the function is designed, this doesn't have to be here at all. Wouldn't it be easier to have list of virResCacheBankPtr *banks; and size_t nbanks; and then just populate each pointer when that socket is on the system, so that you that NULL means the socket info was not filled yet. Or just a list that isn't sparse (like yours is now). The logic here seems hard to read. I'll continue the review tomorrow. Martin

-- Best regards Eli 天涯无处不重逢 a leaf duckweed belongs to the sea, where not to meet in life Sent with Sparrow (http://www.sparrowmailapp.com/?sig) On Friday, 17 February 2017 at 8:47 PM, Martin Kletzander wrote: Done Done Done Not really, here ’s tricky. only host id for VIR_RDT_RESOURCE_L3CODE set to VIR_RDT_RESOURCE_L3DATA Done. Done. I tried to move accessors to .c file, but failed, I can not even reference it outside. Done. Removed

This patch expose cache information to host's capabilites xml. For l3 cache allocation <cache> <bank id='0' type='l3' size='56320' unit='KiB' cpus='0-21,44-65'> <control min='2816' reserved='2816' unit='KiB' scope='L3'/> </bank> <bank id='1' type='l3' size='56320' unit='KiB' cpus='22-43,66-87'> <control min='2816' reserved='2816' unit='KiB' scope='L3'/> </bank> </cache> For l3 cache allocation supported cdp(seperate data/code): <cache> <bank id='0' type='l3' size='56320' unit='KiB' cpus='0-21,44-65'> <control min='2816' reserved='2816' unit='KiB' scope='L3DATA'/> <control min='2816' reserved='2816' unit='KiB' scope='L3CODE'/> </bank> <bank id='1' type='l3' size='56320' unit='KiB' cpus='22-43,66-87'> <control min='2816' reserved='2816' unit='KiB' scope='L3DATA'/> <control min='2816' reserved='2816' unit='KiB' scope='L3CODE'/> </bank> </cache> RFC on mailing list. https://www.redhat.com/archives/libvir-list/2017-January/msg00644.html Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com&gt; --- src/conf/capabilities.c | 56 ++++++++++++++++++++++++++++++++++++++ src/conf/capabilities.h | 23 ++++++++++++++++ src/libvirt_private.syms | 3 +++ src/nodeinfo.c | 64 ++++++++++++++++++++++++++++++++++++++++++++ src/nodeinfo.h | 1 + src/qemu/qemu_capabilities.c | 8 ++++++ src/qemu/qemu_driver.c | 4 +++ 7 files changed, 159 insertions(+) diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c index 9ab343b..23e21e4 100644 --- a/src/conf/capabilities.c +++ b/src/conf/capabilities.c @@ -198,6 +198,18 @@ virCapabilitiesClearSecModel(virCapsHostSecModelPtr secmodel) } static void +virCapabilitiesClearCacheBank(virCapsHostCacheBankPtr cachebank) +{ + size_t i; + for (i = 0; i < cachebank->ncontrol; i++) + VIR_FREE(cachebank->control[i].scope); + + VIR_FREE(cachebank->type); + VIR_FREE(cachebank->cpus); +} + + +static void virCapabilitiesDispose(void *object) { virCapsPtr caps = object; @@ -221,6 +233,10 @@ virCapabilitiesDispose(void *object) virCapabilitiesClearSecModel(&caps->host.secModels[i]); VIR_FREE(caps->host.secModels); + for (i = 0; i < caps->host.ncachebank; i++) + virCapabilitiesClearCacheBank(caps->host.cachebank[i]); + VIR_FREE(caps->host.cachebank); + VIR_FREE(caps->host.netprefix); VIR_FREE(caps->host.pagesSize); virCPUDefFree(caps->host.cpu); @@ -844,6 +860,41 @@ virCapabilitiesFormatNUMATopology(virBufferPtr buf, return 0; } +static int +virCapabilitiesFormatCache(virBufferPtr buf, + size_t ncachebank, + virCapsHostCacheBankPtr *cachebank) +{ + size_t i; + size_t j; + + virBufferAddLit(buf, "<cache>\n"); + virBufferAdjustIndent(buf, 2); + + for (i = 0; i < ncachebank; i++) { + virBufferAsprintf(buf, + "<bank id='%u' type='%s' size='%llu' unit='KiB' cpus='%s'>\n", + cachebank[i]->id, + cachebank[i]->type, + cachebank[i]->size, + cachebank[i]->cpus); + + virBufferAdjustIndent(buf, 2); + for (j = 0; j < cachebank[i]->ncontrol; j++) { + virBufferAsprintf(buf, + "<control min='%llu' reserved='%llu' unit='KiB' scope='%s'/>\n", + cachebank[i]->control[j].min, + cachebank[i]->control[j].reserved, + cachebank[i]->control[j].scope); + } + virBufferAdjustIndent(buf, -2); + virBufferAddLit(buf, "</bank>\n"); + } + virBufferAdjustIndent(buf, -2); + virBufferAddLit(buf, "</cache>\n"); + return 0; +} + /** * virCapabilitiesFormatXML: * @caps: capabilities to format @@ -931,6 +982,11 @@ virCapabilitiesFormatXML(virCapsPtr caps) virBufferAddLit(&buf, "</migration_features>\n"); } + if (caps->host.ncachebank && + virCapabilitiesFormatCache(&buf, caps->host.ncachebank, + caps->host.cachebank) < 0) + return NULL; + if (caps->host.netprefix) virBufferAsprintf(&buf, "<netprefix>%s</netprefix>\n", caps->host.netprefix); diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h index cfdc34a..b446de5 100644 --- a/src/conf/capabilities.h +++ b/src/conf/capabilities.h @@ -138,6 +138,25 @@ struct _virCapsHostSecModel { virCapsHostSecModelLabelPtr labels; }; +typedef struct _virCapsHostCacheControl virCapsHostCacheControl; +typedef virCapsHostCacheControl *virCapsHostCacheControlPtr; +struct _virCapsHostCacheControl { + unsigned long long min; + unsigned long long reserved; + char* scope; +}; + +typedef struct _virCapsHostCacheBank virCapsHostCacheBank; +typedef virCapsHostCacheBank *virCapsHostCacheBankPtr; +struct _virCapsHostCacheBank { + unsigned int id; + char* type; + char* cpus; + unsigned long long size; + size_t ncontrol; + virCapsHostCacheControlPtr control; +}; + typedef struct _virCapsHost virCapsHost; typedef virCapsHost *virCapsHostPtr; struct _virCapsHost { @@ -160,6 +179,10 @@ struct _virCapsHost { size_t nsecModels; virCapsHostSecModelPtr secModels; + size_t ncachebank; + size_t ncachebank_max; + virCapsHostCacheBankPtr *cachebank; + char *netprefix; virCPUDefPtr cpu; int nPagesSize; /* size of pagesSize array */ diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 743e5ac..cc6c433 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1123,6 +1123,7 @@ virLogManagerNew; # nodeinfo.h nodeCapsInitNUMA; nodeGetInfo; +virCapsInitCache; virHostCPUGetCount; virHostCPUGetKVMMaxVCPUs; virHostCPUGetMap; @@ -2315,8 +2316,10 @@ virRandomInt; # util/virresctrl.h virResCtrlAvailable; +virResCtrlGet; virResCtrlInit; + # util/virrotatingfile.h virRotatingFileReaderConsume; virRotatingFileReaderFree; diff --git a/src/nodeinfo.c b/src/nodeinfo.c index f2ded02..a001cd0 100644 --- a/src/nodeinfo.c +++ b/src/nodeinfo.c @@ -48,6 +48,7 @@ #include "virstring.h" #include "virnuma.h" #include "virlog.h" +#include "virresctrl.h" #define VIR_FROM_THIS VIR_FROM_NONE @@ -416,3 +417,66 @@ nodeCapsInitNUMA(virCapsPtr caps) VIR_FREE(pageinfo); return ret; } + +int +virCapsInitCache(virCapsPtr caps) +{ + size_t i, j; + virResCtrlPtr resctrl; + virCapsHostCacheBankPtr bank; + + for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) { + /* L3DATA and L3CODE share L3 resources */ + if (i == VIR_RDT_RESOURCE_L3CODE) + continue; + + resctrl = virResCtrlGet(i); + + if (resctrl->enabled) { + for (j = 0; j < resctrl->num_banks; j++) { + if (VIR_RESIZE_N(caps->host.cachebank, caps->host.ncachebank_max, + caps->host.ncachebank, 1) < 0) + return -1; + + if (VIR_ALLOC(bank) < 0) + return -1; + + bank->id = resctrl->cache_banks[j].host_id; + if (VIR_STRDUP(bank->type, resctrl->cache_level) < 0) + goto err; + if (VIR_STRDUP(bank->cpus, virBitmapFormat(resctrl->cache_banks[j].cpu_mask)) < 0) + goto err; + bank->size = resctrl->cache_banks[j].cache_size; + /*L3DATA and L3CODE shares L3 cache resources, so fill them to the control element*/ + if (i == VIR_RDT_RESOURCE_L3DATA) { + if (VIR_EXPAND_N(bank->control, bank->ncontrol, 2) < 0) + goto err; + + bank->control[0].min = virResCtrlGet(VIR_RDT_RESOURCE_L3DATA)->cache_banks[j].cache_min; + bank->control[0].reserved = bank->control[0].min * (virResCtrlGet(VIR_RDT_RESOURCE_L3DATA)->min_cbm_bits); + if (VIR_STRDUP(bank->control[0].scope, + virResCtrlGet(VIR_RDT_RESOURCE_L3DATA)->name) < 0) + goto err; + + bank->control[1].min = virResCtrlGet(VIR_RDT_RESOURCE_L3CODE)->cache_banks[j].cache_min; + bank->control[1].reserved = bank->control[1].min * (virResCtrlGet(VIR_RDT_RESOURCE_L3CODE)->min_cbm_bits); + if (VIR_STRDUP(bank->control[1].scope, + virResCtrlGet(VIR_RDT_RESOURCE_L3CODE)->name) < 0) + goto err; + } else { + if (VIR_EXPAND_N(bank->control, bank->ncontrol, 1) < 0) + goto err; + bank->control[0].min = resctrl->cache_banks[j].cache_min; + bank->control[0].reserved = bank->control[0].min * resctrl->min_cbm_bits; + if (VIR_STRDUP(bank->control[0].scope, resctrl->name) < 0) + goto err; + } + caps->host.cachebank[caps->host.ncachebank++] = bank; + } + } + } + return 0; + err: + VIR_FREE(bank); + return -1; +} diff --git a/src/nodeinfo.h b/src/nodeinfo.h index 3c4dc46..5eb0f83 100644 --- a/src/nodeinfo.h +++ b/src/nodeinfo.h @@ -28,5 +28,6 @@ int nodeGetInfo(virNodeInfoPtr nodeinfo); int nodeCapsInitNUMA(virCapsPtr caps); +int virCapsInitCache(virCapsPtr caps); #endif /* __VIR_NODEINFO_H__*/ diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 3247d25..662a9ed 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -1098,6 +1098,11 @@ virQEMUCapsInitCPU(virCapsPtr caps, goto cleanup; } +static int +virQEMUCapsInitCache(virCapsPtr caps) +{ + return virCapsInitCache(caps); +} static int virQEMUCapsInitPages(virCapsPtr caps) @@ -1144,6 +1149,9 @@ virCapsPtr virQEMUCapsInit(virQEMUCapsCachePtr cache) if (virQEMUCapsInitCPU(caps, hostarch) < 0) VIR_WARN("Failed to get host CPU"); + if (virQEMUCapsInitCache(caps) < 0) + VIR_WARN("Failed to get host cache"); + /* Add the power management features of the host */ if (virNodeSuspendGetTargetMask(&caps->host.powerMgmt) < 0) VIR_WARN("Failed to get host power management capabilities"); diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 37ccfdf..7995511 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -105,6 +105,7 @@ #include "vircgroup.h" #include "virperf.h" #include "virnuma.h" +#include "virresctrl.h" #include "dirname.h" #include "network/bridge_driver.h" @@ -849,6 +850,9 @@ qemuStateInitialize(bool privileged, run_gid = cfg->group; } + if (virResCtrlAvailable() && virResCtrlInit() < 0) + VIR_WARN("Faild to initialize resource control."); + qemu_driver->qemuCapsCache = virQEMUCapsCacheNew(cfg->libDir, cfg->cacheDir, run_uid, -- 1.9.1

virResCtrlSetCacheBanks: Set cache banks of a libvirt domain. It will create new resource domain under `/sys/fs/resctrl` and fill the schemata according the cache banks configration. virResCtrlUpdate: Update the schemata after libvirt domain destroy. Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com&gt; --- src/libvirt_private.syms | 6 +- src/util/virresctrl.c | 704 ++++++++++++++++++++++++++++++++++++++++++++++- src/util/virresctrl.h | 6 +- 3 files changed, 710 insertions(+), 6 deletions(-) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index cc6c433..85b9e94 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -2318,7 +2318,11 @@ virRandomInt; virResCtrlAvailable; virResCtrlGet; virResCtrlInit; - +virResCtrlSetCacheBanks; +virResCtrlTypeFromString; +virResCtrlTypeToString; +virResCtrlUpdate; +# # util/virrotatingfile.h virRotatingFileReaderConsume; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index b49e965..3da3ec0 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -72,9 +72,59 @@ do { \ #define VIR_RESCTRL_SET_SCHEMATA(p, type, pos, val) \ p->schematas[type]->schemata_items[pos] = val +/** + * a virResSchemata represents a schemata object under a resource control + * domain. + */ +typedef struct _virResSchemataItem virResSchemataItem; +typedef virResSchemataItem *virResSchemataItemPtr; +struct _virResSchemataItem { + unsigned int socket_no; + unsigned schemata; +}; + +typedef struct _virResSchemata virResSchemata; +typedef virResSchemata *virResSchemataPtr; +struct _virResSchemata { + unsigned int n_schemata_items; + virResSchemataItemPtr schemata_items; +}; + +/** + * a virResDomain represents a resource control domain. It's a double linked + * list. + */ + +typedef struct _virResDomain virResDomain; +typedef virResDomain *virResDomainPtr; + +struct _virResDomain { + char *name; + virResSchemataPtr schematas[VIR_RDT_RESOURCE_LAST]; + char **tasks; + size_t n_tasks; + size_t n_sockets; + virResDomainPtr pre; + virResDomainPtr next; +}; + +/* All resource control domains on this host*/ + +typedef struct _virResCtrlDomain virResCtrlDomain; +typedef virResCtrlDomain *virResCtrlDomainPtr; +struct _virResCtrlDomain { + unsigned int num_domains; + virResDomainPtr domains; +}; + static unsigned int host_id; +/* Global static struct to be maintained which is a interface */ +static virResCtrlDomain domainall; + +/* Global static struct array to be maintained which indicate + * resource status on a host */ static virResCtrl resctrlall[] = { { .name = "L3", @@ -94,6 +144,66 @@ static virResCtrl resctrlall[] = { }, }; +/* + * How many bits is set in schemata + * eg: + * virResCtrlBitsNum(1011) = 2 */ +static int virResCtrlBitsContinuesNum(unsigned schemata) +{ + size_t i; + int ret = 0; + for (i = 0; i < MAX_CBM_BIT_LEN; i ++) { + if ((schemata & 0x1) == 0x1) + ret++; + else + break; + schemata = schemata >> 1; + if (schemata == 0) break; + } + return ret; +} + +static int virResCtrlGetStr(const char *domain_name, const char *item_name, char **ret) +{ + char *path; + int rc = 0; + + CONSTRUCT_RESCTRL_PATH(domain_name, item_name); + + if (virFileReadAll(path, MAX_FILE_LEN, ret) < 0) { + rc = -1; + goto cleanup; + } + + cleanup: + VIR_FREE(path); + return rc; +} + +static int virResCtrlGetTasks(const char *domain_name, char **pids) +{ + return virResCtrlGetStr(domain_name, "tasks", pids); +} + +static int virResCtrlGetSchemata(const int type, const char *name, char **schemata) +{ + int rc; + char *tmp, *end; + char *buf; + + if ((rc = virResCtrlGetStr(name, "schemata", &buf)) < 0) + return rc; + + tmp = strstr(buf, resctrlall[type].name); + end = strchr(tmp, '\n'); + *end = '\0'; + if (VIR_STRDUP(*schemata, tmp) < 0) + rc = -1; + + VIR_FREE(buf); + return rc; +} + static int virResCtrlGetInfoStr(const int type, const char *item, char **str) { int ret = 0; @@ -114,6 +224,70 @@ static int virResCtrlGetInfoStr(const int type, const char *item, char **str) return ret; } +/* Return pointer of and ncount of schemata*/ +static virResSchemataPtr virParseSchemata(const char *schemata_str, size_t *ncount) +{ + const char *p, *q; + int pos; + int ischemata; + virResSchemataPtr schemata; + virResSchemataItemPtr schemataitems, tmpitem; + unsigned int socket_no = 0; + char *tmp; + + if (VIR_ALLOC(schemata) < 0) + goto cleanup; + + p = q = schemata_str; + pos = strchr(schemata_str, ':') - p; + + /* calculate cpu socket count */ + *ncount = 1; + while ((q = strchr(p, ';')) != 0) { + p = q + 1; + (*ncount)++; + } + + /* allocat an arrry to store schemata for each socket*/ + if (VIR_ALLOC_N_QUIET(tmpitem, *ncount) < 0) + goto cleanup; + + schemataitems = tmpitem; + + p = q = schemata_str + pos + 1; + + while (*p != '\0') { + if (*p == '=') { + q = p + 1; + + tmpitem->socket_no = socket_no++; + + while (*p != ';' && *p != '\0') p++; + + if (VIR_STRNDUP(tmp, q, p-q) < 0) + goto cleanup; + + if (virStrToLong_i(tmp, NULL, 16, &ischemata) < 0) + goto cleanup; + + VIR_FREE(tmp); + tmp = NULL; + tmpitem->schemata = ischemata; + tmpitem ++; + schemata->n_schemata_items += 1; + } + p++; + } + + schemata->schemata_items = schemataitems; + return schemata; + + cleanup: + VIR_FREE(schemata); + VIR_FREE(tmpitem); + return NULL; +} + static int virResCtrlGetCPUValue(const char *path, char **value) { @@ -251,6 +425,7 @@ static virResCacheBankPtr virResCtrlGetCacheBanks(int type, int *n_sockets) bank[s_id].cache_size = cache_size; bank[s_id].cache_min = cache_size / resctrlall[type].cbm_len; + bank[s_id].cache_left = cache_size - (bank[s_id].cache_min * resctrlall[type].min_cbm_bits); } } return bank; @@ -267,7 +442,7 @@ static int virResCtrlGetConfig(int type) size_t i; char *str; - /* Read min_cbm_bits from resctrl. + /* Read num_closids from resctrl. eg: /sys/fs/resctrl/info/L3/num_closids */ if ((ret = virResCtrlGetInfoStr(type, "num_closids", &str)) < 0) @@ -320,6 +495,524 @@ static int virResCtrlGetConfig(int type) return ret; } +/* Remove the Domain from sysfs, this should only success no pids in tasks + * of a partition. + */ +static +int virResCtrlRemoveDomain(const char *name) +{ + char *path = NULL; + int rc = 0; + + if ((rc = virAsprintf(&path, "%s/%s", RESCTRL_DIR, name)) < 0) + return rc; + rc = rmdir(path); + VIR_FREE(path); + return rc; +} + +static +int virResCtrlDestroyDomain(virResDomainPtr p) +{ + size_t i; + int rc; + if ((rc = virResCtrlRemoveDomain(p->name)) < 0) + VIR_WARN("Failed to removed partition %s", p->name); + + VIR_FREE(p->name); + p->name = NULL; + for (i = 0; i < p->n_tasks; i ++) + VIR_FREE(p->tasks[i]); + VIR_FREE(p); + p = NULL; + return rc; +} + + +/* assemble schemata string*/ +static +char* virResCtrlAssembleSchemata(virResSchemataPtr schemata, int type) +{ + virBuffer buf = VIR_BUFFER_INITIALIZER; + size_t i; + + virBufferAsprintf(&buf, "%s:%u=%x", resctrlall[type].name, + schemata->schemata_items[0].socket_no, + schemata->schemata_items[0].schemata); + + for (i = 1; i < schemata->n_schemata_items; i++) { + virBufferAsprintf(&buf, ";%u=%x", + schemata->schemata_items[i].socket_no, + schemata->schemata_items[i].schemata); + } + + return virBufferContentAndReset(&buf); +} + +/* Refresh default domains' schemata + */ +static +int virResCtrlRefreshSchemata(void) +{ + size_t i, j, k; + unsigned int tmp_schemata; + unsigned int default_schemata; + + virResDomainPtr header, p; + + header = domainall.domains; + + if (!header) + return 0; + + for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) { + if (VIR_RESCTRL_ENABLED(i)) { + for (j = 0; j < header->schematas[i]->n_schemata_items; j ++) { + p = header->next; + default_schemata = VIR_RESCTRL_GET_SCHEMATA(resctrlall[i].cbm_len); + tmp_schemata = 0; + /* NOTEs: if only header domain, the schemata will be set to default one*/ + for (k = 1; k < domainall.num_domains; k++) { + tmp_schemata |= p->schematas[i]->schemata_items[j].schemata; + p = p->next; + } + /* sys fs doens't let us use 0 */ + int min_bits = VIR_RESCTRL_GET_SCHEMATA(resctrlall[i].min_cbm_bits); + if ((tmp_schemata & min_bits) == min_bits) + tmp_schemata -= min_bits; + + default_schemata ^= tmp_schemata; + + int bitsnum = virResCtrlBitsContinuesNum(default_schemata); + // calcuate header's schemata + // NOTES: resctrl sysfs only allow us to set a continues schemata + header->schematas[i]->schemata_items[j].schemata = VIR_RESCTRL_GET_SCHEMATA(bitsnum); + resctrlall[i].cache_banks[j].cache_left = + (bitsnum - resctrlall[i].min_cbm_bits) * resctrlall[i].cache_banks[j].cache_min; + } + } + } + + return 0; + +} + +/* Refresh all domains', remove the domains which has no task ids. + * This will be used after VM pause, restart, destroy etc. + */ +static int +virResCtrlRefresh(void) +{ + size_t i; + char* tasks; + unsigned int origin_count = domainall.num_domains; + virResDomainPtr p, pre, del; + pre = domainall.domains; + p = del = NULL; + if (pre) + p = pre->next; + + for (i = 1; i < origin_count && p; i++) { + if (virResCtrlGetTasks(p->name, &tasks) < 0) { + VIR_WARN("Failed to get tasks from %s", p->name); + pre = p; + p = p->next; + } + if (virStringIsEmpty(tasks)) { + pre->next = p->next; + if (p->next != NULL) + p->next->pre = pre; + + del = p; + p = p->next; + virResCtrlDestroyDomain(del); + domainall.num_domains -= 1; + } else { + pre = p; + p = p->next; + } + VIR_FREE(tasks); + + } + + return virResCtrlRefreshSchemata(); +} + +/* Get a domain ptr by domain's name*/ +static +virResDomainPtr virResCtrlGetDomain(const char* name) { + size_t i; + virResDomainPtr p = domainall.domains; + for (i = 0; i < domainall.num_domains; i++) { + if ((p->name) && STREQ(name, p->name)) + return p; + p = p->next; + } + return NULL; +} + +static int +virResCtrlAddTask(virResDomainPtr dom, pid_t pid) +{ + size_t maxtasks; + + if (!dom->tasks) { + if (VIR_ALLOC_N(dom->tasks, 1) < 0) + return -1; + } else { + if (VIR_RESIZE_N(dom->tasks, maxtasks, dom->n_tasks, 1) < 0) + return -1; + } + + if (virAsprintf(&(dom->tasks[dom->n_tasks]), "%llu", (long long)pid) < 0) + return -1; + + dom->n_tasks += 1; + return 0; +} + +static int +virResCtrlWrite(const char *name, const char *item, const char *content) +{ + char *path; + int writefd; + int rc = -1; + + CONSTRUCT_RESCTRL_PATH(name, item); + + if (!virFileExists(path)) + goto cleanup; + + if ((writefd = open(path, O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR)) < 0) + goto cleanup; + + if (safewrite(writefd, content, strlen(content)) < 0) + goto cleanup; + + rc = 0; + cleanup: + VIR_FREE(path); + VIR_FORCE_CLOSE(writefd); + return rc; +} + +/* if name == NULL we load default schemata */ +static +virResDomainPtr virResCtrlLoadDomain(const char *name) +{ + char *schematas; + virResDomainPtr p; + size_t i; + + if (VIR_ALLOC(p) < 0) + goto cleanup; + + for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) { + if (VIR_RESCTRL_ENABLED(i)) { + if (virResCtrlGetSchemata(i, name, &schematas) < 0) + goto cleanup; + p->schematas[i] = virParseSchemata(schematas, &(p->n_sockets)); + VIR_FREE(schematas); + } + } + + p->tasks = NULL; + p->n_tasks = 0; + + if ((name != NULL) && (VIR_STRDUP(p->name, name)) < 0) + goto cleanup; + + return p; + + cleanup: + VIR_FREE(p); + return NULL; +} + +static +virResDomainPtr virResCtrlCreateDomain(const char *name) +{ + char *path; + mode_t mode = 0755; + virResDomainPtr p; + size_t i, j; + if (virAsprintf(&path, "%s/%s", RESCTRL_DIR, name) < 0) + return NULL; + + if (virDirCreate(path, mode, 0, 0, 0) < 0) + goto cleanup; + + if ((p = virResCtrlLoadDomain(name)) == NULL) + return p; + + /* sys fs doens't let us use 0. + * reset schemata to min_bits*/ + for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) { + if (VIR_RESCTRL_ENABLED(i)) { + int min_bits = VIR_RESCTRL_GET_SCHEMATA(resctrlall[i].min_cbm_bits); + for (j = 0; j < p->n_sockets; j++) + p->schematas[i]->schemata_items[j].schemata = min_bits; + } + } + + VIR_FREE(path); + return p; + + cleanup: + VIR_FREE(path); + return NULL; +} + +/* flush domains's information to sysfs*/ +static int +virResCtrlFlushDomainToSysfs(virResDomainPtr dom) +{ + size_t i; + char* schemata; + char* tmp; + int rc = -1; + virBuffer buf = VIR_BUFFER_INITIALIZER; + + for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) { + if (VIR_RESCTRL_ENABLED(i)) { + tmp = virResCtrlAssembleSchemata(dom->schematas[i], i); + virBufferAsprintf(&buf, "%s\n", tmp); + VIR_FREE(tmp); + } + } + + schemata = virBufferContentAndReset(&buf); + + if (virResCtrlWrite(dom->name, "schemata", schemata) < 0) + goto cleanup; + + if (dom->n_tasks > 0) { + for (i = 0; i < dom->n_tasks; i++) { + if (virResCtrlWrite(dom->name, "tasks", dom->tasks[i]) < 0) + goto cleanup; + } + } + + rc = 0; + + cleanup: + VIR_FREE(schemata); + return rc; +} + +static virResDomainPtr virResCtrlGetAllDomains(unsigned int *len) +{ + struct dirent *ent; + DIR *dp = NULL; + int direrr; + + *len = 0; + virResDomainPtr header, tmp, tmp_pre; + header = tmp = tmp_pre = NULL; + if (virDirOpenQuiet(&dp, RESCTRL_DIR) < 0) { + if (errno == ENOENT) + return NULL; + VIR_ERROR(_("Unable to open %s (%d)"), RESCTRL_DIR, errno); + goto cleanup; + } + + header = virResCtrlLoadDomain(NULL); + if (header == NULL) + goto cleanup; + + header->next = NULL; + + *len = 1; + + while ((direrr = virDirRead(dp, &ent, NULL)) > 0) { + if ((ent->d_type != DT_DIR) || STREQ(ent->d_name, "info")) + continue; + + tmp = virResCtrlLoadDomain(ent->d_name); + if (tmp == NULL) + goto cleanup; + + tmp->next = NULL; + + if (header->next == NULL) + header->next = tmp; + + if (tmp_pre == NULL) { + tmp->pre = header; + } else { + tmp->pre = tmp_pre; + tmp_pre->next = tmp; + } + + tmp_pre = tmp; + (*len) ++; + } + return header; + + cleanup: + VIR_DIR_CLOSE(dp); + tmp_pre = tmp = header; + while (tmp) { + tmp_pre = tmp; + tmp = tmp->next; + VIR_FREE(tmp_pre); + } + return NULL; +} + +static int +virResCtrlAppendDomain(virResDomainPtr dom) +{ + virResDomainPtr p = domainall.domains; + while (p->next != NULL) p = p->next; + p->next = dom; + dom->pre = p; + domainall.num_domains += 1; + return 0; +} + +static int +virResCtrlGetSocketIdByHostID(int type, unsigned int hostid) +{ + size_t i; + for (i = 0; i < resctrlall[type].num_banks; i++) { + if (resctrlall[type].cache_banks[i].host_id == hostid) + return i; + } + return -1; +} + +static int +virResCtrlCalculateSchemata(int type, + int sid, + unsigned hostid, + unsigned long long size) +{ + size_t i; + int count; + virResDomainPtr p; + unsigned int tmp_schemata; + unsigned int schemata_sum = 0; + + if (resctrlall[type].cache_banks[sid].cache_left < size) { + VIR_ERROR(_("Not enough cache left on bank %u"), hostid); + return -1; + } + if ((count = size / resctrlall[type].cache_banks[sid].cache_min) <= 0) { + VIR_ERROR(_("Error cache size %llu"), size); + return -1; + } + + tmp_schemata = VIR_RESCTRL_GET_SCHEMATA(count); + + p = domainall.domains; + p = p->next; + for (i = 1; i < domainall.num_domains; i ++) { + schemata_sum |= p->schematas[type]->schemata_items[sid].schemata; + p = p->next; + } + + tmp_schemata = tmp_schemata << (resctrlall[type].cbm_len - count); + + while ((tmp_schemata & schemata_sum) != 0) + tmp_schemata = tmp_schemata >> 1; + return tmp_schemata; +} + +int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune, + unsigned char *uuid, pid_t *pids, int npid) +{ + size_t i; + char name[VIR_UUID_STRING_BUFLEN]; + virResDomainPtr p; + int type; + int sid; + int schemata; + + virUUIDFormat(uuid, name); + + for (i = 0; i < cachetune->n_banks; i++) { + VIR_DEBUG("cache_banks %u, %u, %llu, %s", + cachetune->cache_banks[i].id, + cachetune->cache_banks[i].host_id, + cachetune->cache_banks[i].size, + cachetune->cache_banks[i].type); + } + + if (cachetune->n_banks < 1) + return 0; + + p = virResCtrlGetDomain(name); + if (p == NULL) { + VIR_DEBUG("no domain name %s found, create new one!", name); + p = virResCtrlCreateDomain(name); + } + + if (p != NULL) { + for (i = 0; i < cachetune->n_banks; i++) { + if ((type = virResCtrlTypeFromString( + cachetune->cache_banks[i].type)) < 0) { + VIR_WARN("Ignore unknown cache type %s.", + cachetune->cache_banks[i].type); + continue; + } + + if ((sid = virResCtrlGetSocketIdByHostID( + type, cachetune->cache_banks[i].host_id)) < 0) { + VIR_WARN("Can not find cache bank host id %u.", + cachetune->cache_banks[i].host_id); + continue; + } + + if ((schemata = virResCtrlCalculateSchemata( + type, sid, cachetune->cache_banks[i].host_id, + cachetune->cache_banks[i].size)) < 0) { + VIR_WARN("Failed to set schemata for cache bank id %u", + cachetune->cache_banks[i].id); + continue; + } + + p->schematas[type]->schemata_items[sid].schemata = schemata; + } + + for (i = 0; i < npid; i++) + virResCtrlAddTask(p, pids[i]); + + if (virResCtrlFlushDomainToSysfs(p) < 0) { + VIR_ERROR(_("failed to flush domain %s to sysfs"), name); + virResCtrlDestroyDomain(p); + return -1; + } + virResCtrlAppendDomain(p); + } else { + VIR_ERROR(_("Failed to create a domain in sysfs")); + return -1; + } + + virResCtrlRefresh(); + /* after refresh, flush header's schemata changes to sys fs */ + if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0) + VIR_WARN("failed to flush domain to sysfs"); + + return 0; +} + +/* Should be called after pid disappeared, we recalculate + * schemata of default and flush it to sys fs. + */ +int virResCtrlUpdate(void) +{ + int rc; + + if ((rc = virResCtrlRefresh()) < 0) + VIR_WARN("failed to refresh rescontrol"); + + if ((rc = virResCtrlFlushDomainToSysfs(domainall.domains)) < 0) + VIR_WARN("failed to flush domain to sysfs"); + + return rc; +} + int virResCtrlInit(void) { @@ -333,13 +1026,18 @@ virResCtrlInit(void) return -1; } if (virFileExists(tmp)) { - if ((rc = virResCtrlGetConfig(i)) < 0) + if ((rc = virResCtrlGetConfig(i)) < 0) { VIR_ERROR(_("Failed to get resource control config")); return -1; + } } - VIR_FREE(tmp); } + + domainall.domains = virResCtrlGetAllDomains(&(domainall.num_domains)); + + if ((rc = virResCtrlRefresh()) < 0) + VIR_ERROR(_("Failed to refresh resource control")); return rc; } diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h index 3f6fcae..ba5697e 100644 --- a/src/util/virresctrl.h +++ b/src/util/virresctrl.h @@ -26,6 +26,7 @@ # include "virutil.h" # include "virbitmap.h" +# include "domain_conf.h" enum { VIR_RDT_RESOURCE_L3, @@ -73,9 +74,10 @@ struct _virResCtrl { virResCacheBankPtr cache_banks; }; - bool virResCtrlAvailable(void); int virResCtrlInit(void); virResCtrlPtr virResCtrlGet(int); - +int virResCtrlSetCacheBanks(virDomainCachetunePtr, + unsigned char *, pid_t *, int); +int virResCtrlUpdate(void); #endif -- 1.9.1

Enable l3code/l3data while doing cache tune. l3code/l3data should use a continus cbm in their seperated schemata and the cache size are shared between them, so we need to deal them differently with l3 cache. This should enable cdp feature while mounting /sys/fs/resctrl, eg: mount -t resctrl resctrl -o cdp /sys/fs/resctrl Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com&gt; --- docs/schemas/domaincommon.rng | 2 +- src/util/virresctrl.c | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index edb2888..f61c57a 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -5489,7 +5489,7 @@ </define> <define name="cachetype"> <data type="string"> - <param name="pattern">(l3)</param> + <param name="pattern">(l3|l3code|l3data)</param> </data> </define> <define name="cacheunit"> diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index 3da3ec0..3ca3600 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -557,6 +557,7 @@ int virResCtrlRefreshSchemata(void) size_t i, j, k; unsigned int tmp_schemata; unsigned int default_schemata; + int pair_type = 0; virResDomainPtr header, p; @@ -567,6 +568,12 @@ int virResCtrlRefreshSchemata(void) for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) { if (VIR_RESCTRL_ENABLED(i)) { + + if (i == VIR_RDT_RESOURCE_L3DATA) + pair_type = VIR_RDT_RESOURCE_L3CODE; + if (i == VIR_RDT_RESOURCE_L3CODE) + pair_type = VIR_RDT_RESOURCE_L3DATA; + for (j = 0; j < header->schematas[i]->n_schemata_items; j ++) { p = header->next; default_schemata = VIR_RESCTRL_GET_SCHEMATA(resctrlall[i].cbm_len); @@ -574,6 +581,8 @@ int virResCtrlRefreshSchemata(void) /* NOTEs: if only header domain, the schemata will be set to default one*/ for (k = 1; k < domainall.num_domains; k++) { tmp_schemata |= p->schematas[i]->schemata_items[j].schemata; + if (pair_type > 0) + tmp_schemata |= p->schematas[pair_type]->schemata_items[j].schemata; p = p->next; } /* sys fs doens't let us use 0 */ @@ -690,6 +699,7 @@ virResCtrlWrite(const char *name, const char *item, const char *content) goto cleanup; rc = 0; + cleanup: VIR_FREE(path); VIR_FORCE_CLOSE(writefd); @@ -893,6 +903,7 @@ virResCtrlCalculateSchemata(int type, virResDomainPtr p; unsigned int tmp_schemata; unsigned int schemata_sum = 0; + int pair_type = 0; if (resctrlall[type].cache_banks[sid].cache_left < size) { VIR_ERROR(_("Not enough cache left on bank %u"), hostid); @@ -907,8 +918,18 @@ virResCtrlCalculateSchemata(int type, p = domainall.domains; p = p->next; + + /* for type is l3code and l3data, we need to deal them specially*/ + if (type == VIR_RDT_RESOURCE_L3DATA) + pair_type = VIR_RDT_RESOURCE_L3CODE; + + if (type == VIR_RDT_RESOURCE_L3CODE) + pair_type = VIR_RDT_RESOURCE_L3DATA; + for (i = 1; i < domainall.num_domains; i ++) { schemata_sum |= p->schematas[type]->schemata_items[sid].schemata; + if (pair_type > 0) + schemata_sum |= p->schematas[pair_type]->schemata_items[sid].schemata; p = p->next; } @@ -949,6 +970,9 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune, } if (p != NULL) { + + virResCtrlAppendDomain(p); + for (i = 0; i < cachetune->n_banks; i++) { if ((type = virResCtrlTypeFromString( cachetune->cache_banks[i].type)) < 0) { @@ -983,7 +1007,6 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune, virResCtrlDestroyDomain(p); return -1; } - virResCtrlAppendDomain(p); } else { VIR_ERROR(_("Failed to create a domain in sysfs")); return -1; -- 1.9.1

This patch support l3 cache allocation compatible mode if cdp enabled on host. In this case l3code/l3data has same schemata. Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com&gt; --- src/conf/domain_conf.c | 15 +++++++++++++-- src/util/virresctrl.c | 10 ++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 0f7f331..ec11792 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -15682,9 +15682,20 @@ virDomainCacheTuneDefParseXML(virDomainDefPtr def, resctrl = virResCtrlGet(type); if (resctrl == NULL || !resctrl->enabled) { - virReportError(VIR_ERR_XML_ERROR, + /* support cdp compatible */ + if (type == VIR_RDT_RESOURCE_L3) { + resctrl = virResCtrlGet(type + 1); + if (resctrl == NULL || !resctrl->enabled) { + virReportError(VIR_ERR_XML_ERROR, _("'host doesn't enabled cache type '%s'"), tmp); - goto cleanup; + goto cleanup; + } + VIR_WARN("Use cdp compatible mode."); + } else { + virReportError(VIR_ERR_XML_ERROR, + _("'host doesn't enabled cache type '%s'"), tmp); + goto cleanup; + } } bool found_host_id = false; diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index d15e985..17d95a8 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -946,6 +946,7 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune, char name[VIR_UUID_STRING_BUFLEN]; virResDomainPtr p; int type; + int pair_type = -1; int sid; int schemata; @@ -979,6 +980,13 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune, cachetune->cache_banks[i].type); continue; } + /* use cdp compatible mode */ + if (!VIR_RESCTRL_ENABLED(type) && + (type == VIR_RDT_RESOURCE_L3) && + VIR_RESCTRL_ENABLED(VIR_RDT_RESOURCE_L3DATA)) { + type = VIR_RDT_RESOURCE_L3DATA; + pair_type = VIR_RDT_RESOURCE_L3CODE; + } if ((sid = virResCtrlGetSocketIdByHostID( type, cachetune->cache_banks[i].host_id)) < 0) { @@ -996,6 +1004,8 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune, } p->schematas[type]->schemata_items[sid].schemata = schemata; + if (pair_type > 0) + p->schematas[pair_type]->schemata_items[sid].schemata = schemata; } for (i = 0; i < npid; i++) -- 1.9.1

Other application may touch resctrl while libvirt's running, scan resctrl again before allocating cache information to the newly created VM. Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com&gt; --- src/util/virresctrl.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c index 2b1ee41..f13bce1 100644 --- a/src/util/virresctrl.c +++ b/src/util/virresctrl.c @@ -897,6 +897,46 @@ virResCtrlAppendDomain(virResDomainPtr dom) return 0; } +/* scan /sys/fs/resctrl again and refresh default schemata */ +static +int virResCtrlScan(void) +{ + struct dirent *ent; + DIR *dp = NULL; + int direrr; + virResDomainPtr p; + int rc = -1; + + if (virDirOpenQuiet(&dp, RESCTRL_DIR) < 0) { + if (errno == ENOENT) + return -1; + VIR_ERROR(_("Unable to open %s (%d)"), RESCTRL_DIR, errno); + goto cleanup; + } + + while ((direrr = virDirRead(dp, &ent, NULL)) > 0) { + if ((ent->d_type != DT_DIR) || STREQ(ent->d_name, "info")) + continue; + /* test if we'v tracked all domains */ + p = virResCtrlGetDomain(ent->d_name); + if (p == NULL) { + p = virResCtrlLoadDomain(ent->d_name); + if (p == NULL) + continue; + /* This domain is not created by libvirt, so we don't care + * about the tasks, add a fake one to prevent virResCtrlRefresh + * remove it from sysfs */ + virResCtrlAddTask(p, 1); + virResCtrlAppendDomain(p); + } + } + rc = 0; + + cleanup: + VIR_DIR_CLOSE(dp); + return rc; +} + static int virResCtrlGetSocketIdByHostID(int type, unsigned int hostid) { @@ -1003,6 +1043,10 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune, goto cleanup; } + if (virResCtrlScan() < 0) { + VIR_ERROR(_("Failed to scan resctrl domain dir")); + return -1; + } p = virResCtrlGetDomain(name); if (p == NULL) { VIR_DEBUG("no domain name %s found, create new one!", name); -- 1.9.1