Re: [libvirt] [V4 RESEND PATCH] Expose resource control capabilites on cache bank

ping On Wednesday, 17 May 2017 at 5:08 PM, taget wrote: > From: Eli Qiao <liyong.qiao(a)intel.com (mailto:liyong.qiao@intel.com)> > > * This patch amends the cache bank capability as follow: > > <cache> > <bank id='0' level='3' type='unified' size='15360' unit='KiB' cpus='0-5'> > <control min='768' unit='KiB' scope='both' max_allocation='4'/> > </bank> > <bank id='1' level='3' type='unified' size='15360' unit='KiB' cpus='6-11'> > <control min='768' unit='KiB' scope='both' max_allocation='4'/> > </bank> > </cache> > > For CDP enabled on x86 arch, we will have: > <cache> > <bank id='0' level='3' type='unified' size='15360' unit='KiB' cpus='0-5'> > <control min='768' unit='KiB' scope='code' max_allocation='4'/> > <control min='768' unit='KiB' scope='data' max_allocation='4'/> > </bank> > ... > > * Added a new testdata directory `linux-resctrl-cdp` to test CDP enabled > case. > > * Along with vircaps2xmltest case updated. > > P.S. here the scope is from /sys/fs/resctrl/info/L3{"" "code" "data"}, I > keep it capital upper as I need to use a macro to convert from enum to > string easily. > > Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com (mailto:liyong.qiao@intel.com)> > --- > docs/schemas/capability.rng | 20 ++++ > src/conf/capabilities.c | 133 ++++++++++++++++++++- > src/conf/capabilities.h | 10 ++ > .../vircaps2xmldata/linux-resctrl-cdp/resctrl/cpus | 1 + > .../linux-resctrl-cdp/resctrl/info/L3CODE/cbm_mask | 1 + > .../resctrl/info/L3CODE/min_cbm_bits | 1 + > .../resctrl/info/L3CODE/num_closids | 1 + > .../linux-resctrl-cdp/resctrl/info/L3DATA/cbm_mask | 1 + > .../resctrl/info/L3DATA/min_cbm_bits | 1 + > .../resctrl/info/L3DATA/num_closids | 1 + > .../linux-resctrl-cdp/resctrl/manualres/cpus | 1 + > .../linux-resctrl-cdp/resctrl/manualres/schemata | 2 + > .../linux-resctrl-cdp/resctrl/manualres/tasks | 0 > .../linux-resctrl-cdp/resctrl/schemata | 2 + > .../linux-resctrl-cdp/resctrl/tasks | 0 > tests/vircaps2xmldata/linux-resctrl-cdp/system | 1 + > .../vircaps2xmldata/vircaps-x86_64-resctrl-cdp.xml | 55 +++++++++ > tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml | 8 +- > tests/vircaps2xmltest.c | 8 ++ > 19 files changed, 244 insertions(+), 3 deletions(-) > create mode 100644 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/cpus > create mode 100755 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/cbm_mask > create mode 100755 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/min_cbm_bits > create mode 100755 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/num_closids > create mode 100755 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/cbm_mask > create mode 100755 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/min_cbm_bits > create mode 100755 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/num_closids > create mode 100644 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/cpus > create mode 100644 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/schemata > create mode 100644 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/tasks > create mode 100644 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/schemata > create mode 100644 tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/tasks > create mode 120000 tests/vircaps2xmldata/linux-resctrl-cdp/system > create mode 100644 tests/vircaps2xmldata/vircaps-x86_64-resctrl-cdp.xml > > diff --git a/docs/schemas/capability.rng b/docs/schemas/capability.rng > index 26f0aa2..927fc18 100644 > --- a/docs/schemas/capability.rng > +++ b/docs/schemas/capability.rng > @@ -277,6 +277,26 @@ > <attribute name='cpus'> > <ref name='cpuset'/> > </attribute> > + <zeroOrMore> > + <element name='control'> > + <attribute name='min'> > + <ref name='unsignedInt'/> > + </attribute> > + <attribute name='unit'> > + <ref name='unit'/> > + </attribute> > + <attribute name='scope'> > + <choice> > + <value>both</value> > + <value>code</value> > + <value>data</value> > + </choice> > + </attribute> > + <attribute name='max_allocation'> > + <ref name='unsignedInt'/> > + </attribute> > + </element> > + </zeroOrMore> > </element> > </oneOrMore> > </element> > diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c > index d699b08..c4a1fdf 100644 > --- a/src/conf/capabilities.c > +++ b/src/conf/capabilities.c > @@ -51,6 +51,7 @@ > #define VIR_FROM_THIS VIR_FROM_CAPABILITIES > > #define SYSFS_SYSTEM_PATH "/sys/devices/system" > +#define SYSFS_RESCTRL_PATH "/sys/fs/resctrl" > > VIR_LOG_INIT("conf.capabilities") > > @@ -872,6 +873,9 @@ virCapabilitiesFormatCaches(virBufferPtr buf, > virCapsHostCacheBankPtr *caches) > { > size_t i = 0; > + size_t j = 0; > + int indent = virBufferGetIndent(buf, false); > + virBuffer controlBuf = VIR_BUFFER_INITIALIZER; > > if (!ncaches) > return 0; > @@ -893,13 +897,35 @@ virCapabilitiesFormatCaches(virBufferPtr buf, > */ > virBufferAsprintf(buf, > "<bank id='%u' level='%u' type='%s' " > - "size='%llu' unit='%s' cpus='%s'/>\n", > + "size='%llu' unit='%s' cpus='%s'", > bank->id, bank->level, > virCacheTypeToString(bank->type), > bank->size >> (kilos * 10), > kilos ? "KiB" : "B", > cpus_str); > > + virBufferAdjustIndent(&controlBuf, indent + 4); > + for (j = 0; j < bank->ncontrols; j++) { > + bool min_kilos = !(bank->controls[j]->min % 1024); > + virBufferAsprintf(&controlBuf, > + "<control min='%llu' unit='%s' " > + "scope='%s' max_allocation='%u'/>\n", > + bank->controls[j]->min >> (min_kilos * 10), > + min_kilos ? "KiB" : "B", > + virCacheTypeToString(bank->controls[j]->scope), > + bank->controls[j]->max_allocation); > + } > + > + if (virBufferUse(&controlBuf)) { > + virBufferAddLit(buf, ">\n"); > + virBufferAddBuffer(buf, &controlBuf); > + virBufferAddLit(buf, "</bank>\n"); > + > + } else { > + virBufferAddLit(buf, "/>\n"); > + } > + > + virBufferFreeAndReset(&controlBuf); > VIR_FREE(cpus_str); > } > > @@ -1519,13 +1545,102 @@ virCapsHostCacheBankEquals(virCapsHostCacheBankPtr a, > void > virCapsHostCacheBankFree(virCapsHostCacheBankPtr ptr) > { > + size_t i; > + > if (!ptr) > return; > > virBitmapFree(ptr->cpus); > + for (i = 0; i < ptr->ncontrols; i++) > + VIR_FREE(ptr->controls[i]); > + VIR_FREE(ptr->controls); > VIR_FREE(ptr); > } > > +/* test which TYPE of cache control supported > + * -1: don't support > + * 0: cat > + * 1: cdp > + */ > +static int > +virCapabilitiesGetCacheControlType(virCapsHostCacheBankPtr bank) > +{ > + int ret = -1; > + char *path = NULL; > + if (virAsprintf(&path, > + SYSFS_RESCTRL_PATH "/info/L%u", > + bank->level) < 0) > + return -1; > + > + if (virFileExists(path)) { > + ret = 0; > + } else { > + VIR_FREE(path); > + /* for CDP enabled case, CODE and DATA will show together */ > + if (virAsprintf(&path, > + SYSFS_RESCTRL_PATH "/info/L%uCODE", > + bank->level) < 0) > + return -1; > + if (virFileExists(path)) > + ret = 1; > + } > + > + VIR_FREE(path); > + return ret; > +} > + > +static int > +virCapabilitiesGetCacheControl(virCapsHostCacheBankPtr bank, > + virCacheType scope) > +{ > + int ret = -1; > + char *path = NULL; > + char *cbm_mask = NULL; > + char *type_upper = NULL; > + virCapsHostCacheControlPtr control; > + > + if (VIR_ALLOC(control) < 0) > + goto cleanup; > + > + if ((scope > VIR_CACHE_TYPE_BOTH) > + && (virStringToUpper(&type_upper, virCacheTypeToString(scope)) < 0)) > + goto cleanup; > + > + if (virFileReadValueUint(&control->max_allocation, > + SYSFS_RESCTRL_PATH "/info/L%u%s/num_closids", > + bank->level, > + type_upper ? type_upper : "") < 0) > + goto cleanup; > + > + if (virFileReadValueString(&cbm_mask, > + SYSFS_RESCTRL_PATH > + "/info/L%u%s/cbm_mask", > + bank->level, > + type_upper ? type_upper: "") < 0) > + goto cleanup; > + > + virStringTrimOptionalNewline(cbm_mask); > + > + /* cbm_mask: cache bit mask, it's in hex, eg: fffff */ > + control->min = bank->size / (strlen(cbm_mask) * 4); > + > + control->scope = scope; > + > + if (VIR_APPEND_ELEMENT(bank->controls, > + bank->ncontrols, > + control) < 0) > + goto cleanup; > + > + ret = 0; > + > + cleanup: > + VIR_FREE(path); > + VIR_FREE(cbm_mask); > + VIR_FREE(type_upper); > + VIR_FREE(control); > + return ret; > +} > + > int > virCapabilitiesInitCaches(virCapsPtr caps) > { > @@ -1534,6 +1649,7 @@ virCapabilitiesInitCaches(virCapsPtr caps) > ssize_t pos = -1; > DIR *dirp = NULL; > int ret = -1; > + int typeret; > char *path = NULL; > char *type = NULL; > struct dirent *ent = NULL; > @@ -1607,12 +1723,27 @@ virCapabilitiesInitCaches(virCapsPtr caps) > SYSFS_SYSTEM_PATH, pos, ent->d_name) < 0) > goto cleanup; > > + typeret = virCapabilitiesGetCacheControlType(bank); > + > + if (typeret == 0) { > + if (virCapabilitiesGetCacheControl(bank, > + VIR_CACHE_TYPE_BOTH) < 0) > + goto cleanup; > + } else if (typeret == 1) { > + if ((virCapabilitiesGetCacheControl(bank, > + VIR_CACHE_TYPE_CODE) < 0) || > + (virCapabilitiesGetCacheControl(bank, > + VIR_CACHE_TYPE_DATA) < 0)) > + goto cleanup; > + } > + > kernel_type = virCacheKernelTypeFromString(type); > if (kernel_type < 0) { > virReportError(VIR_ERR_INTERNAL_ERROR, > _("Unknown cache type '%s'"), type); > goto cleanup; > } > + > bank->type = kernel_type; > VIR_FREE(type); > > diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h > index a8cccf7..ee87d59 100644 > --- a/src/conf/capabilities.h > +++ b/src/conf/capabilities.h > @@ -148,6 +148,14 @@ typedef enum { > > VIR_ENUM_DECL(virCache); > > +typedef struct _virCapsHostCacheControl virCapsHostCacheControl; > +typedef virCapsHostCacheControl *virCapsHostCacheControlPtr; > +struct _virCapsHostCacheControl { > + unsigned long long min; /* minimum cache control size in B */ > + virCacheType scope; /* data, code or both */ > + unsigned int max_allocation; /* max number of supported allocations */ > +}; > + > typedef struct _virCapsHostCacheBank virCapsHostCacheBank; > typedef virCapsHostCacheBank *virCapsHostCacheBankPtr; > struct _virCapsHostCacheBank { > @@ -156,6 +164,8 @@ struct _virCapsHostCacheBank { > unsigned long long size; /* B */ > virCacheType type; /* Data, Instruction or Unified */ > virBitmapPtr cpus; /* All CPUs that share this bank */ > + size_t ncontrols; > + virCapsHostCacheControlPtr *controls; > }; > > typedef struct _virCapsHost virCapsHost; > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/cpus b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/cpus > new file mode 100644 > index 0000000..b3a79aa > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/cpus > @@ -0,0 +1 @@ > +ffffff,ffffffff,ffffffff > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/cbm_mask b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/cbm_mask > new file mode 100755 > index 0000000..78031da > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/cbm_mask > @@ -0,0 +1 @@ > +fffff > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/min_cbm_bits b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/min_cbm_bits > new file mode 100755 > index 0000000..d00491f > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/min_cbm_bits > @@ -0,0 +1 @@ > +1 > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/num_closids b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/num_closids > new file mode 100755 > index 0000000..45a4fb7 > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3CODE/num_closids > @@ -0,0 +1 @@ > +8 > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/cbm_mask b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/cbm_mask > new file mode 100755 > index 0000000..78031da > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/cbm_mask > @@ -0,0 +1 @@ > +fffff > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/min_cbm_bits b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/min_cbm_bits > new file mode 100755 > index 0000000..d00491f > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/min_cbm_bits > @@ -0,0 +1 @@ > +1 > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/num_closids b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/num_closids > new file mode 100755 > index 0000000..45a4fb7 > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/info/L3DATA/num_closids > @@ -0,0 +1 @@ > +8 > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/cpus b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/cpus > new file mode 100644 > index 0000000..ede4cc2 > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/cpus > @@ -0,0 +1 @@ > +000000,00000000,00000000 > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/schemata b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/schemata > new file mode 100644 > index 0000000..a0ef381 > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/schemata > @@ -0,0 +1,2 @@ > +L3DATA:0=c0000;1=c0000 > +L3CODE:0=30000;1=30000 > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/tasks b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/manualres/tasks > new file mode 100644 > index 0000000..e69de29 > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/schemata b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/schemata > new file mode 100644 > index 0000000..89dc76b > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/schemata > @@ -0,0 +1,2 @@ > +L3DATA:0=fffff;1=fffff > +L3CODE:0=fffff;1=fffff > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/tasks b/tests/vircaps2xmldata/linux-resctrl-cdp/resctrl/tasks > new file mode 100644 > index 0000000..e69de29 > diff --git a/tests/vircaps2xmldata/linux-resctrl-cdp/system b/tests/vircaps2xmldata/linux-resctrl-cdp/system > new file mode 120000 > index 0000000..2f3a1d9 > --- /dev/null > +++ b/tests/vircaps2xmldata/linux-resctrl-cdp/system > @@ -0,0 +1 @@ > +../linux-resctrl/system/ > \ No newline at end of file > diff --git a/tests/vircaps2xmldata/vircaps-x86_64-resctrl-cdp.xml b/tests/vircaps2xmldata/vircaps-x86_64-resctrl-cdp.xml > new file mode 100644 > index 0000000..c9f460d > --- /dev/null > +++ b/tests/vircaps2xmldata/vircaps-x86_64-resctrl-cdp.xml > @@ -0,0 +1,55 @@ > +<capabilities> > + > + <host> > + <cpu> > + <arch>x86_64</arch> > + </cpu> > + <power_management/> > + <migration_features> > + <live/> > + </migration_features> > + <topology> > + <cells num='2'> > + <cell id='0'> > + <memory unit='KiB'>1048576</memory> > + <pages unit='KiB' size='4'>2048</pages> > + <pages unit='KiB' size='2048'>4096</pages> > + <pages unit='KiB' size='1048576'>6144</pages> > + <cpus num='6'> > + <cpu id='0' socket_id='0' core_id='0' siblings='0'/> > + <cpu id='1' socket_id='0' core_id='1' siblings='1'/> > + <cpu id='2' socket_id='0' core_id='2' siblings='2'/> > + <cpu id='3' socket_id='0' core_id='3' siblings='3'/> > + <cpu id='4' socket_id='0' core_id='4' siblings='4'/> > + <cpu id='5' socket_id='0' core_id='5' siblings='5'/> > + </cpus> > + </cell> > + <cell id='1'> > + <memory unit='KiB'>2097152</memory> > + <pages unit='KiB' size='4'>4096</pages> > + <pages unit='KiB' size='2048'>6144</pages> > + <pages unit='KiB' size='1048576'>8192</pages> > + <cpus num='6'> > + <cpu id='6' socket_id='1' core_id='0' siblings='6'/> > + <cpu id='7' socket_id='1' core_id='1' siblings='7'/> > + <cpu id='8' socket_id='1' core_id='2' siblings='8'/> > + <cpu id='9' socket_id='1' core_id='3' siblings='9'/> > + <cpu id='10' socket_id='1' core_id='4' siblings='10'/> > + <cpu id='11' socket_id='1' core_id='5' siblings='11'/> > + </cpus> > + </cell> > + </cells> > + </topology> > + <cache> > + <bank id='0' level='3' type='both' size='15360' unit='KiB' cpus='0-5'> > + <control min='768' unit='KiB' scope='code' max_allocation='8'/> > + <control min='768' unit='KiB' scope='data' max_allocation='8'/> > + </bank> > + <bank id='1' level='3' type='both' size='15360' unit='KiB' cpus='6-11'> > + <control min='768' unit='KiB' scope='code' max_allocation='8'/> > + <control min='768' unit='KiB' scope='data' max_allocation='8'/> > + </bank> > + </cache> > + </host> > + > +</capabilities> > diff --git a/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml b/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml > index 6677791..04a5eb8 100644 > --- a/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml > +++ b/tests/vircaps2xmldata/vircaps-x86_64-resctrl.xml > @@ -41,8 +41,12 @@ > </cells> > </topology> > <cache> > - <bank id='0' level='3' type='both' size='15360' unit='KiB' cpus='0-5'/> > - <bank id='1' level='3' type='both' size='15360' unit='KiB' cpus='6-11'/> > + <bank id='0' level='3' type='both' size='15360' unit='KiB' cpus='0-5'> > + <control min='768' unit='KiB' scope='both' max_allocation='4'/> > + </bank> > + <bank id='1' level='3' type='both' size='15360' unit='KiB' cpus='6-11'> > + <control min='768' unit='KiB' scope='both' max_allocation='4'/> > + </bank> > </cache> > </host> > > diff --git a/tests/vircaps2xmltest.c b/tests/vircaps2xmltest.c > index 137598e..ae1cd52 100644 > --- a/tests/vircaps2xmltest.c > +++ b/tests/vircaps2xmltest.c > @@ -47,6 +47,7 @@ test_virCapabilities(const void *opaque) > char *capsXML = NULL; > char *path = NULL; > char *dir = NULL; > + char *resctrl = NULL; > int ret = -1; > > /* > @@ -58,7 +59,12 @@ test_virCapabilities(const void *opaque) > data->resctrl ? "/system" : "") < 0) > goto cleanup; > > + if (virAsprintf(&resctrl, "%s/vircaps2xmldata/linux-%s/resctrl", > + abs_srcdir, data->filename) < 0) > + goto cleanup; > + > virFileWrapperAddPrefix("/sys/devices/system", dir); > + virFileWrapperAddPrefix("/sys/fs/resctrl", resctrl); > caps = virCapabilitiesNew(data->arch, data->offlineMigrate, data->liveMigrate); > > if (!caps) > @@ -84,6 +90,7 @@ test_virCapabilities(const void *opaque) > > cleanup: > VIR_FREE(dir); > + VIR_FREE(resctrl); > VIR_FREE(path); > VIR_FREE(capsXML); > virObjectUnref(caps); > @@ -112,6 +119,7 @@ mymain(void) > DO_TEST("caches", VIR_ARCH_X86_64); > > DO_TEST_FULL("resctrl", VIR_ARCH_X86_64, true, true, true); > + DO_TEST_FULL("resctrl-cdp", VIR_ARCH_X86_64, true, true, true); > > return ret; > } > -- > 1.9.1 > > -- > libvir-list mailing list > libvir-list(a)redhat.com (mailto:libvir-list@redhat.com) > https://www.redhat.com/mailman/listinfo/libvir-list > >