[libvirt] [PATH v9 04/12] Resctrl: Add private interfaces to operate cache bank

Monday, 27 February 2017

virResCtrlSetCacheBanks: Set cache banks of a libvirt domain. It will
                         create new resource domain under
                         `/sys/fs/resctrl` and fill the schemata according
                         the cache bank configration.

virResCtrlUpdate: Destroy resctrl domain directory, and update the schemata
                  after libvirt domain destroy.

Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com&gt;
---
 src/libvirt_private.syms |   6 +-
 src/util/virresctrl.c    | 660 +++++++++++++++++++++++++++++++++++++++++++++++
 src/util/virresctrl.h    |   5 +-
 3 files changed, 669 insertions(+), 2 deletions(-)

diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index cc6c433..85b9e94 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -2318,7 +2318,11 @@ virRandomInt;
 virResCtrlAvailable;
 virResCtrlGet;
 virResCtrlInit;
-
+virResCtrlSetCacheBanks;
+virResCtrlTypeFromString;
+virResCtrlTypeToString;
+virResCtrlUpdate;
+#
 
 # util/virrotatingfile.h
 virRotatingFileReaderConsume;
diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c
index 388249c..5ca9cb0 100644
--- a/src/util/virresctrl.c
+++ b/src/util/virresctrl.c
@@ -19,6 +19,8 @@
  *  Eli Qiao <liyong.qiao(a)intel.com&gt;
  */
 #include <config.h>
+#include <fcntl.h>
+#include <sys/stat.h>
 
 #include "virresctrl.h"
 #include "viralloc.h"
@@ -62,8 +64,58 @@ do { \
 
 #define VIR_RESCTRL_GET_SCHEMATA(count) ((1 << count) - 1)
 
+/**
+ * a virResSchemata represents a schemata object under a resource control
+ * domain.
+ */
+typedef struct _virResSchemataItem virResSchemataItem;
+typedef virResSchemataItem *virResSchemataItemPtr;
+struct _virResSchemataItem {
+    unsigned int socket_no;
+    unsigned schemata;
+};
+
+typedef struct _virResSchemata virResSchemata;
+typedef virResSchemata *virResSchemataPtr;
+struct _virResSchemata {
+    unsigned int n_schemata_items;
+    virResSchemataItemPtr schemata_items;
+};
+
+/**
+ * a virResDomain represents a resource control domain. It's a double linked
+ * list.
+ */
+
+typedef struct _virResDomain virResDomain;
+typedef virResDomain *virResDomainPtr;
+
+struct _virResDomain {
+    char *name;
+    virResSchemataPtr schematas[VIR_RDT_RESOURCE_LAST];
+    char **tasks;
+    size_t n_tasks;
+    size_t n_sockets;
+    virResDomainPtr pre;
+    virResDomainPtr next;
+};
+
+/* All resource control domains on this host*/
+typedef struct _virResCtrlDomain virResCtrlDomain;
+typedef virResCtrlDomain *virResCtrlDomainPtr;
+
+struct _virResCtrlDomain {
+    unsigned int num_domains;
+    virResDomainPtr domains;
+};
+
 static unsigned int host_id;
 
+/* Global static struct to be maintained which is a interface */
+static virResCtrlDomain domainall;
+
+/* Global static struct array to be maintained which indicate
+ * resource status on a host */
 static virResCtrl resctrlall[] = {
     {
         .name = "L3",
@@ -83,6 +135,61 @@ static virResCtrl resctrlall[] = {
     },
 };
 
+/*
+ * How many bits is set in schemata
+ * eg:
+ * virResCtrlBitsNum(1011) = 2 */
+static int virResCtrlBitsContinuesNum(unsigned schemata)
+{
+    size_t i;
+    int ret = 0;
+    for (i = 0; i < MAX_CBM_BIT_LEN; i ++) {
+        if ((schemata & 0x1) == 0x1)
+            ret++;
+        else
+            break;
+        schemata = schemata >> 1;
+        if (schemata == 0) break;
+    }
+    return ret;
+}
+
+static int virResCtrlGetStr(const char *domain_name, const char *item_name, char **ret)
+{
+    char *path;
+    int rc = 0;
+
+    CONSTRUCT_RESCTRL_PATH(domain_name, item_name);
+
+    if (virFileReadAll(path, MAX_FILE_LEN, ret) < 0) {
+        rc = -1;
+        goto cleanup;
+    }
+
+ cleanup:
+    VIR_FREE(path);
+    return rc;
+}
+
+static int virResCtrlGetSchemata(const int type, const char *name, char **schemata)
+{
+    int rc;
+    char *tmp, *end;
+    char *buf;
+
+    if ((rc = virResCtrlGetStr(name, "schemata", &buf)) < 0)
+        return rc;
+
+    tmp = strstr(buf, resctrlall[type].name);
+    end = strchr(tmp, '\n');
+    *end = '\0';
+    if (VIR_STRDUP(*schemata, tmp) < 0)
+        rc = -1;
+
+    VIR_FREE(buf);
+    return rc;
+}
+
 static int virResCtrlGetInfoStr(const int type, const char *item, char **str)
 {
     int ret = 0;
@@ -103,6 +210,71 @@ static int virResCtrlGetInfoStr(const int type, const char *item,
char **str)
     return ret;
 }
 
+/* Return pointer of and ncount of schemata*/
+static virResSchemataPtr virParseSchemata(const char *schemata_str, size_t *ncount)
+{
+    const char *p, *q;
+    int pos;
+    int ischemata;
+    virResSchemataPtr schemata;
+    virResSchemataItemPtr schemataitems, tmpitem;
+    unsigned int socket_no = 0;
+    char *tmp;
+
+    if (VIR_ALLOC(schemata) < 0)
+        goto cleanup;
+
+    p = q = schemata_str;
+    pos = strchr(schemata_str, ':') - p;
+
+    /* calculate cpu socket count */
+    *ncount = 1;
+    while ((q = strchr(p, ';')) != 0) {
+        p = q + 1;
+        (*ncount)++;
+    }
+
+    /* allocat an arrry to store schemata for each socket*/
+    if (VIR_ALLOC_N_QUIET(tmpitem, *ncount) < 0)
+        goto cleanup;
+
+    schemataitems = tmpitem;
+
+    p = q = schemata_str + pos + 1;
+
+    while (*p != '\0') {
+        if (*p == '=') {
+            q = p + 1;
+
+            tmpitem->socket_no = socket_no++;
+
+            while (*p != ';' && *p != '\0') p++;
+
+            if (VIR_STRNDUP(tmp, q, p-q) < 0)
+                goto cleanup;
+
+            if (virStrToLong_i(tmp, NULL, 16, &ischemata) < 0)
+                goto cleanup;
+
+            VIR_FREE(tmp);
+            tmp = NULL;
+            tmpitem->schemata = ischemata;
+            tmpitem ++;
+            schemata->n_schemata_items += 1;
+        }
+        p++;
+    }
+
+    schemata->schemata_items = schemataitems;
+    return schemata;
+
+ cleanup:
+    VIR_FREE(schemata);
+    VIR_FREE(tmpitem);
+    return NULL;
+}
+
+
 static int virResCtrlReadConfig(virArch arch, int type)
 {
     int ret;
@@ -166,6 +338,488 @@ static int virResCtrlReadConfig(virArch arch, int type)
     return ret;
 }
 
+/* Remove the Domain from sysfs, this should only success no pids in tasks
+ * of a partition.
+ */
+static
+int virResCtrlRemoveDomain(const char *name)
+{
+    char *path = NULL;
+    int rc = 0;
+
+    if ((rc = virAsprintf(&path, "%s/%s", RESCTRL_DIR, name)) < 0)
+        return rc;
+    rc = rmdir(path);
+    VIR_FREE(path);
+    return rc;
+}
+
+static
+int virResCtrlDestroyDomain(virResDomainPtr p)
+{
+    size_t i;
+    int rc;
+    if ((rc = virResCtrlRemoveDomain(p->name)) < 0)
+        VIR_WARN("Failed to removed partition %s", p->name);
+
+    VIR_FREE(p->name);
+    p->name = NULL;
+    for (i = 0; i < p->n_tasks; i ++)
+        VIR_FREE(p->tasks[i]);
+    VIR_FREE(p);
+    p = NULL;
+    return rc;
+}
+
+
+/* assemble schemata string*/
+static
+char* virResCtrlAssembleSchemata(virResSchemataPtr schemata, int type)
+{
+    virBuffer buf = VIR_BUFFER_INITIALIZER;
+    size_t i;
+
+    virBufferAsprintf(&buf, "%s:%u=%x", resctrlall[type].name,
+                      schemata->schemata_items[0].socket_no,
+                      schemata->schemata_items[0].schemata);
+
+    for (i = 1; i < schemata->n_schemata_items; i++) {
+        virBufferAsprintf(&buf, ";%u=%x",
+                       schemata->schemata_items[i].socket_no,
+                       schemata->schemata_items[i].schemata);
+    }
+
+    return virBufferContentAndReset(&buf);
+}
+
+/* Refresh default domains' schemata
+ */
+static
+int virResCtrlRefreshSchemata(void)
+{
+    size_t i, j, k;
+    unsigned int tmp_schemata;
+    unsigned int default_schemata;
+
+    virResDomainPtr header, p;
+
+    header = domainall.domains;
+
+    if (!header)
+        return 0;
+
+    for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) {
+        if (VIR_RESCTRL_ENABLED(i)) {
+            for (j = 0; j < header->schematas[i]->n_schemata_items; j ++) {
+                p = header->next;
+                default_schemata = VIR_RESCTRL_GET_SCHEMATA(resctrlall[i].cbm_len);
+                tmp_schemata = 0;
+                /* NOTEs: if only header domain, the schemata will be set to default
one*/
+                for (k = 1; k < domainall.num_domains; k++) {
+                    tmp_schemata |= p->schematas[i]->schemata_items[j].schemata;
+                    p = p->next;
+                }
+                /* sys fs doens't let us use 0 */
+                int min_bits = VIR_RESCTRL_GET_SCHEMATA(resctrlall[i].min_cbm_bits);
+                if ((tmp_schemata & min_bits) == min_bits)
+                    tmp_schemata -= min_bits;
+
+                default_schemata ^= tmp_schemata;
+
+                int bitsnum = virResCtrlBitsContinuesNum(default_schemata);
+                // calcuate header's schemata
+                // NOTES: resctrl sysfs only allow us to set a continues schemata
+                header->schematas[i]->schemata_items[j].schemata =
VIR_RESCTRL_GET_SCHEMATA(bitsnum);
+                resctrlall[i].cache_banks[j].cache_left =
+                    (bitsnum - resctrlall[i].min_cbm_bits) *
resctrlall[i].cache_banks[j].cache_min;
+            }
+        }
+    }
+
+    return 0;
+
+}
+
+/* Get a domain ptr by domain's name*/
+static
+virResDomainPtr virResCtrlGetDomain(const char* name) {
+    size_t i;
+    virResDomainPtr p = domainall.domains;
+    for (i = 0; i < domainall.num_domains; i++) {
+        if ((p->name) && STREQ(name, p->name))
+            return p;
+        p = p->next;
+    }
+    return NULL;
+}
+
+static int
+virResCtrlAddTask(virResDomainPtr dom, pid_t pid)
+{
+    size_t maxtasks;
+
+    if (VIR_RESIZE_N(dom->tasks, maxtasks, dom->n_tasks + 1, 1) < 0)
+        return -1;
+
+    if (virAsprintf(&(dom->tasks[dom->n_tasks]), "%llu", (long
long)pid) < 0)
+        return -1;
+
+    dom->n_tasks += 1;
+    return 0;
+}
+
+static int
+virResCtrlWrite(const char *name, const char *item, const char *content)
+{
+    char *path;
+    int writefd;
+    int rc = -1;
+
+    CONSTRUCT_RESCTRL_PATH(name, item);
+
+    if (!virFileExists(path))
+        goto cleanup;
+
+    if ((writefd = open(path, O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR)) < 0)
+        goto cleanup;
+
+    if (safewrite(writefd, content, strlen(content)) < 0)
+        goto cleanup;
+
+    rc = 0;
+ cleanup:
+    VIR_FREE(path);
+    VIR_FORCE_CLOSE(writefd);
+    return rc;
+}
+
+/* if name == NULL we load default schemata */
+static
+virResDomainPtr virResCtrlLoadDomain(const char *name)
+{
+    char *schematas;
+    virResDomainPtr p;
+    size_t i;
+
+    if (VIR_ALLOC(p) < 0)
+        goto cleanup;
+
+    for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) {
+        if (VIR_RESCTRL_ENABLED(i)) {
+            if (virResCtrlGetSchemata(i, name, &schematas) < 0)
+                goto cleanup;
+            p->schematas[i] = virParseSchemata(schematas, &(p->n_sockets));
+            VIR_FREE(schematas);
+        }
+    }
+
+    p->tasks = NULL;
+    p->n_tasks = 0;
+
+    if ((name != NULL) && (VIR_STRDUP(p->name, name)) < 0)
+        goto cleanup;
+
+    return p;
+
+ cleanup:
+    VIR_FREE(p);
+    return NULL;
+}
+
+static
+virResDomainPtr virResCtrlCreateDomain(const char *name)
+{
+    char *path;
+    mode_t mode = 0755;
+    virResDomainPtr p;
+    size_t i, j;
+
+    if (virAsprintf(&path, "%s/%s", RESCTRL_DIR, name) < 0)
+        return NULL;
+
+    if (virDirCreate(path, mode, 0, 0, 0) < 0)
+        goto cleanup;
+
+    if ((p = virResCtrlLoadDomain(name)) == NULL)
+        return p;
+
+    /* sys fs doens't let us use 0.
+     * reset schemata to min_bits*/
+    for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) {
+        if (VIR_RESCTRL_ENABLED(i)) {
+            int min_bits =  VIR_RESCTRL_GET_SCHEMATA(resctrlall[i].min_cbm_bits);
+            for (j = 0; j < p->n_sockets; j++)
+                p->schematas[i]->schemata_items[j].schemata = min_bits;
+        }
+    }
+
+    VIR_FREE(path);
+    return p;
+
+ cleanup:
+    VIR_FREE(path);
+    return NULL;
+}
+
+/* flush domains's information to sysfs*/
+static int
+virResCtrlFlushDomainToSysfs(virResDomainPtr dom)
+{
+    size_t i;
+    char* schemata;
+    char* tmp;
+    int rc = -1;
+    virBuffer buf = VIR_BUFFER_INITIALIZER;
+
+    for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) {
+        if (VIR_RESCTRL_ENABLED(i)) {
+            tmp = virResCtrlAssembleSchemata(dom->schematas[i], i);
+            virBufferAsprintf(&buf, "%s\n", tmp);
+            VIR_FREE(tmp);
+        }
+    }
+
+    schemata = virBufferContentAndReset(&buf);
+
+    if (virResCtrlWrite(dom->name, "schemata", schemata) < 0)
+        goto cleanup;
+
+    if (dom->n_tasks > 0) {
+        for (i = 0; i < dom->n_tasks; i++) {
+            if (virResCtrlWrite(dom->name, "tasks", dom->tasks[i]) <
0)
+                goto cleanup;
+        }
+    }
+
+    rc = 0;
+
+ cleanup:
+    VIR_FREE(schemata);
+    return rc;
+}
+
+static virResDomainPtr virResCtrlGetAllDomains(unsigned int *len)
+{
+    struct dirent *ent;
+    DIR *dp = NULL;
+    int direrr;
+
+    *len = 0;
+    virResDomainPtr header, tmp, tmp_pre;
+    header = tmp = tmp_pre = NULL;
+    if (virDirOpenQuiet(&dp, RESCTRL_DIR) < 0) {
+        if (errno == ENOENT)
+            return NULL;
+        VIR_ERROR(_("Unable to open %s (%d)"), RESCTRL_DIR, errno);
+        goto cleanup;
+    }
+
+    header = virResCtrlLoadDomain(NULL);
+    if (header == NULL)
+        goto cleanup;
+
+    header->next = NULL;
+
+    *len = 1;
+
+    while ((direrr = virDirRead(dp, &ent, NULL)) > 0) {
+        if ((ent->d_type != DT_DIR) || STREQ(ent->d_name, "info"))
+            continue;
+
+        tmp = virResCtrlLoadDomain(ent->d_name);
+        if (tmp == NULL)
+            goto cleanup;
+
+        tmp->next = NULL;
+
+        if (header->next == NULL)
+            header->next = tmp;
+
+        if (tmp_pre == NULL) {
+            tmp->pre = header;
+        } else {
+            tmp->pre = tmp_pre;
+            tmp_pre->next = tmp;
+        }
+
+        tmp_pre = tmp;
+        (*len) ++;
+    }
+    return header;
+
+ cleanup:
+    VIR_DIR_CLOSE(dp);
+    tmp_pre = tmp = header;
+    while (tmp) {
+        tmp_pre = tmp;
+        tmp = tmp->next;
+        VIR_FREE(tmp_pre);
+    }
+    return NULL;
+}
+
+static int
+virResCtrlAppendDomain(virResDomainPtr dom)
+{
+    virResDomainPtr p = domainall.domains;
+    while (p->next != NULL) p = p->next;
+    p->next = dom;
+    dom->pre = p;
+    domainall.num_domains += 1;
+    return 0;
+}
+
+static int
+virResCtrlGetSocketIdByHostID(int type, unsigned int hostid)
+{
+    size_t i;
+    for (i = 0; i < resctrlall[type].num_banks; i++) {
+        if (resctrlall[type].cache_banks[i].host_id == hostid)
+            return i;
+    }
+    return -1;
+}
+
+static int
+virResCtrlCalculateSchemata(int type,
+                            int sid,
+                            unsigned hostid,
+                            unsigned long long size)
+{
+    size_t i;
+    int count;
+    virResDomainPtr p;
+    unsigned int tmp_schemata;
+    unsigned int schemata_sum = 0;
+
+    if (resctrlall[type].cache_banks[sid].cache_left < size) {
+        VIR_ERROR(_("Not enough cache left on bank %u"), hostid);
+        return -1;
+    }
+    if ((count = size / resctrlall[type].cache_banks[sid].cache_min) <= 0) {
+        VIR_ERROR(_("Error cache size %llu"), size);
+        return -1;
+    }
+
+    tmp_schemata = VIR_RESCTRL_GET_SCHEMATA(count);
+
+    p = domainall.domains;
+    p = p->next;
+    for (i = 1; i < domainall.num_domains; i ++) {
+        schemata_sum |= p->schematas[type]->schemata_items[sid].schemata;
+        p = p->next;
+    }
+
+    tmp_schemata = tmp_schemata << (resctrlall[type].cbm_len - count);
+
+    while ((tmp_schemata & schemata_sum) != 0)
+        tmp_schemata = tmp_schemata >> 1;
+    return tmp_schemata;
+}
+
+int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
+                            unsigned char *uuid, pid_t *pids, int npid)
+{
+    size_t i;
+    char name[VIR_UUID_STRING_BUFLEN];
+    virResDomainPtr p;
+    int type;
+    int sid;
+    int schemata;
+
+    virUUIDFormat(uuid, name);
+
+    for (i = 0; i < cachetune->n_banks; i++) {
+        VIR_DEBUG("cache_banks %u, %u, %llu, %s",
+                 cachetune->cache_banks[i].id,
+                 cachetune->cache_banks[i].host_id,
+                 cachetune->cache_banks[i].size,
+                 cachetune->cache_banks[i].type);
+    }
+
+    if (cachetune->n_banks < 1)
+        return 0;
+
+    p = virResCtrlGetDomain(name);
+    if (p == NULL) {
+        VIR_DEBUG("no domain name %s found, create new one!", name);
+        p = virResCtrlCreateDomain(name);
+    }
+
+    if (p != NULL) {
+        for (i = 0; i < cachetune->n_banks; i++) {
+            if ((type = virResCtrlTypeFromString(
+                            cachetune->cache_banks[i].type)) < 0) {
+                VIR_WARN("Ignore unknown cache type %s.",
+                         cachetune->cache_banks[i].type);
+                continue;
+            }
+
+            if ((sid = virResCtrlGetSocketIdByHostID(
+                            type, cachetune->cache_banks[i].host_id)) < 0) {
+                VIR_WARN("Can not find cache bank host id %u.",
+                         cachetune->cache_banks[i].host_id);
+                continue;
+            }
+
+            if ((schemata = virResCtrlCalculateSchemata(
+                            type, sid, cachetune->cache_banks[i].host_id,
+                            cachetune->cache_banks[i].size)) < 0) {
+                VIR_WARN("Failed to set schemata for cache bank id %u",
+                         cachetune->cache_banks[i].id);
+                continue;
+            }
+
+            p->schematas[type]->schemata_items[sid].schemata = schemata;
+        }
+
+        for (i = 0; i < npid; i++)
+            virResCtrlAddTask(p, pids[i]);
+
+        if (virResCtrlFlushDomainToSysfs(p) < 0) {
+            VIR_ERROR(_("failed to flush domain %s to sysfs"), name);
+            virResCtrlDestroyDomain(p);
+            return -1;
+        }
+        virResCtrlAppendDomain(p);
+    } else {
+        VIR_ERROR(_("Failed to create a domain in sysfs"));
+        return -1;
+    }
+
+    virResCtrlRefreshSchemata();
+    /* after refresh, flush header's schemata changes to sys fs */
+    if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0)
+        VIR_WARN("failed to flush domain to sysfs");
+
+    return 0;
+}
+
+/* Should be called after pid disappeared, we recalculate
+ * schemata of default and flush it to sys fs.
+ */
+int virResCtrlUpdate(unsigned char *uuid)
+{
+    char name[VIR_UUID_STRING_BUFLEN];
+    virResDomainPtr del;
+
+    virUUIDFormat(uuid, name);
+
+    del = virResCtrlGetDomain(name);
+
+    if (del != NULL) {
+        del->pre->next = del->next;
+        if (del->next != NULL)
+            del->next->pre = del->pre;
+        virResCtrlDestroyDomain(del);
+        domainall.num_domains -= 1;
+        virResCtrlRefreshSchemata();
+        if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0)
+            VIR_WARN("failed to flush domain to sysfs");
+    }
+    return 0;
+}
+
 int
 virResCtrlInit(void)
 {
@@ -194,6 +848,12 @@ virResCtrlInit(void)
         VIR_FREE(tmp);
     }
 
+    domainall.domains = virResCtrlGetAllDomains(&(domainall.num_domains));
+
+    if ((rc = virResCtrlRefreshSchemata()) < 0)
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                _("Failed to refresh resource control"));
+
  cleanup:
     VIR_FREE(tmp);
     return rc;
diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h
index 074d307..63dc839 100644
--- a/src/util/virresctrl.h
+++ b/src/util/virresctrl.h
@@ -26,6 +26,7 @@
 
 # include "virbitmap.h"
 # include "virutil.h"
+# include "domain_conf.h"
 
 enum {
     VIR_RDT_RESOURCE_L3,
@@ -76,5 +77,7 @@ struct _virResCtrl {
 bool virResCtrlAvailable(void);
 int virResCtrlInit(void);
 virResCtrlPtr virResCtrlGet(int);
-
+int virResCtrlSetCacheBanks(virDomainCachetunePtr,
+        unsigned char *, pid_t *, int);
+int virResCtrlUpdate(unsigned char *);
 #endif
-- 
1.9.1


    

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

[libvirt] [PATH v9 04/12] Resctrl: Add private interfaces to operate cache bank