3:35 a.m.
The internal struct list domainall is a list which are resctral domain
status shared by all VMs, especiall the default domain, each VM should
access it concomitantly. Ues a mutex to control it.
Each bank's cache_left field is also a global shared resource we need
to be care, add a mutex for each bank.
We need also to add lock to access /sys/fs/resctrl, use flock.
Signed-off-by: Eli Qiao <liyong.qiao(a)intel.com>
---
src/util/virresctrl.c | 97 ++++++++++++++++++++++++++++++++++++++++++---------
src/util/virresctrl.h | 3 ++
2 files changed, 83 insertions(+), 17 deletions(-)
diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c
index 17d95a8..2b1ee41 100644
--- a/src/util/virresctrl.c
+++ b/src/util/virresctrl.c
@@ -24,6 +24,7 @@
#if defined HAVE_SYS_SYSCALL_H
# include <sys/syscall.h>
#endif
+#include <sys/file.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
@@ -68,8 +69,8 @@ do { \
#define VIR_RESCTRL_GET_SCHEMATA(count) ((1 << count) - 1)
-#define VIR_RESCTRL_SET_SCHEMATA(p, type, pos, val) \
- p->schematas[type]->schemata_items[pos] = val
+#define VIR_RESCTRL_LOCK(fd, op) flock(fd, op)
+#define VIR_RESCTRL_UNLOCK(fd) flock(fd, LOCK_UN)
/**
* a virResSchemata represents a schemata object under a resource control
@@ -114,6 +115,8 @@ typedef virResCtrlDomain *virResCtrlDomainPtr;
struct _virResCtrlDomain {
unsigned int num_domains;
virResDomainPtr domains;
+
+ virMutex lock;
};
@@ -169,11 +172,16 @@ static int virResCtrlGetStr(const char *domain_name, const char
*item_name, char
CONSTRUCT_RESCTRL_PATH(domain_name, item_name);
+ if (!virFileExists(path))
+ goto cleanup;
+
if (virFileReadAll(path, MAX_FILE_LEN, ret) < 0) {
rc = -1;
goto cleanup;
}
+ rc = 0;
+
cleanup:
VIR_FREE(path);
return rc;
@@ -616,6 +624,9 @@ virResCtrlRefresh(void)
unsigned int origin_count = domainall.num_domains;
virResDomainPtr p, pre, del;
pre = domainall.domains;
+
+ virMutexLock(&domainall.lock);
+
p = del = NULL;
if (pre)
p = pre->next;
@@ -640,10 +651,11 @@ virResCtrlRefresh(void)
p = p->next;
}
VIR_FREE(tasks);
-
}
- return virResCtrlRefreshSchemata();
+ virResCtrlRefreshSchemata();
+ virMutexUnlock(&domainall.lock);
+ return 0;
}
/* Get a domain ptr by domain's name*/
@@ -873,10 +885,15 @@ static int
virResCtrlAppendDomain(virResDomainPtr dom)
{
virResDomainPtr p = domainall.domains;
+
+ virMutexLock(&domainall.lock);
+
while (p->next != NULL) p = p->next;
p->next = dom;
dom->pre = p;
domainall.num_domains += 1;
+
+ virMutexUnlock(&domainall.lock);
return 0;
}
@@ -899,18 +916,22 @@ virResCtrlCalculateSchemata(int type,
{
size_t i;
int count;
+ int rc = -1;
virResDomainPtr p;
unsigned int tmp_schemata;
unsigned int schemata_sum = 0;
int pair_type = 0;
+ virMutexLock(&resctrlall[type].cache_banks[sid].lock);
+
if (resctrlall[type].cache_banks[sid].cache_left < size) {
VIR_ERROR(_("Not enough cache left on bank %u"), hostid);
- return -1;
+ goto cleanup;
}
+
if ((count = size / resctrlall[type].cache_banks[sid].cache_min) <= 0) {
VIR_ERROR(_("Error cache size %llu"), size);
- return -1;
+ goto cleanup;
}
tmp_schemata = VIR_RESCTRL_GET_SCHEMATA(count);
@@ -925,7 +946,7 @@ virResCtrlCalculateSchemata(int type,
if (type == VIR_RDT_RESOURCE_L3CODE)
pair_type = VIR_RDT_RESOURCE_L3DATA;
- for (i = 1; i < domainall.num_domains; i ++) {
+ for (i = 1; i < domainall.num_domains; i++) {
schemata_sum |= p->schematas[type]->schemata_items[sid].schemata;
if (pair_type > 0)
schemata_sum |= p->schematas[pair_type]->schemata_items[sid].schemata;
@@ -936,7 +957,16 @@ virResCtrlCalculateSchemata(int type,
while ((tmp_schemata & schemata_sum) != 0)
tmp_schemata = tmp_schemata >> 1;
- return tmp_schemata;
+
+ resctrlall[type].cache_banks[sid].cache_left -= size;
+ if (pair_type > 0)
+ resctrlall[pair_type].cache_banks[sid].cache_left =
resctrlall[type].cache_banks[sid].cache_left;
+
+ rc = tmp_schemata;
+
+ cleanup:
+ virMutexUnlock(&resctrlall[type].cache_banks[sid].lock);
+ return rc;
}
int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
@@ -949,8 +979,8 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
int pair_type = -1;
int sid;
int schemata;
-
- virUUIDFormat(uuid, name);
+ int lockfd;
+ int rc = -1;
for (i = 0; i < cachetune->n_banks; i++) {
VIR_DEBUG("cache_banks %u, %u, %llu, %s",
@@ -963,12 +993,21 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
if (cachetune->n_banks < 1)
return 0;
+ virUUIDFormat(uuid, name);
+
+ if ((lockfd = open(RESCTRL_DIR, O_RDONLY)) < 0)
+ goto cleanup;
+
+ if (VIR_RESCTRL_LOCK(lockfd, LOCK_EX) < 0) {
+ virReportSystemError(errno, _("Unable to lock '%s'"),
RESCTRL_DIR);
+ goto cleanup;
+ }
+
p = virResCtrlGetDomain(name);
if (p == NULL) {
VIR_DEBUG("no domain name %s found, create new one!", name);
p = virResCtrlCreateDomain(name);
}
-
if (p != NULL) {
virResCtrlAppendDomain(p);
@@ -1014,19 +1053,25 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
if (virResCtrlFlushDomainToSysfs(p) < 0) {
VIR_ERROR(_("failed to flush domain %s to sysfs"), name);
virResCtrlDestroyDomain(p);
- return -1;
+ goto cleanup;
}
} else {
VIR_ERROR(_("Failed to create a domain in sysfs"));
- return -1;
+ goto cleanup;
}
virResCtrlRefresh();
/* after refresh, flush header's schemata changes to sys fs */
- if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0)
- VIR_WARN("failed to flush domain to sysfs");
+ if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0) {
+ VIR_ERROR(_("failed to flush domain to sysfs"));
+ goto cleanup;
+ }
- return 0;
+ rc = 0;
+
+ cleanup:
+ VIR_RESCTRL_UNLOCK(lockfd);
+ return rc;
}
/* Should be called after pid disappeared, we recalculate
@@ -1048,7 +1093,7 @@ int virResCtrlUpdate(void)
int
virResCtrlInit(void)
{
- size_t i = 0;
+ size_t i, j;
char *tmp;
int rc = 0;
@@ -1068,6 +1113,24 @@ virResCtrlInit(void)
domainall.domains = virResCtrlGetAllDomains(&(domainall.num_domains));
+ for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) {
+ if (VIR_RESCTRL_ENABLED(i)) {
+ for (j = 0; j < resctrlall[i].num_banks; j++) {
+ if (virMutexInit(&resctrlall[i].cache_banks[j].lock) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Unable to initialize mutex"));
+ return -1;
+ }
+ }
+ }
+ }
+
+ if (virMutexInit(&domainall.lock) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Unable to initialize mutex"));
+ return -1;
+ }
+
if ((rc = virResCtrlRefresh()) < 0)
VIR_ERROR(_("Failed to refresh resource control"));
return rc;
diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h
index ee7e115..07b298d 100644
--- a/src/util/virresctrl.h
+++ b/src/util/virresctrl.h
@@ -27,6 +27,7 @@
# include "virutil.h"
# include "virbitmap.h"
# include "domain_conf.h"
+# include "virthread.h"
#define MAX_CPU_SOCKET_NUM 8
@@ -49,6 +50,8 @@ struct _virResCacheBank {
unsigned long long cache_left;
unsigned long long cache_min;
virBitmapPtr cpu_mask;
+
+ virMutex lock;
};
/**
--
1.9.1