Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com> --- src/libvirt_private.syms | 2 + src/qemu/qemu_driver.c | 116 ++------------------------------------- src/util/virprocess.c | 108 ++++++++++++++++++++++++++++++++++++ src/util/virprocess.h | 5 ++ 4 files changed, 120 insertions(+), 111 deletions(-) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index b98cb0f66d..e0c4fba522 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -3099,8 +3099,10 @@ virProcessGetAffinity; virProcessGetMaxMemLock; virProcessGetNamespaces; virProcessGetPids; +virProcessGetSchedInfo; virProcessGetStartTime; virProcessGetStat; +virProcessGetStatInfo; virProcessGroupGet; virProcessGroupKill; virProcessKill; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 4e680bc0a7..a7088d3a66 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -1323,113 +1323,6 @@ qemuGetSchedstatDelay(unsigned long long *cpudelay, return 0; } - -static int -qemuGetSchedInfo(unsigned long long *cpuWait, - pid_t pid, pid_t tid) -{ - g_autofree char *proc = NULL; - g_autofree char *data = NULL; - g_auto(GStrv) lines = NULL; - size_t i; - double val; - - *cpuWait = 0; - - /* In general, we cannot assume pid_t fits in int; but /proc parsing - * is specific to Linux where int works fine. */ - if (tid) - proc = g_strdup_printf("/proc/%d/task/%d/sched", (int)pid, (int)tid); - else - proc = g_strdup_printf("/proc/%d/sched", (int)pid); - if (!proc) - return -1; - - /* The file is not guaranteed to exist (needs CONFIG_SCHED_DEBUG) */ - if (access(proc, R_OK) < 0) { - return 0; - } - - if (virFileReadAll(proc, (1<<16), &data) < 0) - return -1; - - lines = g_strsplit(data, "\n", 0); - if (!lines) - return -1; - - for (i = 0; lines[i] != NULL; i++) { - const char *line = lines[i]; - - /* Needs CONFIG_SCHEDSTATS. The second check - * is the old name the kernel used in past */ - if (STRPREFIX(line, "se.statistics.wait_sum") || - STRPREFIX(line, "se.wait_sum")) { - line = strchr(line, ':'); - if (!line) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Missing separator in sched info '%s'"), - lines[i]); - return -1; - } - line++; - while (*line == ' ') - line++; - - if (virStrToDouble(line, NULL, &val) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Unable to parse sched info value '%s'"), - line); - return -1; - } - - *cpuWait = (unsigned long long)(val * 1000000); - break; - } - } - - return 0; -} - - -static int -qemuGetProcessInfo(unsigned long long *cpuTime, int *lastCpu, long *vm_rss, - pid_t pid, pid_t tid) -{ - g_auto(GStrv) proc_stat = virProcessGetStat(pid, tid); - unsigned long long usertime = 0, systime = 0; - long rss = 0; - int cpu = 0; - - if (!proc_stat || - virStrToLong_ullp(proc_stat[VIR_PROCESS_STAT_UTIME], NULL, 10, &usertime) < 0 || - virStrToLong_ullp(proc_stat[VIR_PROCESS_STAT_STIME], NULL, 10, &systime) < 0 || - virStrToLong_l(proc_stat[VIR_PROCESS_STAT_RSS], NULL, 10, &rss) < 0 || - virStrToLong_i(proc_stat[VIR_PROCESS_STAT_PROCESSOR], NULL, 10, &cpu) < 0) { - VIR_WARN("cannot parse process status data"); - } - - /* We got jiffies - * We want nanoseconds - * _SC_CLK_TCK is jiffies per second - * So calculate thus.... - */ - if (cpuTime) - *cpuTime = 1000ull * 1000ull * 1000ull * (usertime + systime) - / (unsigned long long)sysconf(_SC_CLK_TCK); - if (lastCpu) - *lastCpu = cpu; - - if (vm_rss) - *vm_rss = rss * virGetSystemPageSizeKB(); - - - VIR_DEBUG("Got status for %d/%d user=%llu sys=%llu cpu=%d rss=%ld", - (int)pid, tid, usertime, systime, cpu, rss); - - return 0; -} - - static int qemuDomainHelperGetVcpus(virDomainObj *vm, virVcpuInfoPtr info, @@ -1469,7 +1362,7 @@ qemuDomainHelperGetVcpus(virDomainObj *vm, vcpuinfo->number = i; vcpuinfo->state = VIR_VCPU_RUNNING; - if (qemuGetProcessInfo(&vcpuinfo->cpuTime, + if (virProcessGetStatInfo(&vcpuinfo->cpuTime, &vcpuinfo->cpu, NULL, vm->pid, vcpupid) < 0) { virReportSystemError(errno, "%s", @@ -1490,7 +1383,7 @@ qemuDomainHelperGetVcpus(virDomainObj *vm, } if (cpuwait) { - if (qemuGetSchedInfo(&(cpuwait[ncpuinfo]), vm->pid, vcpupid) < 0) + if (virProcessGetSchedInfo(&(cpuwait[ncpuinfo]), vm->pid, vcpupid) < 0) return -1; } @@ -2638,7 +2531,8 @@ qemuDomainGetInfo(virDomainPtr dom, } if (virDomainObjIsActive(vm)) { - if (qemuGetProcessInfo(&(info->cpuTime), NULL, NULL, vm->pid, 0) < 0) { + if (virProcessGetStatInfo(&(info->cpuTime), NULL, NULL, + vm->pid, 0) < 0) { virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("cannot read cputime for domain")); goto cleanup; @@ -10650,7 +10544,7 @@ qemuDomainMemoryStatsInternal(virQEMUDriver *driver, ret = 0; } - if (qemuGetProcessInfo(NULL, NULL, &rss, vm->pid, 0) < 0) { + if (virProcessGetStatInfo(NULL, NULL, &rss, vm->pid, 0) < 0) { virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("cannot get RSS for domain")); } else { diff --git a/src/util/virprocess.c b/src/util/virprocess.c index 8288e71f67..efa96299fc 100644 --- a/src/util/virprocess.c +++ b/src/util/virprocess.c @@ -1761,3 +1761,111 @@ virProcessGetStat(pid_t pid, pid_t tid) return ret; } + + +int +virProcessGetStatInfo(unsigned long long *cpuTime, int *lastCpu, long *vm_rss, + pid_t pid, pid_t tid) +{ + g_auto(GStrv) proc_stat = virProcessGetStat(pid, tid); + unsigned long long usertime = 0, systime = 0; + long rss = 0; + int cpu = 0; + + if (!proc_stat || + virStrToLong_ullp(proc_stat[VIR_PROCESS_STAT_UTIME], NULL, 10, + &usertime) < 0 + || virStrToLong_ullp(proc_stat[VIR_PROCESS_STAT_STIME], NULL, 10, + &systime) < 0 + || virStrToLong_l(proc_stat[VIR_PROCESS_STAT_RSS], NULL, 10, &rss) < 0 + || virStrToLong_i(proc_stat[VIR_PROCESS_STAT_PROCESSOR], NULL, 10, + &cpu) < 0) { + VIR_WARN("cannot parse process status data"); + } + + /* We got jiffies + * We want nanoseconds + * _SC_CLK_TCK is jiffies per second + * So calculate thus.... + */ + if (cpuTime) + *cpuTime = 1000ull * 1000ull * 1000ull * (usertime + systime) + / (unsigned long long) sysconf(_SC_CLK_TCK); + if (lastCpu) + *lastCpu = cpu; + + if (vm_rss) + *vm_rss = rss * virGetSystemPageSizeKB(); + + + VIR_DEBUG("Got status for %d/%d user=%llu sys=%llu cpu=%d rss=%ld", + (int) pid, tid, usertime, systime, cpu, rss); + + return 0; +} + +int +virProcessGetSchedInfo(unsigned long long *cpuWait, pid_t pid, pid_t tid) +{ + g_autofree char *proc = NULL; + g_autofree char *data = NULL; + + g_auto(GStrv) lines = NULL; + size_t i; + double val; + + *cpuWait = 0; + + /* In general, we cannot assume pid_t fits in int; but /proc parsing + * is specific to Linux where int works fine. */ + if (tid) + proc = g_strdup_printf("/proc/%d/task/%d/sched", (int) pid, (int) tid); + else + proc = g_strdup_printf("/proc/%d/sched", (int) pid); + if (!proc) + return -1; + + /* The file is not guaranteed to exist (needs CONFIG_SCHED_DEBUG) */ + if (access(proc, R_OK) < 0) { + return 0; + } + + if (virFileReadAll(proc, (1 << 16), &data) < 0) + return -1; + + lines = g_strsplit(data, "\n", 0); + if (!lines) + return -1; + + for (i = 0; lines[i] != NULL; i++) { + const char *line = lines[i]; + + /* Needs CONFIG_SCHEDSTATS. The second check + * is the old name the kernel used in past */ + if (STRPREFIX(line, "se.statistics.wait_sum") || + STRPREFIX(line, "se.wait_sum")) { + line = strchr(line, ':'); + if (!line) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Missing separator in sched info '%s'"), + lines[i]); + return -1; + } + line++; + while (*line == ' ') + line++; + + if (virStrToDouble(line, NULL, &val) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to parse sched info value '%s'"), + line); + return -1; + } + + *cpuWait = (unsigned long long) (val * 1000000); + break; + } + } + + return 0; +} diff --git a/src/util/virprocess.h b/src/util/virprocess.h index 82b7403964..d9d27c29b8 100644 --- a/src/util/virprocess.h +++ b/src/util/virprocess.h @@ -193,3 +193,8 @@ typedef enum { } virProcessNamespaceFlags; int virProcessNamespaceAvailable(unsigned int ns); + +int virProcessGetStatInfo(unsigned long long *cpuTime, int *lastCpu, + long *vm_rss, pid_t pid, pid_t tid); +int virProcessGetSchedInfo(unsigned long long *cpuWait, pid_t pid, + pid_t tid); -- 2.27.0

From: Vineeth Pillai <viremana@linux.microsoft.com> Signed-off-by: Vineeth Pillai <viremana@linux.microsoft.com> Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com> --- src/ch/ch_domain.c | 6 ++++++ src/ch/ch_domain.h | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/src/ch/ch_domain.c b/src/ch/ch_domain.c index 44f7d26ca4..7bfe950822 100644 --- a/src/ch/ch_domain.c +++ b/src/ch/ch_domain.c @@ -288,3 +288,9 @@ virDomainDefParserConfig virCHDriverDomainDefParserConfig = { .domainPostParseCallback = virCHDomainDefPostParse, .deviceValidateCallback = chValidateDomainDeviceDef, }; + +virCHMonitor * +virCHDomainGetMonitor(virDomainObj * vm) +{ + return CH_DOMAIN_PRIVATE(vm)->monitor; +} diff --git a/src/ch/ch_domain.h b/src/ch/ch_domain.h index 61b34b0467..04d19398b4 100644 --- a/src/ch/ch_domain.h +++ b/src/ch/ch_domain.h @@ -57,6 +57,11 @@ struct _virCHDomainObjPrivate { virChrdevs *chrdevs; }; +#define CH_DOMAIN_PRIVATE(vm) \ + ((virCHDomainObjPrivate *) (vm)->privateData) + +virCHMonitor *virCHDomainGetMonitor(virDomainObj *vm); + extern virDomainXMLPrivateDataCallbacks virCHDriverPrivateDataCallbacks; extern virDomainDefParserConfig virCHDriverDomainDefParserConfig; -- 2.27.0

Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com> --- src/ch/ch_driver.c | 241 +++++++++++++++++++++++---------------------- 1 file changed, 121 insertions(+), 120 deletions(-) diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 464bcef907..75c6a15dd6 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -50,7 +50,7 @@ VIR_LOG_INIT("ch.ch_driver"); virCHDriver *ch_driver = NULL; static virDomainObj * -chDomObjFromDomain(virDomain *domain) +chDomObjFromDomain(virDomain * domain) { virDomainObj *vm; virCHDriver *driver = domain->conn->privateData; @@ -79,10 +79,10 @@ chConnectURIProbe(char **uri) return 1; } -static virDrvOpenStatus chConnectOpen(virConnectPtr conn, - virConnectAuthPtr auth G_GNUC_UNUSED, - virConf *conf G_GNUC_UNUSED, - unsigned int flags) +static virDrvOpenStatus +chConnectOpen(virConnectPtr conn, + virConnectAuthPtr auth G_GNUC_UNUSED, + virConf * conf G_GNUC_UNUSED, unsigned int flags) { virCheckFlags(VIR_CONNECT_RO, VIR_DRV_OPEN_ERROR); @@ -101,13 +101,15 @@ static virDrvOpenStatus chConnectOpen(virConnectPtr conn, return VIR_DRV_OPEN_SUCCESS; } -static int chConnectClose(virConnectPtr conn) +static int +chConnectClose(virConnectPtr conn) { conn->privateData = NULL; return 0; } -static const char *chConnectGetType(virConnectPtr conn) +static const char * +chConnectGetType(virConnectPtr conn) { if (virConnectGetTypeEnsureACL(conn) < 0) return NULL; @@ -115,8 +117,8 @@ static const char *chConnectGetType(virConnectPtr conn) return "CH"; } -static int chConnectGetVersion(virConnectPtr conn, - unsigned long *version) +static int +chConnectGetVersion(virConnectPtr conn, unsigned long *version) { virCHDriver *driver = conn->privateData; @@ -129,7 +131,8 @@ static int chConnectGetVersion(virConnectPtr conn, return 0; } -static char *chConnectGetHostname(virConnectPtr conn) +static char * +chConnectGetHostname(virConnectPtr conn) { if (virConnectGetHostnameEnsureACL(conn) < 0) return NULL; @@ -137,7 +140,8 @@ static char *chConnectGetHostname(virConnectPtr conn) return virGetHostname(); } -static int chConnectNumOfDomains(virConnectPtr conn) +static int +chConnectNumOfDomains(virConnectPtr conn) { virCHDriver *driver = conn->privateData; @@ -148,7 +152,8 @@ static int chConnectNumOfDomains(virConnectPtr conn) virConnectNumOfDomainsCheckACL, conn); } -static int chConnectListDomains(virConnectPtr conn, int *ids, int nids) +static int +chConnectListDomains(virConnectPtr conn, int *ids, int nids) { virCHDriver *driver = conn->privateData; @@ -156,13 +161,12 @@ static int chConnectListDomains(virConnectPtr conn, int *ids, int nids) return -1; return virDomainObjListGetActiveIDs(driver->domains, ids, nids, - virConnectListDomainsCheckACL, conn); + virConnectListDomainsCheckACL, conn); } static int chConnectListAllDomains(virConnectPtr conn, - virDomainPtr **domains, - unsigned int flags) + virDomainPtr ** domains, unsigned int flags) { virCHDriver *driver = conn->privateData; @@ -172,11 +176,11 @@ chConnectListAllDomains(virConnectPtr conn, return -1; return virDomainObjListExport(driver->domains, conn, domains, - virConnectListAllDomainsCheckACL, flags); + virConnectListAllDomainsCheckACL, flags); } -static int chNodeGetInfo(virConnectPtr conn, - virNodeInfoPtr nodeinfo) +static int +chNodeGetInfo(virConnectPtr conn, virNodeInfoPtr nodeinfo) { if (virNodeGetInfoEnsureACL(conn) < 0) return -1; @@ -184,7 +188,8 @@ static int chNodeGetInfo(virConnectPtr conn, return virCapabilitiesGetNodeInfo(nodeinfo); } -static char *chConnectGetCapabilities(virConnectPtr conn) +static char * +chConnectGetCapabilities(virConnectPtr conn) { virCHDriver *driver = conn->privateData; virCaps *caps; @@ -213,9 +218,7 @@ static char *chConnectGetCapabilities(virConnectPtr conn) * Returns a new domain object or NULL in case of failure. */ static virDomainPtr -chDomainCreateXML(virConnectPtr conn, - const char *xml, - unsigned int flags) +chDomainCreateXML(virConnectPtr conn, const char *xml, unsigned int flags) { virCHDriver *driver = conn->privateData; virDomainDef *vmdef = NULL; @@ -240,8 +243,7 @@ chDomainCreateXML(virConnectPtr conn, &vmdef, driver->xmlopt, VIR_DOMAIN_OBJ_LIST_ADD_LIVE | - VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, - NULL))) + VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, NULL))) goto cleanup; if (virCHDomainObjBeginJob(vm, CH_JOB_MODIFY) < 0) @@ -343,8 +345,7 @@ chDomainDefineXML(virConnectPtr conn, const char *xml) } static int -chDomainUndefineFlags(virDomainPtr dom, - unsigned int flags) +chDomainUndefineFlags(virDomainPtr dom, unsigned int flags) { virCHDriver *driver = dom->conn->privateData; virDomainObj *vm; @@ -383,7 +384,8 @@ chDomainUndefine(virDomainPtr dom) return chDomainUndefineFlags(dom, 0); } -static int chDomainIsActive(virDomainPtr dom) +static int +chDomainIsActive(virDomainPtr dom) { virCHDriver *driver = dom->conn->privateData; virDomainObj *vm; @@ -405,8 +407,7 @@ static int chDomainIsActive(virDomainPtr dom) } static int -chDomainShutdownFlags(virDomainPtr dom, - unsigned int flags) +chDomainShutdownFlags(virDomainPtr dom, unsigned int flags) { virCHDomainObjPrivate *priv; virDomainObj *vm; @@ -437,7 +438,7 @@ chDomainShutdownFlags(virDomainPtr dom, } else { if (virCHMonitorShutdownVM(priv->monitor) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("failed to shutdown guest VM")); + _("failed to shutdown guest VM")); goto endjob; } } @@ -501,7 +502,8 @@ chDomainReboot(virDomainPtr dom, unsigned int flags) if (state == VIR_DOMAIN_RUNNING) virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_BOOTED); else - virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_UNPAUSED); + virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, + VIR_DOMAIN_RUNNING_UNPAUSED); ret = 0; @@ -541,7 +543,7 @@ chDomainSuspend(virDomainPtr dom) } else { if (virCHMonitorSuspendVM(priv->monitor) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("failed to suspend domain")); + _("failed to suspend domain")); goto endjob; } } @@ -586,7 +588,7 @@ chDomainResume(virDomainPtr dom) } else { if (virCHMonitorResumeVM(priv->monitor) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("failed to resume domain")); + _("failed to resume domain")); goto endjob; } } @@ -651,8 +653,8 @@ chDomainDestroy(virDomainPtr dom) return chDomainDestroyFlags(dom, 0); } -static virDomainPtr chDomainLookupByID(virConnectPtr conn, - int id) +static virDomainPtr +chDomainLookupByID(virConnectPtr conn, int id) { virCHDriver *driver = conn->privateData; virDomainObj *vm; @@ -678,8 +680,8 @@ static virDomainPtr chDomainLookupByID(virConnectPtr conn, return dom; } -static virDomainPtr chDomainLookupByName(virConnectPtr conn, - const char *name) +static virDomainPtr +chDomainLookupByName(virConnectPtr conn, const char *name) { virCHDriver *driver = conn->privateData; virDomainObj *vm; @@ -705,8 +707,8 @@ static virDomainPtr chDomainLookupByName(virConnectPtr conn, return dom; } -static virDomainPtr chDomainLookupByUUID(virConnectPtr conn, - const unsigned char *uuid) +static virDomainPtr +chDomainLookupByUUID(virConnectPtr conn, const unsigned char *uuid) { virCHDriver *driver = conn->privateData; virDomainObj *vm; @@ -718,6 +720,7 @@ static virDomainPtr chDomainLookupByUUID(virConnectPtr conn, if (!vm) { char uuidstr[VIR_UUID_STRING_BUFLEN]; + virUUIDFormat(uuid, uuidstr); virReportError(VIR_ERR_NO_DOMAIN, _("no domain with matching uuid '%s'"), uuidstr); @@ -735,10 +738,7 @@ static virDomainPtr chDomainLookupByUUID(virConnectPtr conn, } static int -chDomainGetState(virDomainPtr dom, - int *state, - int *reason, - unsigned int flags) +chDomainGetState(virDomainPtr dom, int *state, int *reason, unsigned int flags) { virDomainObj *vm; int ret = -1; @@ -759,8 +759,8 @@ chDomainGetState(virDomainPtr dom, return ret; } -static char *chDomainGetXMLDesc(virDomainPtr dom, - unsigned int flags) +static char * +chDomainGetXMLDesc(virDomainPtr dom, unsigned int flags) { virCHDriver *driver = dom->conn->privateData; virDomainObj *vm; @@ -782,8 +782,8 @@ static char *chDomainGetXMLDesc(virDomainPtr dom, return ret; } -static int chDomainGetInfo(virDomainPtr dom, - virDomainInfoPtr info) +static int +chDomainGetInfo(virDomainPtr dom, virDomainInfoPtr info) { virDomainObj *vm; int ret = -1; @@ -811,77 +811,76 @@ static int chDomainGetInfo(virDomainPtr dom, static int chDomainOpenConsole(virDomainPtr dom, - const char *dev_name, - virStreamPtr st, - unsigned int flags) -{ - virDomainObj *vm = NULL; - int ret = -1; - size_t i; - virDomainChrDef *chr = NULL; - virCHDomainObjPrivate *priv; - - virCheckFlags(VIR_DOMAIN_CONSOLE_SAFE | VIR_DOMAIN_CONSOLE_FORCE, -1); - - if (!(vm = chDomObjFromDomain(dom))) - goto cleanup; - - if (virDomainOpenConsoleEnsureACL(dom->conn, vm->def) < 0) - goto cleanup; - - if (virDomainObjCheckActive(vm) < 0) - goto cleanup; - - priv = vm->privateData; - - if (dev_name) { - for (i = 0; !chr && i < vm->def->nconsoles; i++) { - if (vm->def->consoles[i]->info.alias && - STREQ(dev_name, vm->def->consoles[i]->info.alias)) - chr = vm->def->consoles[i]; - } - for (i = 0; !chr && i < vm->def->nserials; i++) { - if (STREQ(dev_name, vm->def->serials[i]->info.alias)) - chr = vm->def->serials[i]; - } - } else { - if (vm->def->nconsoles && - vm->def->consoles[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) - chr = vm->def->consoles[0]; - else if (vm->def->nserials && - vm->def->serials[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) - chr = vm->def->serials[0]; - } - - if (!chr) { - virReportError(VIR_ERR_INTERNAL_ERROR, _("cannot find character device %s"), - NULLSTR(dev_name)); - goto cleanup; - } - - if (chr->source->type != VIR_DOMAIN_CHR_TYPE_PTY) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("character device %s is not using a PTY"), - dev_name ? dev_name : NULLSTR(chr->info.alias)); - goto cleanup; - } - - /* handle mutually exclusive access to console devices */ - ret = virChrdevOpen(priv->chrdevs, chr->source, st, - (flags & VIR_DOMAIN_CONSOLE_FORCE) != 0); - - if (ret == 1) { - virReportError(VIR_ERR_OPERATION_FAILED, "%s", - _("Active console session exists for this domain")); - ret = -1; - } + const char *dev_name, virStreamPtr st, unsigned int flags) +{ + virDomainObj *vm = NULL; + int ret = -1; + size_t i; + virDomainChrDef *chr = NULL; + virCHDomainObjPrivate *priv; + + virCheckFlags(VIR_DOMAIN_CONSOLE_SAFE | VIR_DOMAIN_CONSOLE_FORCE, -1); + + if (!(vm = chDomObjFromDomain(dom))) + goto cleanup; + + if (virDomainOpenConsoleEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjCheckActive(vm) < 0) + goto cleanup; + + priv = vm->privateData; + + if (dev_name) { + for (i = 0; !chr && i < vm->def->nconsoles; i++) { + if (vm->def->consoles[i]->info.alias && + STREQ(dev_name, vm->def->consoles[i]->info.alias)) + chr = vm->def->consoles[i]; + } + for (i = 0; !chr && i < vm->def->nserials; i++) { + if (STREQ(dev_name, vm->def->serials[i]->info.alias)) + chr = vm->def->serials[i]; + } + } else { + if (vm->def->nconsoles && + vm->def->consoles[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) + chr = vm->def->consoles[0]; + else if (vm->def->nserials && + vm->def->serials[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) + chr = vm->def->serials[0]; + } + + if (!chr) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("cannot find character device %s"), NULLSTR(dev_name)); + goto cleanup; + } + + if (chr->source->type != VIR_DOMAIN_CHR_TYPE_PTY) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("character device %s is not using a PTY"), + dev_name ? dev_name : NULLSTR(chr->info.alias)); + goto cleanup; + } + + /* handle mutually exclusive access to console devices */ + ret = virChrdevOpen(priv->chrdevs, chr->source, st, + (flags & VIR_DOMAIN_CONSOLE_FORCE) != 0); + + if (ret == 1) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("Active console session exists for this domain")); + ret = -1; + } cleanup: - virDomainObjEndAPI(&vm); - return ret; + virDomainObjEndAPI(&vm); + return ret; } -static int chStateCleanup(void) +static int +chStateCleanup(void) { if (ch_driver == NULL) return -1; @@ -897,10 +896,11 @@ static int chStateCleanup(void) return 0; } -static int chStateInitialize(bool privileged, - const char *root, - virStateInhibitCallback callback G_GNUC_UNUSED, - void *opaque G_GNUC_UNUSED) +static int +chStateInitialize(bool privileged, + const char *root, + virStateInhibitCallback callback G_GNUC_UNUSED, + void *opaque G_GNUC_UNUSED) { int ret = VIR_DRV_STATE_INIT_ERROR; int rv; @@ -984,7 +984,7 @@ static virHypervisorDriver chHypervisorDriver = { static virConnectDriver chConnectDriver = { .localOnly = true, - .uriSchemes = (const char *[]){"ch", NULL}, + .uriSchemes = (const char *[]) {"ch", NULL}, .hypervisorDriver = &chHypervisorDriver, }; @@ -994,7 +994,8 @@ static virStateDriver chStateDriver = { .stateCleanup = chStateCleanup, }; -int chRegister(void) +int +chRegister(void) { if (virRegisterConnectDriver(&chConnectDriver, true) < 0) return -1; -- 2.27.0

Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com> --- src/ch/ch_monitor.c | 301 +++++++++++++++++++++++--------------------- 1 file changed, 161 insertions(+), 140 deletions(-) diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index 12c10da874..68fa5b30aa 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -42,7 +42,8 @@ VIR_LOG_INIT("ch.ch_monitor"); static virClass *virCHMonitorClass; static void virCHMonitorDispose(void *obj); -static int virCHMonitorOnceInit(void) +static int +virCHMonitorOnceInit(void) { if (!VIR_CLASS_NEW(virCHMonitor, virClassForObjectLockable())) return -1; @@ -52,11 +53,11 @@ static int virCHMonitorOnceInit(void) VIR_ONCE_GLOBAL_INIT(virCHMonitor); -int virCHMonitorShutdownVMM(virCHMonitor *mon); -int virCHMonitorPutNoContent(virCHMonitor *mon, const char *endpoint); +int virCHMonitorShutdownVMM(virCHMonitor * mon); +int virCHMonitorPutNoContent(virCHMonitor * mon, const char *endpoint); static int -virCHMonitorBuildCPUJson(virJSONValue *content, virDomainDef *vmdef) +virCHMonitorBuildCPUJson(virJSONValue * content, virDomainDef * vmdef) { g_autoptr(virJSONValue) cpus = NULL; unsigned int maxvcpus = 0; @@ -64,7 +65,7 @@ virCHMonitorBuildCPUJson(virJSONValue *content, virDomainDef *vmdef) virDomainVcpuDef *vcpu; size_t i; - /* count maximum allowed number vcpus and enabled vcpus when boot.*/ + /* count maximum allowed number vcpus and enabled vcpus when boot. */ maxvcpus = virDomainDefGetVcpusMax(vmdef); for (i = 0; i < maxvcpus; i++) { vcpu = virDomainDefGetVcpu(vmdef, i); @@ -76,7 +77,8 @@ virCHMonitorBuildCPUJson(virJSONValue *content, virDomainDef *vmdef) cpus = virJSONValueNewObject(); if (virJSONValueObjectAppendNumberInt(cpus, "boot_vcpus", nvcpus) < 0) return -1; - if (virJSONValueObjectAppendNumberInt(cpus, "max_vcpus", vmdef->maxvcpus) < 0) + if (virJSONValueObjectAppendNumberInt + (cpus, "max_vcpus", vmdef->maxvcpus) < 0) return -1; if (virJSONValueObjectAppend(content, "cpus", &cpus) < 0) return -1; @@ -86,7 +88,7 @@ virCHMonitorBuildCPUJson(virJSONValue *content, virDomainDef *vmdef) } static int -virCHMonitorBuildPTYJson(virJSONValue *content, virDomainDef *vmdef) +virCHMonitorBuildPTYJson(virJSONValue * content, virDomainDef * vmdef) { if (vmdef->nconsoles) { g_autoptr(virJSONValue) pty = virJSONValueNewObject(); @@ -108,7 +110,7 @@ virCHMonitorBuildPTYJson(virJSONValue *content, virDomainDef *vmdef) } static int -virCHMonitorBuildKernelRelatedJson(virJSONValue *content, virDomainDef *vmdef) +virCHMonitorBuildKernelRelatedJson(virJSONValue * content, virDomainDef * vmdef) { g_autoptr(virJSONValue) kernel = virJSONValueNewObject(); g_autoptr(virJSONValue) cmdline = virJSONValueNewObject(); @@ -119,21 +121,24 @@ virCHMonitorBuildKernelRelatedJson(virJSONValue *content, virDomainDef *vmdef) _("Kernel image path in this domain is not defined")); return -1; } else { - if (virJSONValueObjectAppendString(kernel, "path", vmdef->os.kernel) < 0) + if (virJSONValueObjectAppendString(kernel, "path", vmdef->os.kernel) < + 0) return -1; if (virJSONValueObjectAppend(content, "kernel", &kernel) < 0) return -1; } if (vmdef->os.cmdline) { - if (virJSONValueObjectAppendString(cmdline, "args", vmdef->os.cmdline) < 0) + if (virJSONValueObjectAppendString(cmdline, "args", vmdef->os.cmdline) < + 0) return -1; if (virJSONValueObjectAppend(content, "cmdline", &cmdline) < 0) return -1; } if (vmdef->os.initrd != NULL) { - if (virJSONValueObjectAppendString(initramfs, "path", vmdef->os.initrd) < 0) + if (virJSONValueObjectAppendString(initramfs, "path", vmdef->os.initrd) + < 0) return -1; if (virJSONValueObjectAppend(content, "initramfs", &initramfs) < 0) return -1; @@ -143,14 +148,16 @@ virCHMonitorBuildKernelRelatedJson(virJSONValue *content, virDomainDef *vmdef) } static int -virCHMonitorBuildMemoryJson(virJSONValue *content, virDomainDef *vmdef) +virCHMonitorBuildMemoryJson(virJSONValue * content, virDomainDef * vmdef) { - unsigned long long total_memory = virDomainDefGetMemoryInitial(vmdef) * 1024; + unsigned long long total_memory = + virDomainDefGetMemoryInitial(vmdef) * 1024; if (total_memory != 0) { g_autoptr(virJSONValue) memory = virJSONValueNewObject(); - if (virJSONValueObjectAppendNumberUlong(memory, "size", total_memory) < 0) + if (virJSONValueObjectAppendNumberUlong(memory, "size", total_memory) < + 0) return -1; if (virJSONValueObjectAppend(content, "memory", &memory) < 0) @@ -161,7 +168,7 @@ virCHMonitorBuildMemoryJson(virJSONValue *content, virDomainDef *vmdef) } static int -virCHMonitorBuildDiskJson(virJSONValue *disks, virDomainDiskDef *diskdef) +virCHMonitorBuildDiskJson(virJSONValue * disks, virDomainDiskDef * diskdef) { g_autoptr(virJSONValue) disk = virJSONValueNewObject(); @@ -169,44 +176,47 @@ virCHMonitorBuildDiskJson(virJSONValue *disks, virDomainDiskDef *diskdef) return -1; switch (diskdef->src->type) { - case VIR_STORAGE_TYPE_FILE: - if (!diskdef->src->path) { - virReportError(VIR_ERR_INVALID_ARG, "%s", - _("Missing disk file path in domain")); - return -1; - } - if (diskdef->bus != VIR_DOMAIN_DISK_BUS_VIRTIO) { - virReportError(VIR_ERR_INVALID_ARG, - _("Only virtio bus types are supported for '%s'"), diskdef->src->path); - return -1; - } - if (virJSONValueObjectAppendString(disk, "path", diskdef->src->path) < 0) - return -1; - if (diskdef->src->readonly) { - if (virJSONValueObjectAppendBoolean(disk, "readonly", true) < 0) + case VIR_STORAGE_TYPE_FILE: + if (!diskdef->src->path) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Missing disk file path in domain")); + return -1; + } + if (diskdef->bus != VIR_DOMAIN_DISK_BUS_VIRTIO) { + virReportError(VIR_ERR_INVALID_ARG, + _ + ("Only virtio bus types are supported for '%s'"), + diskdef->src->path); + return -1; + } + if (virJSONValueObjectAppendString(disk, "path", diskdef->src->path) + < 0) + return -1; + if (diskdef->src->readonly) { + if (virJSONValueObjectAppendBoolean(disk, "readonly", true) < 0) + return -1; + } + if (virJSONValueArrayAppend(disks, &disk) < 0) return -1; - } - if (virJSONValueArrayAppend(disks, &disk) < 0) - return -1; - break; - case VIR_STORAGE_TYPE_NONE: - case VIR_STORAGE_TYPE_BLOCK: - case VIR_STORAGE_TYPE_DIR: - case VIR_STORAGE_TYPE_NETWORK: - case VIR_STORAGE_TYPE_VOLUME: - case VIR_STORAGE_TYPE_NVME: - case VIR_STORAGE_TYPE_VHOST_USER: - default: - virReportEnumRangeError(virStorageType, diskdef->src->type); - return -1; + break; + case VIR_STORAGE_TYPE_NONE: + case VIR_STORAGE_TYPE_BLOCK: + case VIR_STORAGE_TYPE_DIR: + case VIR_STORAGE_TYPE_NETWORK: + case VIR_STORAGE_TYPE_VOLUME: + case VIR_STORAGE_TYPE_NVME: + case VIR_STORAGE_TYPE_VHOST_USER: + default: + virReportEnumRangeError(virStorageType, diskdef->src->type); + return -1; } return 0; } static int -virCHMonitorBuildDisksJson(virJSONValue *content, virDomainDef *vmdef) +virCHMonitorBuildDisksJson(virJSONValue * content, virDomainDef * vmdef) { g_autoptr(virJSONValue) disks = NULL; size_t i; @@ -226,76 +236,86 @@ virCHMonitorBuildDisksJson(virJSONValue *content, virDomainDef *vmdef) } static int -virCHMonitorBuildNetJson(virJSONValue *nets, virDomainNetDef *netdef) +virCHMonitorBuildNetJson(virJSONValue * nets, virDomainNetDef * netdef) { virDomainNetType netType = virDomainNetGetActualType(netdef); char macaddr[VIR_MAC_STRING_BUFLEN]; + g_autoptr(virJSONValue) net = NULL; // check net type at first net = virJSONValueNewObject(); switch (netType) { - case VIR_DOMAIN_NET_TYPE_ETHERNET: - if (netdef->guestIP.nips == 1) { - const virNetDevIPAddr *ip = netdef->guestIP.ips[0]; - g_autofree char *addr = NULL; - virSocketAddr netmask; - g_autofree char *netmaskStr = NULL; - if (!(addr = virSocketAddrFormat(&ip->address))) - return -1; - if (virJSONValueObjectAppendString(net, "ip", addr) < 0) - return -1; - - if (virSocketAddrPrefixToNetmask(ip->prefix, &netmask, AF_INET) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Failed to translate net prefix %d to netmask"), - ip->prefix); - return -1; + case VIR_DOMAIN_NET_TYPE_ETHERNET: + if (netdef->guestIP.nips == 1) { + const virNetDevIPAddr *ip = netdef->guestIP.ips[0]; + g_autofree char *addr = NULL; + virSocketAddr netmask; + g_autofree char *netmaskStr = NULL; + + if (!(addr = virSocketAddrFormat(&ip->address))) + return -1; + if (virJSONValueObjectAppendString(net, "ip", addr) < 0) + return -1; + + if (virSocketAddrPrefixToNetmask(ip->prefix, &netmask, AF_INET) + < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _ + ("Failed to translate net prefix %d to netmask"), + ip->prefix); + return -1; + } + if (!(netmaskStr = virSocketAddrFormat(&netmask))) + return -1; + if (virJSONValueObjectAppendString(net, "mask", netmaskStr) < 0) + return -1; + } else if (netdef->guestIP.nips > 1) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("ethernet type supports a single guest ip")); } - if (!(netmaskStr = virSocketAddrFormat(&netmask))) - return -1; - if (virJSONValueObjectAppendString(net, "mask", netmaskStr) < 0) + break; + case VIR_DOMAIN_NET_TYPE_VHOSTUSER: + if ((virDomainChrType) netdef->data.vhostuser->type != + VIR_DOMAIN_CHR_TYPE_UNIX) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _ + ("vhost_user type support UNIX socket in this CH")); return -1; - } else if (netdef->guestIP.nips > 1) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("ethernet type supports a single guest ip")); - } - break; - case VIR_DOMAIN_NET_TYPE_VHOSTUSER: - if ((virDomainChrType)netdef->data.vhostuser->type != VIR_DOMAIN_CHR_TYPE_UNIX) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("vhost_user type support UNIX socket in this CH")); + } else { + if (virJSONValueObjectAppendString + (net, "vhost_socket", + netdef->data.vhostuser->data.nix.path) < 0) + return -1; + if (virJSONValueObjectAppendBoolean(net, "vhost_user", true) < + 0) + return -1; + } + break; + case VIR_DOMAIN_NET_TYPE_BRIDGE: + case VIR_DOMAIN_NET_TYPE_NETWORK: + case VIR_DOMAIN_NET_TYPE_DIRECT: + case VIR_DOMAIN_NET_TYPE_USER: + case VIR_DOMAIN_NET_TYPE_SERVER: + case VIR_DOMAIN_NET_TYPE_CLIENT: + case VIR_DOMAIN_NET_TYPE_MCAST: + case VIR_DOMAIN_NET_TYPE_INTERNAL: + case VIR_DOMAIN_NET_TYPE_HOSTDEV: + case VIR_DOMAIN_NET_TYPE_UDP: + case VIR_DOMAIN_NET_TYPE_VDPA: + case VIR_DOMAIN_NET_TYPE_LAST: + default: + virReportEnumRangeError(virDomainNetType, netType); return -1; - } else { - if (virJSONValueObjectAppendString(net, "vhost_socket", netdef->data.vhostuser->data.nix.path) < 0) - return -1; - if (virJSONValueObjectAppendBoolean(net, "vhost_user", true) < 0) - return -1; - } - break; - case VIR_DOMAIN_NET_TYPE_BRIDGE: - case VIR_DOMAIN_NET_TYPE_NETWORK: - case VIR_DOMAIN_NET_TYPE_DIRECT: - case VIR_DOMAIN_NET_TYPE_USER: - case VIR_DOMAIN_NET_TYPE_SERVER: - case VIR_DOMAIN_NET_TYPE_CLIENT: - case VIR_DOMAIN_NET_TYPE_MCAST: - case VIR_DOMAIN_NET_TYPE_INTERNAL: - case VIR_DOMAIN_NET_TYPE_HOSTDEV: - case VIR_DOMAIN_NET_TYPE_UDP: - case VIR_DOMAIN_NET_TYPE_VDPA: - case VIR_DOMAIN_NET_TYPE_LAST: - default: - virReportEnumRangeError(virDomainNetType, netType); - return -1; } if (netdef->ifname != NULL) { if (virJSONValueObjectAppendString(net, "tap", netdef->ifname) < 0) return -1; } - if (virJSONValueObjectAppendString(net, "mac", virMacAddrFormat(&netdef->mac, macaddr)) < 0) + if (virJSONValueObjectAppendString + (net, "mac", virMacAddrFormat(&netdef->mac, macaddr)) < 0) return -1; @@ -306,19 +326,24 @@ virCHMonitorBuildNetJson(virJSONValue *nets, virDomainNetDef *netdef) } } if (netdef->driver.virtio.queues) { - if (virJSONValueObjectAppendNumberInt(net, "num_queues", netdef->driver.virtio.queues) < 0) + if (virJSONValueObjectAppendNumberInt + (net, "num_queues", netdef->driver.virtio.queues) < 0) return -1; } - if (netdef->driver.virtio.rx_queue_size || netdef->driver.virtio.tx_queue_size) { - if (netdef->driver.virtio.rx_queue_size != netdef->driver.virtio.tx_queue_size) { + if (netdef->driver.virtio.rx_queue_size + || netdef->driver.virtio.tx_queue_size) { + if (netdef->driver.virtio.rx_queue_size != + netdef->driver.virtio.tx_queue_size) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("virtio rx_queue_size option %d is not same with tx_queue_size %d"), - netdef->driver.virtio.rx_queue_size, - netdef->driver.virtio.tx_queue_size); + _ + ("virtio rx_queue_size option %d is not same with tx_queue_size %d"), + netdef->driver.virtio.rx_queue_size, + netdef->driver.virtio.tx_queue_size); return -1; } - if (virJSONValueObjectAppendNumberInt(net, "queue_size", netdef->driver.virtio.rx_queue_size) < 0) + if (virJSONValueObjectAppendNumberInt + (net, "queue_size", netdef->driver.virtio.rx_queue_size) < 0) return -1; } @@ -329,7 +354,7 @@ virCHMonitorBuildNetJson(virJSONValue *nets, virDomainNetDef *netdef) } static int -virCHMonitorBuildNetsJson(virJSONValue *content, virDomainDef *vmdef) +virCHMonitorBuildNetsJson(virJSONValue * content, virDomainDef * vmdef) { g_autoptr(virJSONValue) nets = NULL; size_t i; @@ -349,13 +374,12 @@ virCHMonitorBuildNetsJson(virJSONValue *content, virDomainDef *vmdef) } static int -virCHMonitorBuildVMJson(virDomainDef *vmdef, char **jsonstr) +virCHMonitorBuildVMJson(virDomainDef * vmdef, char **jsonstr) { g_autoptr(virJSONValue) content = virJSONValueNewObject(); if (vmdef == NULL) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("VM is not defined")); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("VM is not defined")); return -1; } @@ -391,8 +415,7 @@ chMonitorCreateSocket(const char *socket_path) int fd; if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) { - virReportSystemError(errno, "%s", - _("Unable to create UNIX socket")); + virReportSystemError(errno, "%s", _("Unable to create UNIX socket")); goto error; } @@ -400,19 +423,16 @@ chMonitorCreateSocket(const char *socket_path) addr.sun_family = AF_UNIX; if (virStrcpyStatic(addr.sun_path, socket_path) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, - _("UNIX socket path '%s' too long"), - socket_path); + _("UNIX socket path '%s' too long"), socket_path); goto error; } if (unlink(socket_path) < 0 && errno != ENOENT) { - virReportSystemError(errno, - _("Unable to unlink %s"), - socket_path); + virReportSystemError(errno, _("Unable to unlink %s"), socket_path); goto error; } - if (bind(fd, (struct sockaddr *)&addr, addrlen) < 0) { + if (bind(fd, (struct sockaddr *) &addr, addrlen) < 0) { virReportSystemError(errno, _("Unable to bind to UNIX socket path '%s'"), socket_path); @@ -440,7 +460,7 @@ chMonitorCreateSocket(const char *socket_path) } virCHMonitor * -virCHMonitorNew(virDomainObj *vm, const char *socketdir) +virCHMonitorNew(virDomainObj * vm, const char *socketdir) { g_autoptr(virCHMonitor) mon = NULL; g_autoptr(virCommand) cmd = NULL; @@ -453,8 +473,7 @@ virCHMonitorNew(virDomainObj *vm, const char *socketdir) return NULL; if (!vm->def) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("VM is not defined")); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("VM is not defined")); return NULL; } @@ -472,8 +491,7 @@ virCHMonitorNew(virDomainObj *vm, const char *socketdir) socket_fd = chMonitorCreateSocket(mon->socketpath); if (socket_fd < 0) { virReportSystemError(errno, - _("Cannot create socket '%s'"), - mon->socketpath); + _("Cannot create socket '%s'"), mon->socketpath); return NULL; } @@ -494,7 +512,8 @@ virCHMonitorNew(virDomainObj *vm, const char *socketdir) return g_steal_pointer(&mon); } -static void virCHMonitorDispose(void *opaque) +static void +virCHMonitorDispose(void *opaque) { virCHMonitor *mon = opaque; @@ -502,7 +521,8 @@ static void virCHMonitorDispose(void *opaque) virObjectUnref(mon->vm); } -void virCHMonitorClose(virCHMonitor *mon) +void +virCHMonitorClose(virCHMonitor * mon) { if (!mon) return; @@ -518,8 +538,7 @@ void virCHMonitorClose(virCHMonitor *mon) if (mon->socketpath) { if (virFileRemove(mon->socketpath, -1, -1) < 0) { - VIR_WARN("Unable to remove CH socket file '%s'", - mon->socketpath); + VIR_WARN("Unable to remove CH socket file '%s'", mon->socketpath); } g_free(mon->socketpath); } @@ -528,7 +547,7 @@ void virCHMonitorClose(virCHMonitor *mon) } static int -virCHMonitorCurlPerform(CURL *handle) +virCHMonitorCurlPerform(CURL * handle) { CURLcode errorCode; long responseCode = 0; @@ -547,8 +566,9 @@ virCHMonitorCurlPerform(CURL *handle) if (errorCode != CURLE_OK) { virReportError(VIR_ERR_INTERNAL_ERROR, - _("curl_easy_getinfo(CURLINFO_RESPONSE_CODE) returned an " - "error: %s (%d)"), curl_easy_strerror(errorCode), + _ + ("curl_easy_getinfo(CURLINFO_RESPONSE_CODE) returned an " + "error: %s (%d)"), curl_easy_strerror(errorCode), errorCode); return -1; } @@ -564,7 +584,7 @@ virCHMonitorCurlPerform(CURL *handle) } int -virCHMonitorPutNoContent(virCHMonitor *mon, const char *endpoint) +virCHMonitorPutNoContent(virCHMonitor * mon, const char *endpoint) { g_autofree char *url = NULL; int responseCode = 0; @@ -615,13 +635,14 @@ curl_callback(void *contents, size_t size, size_t nmemb, void *userp) } static int -virCHMonitorGet(virCHMonitor *mon, const char *endpoint, virJSONValue **response) +virCHMonitorGet(virCHMonitor * mon, const char *endpoint, + virJSONValue ** response) { g_autofree char *url = NULL; int responseCode = 0; int ret = -1; struct curl_slist *headers = NULL; - struct curl_data data = {0}; + struct curl_data data = { 0 }; url = g_strdup_printf("%s/%s", URL_ROOT, endpoint); @@ -638,7 +659,7 @@ virCHMonitorGet(virCHMonitor *mon, const char *endpoint, virJSONValue **response headers = curl_slist_append(headers, "Content-Type: application/json"); curl_easy_setopt(mon->handle, CURLOPT_HTTPHEADER, headers); curl_easy_setopt(mon->handle, CURLOPT_WRITEFUNCTION, curl_callback); - curl_easy_setopt(mon->handle, CURLOPT_WRITEDATA, (void *)&data); + curl_easy_setopt(mon->handle, CURLOPT_WRITEDATA, (void *) &data); } responseCode = virCHMonitorCurlPerform(mon->handle); @@ -665,13 +686,13 @@ virCHMonitorGet(virCHMonitor *mon, const char *endpoint, virJSONValue **response } int -virCHMonitorShutdownVMM(virCHMonitor *mon) +virCHMonitorShutdownVMM(virCHMonitor * mon) { return virCHMonitorPutNoContent(mon, URL_VMM_SHUTDOWN); } int -virCHMonitorCreateVM(virCHMonitor *mon) +virCHMonitorCreateVM(virCHMonitor * mon) { g_autofree char *url = NULL; int responseCode = 0; @@ -709,31 +730,31 @@ virCHMonitorCreateVM(virCHMonitor *mon) } int -virCHMonitorBootVM(virCHMonitor *mon) +virCHMonitorBootVM(virCHMonitor * mon) { return virCHMonitorPutNoContent(mon, URL_VM_BOOT); } int -virCHMonitorShutdownVM(virCHMonitor *mon) +virCHMonitorShutdownVM(virCHMonitor * mon) { return virCHMonitorPutNoContent(mon, URL_VM_SHUTDOWN); } int -virCHMonitorRebootVM(virCHMonitor *mon) +virCHMonitorRebootVM(virCHMonitor * mon) { return virCHMonitorPutNoContent(mon, URL_VM_REBOOT); } int -virCHMonitorSuspendVM(virCHMonitor *mon) +virCHMonitorSuspendVM(virCHMonitor * mon) { return virCHMonitorPutNoContent(mon, URL_VM_Suspend); } int -virCHMonitorResumeVM(virCHMonitor *mon) +virCHMonitorResumeVM(virCHMonitor * mon) { return virCHMonitorPutNoContent(mon, URL_VM_RESUME); } @@ -748,7 +769,7 @@ virCHMonitorResumeVM(virCHMonitor *mon) * Returns 0 on success. */ int -virCHMonitorGetInfo(virCHMonitor *mon, virJSONValue **info) +virCHMonitorGetInfo(virCHMonitor * mon, virJSONValue ** info) { return virCHMonitorGet(mon, URL_VM_INFO, info); } -- 2.27.0

Refactor some cgroup management methods from qemu into hypervisor. These methods will be shared by ch driver for cgroup management. Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com> --- src/hypervisor/domain_cgroup.c | 426 ++++++++++++++++++++++++++++++++- src/hypervisor/domain_cgroup.h | 52 ++++ src/libvirt_private.syms | 10 + src/qemu/qemu_cgroup.c | 410 +------------------------------ src/qemu/qemu_cgroup.h | 11 - src/qemu/qemu_driver.c | 14 +- src/qemu/qemu_hotplug.c | 4 +- src/qemu/qemu_process.c | 17 +- 8 files changed, 515 insertions(+), 429 deletions(-) diff --git a/src/hypervisor/domain_cgroup.c b/src/hypervisor/domain_cgroup.c index 61b54f071c..eda46e3421 100644 --- a/src/hypervisor/domain_cgroup.c +++ b/src/hypervisor/domain_cgroup.c @@ -22,11 +22,12 @@ #include "domain_cgroup.h" #include "domain_driver.h" - +#include "util/virnuma.h" +#include "virlog.h" #include "virutil.h" #define VIR_FROM_THIS VIR_FROM_DOMAIN - +VIR_LOG_INIT("domain.cgroup"); int virDomainCgroupSetupBlkio(virCgroup *cgroup, virDomainBlkiotune blkio) @@ -269,3 +270,424 @@ virDomainCgroupSetMemoryLimitParameters(virCgroup *cgroup, return 0; } + + +int +virSetupBlkioCgroup(virDomainObj * vm, virCgroup * cgroup) +{ + + if (!virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_BLKIO)) { + if (vm->def->blkio.weight || vm->def->blkio.ndevices) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Block I/O tuning is not available on this host")); + return -1; + } else { + return 0; + } + } + + return virDomainCgroupSetupBlkio(cgroup, vm->def->blkio); +} + + +int +virSetupMemoryCgroup(virDomainObj * vm, virCgroup * cgroup) +{ + + if (!virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_MEMORY)) { + if (virMemoryLimitIsSet(vm->def->mem.hard_limit) || + virMemoryLimitIsSet(vm->def->mem.soft_limit) || + virMemoryLimitIsSet(vm->def->mem.swap_hard_limit)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Memory cgroup is not available on this host")); + return -1; + } else { + return 0; + } + } + + return virDomainCgroupSetupMemtune(cgroup, vm->def->mem); +} + + +int +virSetupCpusetCgroup(virCgroup * cgroup) +{ + + if (!virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + if (virCgroupSetCpusetMemoryMigrate(cgroup, true) < 0) + return -1; + + return 0; +} + + +int +virSetupCpuCgroup(virDomainObj * vm, virCgroup * cgroup) +{ + + if (!virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + if (vm->def->cputune.sharesSpecified) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("CPU tuning is not available on this host")); + return -1; + } else { + return 0; + } + } + + if (vm->def->cputune.sharesSpecified) { + + if (virCgroupSetCpuShares(cgroup, vm->def->cputune.shares) < 0) + return -1; + + } + + return 0; +} + + +int +virInitCgroup(const char *prefix, + virDomainObj * vm, + size_t nnicindexes, int *nicindexes, + virCgroup * cgroup, int cgroupControllers, + unsigned int maxThreadsPerProc, + bool privileged, char *machineName) +{ + if (!privileged) + return 0; + + if (!virCgroupAvailable()) + return 0; + + virCgroupFree(cgroup); + cgroup = NULL; + + if (!vm->def->resource) + vm->def->resource = g_new0(virDomainResourceDef, 1); + + if (!vm->def->resource->partition) + vm->def->resource->partition = g_strdup("/machine"); + + if (!g_path_is_absolute(vm->def->resource->partition)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Resource partition '%s' must start with '/'"), + vm->def->resource->partition); + return -1; + } + + if (virCgroupNewMachine(machineName, + prefix, + vm->def->uuid, + NULL, + vm->pid, + false, + nnicindexes, nicindexes, + vm->def->resource->partition, + cgroupControllers, + maxThreadsPerProc, &cgroup) < 0) { + if (virCgroupNewIgnoreError()) + return 0; + + return -1; + } + + return 0; +} + + +void +virRestoreCgroupState(virDomainObj * vm, virCgroup * cgroup) +{ + g_autofree char *mem_mask = NULL; + size_t i = 0; + + g_autoptr(virBitmap) all_nodes = NULL; + + if (!virNumaIsAvailable() || + !virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return; + + if (!(all_nodes = virNumaGetHostMemoryNodeset())) + goto error; + + if (!(mem_mask = virBitmapFormat(all_nodes))) + goto error; + + if (virCgroupHasEmptyTasks(cgroup, VIR_CGROUP_CONTROLLER_CPUSET) <= 0) + goto error; + + if (virCgroupSetCpusetMems(cgroup, mem_mask) < 0) + goto error; + + for (i = 0; i < virDomainDefGetVcpusMax(vm->def); i++) { + virDomainVcpuDef *vcpu = virDomainDefGetVcpu(vm->def, i); + + if (!vcpu->online) + continue; + + if (virRestoreCgroupThread(cgroup, VIR_CGROUP_THREAD_VCPU, i) < 0) + return; + } + + for (i = 0; i < vm->def->niothreadids; i++) { + if (virRestoreCgroupThread(cgroup, VIR_CGROUP_THREAD_IOTHREAD, + vm->def->iothreadids[i]->iothread_id) < 0) + return; + } + + if (virRestoreCgroupThread(cgroup, VIR_CGROUP_THREAD_EMULATOR, 0) < 0) + return; + + return; + + error: + virResetLastError(); + VIR_DEBUG("Couldn't restore cgroups to meaningful state"); + return; +} + + +int +virRestoreCgroupThread(virCgroup * cgroup, virCgroupThreadName thread, int id) +{ + g_autoptr(virCgroup) cgroup_temp = NULL; + g_autofree char *nodeset = NULL; + + if (virCgroupNewThread(cgroup, thread, id, false, &cgroup_temp) < 0) + return -1; + + if (virCgroupSetCpusetMemoryMigrate(cgroup_temp, true) < 0) + return -1; + + if (virCgroupGetCpusetMems(cgroup_temp, &nodeset) < 0) + return -1; + + if (virCgroupSetCpusetMems(cgroup_temp, nodeset) < 0) + return -1; + + return 0; +} + + +int +virConnectCgroup(const char *prefix, virDomainObj * vm, virCgroup * cgroup, + int cgroupControllers, bool privileged, char *machineName) +{ + if (privileged) + return 0; + + if (!virCgroupAvailable()) + return 0; + + virCgroupFree(cgroup); + + if (virCgroupNewDetectMachine(vm->def->name, + prefix, + vm->pid, + cgroupControllers, machineName, &cgroup) < 0) + return -1; + + virRestoreCgroupState(vm, cgroup); + return 0; +} + + +int +virSetupCgroup(const char *prefix, + virDomainObj * vm, + size_t nnicindexes, int *nicindexes, + virCgroup * cgroup, + int cgroupControllers, + unsigned int maxThreadsPerProc, + bool privileged, char *machineName) +{ + if (!vm->pid) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot setup cgroups until process is started")); + return -1; + } + + if (virInitCgroup(prefix, vm, nnicindexes, nicindexes, + cgroup, cgroupControllers, + maxThreadsPerProc, privileged, machineName) < 0) + return -1; + + if (!cgroup) + return 0; + + if (virSetupBlkioCgroup(vm, cgroup) < 0) + return -1; + + if (virSetupMemoryCgroup(vm, cgroup) < 0) + return -1; + + if (virSetupCpuCgroup(vm, cgroup) < 0) + return -1; + + if (virSetupCpusetCgroup(cgroup) < 0) + return -1; + + return 0; +} + + +int +virSetupCgroupVcpuBW(virCgroup * cgroup, + unsigned long long period, long long quota) +{ + return virCgroupSetupCpuPeriodQuota(cgroup, period, quota); +} + + +int +virSetupCgroupCpusetCpus(virCgroup * cgroup, virBitmap * cpumask) +{ + return virCgroupSetupCpusetCpus(cgroup, cpumask); +} + + +int +virSetupGlobalCpuCgroup(virDomainObj * vm, virCgroup * cgroup, + virBitmap * autoNodeset) +{ + unsigned long long period = vm->def->cputune.global_period; + long long quota = vm->def->cputune.global_quota; + g_autofree char *mem_mask = NULL; + virDomainNumatuneMemMode mem_mode; + + if ((period || quota) && + !virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + return -1; + } + + /* + * If CPU cgroup controller is not initialized here, then we need + * neither period nor quota settings. And if CPUSET controller is + * not initialized either, then there's nothing to do anyway. + */ + if (!virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPU) && + !virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && + virDomainNumatuneMaybeFormatNodeset(vm->def->numa, + autoNodeset, &mem_mask, -1) < 0) + return -1; + + if (period || quota) { + if (virSetupCgroupVcpuBW(cgroup, period, quota) < 0) + return -1; + } + + return 0; +} + + +int +virRemoveCgroup(virDomainObj * vm, virCgroup * cgroup, char *machineName) +{ + if (cgroup == NULL) + return 0; /* Not supported, so claim success */ + + if (virCgroupTerminateMachine(machineName) < 0) { + if (!virCgroupNewIgnoreError()) + VIR_DEBUG("Failed to terminate cgroup for %s", vm->def->name); + } + + return virCgroupRemove(cgroup); +} + + +void +virCgroupEmulatorAllNodesDataFree(virCgroupEmulatorAllNodesData * data) +{ + if (!data) + return; + + virCgroupFree(data->emulatorCgroup); + g_free(data->emulatorMemMask); + g_free(data); +} + + +/** + * virCgroupEmulatorAllNodesAllow: + * @cgroup: domain cgroup pointer + * @retData: filled with structure used to roll back the operation + * + * Allows all NUMA nodes for the cloud hypervisor thread temporarily. This is + * necessary when hotplugging cpus since it requires memory allocated in the + * DMA region. Afterwards the operation can be reverted by + * virCgroupEmulatorAllNodesRestore. + * + * Returns 0 on success -1 on error + */ +int +virCgroupEmulatorAllNodesAllow(virCgroup * cgroup, + virCgroupEmulatorAllNodesData ** retData) +{ + virCgroupEmulatorAllNodesData *data = NULL; + g_autofree char *all_nodes_str = NULL; + + g_autoptr(virBitmap) all_nodes = NULL; + int ret = -1; + + if (!virNumaIsAvailable() || + !virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + if (!(all_nodes = virNumaGetHostMemoryNodeset())) + goto cleanup; + + if (!(all_nodes_str = virBitmapFormat(all_nodes))) + goto cleanup; + + data = g_new0(virCgroupEmulatorAllNodesData, 1); + + if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_EMULATOR, 0, + false, &data->emulatorCgroup) < 0) + goto cleanup; + + if (virCgroupGetCpusetMems(data->emulatorCgroup, &data->emulatorMemMask) < 0 + || virCgroupSetCpusetMems(data->emulatorCgroup, all_nodes_str) < 0) + goto cleanup; + + *retData = g_steal_pointer(&data); + ret = 0; + + cleanup: + virCgroupEmulatorAllNodesDataFree(data); + + return ret; +} + + +/** + * virCgroupEmulatorAllNodesRestore: + * @data: data structure created by virCgroupEmulatorAllNodesAllow + * + * Rolls back the setting done by virCgroupEmulatorAllNodesAllow and frees the + * associated data. + */ +void +virCgroupEmulatorAllNodesRestore(virCgroupEmulatorAllNodesData * data) +{ + virError *err; + + if (!data) + return; + + virErrorPreserveLast(&err); + virCgroupSetCpusetMems(data->emulatorCgroup, data->emulatorMemMask); + virErrorRestore(&err); + + virCgroupEmulatorAllNodesDataFree(data); +} diff --git a/src/hypervisor/domain_cgroup.h b/src/hypervisor/domain_cgroup.h index f93e5f74fe..fd8b74a4ee 100644 --- a/src/hypervisor/domain_cgroup.h +++ b/src/hypervisor/domain_cgroup.h @@ -23,6 +23,11 @@ #include "vircgroup.h" #include "domain_conf.h" +typedef struct _virCgroupEmulatorAllNodesData virCgroupEmulatorAllNodesData; +struct _virCgroupEmulatorAllNodesData { + virCgroup *emulatorCgroup; + char *emulatorMemMask; +}; int virDomainCgroupSetupBlkio(virCgroup *cgroup, virDomainBlkiotune blkio); int virDomainCgroupSetupMemtune(virCgroup *cgroup, virDomainMemtune mem); @@ -36,3 +41,50 @@ int virDomainCgroupSetMemoryLimitParameters(virCgroup *cgroup, virDomainDef *persistentDef, virTypedParameterPtr params, int nparams); +int +virSetupBlkioCgroup(virDomainObj * vm, virCgroup *cgroup); +int +virSetupMemoryCgroup(virDomainObj * vm, virCgroup *cgroup); +int +virSetupCpusetCgroup(virCgroup *cgroup); +int +virSetupCpuCgroup(virDomainObj * vm, virCgroup *cgroup); +int +virInitCgroup(const char *prefix, virDomainObj * vm, + size_t nnicindexes, int *nicindexes, + virCgroup *cgroup, int cgroupControllers, + unsigned int maxThreadsPerProc, + bool privileged, + char *machineName); +void +virRestoreCgroupState(virDomainObj * vm, virCgroup *cgroup); +int +virConnectCgroup(const char *prefix, virDomainObj * vm, virCgroup *cgroup, + int cgroupControllers, bool privileged, char *machineName); +int +virSetupCgroup(const char *prefix, virDomainObj * vm, + size_t nnicindexes, int *nicindexes, + virCgroup *cgroup, int cgroupControllers, + unsigned int maxThreadsPerProc, + bool privileged, + char *machineName); +void +virCgroupEmulatorAllNodesDataFree(virCgroupEmulatorAllNodesData * data); +int +virCgroupEmulatorAllNodesAllow(virCgroup * cgroup, + virCgroupEmulatorAllNodesData ** retData); +void +virCgroupEmulatorAllNodesRestore(virCgroupEmulatorAllNodesData * data); +int +virSetupCgroupVcpuBW(virCgroup * cgroup, + unsigned long long period, long long quota); +int +virSetupCgroupCpusetCpus(virCgroup * cgroup, virBitmap * cpumask); +int +virSetupGlobalCpuCgroup(virDomainObj * vm, virCgroup * cgroup, + virBitmap *autoNodeset); +int +virRemoveCgroup(virDomainObj * vm, virCgroup * cgroup, char *machineName); +int +virRestoreCgroupThread(virCgroup *cgroup, virCgroupThreadName thread, + int id); diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index e0c4fba522..2e510f5b91 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1545,6 +1545,16 @@ virDomainCgroupSetMemoryLimitParameters; virDomainCgroupSetupBlkio; virDomainCgroupSetupDomainBlkioParameters; virDomainCgroupSetupMemtune; +virInitCgroup; +virRemoveCgroup; +virSetupBlkioCgroup; +virSetupCgroup; +virSetupCgroupCpusetCpus; +virSetupCgroupVcpuBW; +virSetupCpuCgroup; +virSetupCpusetCgroup; +virSetupGlobalCpuCgroup; +virSetupMemoryCgroup; # hypervisor/domain_driver.h diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 308be5b00f..b1d2875959 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -593,46 +593,6 @@ qemuSetupVideoCgroup(virDomainObj *vm, return ret; } - -static int -qemuSetupBlkioCgroup(virDomainObj *vm) -{ - qemuDomainObjPrivate *priv = vm->privateData; - - if (!virCgroupHasController(priv->cgroup, - VIR_CGROUP_CONTROLLER_BLKIO)) { - if (vm->def->blkio.weight || vm->def->blkio.ndevices) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("Block I/O tuning is not available on this host")); - return -1; - } - return 0; - } - - return virDomainCgroupSetupBlkio(priv->cgroup, vm->def->blkio); -} - - -static int -qemuSetupMemoryCgroup(virDomainObj *vm) -{ - qemuDomainObjPrivate *priv = vm->privateData; - - if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_MEMORY)) { - if (virMemoryLimitIsSet(vm->def->mem.hard_limit) || - virMemoryLimitIsSet(vm->def->mem.soft_limit) || - virMemoryLimitIsSet(vm->def->mem.swap_hard_limit)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("Memory cgroup is not available on this host")); - return -1; - } - return 0; - } - - return virDomainCgroupSetupMemtune(priv->cgroup, vm->def->mem); -} - - static int qemuSetupFirmwareCgroup(virDomainObj *vm) { @@ -861,44 +821,6 @@ qemuSetupDevicesCgroup(virDomainObj *vm) } -static int -qemuSetupCpusetCgroup(virDomainObj *vm) -{ - qemuDomainObjPrivate *priv = vm->privateData; - - if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) - return 0; - - if (virCgroupSetCpusetMemoryMigrate(priv->cgroup, true) < 0) - return -1; - - return 0; -} - - -static int -qemuSetupCpuCgroup(virDomainObj *vm) -{ - qemuDomainObjPrivate *priv = vm->privateData; - - if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { - if (vm->def->cputune.sharesSpecified) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("CPU tuning is not available on this host")); - return -1; - } - return 0; - } - - if (vm->def->cputune.sharesSpecified) { - if (virCgroupSetCpuShares(priv->cgroup, vm->def->cputune.shares) < 0) - return -1; - } - - return 0; -} - - static int qemuSetupCgroupAppid(virDomainObj *vm) { @@ -927,166 +849,13 @@ qemuSetupCgroupAppid(virDomainObj *vm) } -static int -qemuInitCgroup(virDomainObj *vm, - size_t nnicindexes, - int *nicindexes) -{ - qemuDomainObjPrivate *priv = vm->privateData; - g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(priv->driver); - - if (!priv->driver->privileged) - return 0; - - if (!virCgroupAvailable()) - return 0; - - virCgroupFree(priv->cgroup); - priv->cgroup = NULL; - - if (!vm->def->resource) - vm->def->resource = g_new0(virDomainResourceDef, 1); - - if (!vm->def->resource->partition) - vm->def->resource->partition = g_strdup("/machine"); - - if (!g_path_is_absolute(vm->def->resource->partition)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("Resource partition '%s' must start with '/'"), - vm->def->resource->partition); - return -1; - } - - if (virCgroupNewMachine(priv->machineName, - "qemu", - vm->def->uuid, - NULL, - vm->pid, - false, - nnicindexes, nicindexes, - vm->def->resource->partition, - cfg->cgroupControllers, - cfg->maxThreadsPerProc, - &priv->cgroup) < 0) { - if (virCgroupNewIgnoreError()) - return 0; - - return -1; - } - - return 0; -} - -static int -qemuRestoreCgroupThread(virCgroup *cgroup, - virCgroupThreadName thread, - int id) -{ - g_autoptr(virCgroup) cgroup_temp = NULL; - g_autofree char *nodeset = NULL; - - if (virCgroupNewThread(cgroup, thread, id, false, &cgroup_temp) < 0) - return -1; - - if (virCgroupSetCpusetMemoryMigrate(cgroup_temp, true) < 0) - return -1; - - if (virCgroupGetCpusetMems(cgroup_temp, &nodeset) < 0) - return -1; - - if (virCgroupSetCpusetMems(cgroup_temp, nodeset) < 0) - return -1; - - return 0; -} - -static void -qemuRestoreCgroupState(virDomainObj *vm) -{ - g_autofree char *mem_mask = NULL; - qemuDomainObjPrivate *priv = vm->privateData; - size_t i = 0; - g_autoptr(virBitmap) all_nodes = NULL; - - if (!virNumaIsAvailable() || - !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) - return; - - if (!(all_nodes = virNumaGetHostMemoryNodeset())) - goto error; - - if (!(mem_mask = virBitmapFormat(all_nodes))) - goto error; - - if (virCgroupHasEmptyTasks(priv->cgroup, - VIR_CGROUP_CONTROLLER_CPUSET) <= 0) - goto error; - - if (virCgroupSetCpusetMems(priv->cgroup, mem_mask) < 0) - goto error; - - for (i = 0; i < virDomainDefGetVcpusMax(vm->def); i++) { - virDomainVcpuDef *vcpu = virDomainDefGetVcpu(vm->def, i); - - if (!vcpu->online) - continue; - - if (qemuRestoreCgroupThread(priv->cgroup, - VIR_CGROUP_THREAD_VCPU, i) < 0) - return; - } - - for (i = 0; i < vm->def->niothreadids; i++) { - if (qemuRestoreCgroupThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD, - vm->def->iothreadids[i]->iothread_id) < 0) - return; - } - - if (qemuRestoreCgroupThread(priv->cgroup, - VIR_CGROUP_THREAD_EMULATOR, 0) < 0) - return; - - return; - - error: - virResetLastError(); - VIR_DEBUG("Couldn't restore cgroups to meaningful state"); - return; -} - -int -qemuConnectCgroup(virDomainObj *vm) -{ - qemuDomainObjPrivate *priv = vm->privateData; - g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(priv->driver); - - if (!priv->driver->privileged) - return 0; - - if (!virCgroupAvailable()) - return 0; - - virCgroupFree(priv->cgroup); - priv->cgroup = NULL; - - if (virCgroupNewDetectMachine(vm->def->name, - "qemu", - vm->pid, - cfg->cgroupControllers, - priv->machineName, - &priv->cgroup) < 0) - return -1; - - qemuRestoreCgroupState(vm); - return 0; -} - int qemuSetupCgroup(virDomainObj *vm, size_t nnicindexes, int *nicindexes) { qemuDomainObjPrivate *priv = vm->privateData; + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(priv->driver); if (!vm->pid) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", @@ -1094,7 +863,12 @@ qemuSetupCgroup(virDomainObj *vm, return -1; } - if (qemuInitCgroup(vm, nnicindexes, nicindexes) < 0) + if (virInitCgroup("qemu", vm, nnicindexes, nicindexes, + priv->cgroup, + cfg->cgroupControllers, + cfg->maxThreadsPerProc, + priv->driver->privileged, + priv->machineName) < 0) return -1; if (!priv->cgroup) @@ -1103,16 +877,16 @@ qemuSetupCgroup(virDomainObj *vm, if (qemuSetupDevicesCgroup(vm) < 0) return -1; - if (qemuSetupBlkioCgroup(vm) < 0) + if (virSetupBlkioCgroup(vm, priv->cgroup) < 0) return -1; - if (qemuSetupMemoryCgroup(vm) < 0) + if (virSetupMemoryCgroup(vm, priv->cgroup) < 0) return -1; - if (qemuSetupCpuCgroup(vm) < 0) + if (virSetupCpuCgroup(vm, priv->cgroup) < 0) return -1; - if (qemuSetupCpusetCgroup(vm) < 0) + if (virSetupCpusetCgroup(priv->cgroup) < 0) return -1; if (qemuSetupCgroupAppid(vm) < 0) @@ -1121,23 +895,6 @@ qemuSetupCgroup(virDomainObj *vm, return 0; } -int -qemuSetupCgroupVcpuBW(virCgroup *cgroup, - unsigned long long period, - long long quota) -{ - return virCgroupSetupCpuPeriodQuota(cgroup, period, quota); -} - - -int -qemuSetupCgroupCpusetCpus(virCgroup *cgroup, - virBitmap *cpumask) -{ - return virCgroupSetupCpusetCpus(cgroup, cpumask); -} - - int qemuSetupCgroupForExtDevices(virDomainObj *vm, virQEMUDriver *driver) @@ -1164,148 +921,3 @@ qemuSetupCgroupForExtDevices(virDomainObj *vm, return qemuExtDevicesSetupCgroup(driver, vm, cgroup_temp); } - - -int -qemuSetupGlobalCpuCgroup(virDomainObj *vm) -{ - qemuDomainObjPrivate *priv = vm->privateData; - unsigned long long period = vm->def->cputune.global_period; - long long quota = vm->def->cputune.global_quota; - g_autofree char *mem_mask = NULL; - virDomainNumatuneMemMode mem_mode; - - if ((period || quota) && - !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("cgroup cpu is required for scheduler tuning")); - return -1; - } - - /* - * If CPU cgroup controller is not initialized here, then we need - * neither period nor quota settings. And if CPUSET controller is - * not initialized either, then there's nothing to do anyway. - */ - if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) && - !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) - return 0; - - - if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && - mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && - virDomainNumatuneMaybeFormatNodeset(vm->def->numa, - priv->autoNodeset, - &mem_mask, -1) < 0) - return -1; - - if (period || quota) { - if (qemuSetupCgroupVcpuBW(priv->cgroup, period, quota) < 0) - return -1; - } - - return 0; -} - - -int -qemuRemoveCgroup(virDomainObj *vm) -{ - qemuDomainObjPrivate *priv = vm->privateData; - - if (priv->cgroup == NULL) - return 0; /* Not supported, so claim success */ - - if (virCgroupTerminateMachine(priv->machineName) < 0) { - if (!virCgroupNewIgnoreError()) - VIR_DEBUG("Failed to terminate cgroup for %s", vm->def->name); - } - - return virCgroupRemove(priv->cgroup); -} - - -static void -qemuCgroupEmulatorAllNodesDataFree(qemuCgroupEmulatorAllNodesData *data) -{ - if (!data) - return; - - virCgroupFree(data->emulatorCgroup); - g_free(data->emulatorMemMask); - g_free(data); -} - - -/** - * qemuCgroupEmulatorAllNodesAllow: - * @cgroup: domain cgroup pointer - * @retData: filled with structure used to roll back the operation - * - * Allows all NUMA nodes for the qemu emulator thread temporarily. This is - * necessary when hotplugging cpus since it requires memory allocated in the - * DMA region. Afterwards the operation can be reverted by - * qemuCgroupEmulatorAllNodesRestore. - * - * Returns 0 on success -1 on error - */ -int -qemuCgroupEmulatorAllNodesAllow(virCgroup *cgroup, - qemuCgroupEmulatorAllNodesData **retData) -{ - qemuCgroupEmulatorAllNodesData *data = NULL; - g_autofree char *all_nodes_str = NULL; - g_autoptr(virBitmap) all_nodes = NULL; - int ret = -1; - - if (!virNumaIsAvailable() || - !virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) - return 0; - - if (!(all_nodes = virNumaGetHostMemoryNodeset())) - goto cleanup; - - if (!(all_nodes_str = virBitmapFormat(all_nodes))) - goto cleanup; - - data = g_new0(qemuCgroupEmulatorAllNodesData, 1); - - if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_EMULATOR, 0, - false, &data->emulatorCgroup) < 0) - goto cleanup; - - if (virCgroupGetCpusetMems(data->emulatorCgroup, &data->emulatorMemMask) < 0 || - virCgroupSetCpusetMems(data->emulatorCgroup, all_nodes_str) < 0) - goto cleanup; - - *retData = g_steal_pointer(&data); - ret = 0; - - cleanup: - qemuCgroupEmulatorAllNodesDataFree(data); - - return ret; -} - - -/** - * qemuCgroupEmulatorAllNodesRestore: - * @data: data structure created by qemuCgroupEmulatorAllNodesAllow - * - * Rolls back the setting done by qemuCgroupEmulatorAllNodesAllow and frees the - * associated data. - */ -void -qemuCgroupEmulatorAllNodesRestore(qemuCgroupEmulatorAllNodesData *data) -{ - virErrorPtr err; - - if (!data) - return; - - virErrorPreserveLast(&err); - virCgroupSetCpusetMems(data->emulatorCgroup, data->emulatorMemMask); - virErrorRestore(&err); - - qemuCgroupEmulatorAllNodesDataFree(data); -} diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h index cd537ebd82..f09134947f 100644 --- a/src/qemu/qemu_cgroup.h +++ b/src/qemu/qemu_cgroup.h @@ -56,18 +56,11 @@ int qemuSetupChardevCgroup(virDomainObj *vm, virDomainChrDef *dev); int qemuTeardownChardevCgroup(virDomainObj *vm, virDomainChrDef *dev); -int qemuConnectCgroup(virDomainObj *vm); int qemuSetupCgroup(virDomainObj *vm, size_t nnicindexes, int *nicindexes); -int qemuSetupCgroupVcpuBW(virCgroup *cgroup, - unsigned long long period, - long long quota); -int qemuSetupCgroupCpusetCpus(virCgroup *cgroup, virBitmap *cpumask); -int qemuSetupGlobalCpuCgroup(virDomainObj *vm); int qemuSetupCgroupForExtDevices(virDomainObj *vm, virQEMUDriver *driver); -int qemuRemoveCgroup(virDomainObj *vm); typedef struct _qemuCgroupEmulatorAllNodesData qemuCgroupEmulatorAllNodesData; struct _qemuCgroupEmulatorAllNodesData { @@ -75,8 +68,4 @@ struct _qemuCgroupEmulatorAllNodesData { char *emulatorMemMask; }; -int qemuCgroupEmulatorAllNodesAllow(virCgroup *cgroup, - qemuCgroupEmulatorAllNodesData **data); -void qemuCgroupEmulatorAllNodesRestore(qemuCgroupEmulatorAllNodesData *data); - extern const char *const defaultDeviceACL[]; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index a7088d3a66..316ea4642f 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -4438,7 +4438,7 @@ qemuDomainPinVcpuLive(virDomainObj *vm, if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, vcpu, false, &cgroup_vcpu) < 0) goto cleanup; - if (qemuSetupCgroupCpusetCpus(cgroup_vcpu, cpumap) < 0) + if (virSetupCgroupCpusetCpus(cgroup_vcpu, cpumap) < 0) goto cleanup; } @@ -4651,7 +4651,7 @@ qemuDomainPinEmulator(virDomainPtr dom, 0, false, &cgroup_emulator) < 0) goto endjob; - if (qemuSetupCgroupCpusetCpus(cgroup_emulator, pcpumap) < 0) { + if (virSetupCgroupCpusetCpus(cgroup_emulator, pcpumap) < 0) { virReportError(VIR_ERR_OPERATION_INVALID, "%s", _("failed to set cpuset.cpus in cgroup" " for emulator threads")); @@ -5062,7 +5062,7 @@ qemuDomainPinIOThread(virDomainPtr dom, if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD, iothread_id, false, &cgroup_iothread) < 0) goto endjob; - if (qemuSetupCgroupCpusetCpus(cgroup_iothread, pcpumap) < 0) { + if (virSetupCgroupCpusetCpus(cgroup_iothread, pcpumap) < 0) { virReportError(VIR_ERR_OPERATION_INVALID, _("failed to set cpuset.cpus in cgroup" " for iothread %d"), iothread_id); @@ -8948,7 +8948,7 @@ qemuSetGlobalBWLive(virCgroup *cgroup, unsigned long long period, if (period == 0 && quota == 0) return 0; - if (qemuSetupCgroupVcpuBW(cgroup, period, quota) < 0) + if (virSetupCgroupVcpuBW(cgroup, period, quota) < 0) return -1; return 0; @@ -9143,7 +9143,7 @@ qemuSetVcpusBWLive(virDomainObj *vm, virCgroup *cgroup, false, &cgroup_vcpu) < 0) return -1; - if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0) + if (virSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0) return -1; } @@ -9164,7 +9164,7 @@ qemuSetEmulatorBandwidthLive(virCgroup *cgroup, false, &cgroup_emulator) < 0) return -1; - if (qemuSetupCgroupVcpuBW(cgroup_emulator, period, quota) < 0) + if (virSetupCgroupVcpuBW(cgroup_emulator, period, quota) < 0) return -1; return 0; @@ -9191,7 +9191,7 @@ qemuSetIOThreadsBWLive(virDomainObj *vm, virCgroup *cgroup, false, &cgroup_iothread) < 0) return -1; - if (qemuSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0) + if (virSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0) return -1; } diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 426710786b..f930a9eb7a 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -6634,7 +6634,7 @@ qemuDomainSetVcpusLive(virQEMUDriver *driver, ssize_t nextvcpu = -1; int ret = -1; - if (qemuCgroupEmulatorAllNodesAllow(priv->cgroup, &emulatorCgroup) < 0) + if (virCgroupEmulatorAllNodesAllow(priv->cgroup, &emulatorCgroup) < 0) goto cleanup; if (enable) { @@ -6655,7 +6655,7 @@ qemuDomainSetVcpusLive(virQEMUDriver *driver, ret = 0; cleanup: - qemuCgroupEmulatorAllNodesRestore(emulatorCgroup); + virCgroupEmulatorAllNodesRestore(emulatorCgroup); return ret; } diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index c355a39e15..53662c3d52 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -2756,7 +2756,7 @@ qemuProcessSetupPid(virDomainObj *vm, if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { if (use_cpumask && - qemuSetupCgroupCpusetCpus(cgroup, use_cpumask) < 0) + virSetupCgroupCpusetCpus(cgroup, use_cpumask) < 0) goto cleanup; if (mem_mask && virCgroupSetCpusetMems(cgroup, mem_mask) < 0) @@ -2765,7 +2765,7 @@ qemuProcessSetupPid(virDomainObj *vm, } if ((period || quota) && - qemuSetupCgroupVcpuBW(cgroup, period, quota) < 0) + virSetupCgroupVcpuBW(cgroup, period, quota) < 0) goto cleanup; /* Move the thread to the sub dir */ @@ -6100,7 +6100,7 @@ qemuProcessSetupHotpluggableVcpus(virQEMUDriver *driver, qsort(bootHotplug, nbootHotplug, sizeof(*bootHotplug), qemuProcessVcpusSortOrder); - if (qemuCgroupEmulatorAllNodesAllow(priv->cgroup, &emulatorCgroup) < 0) + if (virCgroupEmulatorAllNodesAllow(priv->cgroup, &emulatorCgroup) < 0) goto cleanup; for (i = 0; i < nbootHotplug; i++) { @@ -6125,7 +6125,7 @@ qemuProcessSetupHotpluggableVcpus(virQEMUDriver *driver, ret = 0; cleanup: - qemuCgroupEmulatorAllNodesRestore(emulatorCgroup); + virCgroupEmulatorAllNodesRestore(emulatorCgroup); return ret; } @@ -6887,7 +6887,7 @@ qemuProcessPrepareHost(virQEMUDriver *driver, /* Ensure no historical cgroup for this VM is lying around bogus * settings */ VIR_DEBUG("Ensuring no historical cgroup is lying around"); - qemuRemoveCgroup(vm); + virRemoveCgroup(vm, priv->cgroup, priv->machineName); if (g_mkdir_with_parents(cfg->logDir, 0777) < 0) { virReportSystemError(errno, @@ -7505,7 +7505,7 @@ qemuProcessLaunch(virConnectPtr conn, goto cleanup; VIR_DEBUG("Setting global CPU cgroup (if required)"); - if (qemuSetupGlobalCpuCgroup(vm) < 0) + if (virSetupGlobalCpuCgroup(vm, priv->cgroup, priv->autoNodeset) < 0) goto cleanup; VIR_DEBUG("Setting vCPU tuning/settings"); @@ -8119,7 +8119,7 @@ void qemuProcessStop(virQEMUDriver *driver, } retry: - if ((ret = qemuRemoveCgroup(vm)) < 0) { + if ((ret = virRemoveCgroup(vm, priv->cgroup, priv->machineName)) < 0) { if (ret == -EBUSY && (retries++ < 5)) { g_usleep(200*1000); goto retry; @@ -8686,7 +8686,8 @@ qemuProcessReconnect(void *opaque) if (!priv->machineName) goto error; - if (qemuConnectCgroup(obj) < 0) + if (virConnectCgroup("qemu", obj, priv->cgroup, cfg->cgroupControllers, + priv->driver->privileged, priv->machineName) < 0) goto error; if (qemuDomainPerfRestart(obj) < 0) -- 2.27.0

From: Vineeth Pillai <viremana@linux.microsoft.com> Signed-off-by: Vineeth Pillai <viremana@linux.microsoft.com> Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com> --- src/ch/ch_conf.c | 2 + src/ch/ch_conf.h | 4 +- src/ch/ch_domain.c | 34 +++++ src/ch/ch_domain.h | 3 +- src/ch/ch_monitor.c | 96 ++++++++++++++ src/ch/ch_monitor.h | 54 +++++++- src/ch/ch_process.c | 306 ++++++++++++++++++++++++++++++++++++++++++-- src/ch/ch_process.h | 3 + 8 files changed, 484 insertions(+), 18 deletions(-) diff --git a/src/ch/ch_conf.c b/src/ch/ch_conf.c index ef6f4b5ba8..cfc1174354 100644 --- a/src/ch/ch_conf.c +++ b/src/ch/ch_conf.c @@ -129,6 +129,8 @@ virCHDriverConfigNew(bool privileged) if (!(cfg = virObjectNew(virCHDriverConfigClass))) return NULL; + cfg->cgroupControllers = -1; /* Auto detect */ + if (privileged) { if (virGetUserID(CH_USER, &cfg->user) < 0) return NULL; diff --git a/src/ch/ch_conf.h b/src/ch/ch_conf.h index 8fe69c8545..1790295ede 100644 --- a/src/ch/ch_conf.h +++ b/src/ch/ch_conf.h @@ -35,11 +35,13 @@ struct _virCHDriverConfig { char *stateDir; char *logDir; - + int cgroupControllers; uid_t user; gid_t group; }; +G_DEFINE_AUTOPTR_CLEANUP_FUNC(virCHDriverConfig, virObjectUnref); + struct _virCHDriver { virMutex lock; diff --git a/src/ch/ch_domain.c b/src/ch/ch_domain.c index 326c3802c3..0a0b1206d6 100644 --- a/src/ch/ch_domain.c +++ b/src/ch/ch_domain.c @@ -322,6 +322,40 @@ chValidateDomainDeviceDef(const virDomainDeviceDef * dev, _("Serial can only be enabled for a PTY")); return -1; } + return 0; +} + +int +virCHDomainRefreshThreadInfo(virDomainObj * vm) +{ + size_t maxvcpus = virDomainDefGetVcpusMax(vm->def); + virCHMonitorThreadInfo *info = NULL; + size_t nthreads, ncpus = 0; + size_t i; + + nthreads = virCHMonitorGetThreadInfo(virCHDomainGetMonitor(vm), + true, &info); + + for (i = 0; i < nthreads; i++) { + virCHDomainVcpuPrivate *vcpupriv; + virDomainVcpuDef *vcpu; + virCHMonitorCPUInfo *vcpuInfo; + + if (info[i].type != virCHThreadTypeVcpu) + continue; + + // TODO: hotplug support + vcpuInfo = &info[i].vcpuInfo; + vcpu = virDomainDefGetVcpu(vm->def, vcpuInfo->cpuid); + vcpupriv = CH_DOMAIN_VCPU_PRIVATE(vcpu); + vcpupriv->tid = vcpuInfo->tid; + ncpus++; + } + + // TODO: Remove the warning when hotplug is implemented. + if (ncpus != maxvcpus) + VIR_WARN("Mismatch in the number of cpus, expected: %ld, actual: %ld", + maxvcpus, ncpus); return 0; } diff --git a/src/ch/ch_domain.h b/src/ch/ch_domain.h index 3ac3421015..2ce3e2cef3 100644 --- a/src/ch/ch_domain.h +++ b/src/ch/ch_domain.h @@ -89,7 +89,8 @@ virCHDomainObjBeginJob(virDomainObj *obj, enum virCHDomainJob job) void virCHDomainObjEndJob(virDomainObj *obj); -int virCHDomainRefreshVcpuInfo(virDomainObj *vm); +int virCHDomainRefreshThreadInfo(virDomainObj *vm); + pid_t virCHDomainGetVcpuPid(virDomainObj *vm, unsigned int vcpuid); bool virCHDomainHasVcpuPids(virDomainObj *vm); diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index 6bc7d035ad..d7593d4aa6 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -41,6 +41,7 @@ VIR_LOG_INIT("ch.ch_monitor"); static virClass *virCHMonitorClass; static void virCHMonitorDispose(void *obj); +static void virCHMonitorThreadInfoFree(virCHMonitor * mon); static int virCHMonitorOnceInit(void) @@ -589,6 +590,7 @@ virCHMonitorDispose(void *opaque) virCHMonitor *mon = opaque; VIR_DEBUG("mon=%p", mon); + virCHMonitorThreadInfoFree(mon); virObjectUnref(mon->vm); } @@ -756,6 +758,100 @@ virCHMonitorGet(virCHMonitor * mon, const char *endpoint, return ret; } +static void +virCHMonitorThreadInfoFree(virCHMonitor * mon) +{ + mon->nthreads = 0; + if (mon->threads) + VIR_FREE(mon->threads); +} + +static size_t +virCHMonitorRefreshThreadInfo(virCHMonitor * mon) +{ + virCHMonitorThreadInfo *info = NULL; + g_autofree pid_t *tids = NULL; + virDomainObj *vm = mon->vm; + size_t ntids = 0; + size_t i; + + + virCHMonitorThreadInfoFree(mon); + if (virProcessGetPids(vm->pid, &ntids, &tids) < 0) { + mon->threads = NULL; + return 0; + } + + info = g_new0(virCHMonitorThreadInfo, ntids); + for (i = 0; i < ntids; i++) { + g_autofree char *proc = NULL; + g_autofree char *data = NULL; + + proc = g_strdup_printf("/proc/%d/task/%d/comm", + (int) vm->pid, (int) tids[i]); + + if (virFileReadAll(proc, (1 << 16), &data) < 0) { + continue; + } + + VIR_DEBUG("VM PID: %d, TID %d, COMM: %s", + (int) vm->pid, (int) tids[i], data); + if (STRPREFIX(data, "vcpu")) { + int cpuid; + char *tmp; + + if (virStrToLong_i(data + 4, &tmp, 0, &cpuid) < 0) { + VIR_WARN("Index is not specified correctly"); + continue; + } + info[i].type = virCHThreadTypeVcpu; + info[i].vcpuInfo.tid = tids[i]; + info[i].vcpuInfo.online = true; + info[i].vcpuInfo.cpuid = cpuid; + VIR_DEBUG("vcpu%d -> tid: %d", cpuid, tids[i]); + } else if (STRPREFIX(data, "_disk") || STRPREFIX(data, "_net") || + STRPREFIX(data, "_rng")) { + /* Prefixes used by cloud-hypervisor for IO Threads are captured at + * https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/vmm/src/devic... */ + info[i].type = virCHThreadTypeIO; + info[i].ioInfo.tid = tids[i]; + virStrcpy(info[i].ioInfo.thrName, data, VIRCH_THREAD_NAME_LEN - 1); + } else { + info[i].type = virCHThreadTypeEmulator; + info[i].emuInfo.tid = tids[i]; + virStrcpy(info[i].emuInfo.thrName, data, VIRCH_THREAD_NAME_LEN - 1); + } + mon->nthreads++; + + } + mon->threads = info; + + return mon->nthreads; +} + +/** + * virCHMonitorGetThreadInfo: + * @mon: Pointer to the monitor + * @refresh: Refresh thread info or not + * + * Retrive thread info and store to @threads + * + * Returns count of threads on success. + */ +size_t +virCHMonitorGetThreadInfo(virCHMonitor * mon, bool refresh, + virCHMonitorThreadInfo ** threads) +{ + int nthreads = 0; + + if (refresh) + nthreads = virCHMonitorRefreshThreadInfo(mon); + + *threads = mon->threads; + + return nthreads; +} + int virCHMonitorShutdownVMM(virCHMonitor * mon) { diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index 8ca9e17a9a..f8c3fa75e8 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -37,6 +37,50 @@ #define URL_VM_RESUME "vm.resume" #define URL_VM_INFO "vm.info" +#define VIRCH_THREAD_NAME_LEN 16 + +typedef enum { + virCHThreadTypeEmulator, + virCHThreadTypeVcpu, + virCHThreadTypeIO, + virCHThreadTypeMax +} virCHThreadType; + +typedef struct _virCHMonitorCPUInfo virCHMonitorCPUInfo; + +struct _virCHMonitorCPUInfo { + int cpuid; + pid_t tid; + + bool online; +}; + +typedef struct _virCHMonitorEmuThreadInfo virCHMonitorEmuThreadInfo; + +struct _virCHMonitorEmuThreadInfo { + char thrName[VIRCH_THREAD_NAME_LEN]; + pid_t tid; +}; + +typedef struct _virCHMonitorIOThreadInfo virCHMonitorIOThreadInfo; + +struct _virCHMonitorIOThreadInfo { + char thrName[VIRCH_THREAD_NAME_LEN]; + pid_t tid; +}; + +typedef struct _virCHMonitorThreadInfo virCHMonitorThreadInfo; + +struct _virCHMonitorThreadInfo { + virCHThreadType type; + + union { + virCHMonitorCPUInfo vcpuInfo; + virCHMonitorEmuThreadInfo emuInfo; + virCHMonitorIOThreadInfo ioInfo; + }; +}; + typedef struct _virCHMonitor virCHMonitor; struct _virCHMonitor { @@ -49,6 +93,9 @@ struct _virCHMonitor { pid_t pid; virDomainObj *vm; + + size_t nthreads; + virCHMonitorThreadInfo *threads; }; virCHMonitor *virCHMonitorNew(virDomainObj *vm, const char *socketdir); @@ -65,12 +112,9 @@ int virCHMonitorSuspendVM(virCHMonitor *mon); int virCHMonitorResumeVM(virCHMonitor *mon); int virCHMonitorGetInfo(virCHMonitor *mon, virJSONValue **info); -typedef struct _virCHMonitorCPUInfo virCHMonitorCPUInfo; -struct _virCHMonitorCPUInfo { - pid_t tid; - bool online; -}; void virCHMonitorCPUInfoFree(virCHMonitorCPUInfo *cpus); int virCHMonitorGetCPUInfo(virCHMonitor *mon, virCHMonitorCPUInfo **vcpus, size_t maxvcpus); +size_t virCHMonitorGetThreadInfo(virCHMonitor *mon, bool refresh, + virCHMonitorThreadInfo **threads); diff --git a/src/ch/ch_process.c b/src/ch/ch_process.c index 3b7f6fcddf..7a51531224 100644 --- a/src/ch/ch_process.c +++ b/src/ch/ch_process.c @@ -26,6 +26,8 @@ #include "ch_domain.h" #include "ch_monitor.h" #include "ch_process.h" +#include "domain_cgroup.h" +#include "virnuma.h" #include "viralloc.h" #include "virerror.h" #include "virjson.h" @@ -133,6 +135,254 @@ virCHProcessUpdateInfo(virDomainObj *vm) return 0; } +static int +virCHProcessGetAllCpuAffinity(virBitmap ** cpumapRet) +{ + *cpumapRet = NULL; + + if (!virHostCPUHasBitmap()) + return 0; + + if (!(*cpumapRet = virHostCPUGetOnlineBitmap())) + return -1; + + return 0; +} + +#if defined(WITH_SCHED_GETAFFINITY) || defined(WITH_BSD_CPU_AFFINITY) +static int +virCHProcessInitCpuAffinity(virDomainObj * vm) +{ + g_autoptr(virBitmap) cpumapToSet = NULL; + virDomainNumatuneMemMode mem_mode; + virCHDomainObjPrivate *priv = vm->privateData; + + if (!vm->pid) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot setup CPU affinity until process is started")); + return -1; + } + + if (virDomainNumaGetNodeCount(vm->def->numa) <= 1 && + virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { + virBitmap *nodeset = NULL; + + if (virDomainNumatuneMaybeGetNodeset(vm->def->numa, + priv->autoNodeset, + &nodeset, -1) < 0) + return -1; + + if (virNumaNodesetToCPUset(nodeset, &cpumapToSet) < 0) + return -1; + } else if (vm->def->cputune.emulatorpin) { + if (!(cpumapToSet = virBitmapNewCopy(vm->def->cputune.emulatorpin))) + return -1; + } else { + if (virCHProcessGetAllCpuAffinity(&cpumapToSet) < 0) + return -1; + } + + if (cpumapToSet && virProcessSetAffinity(vm->pid, cpumapToSet, false) < 0) { + return -1; + } + + return 0; +} +#else /* !defined(WITH_SCHED_GETAFFINITY) && !defined(WITH_BSD_CPU_AFFINITY) */ +static int +virCHProcessInitCpuAffinity(virDomainObj * vm G_GNUC_UNUSED) +{ + return 0; +} +#endif /* !defined(WITH_SCHED_GETAFFINITY) && !defined(WITH_BSD_CPU_AFFINITY) */ + +/** + * virCHProcessSetupPid: + * + * This function sets resource properties (affinity, cgroups, + * scheduler) for any PID associated with a domain. It should be used + * to set up emulator PIDs as well as vCPU and I/O thread pids to + * ensure they are all handled the same way. + * + * Returns 0 on success, -1 on error. + */ +static int +virCHProcessSetupPid(virDomainObj *vm, + pid_t pid, + virCgroupThreadName nameval, + int id, + virBitmap *cpumask, + unsigned long long period, + long long quota, + virDomainThreadSchedParam *sched) +{ + virCHDomainObjPrivate *priv = vm->privateData; + virDomainNumatuneMemMode mem_mode; + virCgroup *cgroup = NULL; + virBitmap *use_cpumask = NULL; + virBitmap *affinity_cpumask = NULL; + g_autoptr(virBitmap) hostcpumap = NULL; + g_autofree char *mem_mask = NULL; + int ret = -1; + + if ((period || quota) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + goto cleanup; + } + + /* Infer which cpumask shall be used. */ + if (cpumask) { + use_cpumask = cpumask; + } else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) { + use_cpumask = priv->autoCpuset; + } else if (vm->def->cpumask) { + use_cpumask = vm->def->cpumask; + } else { + /* we can't assume cloud-hypervisor itself is running on all pCPUs, + * so we need to explicitly set the spawned instance to all pCPUs. */ + if (virCHProcessGetAllCpuAffinity(&hostcpumap) < 0) + goto cleanup; + affinity_cpumask = hostcpumap; + } + + /* + * If CPU cgroup controller is not initialized here, then we need + * neither period nor quota settings. And if CPUSET controller is + * not initialized either, then there's nothing to do anyway. + */ + if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) || + virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && + virDomainNumatuneMaybeFormatNodeset(vm->def->numa, + priv->autoNodeset, + &mem_mask, -1) < 0) + goto cleanup; + + if (virCgroupNewThread(priv->cgroup, nameval, id, true, &cgroup) < 0) + goto cleanup; + + if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { + if (use_cpumask && + virSetupCgroupCpusetCpus(cgroup, use_cpumask) < 0) + goto cleanup; + + if (mem_mask && virCgroupSetCpusetMems(cgroup, mem_mask) < 0) + goto cleanup; + + } + + if ((period || quota) && + virSetupCgroupVcpuBW(cgroup, period, quota) < 0) + goto cleanup; + + /* Move the thread to the sub dir */ + VIR_INFO("Adding pid %d to cgroup", pid); + if (virCgroupAddThread(cgroup, pid) < 0) + goto cleanup; + + } + + if (!affinity_cpumask) + affinity_cpumask = use_cpumask; + + /* Setup legacy affinity. */ + if (affinity_cpumask + && virProcessSetAffinity(pid, affinity_cpumask, false) < 0) + goto cleanup; + + /* Set scheduler type and priority, but not for the main thread. */ + if (sched && + nameval != VIR_CGROUP_THREAD_EMULATOR && + virProcessSetScheduler(pid, sched->policy, sched->priority) < 0) + goto cleanup; + + ret = 0; + cleanup: + if (cgroup) { + if (ret < 0) + virCgroupRemove(cgroup); + virCgroupFree(cgroup); + } + + return ret; +} + +/** + * virCHProcessSetupVcpu: + * @vm: domain object + * @vcpuid: id of VCPU to set defaults + * + * This function sets resource properties (cgroups, affinity, scheduler) for a + * vCPU. This function expects that the vCPU is online and the vCPU pids were + * correctly detected at the point when it's called. + * + * Returns 0 on success, -1 on error. + */ +int +virCHProcessSetupVcpu(virDomainObj * vm, unsigned int vcpuid) +{ + pid_t vcpupid = virCHDomainGetVcpuPid(vm, vcpuid); + virDomainVcpuDef *vcpu = virDomainDefGetVcpu(vm->def, vcpuid); + + return virCHProcessSetupPid(vm, vcpupid, VIR_CGROUP_THREAD_VCPU, + vcpuid, vcpu->cpumask, + vm->def->cputune.period, + vm->def->cputune.quota, &vcpu->sched); +} + +static int +virCHProcessSetupVcpus(virDomainObj * vm) +{ + virDomainVcpuDef *vcpu; + unsigned int maxvcpus = virDomainDefGetVcpusMax(vm->def); + size_t i; + + if ((vm->def->cputune.period || vm->def->cputune.quota) && + !virCgroupHasController(((virCHDomainObjPrivate *) vm->privateData)-> + cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + return -1; + } + + if (!virCHDomainHasVcpuPids(vm)) { + /* If any CPU has custom affinity that differs from the + * VM default affinity, we must reject it */ + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(vm->def, i); + + if (!vcpu->online) + continue; + + if (vcpu->cpumask && + !virBitmapEqual(vm->def->cpumask, vcpu->cpumask)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cpu affinity is not supported")); + return -1; + } + } + + return 0; + } + + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(vm->def, i); + + if (!vcpu->online) + continue; + + if (virCHProcessSetupVcpu(vm, i) < 0) + return -1; + } + + return 0; +} + /** * virCHProcessStart: * @driver: pointer to driver structure @@ -143,12 +393,13 @@ virCHProcessUpdateInfo(virDomainObj *vm) * * Returns 0 on success or -1 in case of error */ -int virCHProcessStart(virCHDriver *driver, - virDomainObj *vm, - virDomainRunningReason reason) +int +virCHProcessStart(virCHDriver * driver, + virDomainObj * vm, virDomainRunningReason reason) { int ret = -1; virCHDomainObjPrivate *priv = vm->privateData; + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(priv->driver); g_autofree int *nicindexes = NULL; size_t nnicindexes = 0; @@ -168,18 +419,39 @@ int virCHProcessStart(virCHDriver *driver, } } + vm->pid = priv->monitor->pid; + vm->def->id = vm->pid; + priv->machineName = virCHDomainGetMachineName(vm); + + if (virSetupCgroup("ch", vm, + nnicindexes, nicindexes, + priv->cgroup, + cfg->cgroupControllers, + 0, /*maxThreadsPerProc*/ + priv->driver->privileged, + priv->machineName) < 0) + goto cleanup; + + if (virCHProcessInitCpuAffinity(vm) < 0) + goto cleanup; + if (virCHMonitorBootVM(priv->monitor) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("failed to boot guest VM")); goto cleanup; } - priv->machineName = virCHDomainGetMachineName(vm); - vm->pid = priv->monitor->pid; - vm->def->id = vm->pid; + virCHDomainRefreshThreadInfo(vm); - virCHProcessUpdateInfo(vm); + VIR_DEBUG("Setting global CPU cgroup (if required)"); + if (virSetupGlobalCpuCgroup(vm, priv->cgroup, priv->autoNodeset) < 0) + goto cleanup; + + VIR_DEBUG("Setting vCPU tuning/settings"); + if (virCHProcessSetupVcpus(vm) < 0) + goto cleanup; + virCHProcessUpdateInfo(vm); virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason); return 0; @@ -191,20 +463,32 @@ int virCHProcessStart(virCHDriver *driver, return ret; } -int virCHProcessStop(virCHDriver *driver G_GNUC_UNUSED, - virDomainObj *vm, - virDomainShutoffReason reason) +int +virCHProcessStop(virCHDriver * driver G_GNUC_UNUSED, + virDomainObj * vm, virDomainShutoffReason reason) { + int ret; + int retries = 0; virCHDomainObjPrivate *priv = vm->privateData; VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d", - vm->def->name, (int)vm->pid, (int)reason); + vm->def->name, (int) vm->pid, (int) reason); if (priv->monitor) { virCHMonitorClose(priv->monitor); priv->monitor = NULL; } + retry: + if ((ret = virRemoveCgroup(vm, priv->cgroup, priv->machineName)) < 0) { + if (ret == -EBUSY && (retries++ < 5)) { + g_usleep(200*1000); + goto retry; + } + VIR_WARN("Failed to remove cgroup for %s", + vm->def->name); + } + vm->pid = -1; vm->def->id = -1; diff --git a/src/ch/ch_process.h b/src/ch/ch_process.h index abc4915979..800e3f4e23 100644 --- a/src/ch/ch_process.h +++ b/src/ch/ch_process.h @@ -29,3 +29,6 @@ int virCHProcessStart(virCHDriver *driver, int virCHProcessStop(virCHDriver *driver, virDomainObj *vm, virDomainShutoffReason reason); + +int virCHProcessSetupVcpu(virDomainObj *vm, + unsigned int vcpuid); -- 2.27.0

From: Vineeth Pillai <viremana@linux.microsoft.com> Signed-off-by: Vineeth Pillai <viremana@linux.microsoft.com> Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com> --- src/ch/ch_driver.c | 316 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 297 insertions(+), 19 deletions(-) diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index c6dcc33f5e..8c14829d2a 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -43,6 +43,7 @@ #include "viruri.h" #include "virutil.h" #include "viruuid.h" +#include "virnuma.h" #define VIR_FROM_THIS VIR_FROM_CH @@ -954,25 +955,25 @@ chConnectSupportsFeature(virConnectPtr conn, int feature) return -1; switch ((virDrvFeature) feature) { - case VIR_DRV_FEATURE_TYPED_PARAM_STRING: - return 1; - case VIR_DRV_FEATURE_MIGRATION_V2: - case VIR_DRV_FEATURE_MIGRATION_V3: - case VIR_DRV_FEATURE_MIGRATION_P2P: - case VIR_DRV_FEATURE_MIGRATE_CHANGE_PROTECTION: - case VIR_DRV_FEATURE_FD_PASSING: - case VIR_DRV_FEATURE_XML_MIGRATABLE: - case VIR_DRV_FEATURE_MIGRATION_OFFLINE: - case VIR_DRV_FEATURE_MIGRATION_PARAMS: - case VIR_DRV_FEATURE_MIGRATION_DIRECT: - case VIR_DRV_FEATURE_MIGRATION_V1: - case VIR_DRV_FEATURE_PROGRAM_KEEPALIVE: - case VIR_DRV_FEATURE_REMOTE: - case VIR_DRV_FEATURE_REMOTE_CLOSE_CALLBACK: - case VIR_DRV_FEATURE_REMOTE_EVENT_CALLBACK: - case VIR_DRV_FEATURE_NETWORK_UPDATE_HAS_CORRECT_ORDER: - default: - return 0; + case VIR_DRV_FEATURE_TYPED_PARAM_STRING: + return 1; + case VIR_DRV_FEATURE_MIGRATION_V2: + case VIR_DRV_FEATURE_MIGRATION_V3: + case VIR_DRV_FEATURE_MIGRATION_P2P: + case VIR_DRV_FEATURE_MIGRATE_CHANGE_PROTECTION: + case VIR_DRV_FEATURE_FD_PASSING: + case VIR_DRV_FEATURE_XML_MIGRATABLE: + case VIR_DRV_FEATURE_MIGRATION_OFFLINE: + case VIR_DRV_FEATURE_MIGRATION_PARAMS: + case VIR_DRV_FEATURE_MIGRATION_DIRECT: + case VIR_DRV_FEATURE_MIGRATION_V1: + case VIR_DRV_FEATURE_PROGRAM_KEEPALIVE: + case VIR_DRV_FEATURE_REMOTE: + case VIR_DRV_FEATURE_REMOTE_CLOSE_CALLBACK: + case VIR_DRV_FEATURE_REMOTE_EVENT_CALLBACK: + case VIR_DRV_FEATURE_NETWORK_UPDATE_HAS_CORRECT_ORDER: + default: + return 0; } } @@ -1308,6 +1309,281 @@ chDomainPinVcpu(virDomainPtr dom, VIR_DOMAIN_AFFECT_LIVE); } +#define CH_NB_NUMA_PARAM 2 + +static int +chDomainGetNumaParameters(virDomainPtr dom, + virTypedParameterPtr params, + int *nparams, + unsigned int flags) +{ + size_t i; + virDomainObj *vm = NULL; + virDomainNumatuneMemMode tmpmode = VIR_DOMAIN_NUMATUNE_MEM_STRICT; + virCHDomainObjPrivate *priv; + g_autofree char *nodeset = NULL; + int ret = -1; + virDomainDef *def = NULL; + bool live = false; + virBitmap *autoNodeset = NULL; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG | + VIR_TYPED_PARAM_STRING_OKAY, -1); + + if (!(vm = virCHDomainObjFromDomain(dom))) + return -1; + priv = vm->privateData; + + if (virDomainGetNumaParametersEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (!(def = virDomainObjGetOneDefState(vm, flags, &live))) + goto cleanup; + + if (live) + autoNodeset = priv->autoNodeset; + + if ((*nparams) == 0) { + *nparams = CH_NB_NUMA_PARAM; + ret = 0; + goto cleanup; + } + + for (i = 0; i < CH_NB_NUMA_PARAM && i < *nparams; i++) { + virMemoryParameterPtr param = ¶ms[i]; + + switch (i) { + case 0: /* fill numa mode here */ + ignore_value(virDomainNumatuneGetMode(def->numa, -1, &tmpmode)); + + if (virTypedParameterAssign(param, VIR_DOMAIN_NUMA_MODE, + VIR_TYPED_PARAM_INT, tmpmode) < 0) + goto cleanup; + + break; + + case 1: /* fill numa nodeset here */ + nodeset = virDomainNumatuneFormatNodeset(def->numa, autoNodeset, -1); + + if (!nodeset || + virTypedParameterAssign(param, VIR_DOMAIN_NUMA_NODESET, + VIR_TYPED_PARAM_STRING, nodeset) < 0) + goto cleanup; + + nodeset = NULL; + break; + + /* coverity[dead_error_begin] */ + default: + break; + /* should not hit here */ + } + } + + if (*nparams > CH_NB_NUMA_PARAM) + *nparams = CH_NB_NUMA_PARAM; + ret = 0; + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + +static int +chDomainSetNumaParamsLive(virDomainObj *vm, + virBitmap *nodeset) +{ + virCgroup *cgroup_temp = NULL; + virCHDomainObjPrivate *priv = vm->privateData; + g_autofree char *nodeset_str = NULL; + virDomainNumatuneMemMode mode; + size_t i = 0; + int ret = -1; + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mode) == 0 && + mode != VIR_DOMAIN_NUMATUNE_MEM_STRICT) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("change of nodeset for running domain " + "requires strict numa mode")); + goto cleanup; + } + + if (!virNumaNodesetIsAvailable(nodeset)) + goto cleanup; + + /* Ensure the cpuset string is formatted before passing to cgroup */ + if (!(nodeset_str = virBitmapFormat(nodeset))) + goto cleanup; + + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset_str) < 0) + goto cleanup; + virCgroupFree(cgroup_temp); + + for (i = 0; i < virDomainDefGetVcpusMax(vm->def); i++) { + virDomainVcpuDef *vcpu = virDomainDefGetVcpu(vm->def, i); + + if (!vcpu->online) + continue; + + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, i, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset_str) < 0) + goto cleanup; + virCgroupFree(cgroup_temp); + } + + for (i = 0; i < vm->def->niothreadids; i++) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD, + vm->def->iothreadids[i]->iothread_id, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset_str) < 0) + goto cleanup; + virCgroupFree(cgroup_temp); + } + + /* set nodeset for root cgroup */ + if (virCgroupSetCpusetMems(priv->cgroup, nodeset_str) < 0) + goto cleanup; + + ret = 0; + cleanup: + virCgroupFree(cgroup_temp); + + return ret; +} + +static int +chDomainSetNumaParameters(virDomainPtr dom, + virTypedParameterPtr params, + int nparams, + unsigned int flags) +{ + virCHDriver *driver = dom->conn->privateData; + size_t i; + virDomainDef *def; + virDomainDef *persistentDef; + virDomainObj *vm = NULL; + int ret = -1; + g_autoptr(virCHDriverConfig) cfg = NULL; + virCHDomainObjPrivate *priv; + virBitmap *nodeset = NULL; + virDomainNumatuneMemMode config_mode; + int mode = -1; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG, -1); + + if (virTypedParamsValidate(params, nparams, + VIR_DOMAIN_NUMA_MODE, + VIR_TYPED_PARAM_INT, + VIR_DOMAIN_NUMA_NODESET, + VIR_TYPED_PARAM_STRING, + NULL) < 0) + return -1; + + if (!(vm = virCHDomainObjFromDomain(dom))) + return -1; + + priv = vm->privateData; + cfg = virCHDriverGetConfig(driver); + + if (virDomainSetNumaParametersEnsureACL(dom->conn, vm->def, flags) < 0) + goto cleanup; + + for (i = 0; i < nparams; i++) { + virTypedParameterPtr param = ¶ms[i]; + + if (STREQ(param->field, VIR_DOMAIN_NUMA_MODE)) { + mode = param->value.i; + + if (mode < 0 || mode >= VIR_DOMAIN_NUMATUNE_MEM_LAST) { + virReportError(VIR_ERR_INVALID_ARG, + _("unsupported numatune mode: '%d'"), mode); + goto cleanup; + } + + } else if (STREQ(param->field, VIR_DOMAIN_NUMA_NODESET)) { + if (virBitmapParse(param->value.s, &nodeset, + VIR_DOMAIN_CPUMASK_LEN) < 0) + goto cleanup; + + if (virBitmapIsAllClear(nodeset)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("Invalid nodeset of 'numatune': %s"), + param->value.s); + goto cleanup; + } + } + } + + if (virCHDomainObjBeginJob(vm, CH_JOB_MODIFY) < 0) + goto cleanup; + + if (virDomainObjGetDefs(vm, flags, &def, &persistentDef) < 0) + goto endjob; + + if (def) { + if (!driver->privileged) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("NUMA tuning is not available in session mode")); + goto endjob; + } + + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cgroup cpuset controller is not mounted")); + goto endjob; + } + + if (mode != -1 && + virDomainNumatuneGetMode(def->numa, -1, &config_mode) == 0 && + config_mode != mode) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("can't change numatune mode for running domain")); + goto endjob; + } + + if (nodeset && + chDomainSetNumaParamsLive(vm, nodeset) < 0) + goto endjob; + + if (virDomainNumatuneSet(def->numa, + def->placement_mode == + VIR_DOMAIN_CPU_PLACEMENT_MODE_STATIC, + -1, mode, nodeset) < 0) + goto endjob; + + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) + goto endjob; + } + + /* + if (persistentDef) { + if (virDomainNumatuneSet(persistentDef->numa, + persistentDef->placement_mode == + VIR_DOMAIN_CPU_PLACEMENT_MODE_STATIC, + -1, mode, nodeset) < 0) + goto endjob; + + if (virDomainDefSave(persistentDef, driver->xmlopt, cfg->configDir) < 0) + goto endjob; + } + */ + + ret = 0; + + endjob: + virCHDomainObjEndJob(vm); + + cleanup: + virBitmapFree(nodeset); + virDomainObjEndAPI(&vm); + return ret; +} + /* Function Tables */ static virHypervisorDriver chHypervisorDriver = { .name = "CH", @@ -1352,6 +1628,8 @@ static virHypervisorDriver chHypervisorDriver = { .domainPinVcpu = chDomainPinVcpu, /* 7.11.0 */ .domainPinVcpuFlags = chDomainPinVcpuFlags, /* 7.11.0 */ .nodeGetCPUMap = chNodeGetCPUMap, /* 7.11.0 */ + .domainSetNumaParameters = chDomainSetNumaParameters, /* 7.11.0 */ + .domainGetNumaParameters = chDomainGetNumaParameters, /* 7.11.0 */ }; static virConnectDriver chConnectDriver = { -- 2.27.0

Signed-off-by: Praveen K Paladugu <prapal@linux.microsoft.com> --- src/ch/ch_driver.c | 154 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 8c14829d2a..100c54163e 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -1309,6 +1309,158 @@ chDomainPinVcpu(virDomainPtr dom, VIR_DOMAIN_AFFECT_LIVE); } + + +static int +chDomainGetEmulatorPinInfo(virDomainPtr dom, + unsigned char *cpumaps, + int maplen, + unsigned int flags) +{ + virDomainObj *vm = NULL; + virDomainDef *def; + virCHDomainObjPrivate *priv; + bool live; + int ret = -1; + virBitmap *cpumask = NULL; + g_autoptr(virBitmap) bitmap = NULL; + virBitmap *autoCpuset = NULL; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG, -1); + + if (!(vm = chDomObjFromDomain(dom))) + goto cleanup; + + if (virDomainGetEmulatorPinInfoEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (!(def = virDomainObjGetOneDefState(vm, flags, &live))) + goto cleanup; + + if (live) { + priv = vm->privateData; + autoCpuset = priv->autoCpuset; + } + if (def->cputune.emulatorpin) { + cpumask = def->cputune.emulatorpin; + } else if (def->cpumask) { + cpumask = def->cpumask; + } else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO && + autoCpuset) { + cpumask = autoCpuset; + } else { + if (!(bitmap = virHostCPUGetAvailableCPUsBitmap())) + goto cleanup; + cpumask = bitmap; + } + + virBitmapToDataBuf(cpumask, cpumaps, maplen); + + ret = 1; + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + +static int +chDomainPinEmulator(virDomainPtr dom, + unsigned char *cpumap, + int maplen, + unsigned int flags) +{ + virCHDriver *driver = dom->conn->privateData; + virDomainObj *vm; + virCgroup *cgroup_emulator = NULL; + virDomainDef *def; + virDomainDef *persistentDef; + int ret = -1; + virCHDomainObjPrivate *priv; + virBitmap *pcpumap = NULL; + g_autoptr(virCHDriverConfig) cfg = NULL; + g_autofree char *str = NULL; + virTypedParameterPtr eventParams = NULL; + int eventNparams = 0; + int eventMaxparams = 0; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG, -1); + + cfg = virCHDriverGetConfig(driver); + + if (!(vm = chDomObjFromDomain(dom))) + goto cleanup; + + if (virDomainPinEmulatorEnsureACL(dom->conn, vm->def, flags) < 0) + goto cleanup; + + if (virCHDomainObjBeginJob(vm, CH_JOB_MODIFY) < 0) + goto cleanup; + + if (virDomainObjGetDefs(vm, flags, &def, &persistentDef) < 0) + goto endjob; + + priv = vm->privateData; + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto endjob; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty cpu list for pinning")); + goto endjob; + } + + if (def) { + if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, + 0, false, &cgroup_emulator) < 0) + goto endjob; + + if (virSetupCgroupCpusetCpus(cgroup_emulator, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("failed to set cpuset.cpus in cgroup" + " for emulator threads")); + goto endjob; + } + } + + if (virProcessSetAffinity(vm->pid, pcpumap, false) < 0) + goto endjob; + + virBitmapFree(def->cputune.emulatorpin); + def->cputune.emulatorpin = NULL; + + if (!(def->cputune.emulatorpin = virBitmapNewCopy(pcpumap))) + goto endjob; + + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) + goto endjob; + + str = virBitmapFormat(pcpumap); + if (virTypedParamsAddString(&eventParams, &eventNparams, + &eventMaxparams, + VIR_DOMAIN_TUNABLE_CPU_EMULATORPIN, + str) < 0) + goto endjob; + + } + + + ret = 0; + + endjob: + virCHDomainObjEndJob(vm); + + cleanup: + if (cgroup_emulator) + virCgroupFree(cgroup_emulator); + virBitmapFree(pcpumap); + virDomainObjEndAPI(&vm); + return ret; +} + #define CH_NB_NUMA_PARAM 2 static int @@ -1627,6 +1779,8 @@ static virHypervisorDriver chHypervisorDriver = { .domainGetVcpuPinInfo = chDomainGetVcpuPinInfo, /* 7.11.0 */ .domainPinVcpu = chDomainPinVcpu, /* 7.11.0 */ .domainPinVcpuFlags = chDomainPinVcpuFlags, /* 7.11.0 */ + .domainPinEmulator = chDomainPinEmulator, /* 7.11.0 */ + .domainGetEmulatorPinInfo = chDomainGetEmulatorPinInfo, /* 7.11.0 */ .nodeGetCPUMap = chNodeGetCPUMap, /* 7.11.0 */ .domainSetNumaParameters = chDomainSetNumaParameters, /* 7.11.0 */ .domainGetNumaParameters = chDomainGetNumaParameters, /* 7.11.0 */ -- 2.27.0