These helper methods are used to capture vcpu information in ch driver. Signed-off-by: Vineeth Pillai <viremana(a)linux.microsoft.com&gt; Signed-off-by: Praveen K Paladugu <prapal(a)linux.microsoft.com&gt; --- src/util/virprocess.c | 136 ++++++++++++++++++++++++++++++++++++++++++ src/util/virprocess.h | 5 ++ 2 files changed, 141 insertions(+) diff --git a/src/util/virprocess.c b/src/util/virprocess.c index 6de3f36f52..0164d70df6 100644 --- a/src/util/virprocess.c +++ b/src/util/virprocess.c @@ -1721,3 +1721,139 @@ virProcessSetScheduler(pid_t pid G_GNUC_UNUSED, } #endif /* !WITH_SCHED_SETSCHEDULER */ + +/* +TODO: This method was cloned from qemuGetProcessInfo in src/qemu/qemu_driver.c. +Need to refactor qemu driver to use this shared function. +*/ +int +virProcessGetStatInfo(unsigned long long *cpuTime, int *lastCpu, long *vm_rss, + pid_t pid, pid_t tid) +{ + g_autofree char *proc = NULL; + FILE *pidinfo; + unsigned long long usertime = 0, systime = 0; + long rss = 0; + int cpu = 0; + + /* In general, we cannot assume pid_t fits in int; but /proc parsing + * is specific to Linux where int works fine. */ + if (tid) + proc = g_strdup_printf("/proc/%d/task/%d/stat", (int)pid, tid); + else + proc = g_strdup_printf("/proc/%d/stat", (int)pid); + if (!proc) + return -1; + + pidinfo = fopen(proc, "r"); + + /* See 'man proc' for information about what all these fields are. We're + * only interested in a very few of them */ + if (!pidinfo || + fscanf(pidinfo, + /* pid -> stime */ + "%*d (%*[^)]) %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %llu %llu" + /* cutime -> endcode */ + "%*d %*d %*d %*d %*d %*d %*u %*u %ld %*u %*u %*u" + /* startstack -> processor */ + "%*u %*u %*u %*u %*u %*u %*u %*u %*u %*u %*d %d", + &usertime, &systime, &rss, &cpu) != 4) { + VIR_WARN("cannot parse process status data"); + } + + /* We got jiffies + * We want nanoseconds + * _SC_CLK_TCK is jiffies per second + * So calculate thus.... + */ + if (cpuTime) + *cpuTime = 1000ull * 1000ull * 1000ull * (usertime + systime) + / (unsigned long long)sysconf(_SC_CLK_TCK); + if (lastCpu) + *lastCpu = cpu; + + if (vm_rss) + *vm_rss = rss * virGetSystemPageSizeKB(); + + + VIR_DEBUG("Got status for %d/%d user=%llu sys=%llu cpu=%d rss=%ld", + (int)pid, tid, usertime, systime, cpu, rss); + + VIR_FORCE_FCLOSE(pidinfo); + + return 0; +} +/* +TODO: This method was cloned from qemuGetSchedInfo in src/qemu/qemu_driver.c. +Need to refactor qemu driver to use this shared function. +*/ +int virProcessGetSchedInfo(unsigned long long *cpuWait, pid_t pid, pid_t tid) +{ + g_autofree char *proc = NULL; + g_autofree char *data = NULL; + char **lines = NULL; + size_t i; + int ret = -1; + double val; + + *cpuWait = 0; + + /* In general, we cannot assume pid_t fits in int; but /proc parsing + * is specific to Linux where int works fine. */ + if (tid) + proc = g_strdup_printf("/proc/%d/task/%d/sched", (int)pid, (int)tid); + else + proc = g_strdup_printf("/proc/%d/sched", (int)pid); + if (!proc) + goto cleanup; + ret = -1; + + /* The file is not guaranteed to exist (needs CONFIG_SCHED_DEBUG) */ + if (access(proc, R_OK) < 0) { + ret = 0; + goto cleanup; + } + + if (virFileReadAll(proc, (1<<16), &data) < 0) + goto cleanup; + + lines = g_strsplit(data, "\n", 0); + if (!lines) + goto cleanup; + + for (i = 0; lines[i] != NULL; i++) { + const char *line = lines[i]; + + /* Needs CONFIG_SCHEDSTATS. The second check + * is the old name the kernel used in past */ + if (STRPREFIX(line, "se.statistics.wait_sum") || + STRPREFIX(line, "se.wait_sum")) { + line = strchr(line, ':'); + if (!line) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Missing separator in sched info '%s'"), + lines[i]); + goto cleanup; + } + line++; + while (*line == ' ') + line++; + + if (virStrToDouble(line, NULL, &val) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to parse sched info value '%s'"), + line); + goto cleanup; + } + + *cpuWait = (unsigned long long)(val * 1000000); + break; + } + } + + ret = 0; + + cleanup: + g_strfreev(lines); + return ret; +} diff --git a/src/util/virprocess.h b/src/util/virprocess.h index 9910331a0c..3a7c4c2d64 100644 --- a/src/util/virprocess.h +++ b/src/util/virprocess.h @@ -127,3 +127,8 @@ typedef enum { } virProcessNamespaceFlags; int virProcessNamespaceAvailable(unsigned int ns); + +int virProcessGetStatInfo(unsigned long long *cpuTime, int *lastCpu, + long *vm_rss, pid_t pid, pid_t tid); +int virProcessGetSchedInfo(unsigned long long *cpuWait, pid_t pid, + pid_t tid); -- 2.27.0

link to virt_util_lib while building ch driver. Signed-off-by: Vineeth Pillai <viremana(a)linux.microsoft.com&gt; --- src/ch/meson.build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ch/meson.build b/src/ch/meson.build index e34974d56c..5c6cab2a9f 100644 --- a/src/ch/meson.build +++ b/src/ch/meson.build @@ -30,6 +30,9 @@ if conf.has('WITH_CH') include_directories: [ conf_inc_dir, ], + link_with: [ + virt_util_lib, + ] ) virt_modules += { -- 2.27.0

Signed-off-by: Vineeth Pillai <viremana(a)linux.microsoft.com&gt; Signed-off-by: Praveen K Paladugu <prapal(a)linux.microsoft.com&gt; --- src/ch/ch_domain.c | 50 +++++++++++++++++++++++++++++++++++++++++----- src/ch/ch_domain.h | 11 ++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/ch/ch_domain.c b/src/ch/ch_domain.c index ae79b47253..fedde4581b 100644 --- a/src/ch/ch_domain.c +++ b/src/ch/ch_domain.c @@ -166,11 +166,6 @@ virCHDomainObjPrivateFree(void *data) g_free(priv); } -virDomainXMLPrivateDataCallbacks virCHDriverPrivateDataCallbacks = { - .alloc = virCHDomainObjPrivateAlloc, - .free = virCHDomainObjPrivateFree, -}; - static int virCHDomainDefPostParseBasic(virDomainDef *def, void *opaque G_GNUC_UNUSED) @@ -187,6 +182,45 @@ virCHDomainDefPostParseBasic(virDomainDef *def, return 0; } +static virClass *virCHDomainVcpuPrivateClass; +static void virCHDomainVcpuPrivateDispose(void *obj); + +static int +virCHDomainVcpuPrivateOnceInit(void) +{ + if (!VIR_CLASS_NEW(virCHDomainVcpuPrivate, virClassForObject())) + return -1; + + return 0; +} + +VIR_ONCE_GLOBAL_INIT(virCHDomainVcpuPrivate); + +static virObject * +virCHDomainVcpuPrivateNew(void) +{ + virCHDomainVcpuPrivate *priv; + + if (virCHDomainVcpuPrivateInitialize() < 0) + return NULL; + + if (!(priv = virObjectNew(virCHDomainVcpuPrivateClass))) + return NULL; + + return (virObject *) priv; +} + + +static void +virCHDomainVcpuPrivateDispose(void *obj) +{ + virCHDomainVcpuPrivate *priv = obj; + + priv->tid = 0; + + return; +} + static int virCHDomainDefPostParse(virDomainDef *def, unsigned int parseFlags G_GNUC_UNUSED, @@ -205,6 +239,12 @@ virCHDomainDefPostParse(virDomainDef *def, return 0; } +virDomainXMLPrivateDataCallbacks virCHDriverPrivateDataCallbacks = { + .alloc = virCHDomainObjPrivateAlloc, + .free = virCHDomainObjPrivateFree, + .vcpuNew = virCHDomainVcpuPrivateNew, +}; + static int chValidateDomainDeviceDef(const virDomainDeviceDef *dev, const virDomainDef *def G_GNUC_UNUSED, diff --git a/src/ch/ch_domain.h b/src/ch/ch_domain.h index 04d19398b4..75b9933130 100644 --- a/src/ch/ch_domain.h +++ b/src/ch/ch_domain.h @@ -62,6 +62,17 @@ struct _virCHDomainObjPrivate { virCHMonitor *virCHDomainGetMonitor(virDomainObj *vm); +typedef struct _virCHDomainVcpuPrivate virCHDomainVcpuPrivate; +struct _virCHDomainVcpuPrivate { + virObject parent; + + pid_t tid; /* vcpu thread id */ + virTristateBool halted; +}; + +#define CH_DOMAIN_VCPU_PRIVATE(vcpu) \ + ((virCHDomainVcpuPrivate *) (vcpu)->privateData) + extern virDomainXMLPrivateDataCallbacks virCHDriverPrivateDataCallbacks; extern virDomainDefParserConfig virCHDriverDomainDefParserConfig; -- 2.27.0

On 10/21/21 16:31, Vineeth Pillai wrote: Label indentation in this case is one space only, without changing the indentation level of the code that follows it. See docs/coding-style.rst for an example. chStateInitialize() in ch_driver.c has another example as well: ------- cleanup: if (ret != VIR_DRV_STATE_INIT_COMPLETE) chStateCleanup(); return ret; } ------- Strange label indentation here as well. Thanks, Daniel

On 10/21/21 16:31, Vineeth Pillai wrote: Wrong label indentation. LGTM otherwise. Daniel

Signed-off-by: Vineeth Pillai <viremana(a)linux.microsoft.com&gt; Signed-off-by: Praveen K Paladugu <prapal(a)linux.microsoft.com&gt; --- po/POTFILES.in | 1 + src/ch/ch_cgroup.c | 457 ++++++++++++++++++++++++++++++++++++++++++++ src/ch/ch_cgroup.h | 45 +++++ src/ch/ch_conf.c | 2 + src/ch/ch_conf.h | 4 +- src/ch/ch_domain.c | 33 ++++ src/ch/ch_domain.h | 3 +- src/ch/ch_monitor.c | 125 ++++++++++-- src/ch/ch_monitor.h | 54 +++++- src/ch/ch_process.c | 288 +++++++++++++++++++++++++++- src/ch/ch_process.h | 3 + src/ch/meson.build | 2 + 12 files changed, 991 insertions(+), 26 deletions(-) create mode 100644 src/ch/ch_cgroup.c create mode 100644 src/ch/ch_cgroup.h diff --git a/po/POTFILES.in b/po/POTFILES.in index b554cf08ca..3a8db501bc 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -19,6 +19,7 @@ @SRCDIR(a)src/bhyve/bhyve_parse_command.c @SRCDIR(a)src/bhyve/bhyve_process.c @SRCDIR(a)src/ch/ch_conf.c +@SRCDIR(a)src/ch/ch_cgroup.c @SRCDIR(a)src/ch/ch_domain.c @SRCDIR(a)src/ch/ch_driver.c @SRCDIR(a)src/ch/ch_monitor.c diff --git a/src/ch/ch_cgroup.c b/src/ch/ch_cgroup.c new file mode 100644 index 0000000000..6be2184cf1 --- /dev/null +++ b/src/ch/ch_cgroup.c @@ -0,0 +1,457 @@ +/* + * ch_cgroup.c: CH cgroup management + * + * Copyright Microsoft Corp. 2020-2021 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ + +#include <config.h> + +#include "ch_cgroup.h" +#include "ch_domain.h" +#include "ch_process.h" +#include "vircgroup.h" +#include "virlog.h" +#include "viralloc.h" +#include "virerror.h" +#include "domain_audit.h" +#include "domain_cgroup.h" +#include "virscsi.h" +#include "virstring.h" +#include "virfile.h" +#include "virtypedparam.h" +#include "virnuma.h" +#include "virdevmapper.h" +#include "virutil.h" + +#define VIR_FROM_THIS VIR_FROM_CH + +VIR_LOG_INIT("ch.ch_cgroup"); + +static int +chSetupBlkioCgroup(virDomainObj * vm) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_BLKIO)) { + if (vm->def->blkio.weight || vm->def->blkio.ndevices) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Block I/O tuning is not available on this host")); + return -1; + } else { + return 0; + } + } + + return virDomainCgroupSetupBlkio(priv->cgroup, vm->def->blkio); +} + + +static int +chSetupMemoryCgroup(virDomainObj * vm) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_MEMORY)) { + if (virMemoryLimitIsSet(vm->def->mem.hard_limit) || + virMemoryLimitIsSet(vm->def->mem.soft_limit) || + virMemoryLimitIsSet(vm->def->mem.swap_hard_limit)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Memory cgroup is not available on this host")); + return -1; + } else { + return 0; + } + } + + return virDomainCgroupSetupMemtune(priv->cgroup, vm->def->mem); +} + +static int +chSetupCpusetCgroup(virDomainObj * vm) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + if (virCgroupSetCpusetMemoryMigrate(priv->cgroup, true) < 0) + return -1; + + return 0; +} + + +static int +chSetupCpuCgroup(virDomainObj * vm) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + if (vm->def->cputune.sharesSpecified) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("CPU tuning is not available on this host")); + return -1; + } else { + return 0; + } + } + + if (vm->def->cputune.sharesSpecified) { + + if (virCgroupSetCpuShares(priv->cgroup, vm->def->cputune.shares) < 0) + return -1; + + } + + return 0; +} + + +static int +chInitCgroup(virDomainObj * vm, size_t nnicindexes, int *nicindexes) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(priv->driver); + + if (!priv->driver->privileged) + return 0; + + if (!virCgroupAvailable()) + return 0; + + virCgroupFree(priv->cgroup); + + if (!vm->def->resource) { + virDomainResourceDef *res; + + res = g_new0(virDomainResourceDef, 1); + + res->partition = g_strdup("/machine"); + + vm->def->resource = res; + } + + if (vm->def->resource->partition[0] != '/') { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Resource partition '%s' must start with '/'"), + vm->def->resource->partition); + return -1; + } + + if (virCgroupNewMachine(priv->machineName, "ch", vm->def->uuid, NULL, vm->pid, false, nnicindexes, nicindexes, vm->def->resource->partition, cfg->cgroupControllers, 0, /* maxThreadsPerProc */ + &priv->cgroup) < 0) { + if (virCgroupNewIgnoreError()) + return 0; + + return -1; + } + + return 0; +} + +static void +chRestoreCgroupState(virDomainObj * vm) +{ + g_autofree char *mem_mask = NULL; + g_autofree char *nodeset = NULL; + virCHDomainObjPrivate *priv = vm->privateData; + size_t i = 0; + + g_autoptr(virBitmap) all_nodes = NULL; + virCgroup *cgroup_temp = NULL; + + if (!virNumaIsAvailable() || + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return; + + if (!(all_nodes = virNumaGetHostMemoryNodeset())) + goto error; + + if (!(mem_mask = virBitmapFormat(all_nodes))) + goto error; + + if ((virCgroupHasEmptyTasks(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) <= 0) + goto error; + + if (virCgroupSetCpusetMems(priv->cgroup, mem_mask) < 0) + goto error; + + for (i = 0; i < virDomainDefGetVcpusMax(vm->def); i++) { + virDomainVcpuDef *vcpu = virDomainDefGetVcpu(vm->def, i); + + if (!vcpu->online) + continue; + + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, i, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMemoryMigrate(cgroup_temp, true) < 0 || + virCgroupGetCpusetMems(cgroup_temp, &nodeset) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset) < 0) + goto cleanup; + + g_free(nodeset); + virCgroupFree(cgroup_temp); + } + + for (i = 0; i < vm->def->niothreadids; i++) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD, + vm->def->iothreadids[i]->iothread_id, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMemoryMigrate(cgroup_temp, true) < 0 || + virCgroupGetCpusetMems(cgroup_temp, &nodeset) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset) < 0) + goto cleanup; + + g_free(nodeset); + virCgroupFree(cgroup_temp); + } + + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMemoryMigrate(cgroup_temp, true) < 0 || + virCgroupGetCpusetMems(cgroup_temp, &nodeset) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset) < 0) + goto cleanup; + + cleanup: + virCgroupFree(cgroup_temp); + return; + + error: + virResetLastError(); + VIR_DEBUG("Couldn't restore cgroups to meaningful state"); + goto cleanup; +} + +int +chConnectCgroup(virDomainObj * vm) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(priv->driver); + + if (!priv->driver->privileged) + return 0; + + if (!virCgroupAvailable()) + return 0; + + virCgroupFree(priv->cgroup); + + if (virCgroupNewDetectMachine(vm->def->name, + "ch", + vm->pid, + cfg->cgroupControllers, + priv->machineName, &priv->cgroup) < 0) + return -1; + + chRestoreCgroupState(vm); + return 0; +} + +int +chSetupCgroup(virDomainObj * vm, size_t nnicindexes, int *nicindexes) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + if (!vm->pid) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot setup cgroups until process is started")); + return -1; + } + + if (chInitCgroup(vm, nnicindexes, nicindexes) < 0) + return -1; + + if (!priv->cgroup) + return 0; + + if (chSetupBlkioCgroup(vm) < 0) + return -1; + + if (chSetupMemoryCgroup(vm) < 0) + return -1; + + if (chSetupCpuCgroup(vm) < 0) + return -1; + + if (chSetupCpusetCgroup(vm) < 0) + return -1; + + return 0; +} + +int +chSetupCgroupVcpuBW(virCgroup * cgroup, + unsigned long long period, long long quota) +{ + return virCgroupSetupCpuPeriodQuota(cgroup, period, quota); +} + + +int +chSetupCgroupCpusetCpus(virCgroup * cgroup, virBitmap * cpumask) +{ + return virCgroupSetupCpusetCpus(cgroup, cpumask); +} + +int +chSetupGlobalCpuCgroup(virDomainObj * vm) +{ + virCHDomainObjPrivate *priv = vm->privateData; + unsigned long long period = vm->def->cputune.global_period; + long long quota = vm->def->cputune.global_quota; + g_autofree char *mem_mask = NULL; + virDomainNumatuneMemMode mem_mode; + + if ((period || quota) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + return -1; + } + + /* + * If CPU cgroup controller is not initialized here, then we need + * neither period nor quota settings. And if CPUSET controller is + * not initialized either, then there's nothing to do anyway. + */ + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && + virDomainNumatuneMaybeFormatNodeset(vm->def->numa, + priv->autoNodeset, + &mem_mask, -1) < 0) + return -1; + + if (period || quota) { + if (chSetupCgroupVcpuBW(priv->cgroup, period, quota) < 0) + return -1; + } + + return 0; +} + + +int +chRemoveCgroup(virDomainObj * vm) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + if (priv->cgroup == NULL) + return 0; /* Not supported, so claim success */ + + if (virCgroupTerminateMachine(priv->machineName) < 0) { + if (!virCgroupNewIgnoreError()) + VIR_DEBUG("Failed to terminate cgroup for %s", vm->def->name); + } + + return virCgroupRemove(priv->cgroup); +} + + +static void +chCgroupEmulatorAllNodesDataFree(chCgroupEmulatorAllNodesData * data) +{ + if (!data) + return; + + virCgroupFree(data->emulatorCgroup); + g_free(data->emulatorMemMask); + g_free(data); +} + + +/** + * chCgroupEmulatorAllNodesAllow: + * @cgroup: domain cgroup pointer + * @retData: filled with structure used to roll back the operation + * + * Allows all NUMA nodes for the cloud hypervisor thread temporarily. This is + * necessary when hotplugging cpus since it requires memory allocated in the + * DMA region. Afterwards the operation can be reverted by + * chCgroupEmulatorAllNodesRestore. + * + * Returns 0 on success -1 on error + */ +int +chCgroupEmulatorAllNodesAllow(virCgroup * cgroup, + chCgroupEmulatorAllNodesData ** retData) +{ + chCgroupEmulatorAllNodesData *data = NULL; + g_autofree char *all_nodes_str = NULL; + + g_autoptr(virBitmap) all_nodes = NULL; + int ret = -1; + + if (!virNumaIsAvailable() || + !virCgroupHasController(cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) + return 0; + + if (!(all_nodes = virNumaGetHostMemoryNodeset())) + goto cleanup; + + if (!(all_nodes_str = virBitmapFormat(all_nodes))) + goto cleanup; + + data = g_new0(chCgroupEmulatorAllNodesData, 1); + + if (virCgroupNewThread(cgroup, VIR_CGROUP_THREAD_EMULATOR, 0, + false, &data->emulatorCgroup) < 0) + goto cleanup; + + if (virCgroupGetCpusetMems(data->emulatorCgroup, &data->emulatorMemMask) < 0 + || virCgroupSetCpusetMems(data->emulatorCgroup, all_nodes_str) < 0) + goto cleanup; + + *retData = g_steal_pointer(&data); + ret = 0; + + cleanup: + chCgroupEmulatorAllNodesDataFree(data); + + return ret; +} + + +/** + * chCgroupEmulatorAllNodesRestore: + * @data: data structure created by chCgroupEmulatorAllNodesAllow + * + * Rolls back the setting done by chCgroupEmulatorAllNodesAllow and frees the + * associated data. + */ +void +chCgroupEmulatorAllNodesRestore(chCgroupEmulatorAllNodesData * data) +{ + virError *err; + + if (!data) + return; + + virErrorPreserveLast(&err); + virCgroupSetCpusetMems(data->emulatorCgroup, data->emulatorMemMask); + virErrorRestore(&err); + + chCgroupEmulatorAllNodesDataFree(data); +} diff --git a/src/ch/ch_cgroup.h b/src/ch/ch_cgroup.h new file mode 100644 index 0000000000..0152b5477c --- /dev/null +++ b/src/ch/ch_cgroup.h @@ -0,0 +1,45 @@ +/* + * ch_cgroup.h: CH cgroup management + * + * Copyright Microsoft Corp. 2020-2021 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ + +#pragma once + +#include "virusb.h" +#include "vircgroup.h" +#include "domain_conf.h" +#include "ch_conf.h" + +int chConnectCgroup(virDomainObj * vm); +int chSetupCgroup(virDomainObj * vm, size_t nnicindexes, int *nicindexes); +int chSetupCgroupVcpuBW(virCgroup * cgroup, + unsigned long long period, long long quota); +int chSetupCgroupCpusetCpus(virCgroup * cgroup, virBitmap * cpumask); +int chSetupGlobalCpuCgroup(virDomainObj * vm); +int chRemoveCgroup(virDomainObj * vm); + +typedef struct _chCgroupEmulatorAllNodesData chCgroupEmulatorAllNodesData; + +struct _chCgroupEmulatorAllNodesData { + virCgroup *emulatorCgroup; + char *emulatorMemMask; +}; + +int chCgroupEmulatorAllNodesAllow(virCgroup * cgroup, + chCgroupEmulatorAllNodesData ** data); +void chCgroupEmulatorAllNodesRestore(chCgroupEmulatorAllNodesData * data); diff --git a/src/ch/ch_conf.c b/src/ch/ch_conf.c index ed0fffe5d6..7f70452296 100644 --- a/src/ch/ch_conf.c +++ b/src/ch/ch_conf.c @@ -141,6 +141,8 @@ virCHDriverConfigNew(bool privileged) if (!(cfg = virObjectNew(virCHDriverConfigClass))) return NULL; + cfg->cgroupControllers = -1; /* Auto detect */ + if (privileged) { if (virGetUserID(CH_USER, &cfg->user) < 0) return NULL; diff --git a/src/ch/ch_conf.h b/src/ch/ch_conf.h index 49f286f97a..19deb8e568 100644 --- a/src/ch/ch_conf.h +++ b/src/ch/ch_conf.h @@ -35,11 +35,13 @@ struct _virCHDriverConfig { char *stateDir; char *logDir; - + int cgroupControllers; uid_t user; gid_t group; }; +G_DEFINE_AUTOPTR_CLEANUP_FUNC(virCHDriverConfig, virObjectUnref); + struct _virCHDriver { virMutex lock; diff --git a/src/ch/ch_domain.c b/src/ch/ch_domain.c index e1030800aa..d0aaeed1f4 100644 --- a/src/ch/ch_domain.c +++ b/src/ch/ch_domain.c @@ -326,6 +326,39 @@ chValidateDomainDeviceDef(const virDomainDeviceDef *dev, _("Serial can only be enabled for a PTY")); return -1; } + return 0; +} +int +virCHDomainRefreshThreadInfo(virDomainObj *vm) +{ + size_t maxvcpus = virDomainDefGetVcpusMax(vm->def); + virCHMonitorThreadInfo *info = NULL; + size_t nthreads, ncpus = 0; + size_t i; + + nthreads = virCHMonitorGetThreadInfo(virCHDomainGetMonitor(vm), + true, &info); + + for (i = 0; i < nthreads; i++) { + virCHDomainVcpuPrivate *vcpupriv; + virDomainVcpuDef *vcpu; + virCHMonitorCPUInfo *vcpuInfo; + + if (info[i].type != virCHThreadTypeVcpu) + continue; + + // TODO: hotplug support + vcpuInfo = &info[i].vcpuInfo; + vcpu = virDomainDefGetVcpu(vm->def, vcpuInfo->cpuid); + vcpupriv = CH_DOMAIN_VCPU_PRIVATE(vcpu); + vcpupriv->tid = vcpuInfo->tid; + ncpus++; + } + + // TODO: Remove the warning when hotplug is implemented. + if (ncpus != maxvcpus) + VIR_WARN("Mismatch in the number of cpus, expected: %ld, actual: %ld", + maxvcpus, ncpus); return 0; } diff --git a/src/ch/ch_domain.h b/src/ch/ch_domain.h index 3ac3421015..2ce3e2cef3 100644 --- a/src/ch/ch_domain.h +++ b/src/ch/ch_domain.h @@ -89,7 +89,8 @@ virCHDomainObjBeginJob(virDomainObj *obj, enum virCHDomainJob job) void virCHDomainObjEndJob(virDomainObj *obj); -int virCHDomainRefreshVcpuInfo(virDomainObj *vm); +int virCHDomainRefreshThreadInfo(virDomainObj *vm); + pid_t virCHDomainGetVcpuPid(virDomainObj *vm, unsigned int vcpuid); bool virCHDomainHasVcpuPids(virDomainObj *vm); diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index c0ae031200..095779cb3f 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -41,6 +41,7 @@ VIR_LOG_INIT("ch.ch_monitor"); static virClass *virCHMonitorClass; static void virCHMonitorDispose(void *obj); +static void virCHMonitorThreadInfoFree(virCHMonitor *mon); static int virCHMonitorOnceInit(void) { @@ -571,6 +572,7 @@ static void virCHMonitorDispose(void *opaque) virCHMonitor *mon = opaque; VIR_DEBUG("mon=%p", mon); + virCHMonitorThreadInfoFree(mon); virObjectUnref(mon->vm); } @@ -736,6 +738,114 @@ virCHMonitorGet(virCHMonitor *mon, const char *endpoint, virJSONValue **response return ret; } +/** + * virCHMonitorGetInfo: + * @mon: Pointer to the monitor + * @info: Get VM info + * + * Retrieve the VM info and store in @info + * + * Returns 0 on success. + */ +int +virCHMonitorGetInfo(virCHMonitor *mon, virJSONValue **info) +{ + return virCHMonitorGet(mon, URL_VM_INFO, info); +} + +static void +virCHMonitorThreadInfoFree(virCHMonitor *mon) +{ + mon->nthreads = 0; + if (mon->threads) + VIR_FREE(mon->threads); +} + +static size_t +virCHMonitorRefreshThreadInfo(virCHMonitor *mon) +{ + virCHMonitorThreadInfo *info = NULL; + g_autofree pid_t *tids = NULL; + virDomainObj *vm = mon->vm; + size_t ntids = 0; + size_t i; + + + virCHMonitorThreadInfoFree(mon); + if (virProcessGetPids(vm->pid, &ntids, &tids) < 0) { + mon->threads = NULL; + return 0; + } + + info = g_new0(virCHMonitorThreadInfo, ntids); + for (i = 0; i < ntids; i++) { + g_autofree char *proc = NULL; + g_autofree char *data = NULL; + + proc = g_strdup_printf("/proc/%d/task/%d/comm", + (int)vm->pid, (int)tids[i]); + + if (virFileReadAll(proc, (1<<16), &data) < 0) { + continue; + } + + VIR_DEBUG("VM PID: %d, TID %d, COMM: %s", + (int)vm->pid, (int)tids[i], data); + if (STRPREFIX(data, "vcpu")) { + int cpuid; + char *tmp; + if (virStrToLong_i(data + 4, &tmp, 0, &cpuid) < 0) { + VIR_WARN("Index is not specified correctly"); + continue; + } + info[i].type = virCHThreadTypeVcpu; + info[i].vcpuInfo.tid = tids[i]; + info[i].vcpuInfo.online = true; + info[i].vcpuInfo.cpuid = cpuid; + VIR_DEBUG("vcpu%d -> tid: %d", cpuid, tids[i]); + } else if (STRPREFIX(data, "_disk") || STRPREFIX(data, "_net") || + STRPREFIX(data, "_rng")) { + /* Prefixes used by cloud-hypervisor for IO Threads are captured at + https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/vmm/src/de... */ + info[i].type = virCHThreadTypeIO; + info[i].ioInfo.tid = tids[i]; + virStrcpy(info[i].ioInfo.thrName, data, VIRCH_THREAD_NAME_LEN - 1); + }else { + info[i].type = virCHThreadTypeEmulator; + info[i].emuInfo.tid = tids[i]; + virStrcpy(info[i].emuInfo.thrName, data, VIRCH_THREAD_NAME_LEN - 1); + } + mon->nthreads++; + + } + mon->threads = info; + + return mon->nthreads; +} + +/** + * virCHMonitorGetThreadInfo: + * @mon: Pointer to the monitor + * @refresh: Refresh thread info or not + * + * Retrive thread info and store to @threads + * + * Returns count of threads on success. + */ +size_t +virCHMonitorGetThreadInfo(virCHMonitor *mon, bool refresh, + virCHMonitorThreadInfo **threads) +{ + int nthreads = 0; + + if (refresh) + nthreads = virCHMonitorRefreshThreadInfo(mon); + + *threads = mon->threads; + + return nthreads; +} + int virCHMonitorShutdownVMM(virCHMonitor *mon) { @@ -810,18 +920,3 @@ virCHMonitorResumeVM(virCHMonitor *mon) { return virCHMonitorPutNoContent(mon, URL_VM_RESUME); } - -/** - * virCHMonitorGetInfo: - * @mon: Pointer to the monitor - * @info: Get VM info - * - * Retrieve the VM info and store in @info - * - * Returns 0 on success. - */ -int -virCHMonitorGetInfo(virCHMonitor *mon, virJSONValue **info) -{ - return virCHMonitorGet(mon, URL_VM_INFO, info); -} diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index 8ca9e17a9a..f8c3fa75e8 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -37,6 +37,50 @@ #define URL_VM_RESUME "vm.resume" #define URL_VM_INFO "vm.info" +#define VIRCH_THREAD_NAME_LEN 16 + +typedef enum { + virCHThreadTypeEmulator, + virCHThreadTypeVcpu, + virCHThreadTypeIO, + virCHThreadTypeMax +} virCHThreadType; + +typedef struct _virCHMonitorCPUInfo virCHMonitorCPUInfo; + +struct _virCHMonitorCPUInfo { + int cpuid; + pid_t tid; + + bool online; +}; + +typedef struct _virCHMonitorEmuThreadInfo virCHMonitorEmuThreadInfo; + +struct _virCHMonitorEmuThreadInfo { + char thrName[VIRCH_THREAD_NAME_LEN]; + pid_t tid; +}; + +typedef struct _virCHMonitorIOThreadInfo virCHMonitorIOThreadInfo; + +struct _virCHMonitorIOThreadInfo { + char thrName[VIRCH_THREAD_NAME_LEN]; + pid_t tid; +}; + +typedef struct _virCHMonitorThreadInfo virCHMonitorThreadInfo; + +struct _virCHMonitorThreadInfo { + virCHThreadType type; + + union { + virCHMonitorCPUInfo vcpuInfo; + virCHMonitorEmuThreadInfo emuInfo; + virCHMonitorIOThreadInfo ioInfo; + }; +}; + typedef struct _virCHMonitor virCHMonitor; struct _virCHMonitor { @@ -49,6 +93,9 @@ struct _virCHMonitor { pid_t pid; virDomainObj *vm; + + size_t nthreads; + virCHMonitorThreadInfo *threads; }; virCHMonitor *virCHMonitorNew(virDomainObj *vm, const char *socketdir); @@ -65,12 +112,9 @@ int virCHMonitorSuspendVM(virCHMonitor *mon); int virCHMonitorResumeVM(virCHMonitor *mon); int virCHMonitorGetInfo(virCHMonitor *mon, virJSONValue **info); -typedef struct _virCHMonitorCPUInfo virCHMonitorCPUInfo; -struct _virCHMonitorCPUInfo { - pid_t tid; - bool online; -}; void virCHMonitorCPUInfoFree(virCHMonitorCPUInfo *cpus); int virCHMonitorGetCPUInfo(virCHMonitor *mon, virCHMonitorCPUInfo **vcpus, size_t maxvcpus); +size_t virCHMonitorGetThreadInfo(virCHMonitor *mon, bool refresh, + virCHMonitorThreadInfo **threads); diff --git a/src/ch/ch_process.c b/src/ch/ch_process.c index 3b7f6fcddf..8dce737adb 100644 --- a/src/ch/ch_process.c +++ b/src/ch/ch_process.c @@ -26,6 +26,8 @@ #include "ch_domain.h" #include "ch_monitor.h" #include "ch_process.h" +#include "ch_cgroup.h" +#include "virnuma.h" #include "viralloc.h" #include "virerror.h" #include "virjson.h" @@ -133,6 +135,257 @@ virCHProcessUpdateInfo(virDomainObj *vm) return 0; } +static int +virCHProcessGetAllCpuAffinity(virBitmap **cpumapRet) +{ + *cpumapRet = NULL; + + if (!virHostCPUHasBitmap()) + return 0; + + if (!(*cpumapRet = virHostCPUGetOnlineBitmap())) + return -1; + + return 0; +} + +#if defined(WITH_SCHED_GETAFFINITY) || defined(WITH_BSD_CPU_AFFINITY) +static int +virCHProcessInitCpuAffinity(virDomainObj *vm) +{ + g_autoptr(virBitmap) cpumapToSet = NULL; + virDomainNumatuneMemMode mem_mode; + virCHDomainObjPrivate *priv = vm->privateData; + + if (!vm->pid) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot setup CPU affinity until process is started")); + return -1; + } + + if (virDomainNumaGetNodeCount(vm->def->numa) <= 1 && + virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { + virBitmap *nodeset = NULL; + + if (virDomainNumatuneMaybeGetNodeset(vm->def->numa, + priv->autoNodeset, + &nodeset, + -1) < 0) + return -1; + + if (virNumaNodesetToCPUset(nodeset, &cpumapToSet) < 0) + return -1; + } else if (vm->def->cputune.emulatorpin) { + if (!(cpumapToSet = virBitmapNewCopy(vm->def->cputune.emulatorpin))) + return -1; + } else { + if (virCHProcessGetAllCpuAffinity(&cpumapToSet) < 0) + return -1; + } + + if (cpumapToSet && + virProcessSetAffinity(vm->pid, cpumapToSet, false) < 0) { + return -1; + } + + return 0; +} +#else /* !defined(WITH_SCHED_GETAFFINITY) && !defined(WITH_BSD_CPU_AFFINITY) */ +static int +virCHProcessInitCpuAffinity(virDomainObj *vm G_GNUC_UNUSED) +{ + return 0; +} +#endif /* !defined(WITH_SCHED_GETAFFINITY) && !defined(WITH_BSD_CPU_AFFINITY) */ + +/** + * virCHProcessSetupPid: + * + * This function sets resource properties (affinity, cgroups, + * scheduler) for any PID associated with a domain. It should be used + * to set up emulator PIDs as well as vCPU and I/O thread pids to + * ensure they are all handled the same way. + * + * Returns 0 on success, -1 on error. + */ +static int +virCHProcessSetupPid(virDomainObj *vm, + pid_t pid, + virCgroupThreadName nameval, + int id, + virBitmap *cpumask, + unsigned long long period, + long long quota, + virDomainThreadSchedParam *sched) +{ + virCHDomainObjPrivate *priv = vm->privateData; + virDomainNumatuneMemMode mem_mode; + virCgroup *cgroup = NULL; + virBitmap *use_cpumask = NULL; + virBitmap *affinity_cpumask = NULL; + g_autoptr(virBitmap) hostcpumap = NULL; + g_autofree char *mem_mask = NULL; + int ret = -1; + + if ((period || quota) && + !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + goto cleanup; + } + + /* Infer which cpumask shall be used. */ + if (cpumask) { + use_cpumask = cpumask; + } else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) { + use_cpumask = priv->autoCpuset; + } else if (vm->def->cpumask) { + use_cpumask = vm->def->cpumask; + } else { + /* we can't assume cloud-hypervisor itself is running on all pCPUs, + * so we need to explicitly set the spawned instance to all pCPUs. */ + if (virCHProcessGetAllCpuAffinity(&hostcpumap) < 0) + goto cleanup; + affinity_cpumask = hostcpumap; + } + + /* + * If CPU cgroup controller is not initialized here, then we need + * neither period nor quota settings. And if CPUSET controller is + * not initialized either, then there's nothing to do anyway. + */ + if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) || + virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && + virDomainNumatuneMaybeFormatNodeset(vm->def->numa, + priv->autoNodeset, + &mem_mask, -1) < 0) + goto cleanup; + + if (virCgroupNewThread(priv->cgroup, nameval, id, true, &cgroup) < 0) + goto cleanup; + + if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { + if (use_cpumask && + chSetupCgroupCpusetCpus(cgroup, use_cpumask) < 0) + goto cleanup; + + if (mem_mask && virCgroupSetCpusetMems(cgroup, mem_mask) < 0) + goto cleanup; + + } + + if ((period || quota) && + chSetupCgroupVcpuBW(cgroup, period, quota) < 0) + goto cleanup; + + /* Move the thread to the sub dir */ + VIR_INFO("Adding pid %d to cgroup", pid); + if (virCgroupAddThread(cgroup, pid) < 0) + goto cleanup; + + } + + if (!affinity_cpumask) + affinity_cpumask = use_cpumask; + + /* Setup legacy affinity. */ + if (affinity_cpumask && virProcessSetAffinity(pid, affinity_cpumask, false) < 0) + goto cleanup; + + /* Set scheduler type and priority, but not for the main thread. */ + if (sched && + nameval != VIR_CGROUP_THREAD_EMULATOR && + virProcessSetScheduler(pid, sched->policy, sched->priority) < 0) + goto cleanup; + + ret = 0; + cleanup: + if (cgroup) { + if (ret < 0) + virCgroupRemove(cgroup); + virCgroupFree(cgroup); + } + + return ret; +} + +/** + * virCHProcessSetupVcpu: + * @vm: domain object + * @vcpuid: id of VCPU to set defaults + * + * This function sets resource properties (cgroups, affinity, scheduler) for a + * vCPU. This function expects that the vCPU is online and the vCPU pids were + * correctly detected at the point when it's called. + * + * Returns 0 on success, -1 on error. + */ +int +virCHProcessSetupVcpu(virDomainObj *vm, + unsigned int vcpuid) +{ + pid_t vcpupid = virCHDomainGetVcpuPid(vm, vcpuid); + virDomainVcpuDef *vcpu = virDomainDefGetVcpu(vm->def, vcpuid); + + return virCHProcessSetupPid(vm, vcpupid, VIR_CGROUP_THREAD_VCPU, + vcpuid, vcpu->cpumask, + vm->def->cputune.period, + vm->def->cputune.quota, + &vcpu->sched); +} + +static int +virCHProcessSetupVcpus(virDomainObj *vm) +{ + virDomainVcpuDef *vcpu; + unsigned int maxvcpus = virDomainDefGetVcpusMax(vm->def); + size_t i; + + if ((vm->def->cputune.period || vm->def->cputune.quota) && + !virCgroupHasController(((virCHDomainObjPrivate *) vm->privateData)->cgroup, + VIR_CGROUP_CONTROLLER_CPU)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("cgroup cpu is required for scheduler tuning")); + return -1; + } + + if (!virCHDomainHasVcpuPids(vm)) { + /* If any CPU has custom affinity that differs from the + * VM default affinity, we must reject it */ + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(vm->def, i); + + if (!vcpu->online) + continue; + + if (vcpu->cpumask && + !virBitmapEqual(vm->def->cpumask, vcpu->cpumask)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cpu affinity is not supported")); + return -1; + } + } + + return 0; + } + + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(vm->def, i); + + if (!vcpu->online) + continue; + + if (virCHProcessSetupVcpu(vm, i) < 0) + return -1; + } + + return 0; +} + /** * virCHProcessStart: * @driver: pointer to driver structure @@ -168,18 +421,33 @@ int virCHProcessStart(virCHDriver *driver, } } + vm->pid = priv->monitor->pid; + vm->def->id = vm->pid; + priv->machineName = virCHDomainGetMachineName(vm); + + if (chSetupCgroup(vm, nnicindexes, nicindexes) < 0) + goto cleanup; + + if (virCHProcessInitCpuAffinity(vm) < 0) + goto cleanup; + if (virCHMonitorBootVM(priv->monitor) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("failed to boot guest VM")); goto cleanup; } - priv->machineName = virCHDomainGetMachineName(vm); - vm->pid = priv->monitor->pid; - vm->def->id = vm->pid; + virCHDomainRefreshThreadInfo(vm); - virCHProcessUpdateInfo(vm); + VIR_DEBUG("Setting global CPU cgroup (if required)"); + if (chSetupGlobalCpuCgroup(vm) < 0) + goto cleanup; + + VIR_DEBUG("Setting vCPU tuning/settings"); + if (virCHProcessSetupVcpus(vm) < 0) + goto cleanup; + virCHProcessUpdateInfo(vm); virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason); return 0; @@ -195,6 +463,8 @@ int virCHProcessStop(virCHDriver *driver G_GNUC_UNUSED, virDomainObj *vm, virDomainShutoffReason reason) { + int ret; + int retries = 0; virCHDomainObjPrivate *priv = vm->privateData; VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d", @@ -205,6 +475,16 @@ int virCHProcessStop(virCHDriver *driver G_GNUC_UNUSED, priv->monitor = NULL; } + retry: + if ((ret = chRemoveCgroup(vm)) < 0) { + if (ret == -EBUSY && (retries++ < 5)) { + g_usleep(200*1000); + goto retry; + } + VIR_WARN("Failed to remove cgroup for %s", + vm->def->name); + } + vm->pid = -1; vm->def->id = -1; diff --git a/src/ch/ch_process.h b/src/ch/ch_process.h index abc4915979..800e3f4e23 100644 --- a/src/ch/ch_process.h +++ b/src/ch/ch_process.h @@ -29,3 +29,6 @@ int virCHProcessStart(virCHDriver *driver, int virCHProcessStop(virCHDriver *driver, virDomainObj *vm, virDomainShutoffReason reason); + +int virCHProcessSetupVcpu(virDomainObj *vm, + unsigned int vcpuid); diff --git a/src/ch/meson.build b/src/ch/meson.build index 2b2bdda26c..0b20de56fd 100644 --- a/src/ch/meson.build +++ b/src/ch/meson.build @@ -1,6 +1,8 @@ ch_driver_sources = [ 'ch_conf.c', 'ch_conf.h', + 'ch_cgroup.c', + 'ch_cgroup.h', 'ch_domain.c', 'ch_domain.h', 'ch_driver.c', -- 2.27.0

Signed-off-by: Vineeth Pillai <viremana(a)linux.microsoft.com&gt; Signed-off-by: Praveen K Paladugu <prapal(a)linux.microsoft.com&gt; --- src/ch/ch_domain.c | 30 +++++++++ src/ch/ch_domain.h | 1 + src/ch/ch_driver.c | 151 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+) diff --git a/src/ch/ch_domain.c b/src/ch/ch_domain.c index d0aaeed1f4..9ad00583aa 100644 --- a/src/ch/ch_domain.c +++ b/src/ch/ch_domain.c @@ -20,6 +20,7 @@ #include <config.h> +#include "datatypes.h" #include "ch_domain.h" #include "domain_driver.h" #include "viralloc.h" @@ -420,3 +421,32 @@ char *virCHDomainGetMachineName(virDomainObj *vm) return ret; } + +/** + * virCHDomainObjFromDomain: + * @domain: Domain pointer that has to be looked up + * + * This function looks up @domain and returns the appropriate virDomainObjPtr + * that has to be released by calling virDomainObjEndAPI(). + * + * Returns the domain object with incremented reference counter which is locked + * on success, NULL otherwise. + */ +virDomainObj * +virCHDomainObjFromDomain(virDomainPtr domain) +{ + virDomainObj *vm; + virCHDriver *driver = domain->conn->privateData; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + vm = virDomainObjListFindByUUID(driver->domains, domain->uuid); + if (!vm) { + virUUIDFormat(domain->uuid, uuidstr); + virReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s' (%s)"), + uuidstr, domain->name); + return NULL; + } + + return vm; +} diff --git a/src/ch/ch_domain.h b/src/ch/ch_domain.h index 2ce3e2cef3..db1451405b 100644 --- a/src/ch/ch_domain.h +++ b/src/ch/ch_domain.h @@ -95,3 +95,4 @@ pid_t virCHDomainGetVcpuPid(virDomainObj *vm, unsigned int vcpuid); bool virCHDomainHasVcpuPids(virDomainObj *vm); char *virCHDomainGetMachineName(virDomainObj *vm); +virDomainObj *virCHDomainObjFromDomain(virDomain *domain); diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index ca854da123..7f3cf6dbef 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -25,6 +25,7 @@ #include "ch_driver.h" #include "ch_monitor.h" #include "ch_process.h" +#include "ch_cgroup.h" #include "datatypes.h" #include "driver.h" #include "viraccessapicheck.h" @@ -1141,6 +1142,154 @@ chDomainGetVcpus(virDomainPtr dom, return ret; } +static int +chDomainPinVcpuLive(virDomainObj *vm, + virDomainDef *def, + int vcpu, + virCHDriver *driver, + virCHDriverConfig *cfg, + virBitmap *cpumap) +{ + virBitmap *tmpmap = NULL; + virDomainVcpuDef *vcpuinfo; + virCHDomainObjPrivate *priv = vm->privateData; + virCgroup *cgroup_vcpu = NULL; + g_autofree char *str = NULL; + int ret = -1; + + if (!virCHDomainHasVcpuPids(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + "%s", _("cpu affinity is not supported")); + goto cleanup; + } + + if (!(vcpuinfo = virDomainDefGetVcpu(def, vcpu))) { + virReportError(VIR_ERR_INVALID_ARG, + _("vcpu %d is out of range of live cpu count %d"), + vcpu, virDomainDefGetVcpusMax(def)); + goto cleanup; + } + + if (!(tmpmap = virBitmapNewCopy(cpumap))) + goto cleanup; + + if (!(str = virBitmapFormat(cpumap))) + goto cleanup; + + if (vcpuinfo->online) { + /* Configure the corresponding cpuset cgroup before set affinity. */ + if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, vcpu, + false, &cgroup_vcpu) < 0) + goto cleanup; + if (chSetupCgroupCpusetCpus(cgroup_vcpu, cpumap) < 0) + goto cleanup; + } + + if (virProcessSetAffinity(virCHDomainGetVcpuPid(vm, vcpu), cpumap, false) < 0) + goto cleanup; + } + + virBitmapFree(vcpuinfo->cpumask); + vcpuinfo->cpumask = tmpmap; + tmpmap = NULL; + + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) + goto cleanup; + + ret = 0; + + cleanup: + virBitmapFree(tmpmap); + virCgroupFree(cgroup_vcpu); + return ret; +} + + +static int +chDomainPinVcpuFlags(virDomainPtr dom, + unsigned int vcpu, + unsigned char *cpumap, + int maplen, + unsigned int flags) +{ + virCHDriver *driver = dom->conn->privateData; + virDomainObj *vm; + virDomainDef *def; + virDomainDef *persistentDef; + int ret = -1; + virBitmap *pcpumap = NULL; + virDomainVcpuDef *vcpuinfo = NULL; + g_autoptr(virCHDriverConfig) cfg = NULL; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG, -1); + + cfg = virCHDriverGetConfig(driver); + + if (!(vm = virCHDomainObjFromDomain(dom))) + goto cleanup; + + if (virDomainPinVcpuFlagsEnsureACL(dom->conn, vm->def, flags) < 0) + goto cleanup; + + if (virCHDomainObjBeginJob(vm, CH_JOB_MODIFY) < 0) + goto cleanup; + + if (virDomainObjGetDefs(vm, flags, &def, &persistentDef) < 0) + goto endjob; + + if (persistentDef && + !(vcpuinfo = virDomainDefGetVcpu(persistentDef, vcpu))) { + virReportError(VIR_ERR_INVALID_ARG, + _("vcpu %d is out of range of persistent cpu count %d"), + vcpu, virDomainDefGetVcpus(persistentDef)); + goto endjob; + } + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto endjob; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty cpu list for pinning")); + goto endjob; + } + + if (def && + chDomainPinVcpuLive(vm, def, vcpu, driver, cfg, pcpumap) < 0) + goto endjob; + + if (persistentDef) { + virBitmapFree(vcpuinfo->cpumask); + vcpuinfo->cpumask = pcpumap; + pcpumap = NULL; + + // ret = virDomainDefSave(persistentDef, driver->xmlopt, cfg->configDir); + goto endjob; + } + + ret = 0; + + endjob: + virCHDomainObjEndJob(vm); + + cleanup: + virDomainObjEndAPI(&vm); + virBitmapFree(pcpumap); + return ret; +} + +static int +chDomainPinVcpu(virDomainPtr dom, + unsigned int vcpu, + unsigned char *cpumap, + int maplen) +{ + return chDomainPinVcpuFlags(dom, vcpu, cpumap, maplen, + VIR_DOMAIN_AFFECT_LIVE); +} + /* Function Tables */ static virHypervisorDriver chHypervisorDriver = { .name = "CH", @@ -1181,6 +1330,8 @@ static virHypervisorDriver chHypervisorDriver = { .domainGetVcpusFlags = chDomainGetVcpusFlags, /* 7.9.0 */ .domainGetMaxVcpus = chDomainGetMaxVcpus, /* 7.9.0 */ .domainGetVcpuPinInfo = chDomainGetVcpuPinInfo, /* 7.9.0 */ + .domainPinVcpu = chDomainPinVcpu, /* 7.9.0 */ + .domainPinVcpuFlags = chDomainPinVcpuFlags, /* 7.9.0 */ .nodeGetCPUMap = chNodeGetCPUMap, /* 7.9.0 */ }; -- 2.27.0

Signed-off-by: Vineeth Pillai <viremana(a)linux.microsoft.com&gt; Signed-off-by: Praveen K Paladugu <prapal(a)linux.microsoft.com&gt; --- src/ch/ch_driver.c | 278 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 9ad23c1710..97dfda85e1 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -43,6 +43,7 @@ #include "viruri.h" #include "virutil.h" #include "viruuid.h" +#include "virnuma.h" #define VIR_FROM_THIS VIR_FROM_CH @@ -1320,6 +1321,281 @@ chDomainPinVcpu(virDomainPtr dom, VIR_DOMAIN_AFFECT_LIVE); } +#define CH_NB_NUMA_PARAM 2 + +static int +chDomainGetNumaParameters(virDomainPtr dom, + virTypedParameterPtr params, + int *nparams, + unsigned int flags) +{ + size_t i; + virDomainObj *vm = NULL; + virDomainNumatuneMemMode tmpmode = VIR_DOMAIN_NUMATUNE_MEM_STRICT; + virCHDomainObjPrivate *priv; + g_autofree char *nodeset = NULL; + int ret = -1; + virDomainDef *def = NULL; + bool live = false; + virBitmap *autoNodeset = NULL; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG | + VIR_TYPED_PARAM_STRING_OKAY, -1); + + if (!(vm = virCHDomainObjFromDomain(dom))) + return -1; + priv = vm->privateData; + + if (virDomainGetNumaParametersEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (!(def = virDomainObjGetOneDefState(vm, flags, &live))) + goto cleanup; + + if (live) + autoNodeset = priv->autoNodeset; + + if ((*nparams) == 0) { + *nparams = CH_NB_NUMA_PARAM; + ret = 0; + goto cleanup; + } + + for (i = 0; i < CH_NB_NUMA_PARAM && i < *nparams; i++) { + virMemoryParameterPtr param = &params[i]; + + switch (i) { + case 0: /* fill numa mode here */ + ignore_value(virDomainNumatuneGetMode(def->numa, -1, &tmpmode)); + + if (virTypedParameterAssign(param, VIR_DOMAIN_NUMA_MODE, + VIR_TYPED_PARAM_INT, tmpmode) < 0) + goto cleanup; + + break; + + case 1: /* fill numa nodeset here */ + nodeset = virDomainNumatuneFormatNodeset(def->numa, autoNodeset, -1); + + if (!nodeset || + virTypedParameterAssign(param, VIR_DOMAIN_NUMA_NODESET, + VIR_TYPED_PARAM_STRING, nodeset) < 0) + goto cleanup; + + nodeset = NULL; + break; + + /* coverity[dead_error_begin] */ + default: + break; + /* should not hit here */ + } + } + + if (*nparams > CH_NB_NUMA_PARAM) + *nparams = CH_NB_NUMA_PARAM; + ret = 0; + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + +static int +chDomainSetNumaParamsLive(virDomainObj *vm, + virBitmap *nodeset) +{ + virCgroup *cgroup_temp = NULL; + virCHDomainObjPrivate *priv = vm->privateData; + g_autofree char *nodeset_str = NULL; + virDomainNumatuneMemMode mode; + size_t i = 0; + int ret = -1; + + if (virDomainNumatuneGetMode(vm->def->numa, -1, &mode) == 0 && + mode != VIR_DOMAIN_NUMATUNE_MEM_STRICT) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("change of nodeset for running domain " + "requires strict numa mode")); + goto cleanup; + } + + if (!virNumaNodesetIsAvailable(nodeset)) + goto cleanup; + + /* Ensure the cpuset string is formatted before passing to cgroup */ + if (!(nodeset_str = virBitmapFormat(nodeset))) + goto cleanup; + + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset_str) < 0) + goto cleanup; + virCgroupFree(cgroup_temp); + + for (i = 0; i < virDomainDefGetVcpusMax(vm->def); i++) { + virDomainVcpuDef *vcpu = virDomainDefGetVcpu(vm->def, i); + + if (!vcpu->online) + continue; + + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, i, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset_str) < 0) + goto cleanup; + virCgroupFree(cgroup_temp); + } + + for (i = 0; i < vm->def->niothreadids; i++) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD, + vm->def->iothreadids[i]->iothread_id, + false, &cgroup_temp) < 0 || + virCgroupSetCpusetMems(cgroup_temp, nodeset_str) < 0) + goto cleanup; + virCgroupFree(cgroup_temp); + } + + /* set nodeset for root cgroup */ + if (virCgroupSetCpusetMems(priv->cgroup, nodeset_str) < 0) + goto cleanup; + + ret = 0; + cleanup: + virCgroupFree(cgroup_temp); + + return ret; +} + +static int +chDomainSetNumaParameters(virDomainPtr dom, + virTypedParameterPtr params, + int nparams, + unsigned int flags) +{ + virCHDriver *driver = dom->conn->privateData; + size_t i; + virDomainDef *def; + virDomainDef *persistentDef; + virDomainObj *vm = NULL; + int ret = -1; + g_autoptr(virCHDriverConfig) cfg = NULL; + virCHDomainObjPrivate *priv; + virBitmap *nodeset = NULL; + virDomainNumatuneMemMode config_mode; + int mode = -1; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG, -1); + + if (virTypedParamsValidate(params, nparams, + VIR_DOMAIN_NUMA_MODE, + VIR_TYPED_PARAM_INT, + VIR_DOMAIN_NUMA_NODESET, + VIR_TYPED_PARAM_STRING, + NULL) < 0) + return -1; + + if (!(vm = virCHDomainObjFromDomain(dom))) + return -1; + + priv = vm->privateData; + cfg = virCHDriverGetConfig(driver); + + if (virDomainSetNumaParametersEnsureACL(dom->conn, vm->def, flags) < 0) + goto cleanup; + + for (i = 0; i < nparams; i++) { + virTypedParameterPtr param = &params[i]; + + if (STREQ(param->field, VIR_DOMAIN_NUMA_MODE)) { + mode = param->value.i; + + if (mode < 0 || mode >= VIR_DOMAIN_NUMATUNE_MEM_LAST) { + virReportError(VIR_ERR_INVALID_ARG, + _("unsupported numatune mode: '%d'"), mode); + goto cleanup; + } + + } else if (STREQ(param->field, VIR_DOMAIN_NUMA_NODESET)) { + if (virBitmapParse(param->value.s, &nodeset, + VIR_DOMAIN_CPUMASK_LEN) < 0) + goto cleanup; + + if (virBitmapIsAllClear(nodeset)) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("Invalid nodeset of 'numatune': %s"), + param->value.s); + goto cleanup; + } + } + } + + if (virCHDomainObjBeginJob(vm, CH_JOB_MODIFY) < 0) + goto cleanup; + + if (virDomainObjGetDefs(vm, flags, &def, &persistentDef) < 0) + goto endjob; + + if (def) { + if (!driver->privileged) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("NUMA tuning is not available in session mode")); + goto endjob; + } + + if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cgroup cpuset controller is not mounted")); + goto endjob; + } + + if (mode != -1 && + virDomainNumatuneGetMode(def->numa, -1, &config_mode) == 0 && + config_mode != mode) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("can't change numatune mode for running domain")); + goto endjob; + } + + if (nodeset && + chDomainSetNumaParamsLive(vm, nodeset) < 0) + goto endjob; + + if (virDomainNumatuneSet(def->numa, + def->placement_mode == + VIR_DOMAIN_CPU_PLACEMENT_MODE_STATIC, + -1, mode, nodeset) < 0) + goto endjob; + + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) + goto endjob; + } + + /* + if (persistentDef) { + if (virDomainNumatuneSet(persistentDef->numa, + persistentDef->placement_mode == + VIR_DOMAIN_CPU_PLACEMENT_MODE_STATIC, + -1, mode, nodeset) < 0) + goto endjob; + + if (virDomainDefSave(persistentDef, driver->xmlopt, cfg->configDir) < 0) + goto endjob; + } + */ + + ret = 0; + + endjob: + virCHDomainObjEndJob(vm); + + cleanup: + virBitmapFree(nodeset); + virDomainObjEndAPI(&vm); + return ret; +} + /* Function Tables */ static virHypervisorDriver chHypervisorDriver = { .name = "CH", @@ -1364,6 +1640,8 @@ static virHypervisorDriver chHypervisorDriver = { .domainPinVcpu = chDomainPinVcpu, /* 7.9.0 */ .domainPinVcpuFlags = chDomainPinVcpuFlags, /* 7.9.0 */ .nodeGetCPUMap = chNodeGetCPUMap, /* 7.9.0 */ + .domainSetNumaParameters = chDomainSetNumaParameters, /* 7.9.0 */ + .domainGetNumaParameters = chDomainGetNumaParameters, /* 7.9.0 */ }; static virConnectDriver chConnectDriver = { -- 2.27.0

On 10/21/21 16:31, Vineeth Pillai wrote: Please use "*" at the start of each new line in the block comment: Daniel

On 10/21/21 16:31, Vineeth Pillai wrote: Wrong label indentation. Same here. Thanks, Daniel