Some VM configurations may result in a large number of threads created by the associated qemu process which can exceed the system default limit. The maximum number of threads allowed per process is controlled by the pids cgroup controller and is set to 16k when creating VMs with systemd's machined service. The maximum number of threads per process is recorded in the pids.max file under the machine's pids controller cgroup hierarchy, e.g. $cgrp-mnt/pids/machine.slice/machine-qemu\\x2d1\\x2dtest.scope/pids.max Maximum threads per process is controlled with the TasksMax property of the systemd scope for the machine. This patch adds an option to qemu.conf which can be used to override the maximum number of threads allowed per qemu process. If the value of option is greater than zero, it will be set in the TasksMax property of the machine's scope after creating the machine. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- Rebase and resend of https://www.redhat.com/archives/libvir-list/2019-June/msg00185.html src/lxc/lxc_cgroup.c | 1 + src/qemu/libvirtd_qemu.aug | 1 + src/qemu/qemu.conf | 10 ++++++++++ src/qemu/qemu_cgroup.c | 1 + src/qemu/qemu_conf.c | 2 ++ src/qemu/qemu_conf.h | 1 + src/qemu/test_libvirtd_qemu.aug.in | 1 + src/util/vircgroup.c | 6 +++++- src/util/vircgroup.h | 1 + src/util/virsystemd.c | 24 +++++++++++++++++++++++- src/util/virsystemd.h | 3 ++- tests/virsystemdtest.c | 12 ++++++------ 12 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index d93a19d684..76014f3bfd 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -455,6 +455,7 @@ virCgroupPtr virLXCCgroupCreate(virDomainDefPtr def, nnicindexes, nicindexes, def->resource->partition, -1, + 0, &cgroup) < 0) goto cleanup; diff --git a/src/qemu/libvirtd_qemu.aug b/src/qemu/libvirtd_qemu.aug index eea9094d39..2a99a0c55f 100644 --- a/src/qemu/libvirtd_qemu.aug +++ b/src/qemu/libvirtd_qemu.aug @@ -95,6 +95,7 @@ module Libvirtd_qemu = | limits_entry "max_core" | bool_entry "dump_guest_core" | str_entry "stdio_handler" + | int_entry "max_threads_per_process" let device_entry = bool_entry "mac_filter" | bool_entry "relaxed_acs_check" diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf index fd2ed9dc21..8cabeccacb 100644 --- a/src/qemu/qemu.conf +++ b/src/qemu/qemu.conf @@ -613,6 +613,16 @@ #max_processes = 0 #max_files = 0 +# If max_threads_per_process is set to a positive integer, libvirt +# will use it to set the maximum number of threads that can be +# created by a qemu process. Some VM configurations can result in +# qemu processes with tens of thousands of threads. systemd-based +# systems typically limit the number of threads per process to +# 16k. max_threads_per_process can be used to override default +# limits in the host OS. +# +#max_threads_per_process = 0 + # If max_core is set to a non-zero integer, then QEMU will be # permitted to create core dumps when it crashes, provided its # RAM size is smaller than the limit set. diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 19ca60905a..ecd96efb0a 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -929,6 +929,7 @@ qemuInitCgroup(virDomainObjPtr vm, nnicindexes, nicindexes, vm->def->resource->partition, cfg->cgroupControllers, + cfg->maxThreadsPerProc, &priv->cgroup) < 0) { if (virCgroupNewIgnoreError()) goto done; diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c index e0195dac29..71d0464c0d 100644 --- a/src/qemu/qemu_conf.c +++ b/src/qemu/qemu_conf.c @@ -670,6 +670,8 @@ virQEMUDriverConfigLoadProcessEntry(virQEMUDriverConfigPtr cfg, return -1; if (virConfGetValueUInt(conf, "max_files", &cfg->maxFiles) < 0) return -1; + if (virConfGetValueUInt(conf, "max_threads_per_process", &cfg->maxThreadsPerProc) < 0) + return -1; if (virConfGetValueType(conf, "max_core") == VIR_CONF_STRING) { if (virConfGetValueString(conf, "max_core", &corestr) < 0) diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index 2229b76e89..d8e3bfe87c 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -162,6 +162,7 @@ struct _virQEMUDriverConfig { unsigned int maxProcesses; unsigned int maxFiles; + unsigned int maxThreadsPerProc; unsigned long long maxCore; bool dumpGuestCore; diff --git a/src/qemu/test_libvirtd_qemu.aug.in b/src/qemu/test_libvirtd_qemu.aug.in index 388ba24b8b..b3b44d42d9 100644 --- a/src/qemu/test_libvirtd_qemu.aug.in +++ b/src/qemu/test_libvirtd_qemu.aug.in @@ -76,6 +76,7 @@ module Test_libvirtd_qemu = { "set_process_name" = "1" } { "max_processes" = "0" } { "max_files" = "0" } +{ "max_threads_per_process" = "0" } { "max_core" = "unlimited" } { "dump_guest_core" = "1" } { "mac_filter" = "1" } diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c index f7afc2964d..9daf62795e 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -1118,6 +1118,7 @@ virCgroupNewMachineSystemd(const char *name, int *nicindexes, const char *partition, int controllers, + unsigned int maxthreads, virCgroupPtr *group) { int rv; @@ -1134,7 +1135,8 @@ virCgroupNewMachineSystemd(const char *name, isContainer, nnicindexes, nicindexes, - partition)) < 0) + partition, + maxthreads)) < 0) return rv; if (controllers != -1) @@ -1246,6 +1248,7 @@ virCgroupNewMachine(const char *name, int *nicindexes, const char *partition, int controllers, + unsigned int maxthreads, virCgroupPtr *group) { int rv; @@ -1262,6 +1265,7 @@ virCgroupNewMachine(const char *name, nicindexes, partition, controllers, + maxthreads, group)) == 0) return 0; diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h index 2f68fdb685..3eefe78787 100644 --- a/src/util/vircgroup.h +++ b/src/util/vircgroup.h @@ -98,6 +98,7 @@ int virCgroupNewMachine(const char *name, int *nicindexes, const char *partition, int controllers, + unsigned int maxthreads, virCgroupPtr *group) ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(3); diff --git a/src/util/virsystemd.c b/src/util/virsystemd.c index f6c5adc5ef..1cb8874403 100644 --- a/src/util/virsystemd.c +++ b/src/util/virsystemd.c @@ -252,12 +252,14 @@ int virSystemdCreateMachine(const char *name, bool iscontainer, size_t nnicindexes, int *nicindexes, - const char *partition) + const char *partition, + unsigned int maxthreads) { int ret; DBusConnection *conn; char *creatorname = NULL; char *slicename = NULL; + char *scopename = NULL; static int hasCreateWithNetwork = 1; if ((ret = virSystemdHasMachined()) < 0) @@ -403,11 +405,31 @@ int virSystemdCreateMachine(const char *name, goto cleanup; } + if (maxthreads > 0) { + if (!(scopename = virSystemdMakeScopeName(name, drivername, false))) + goto cleanup; + + if (virDBusCallMethod(conn, + NULL, + NULL, + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "SetUnitProperties", + "sba(sv)", + scopename, + true, + 1, + "TasksMax", "t", (uint64_t)maxthreads) < 0) + goto cleanup; + } + ret = 0; cleanup: VIR_FREE(creatorname); VIR_FREE(slicename); + VIR_FREE(scopename); return ret; } diff --git a/src/util/virsystemd.h b/src/util/virsystemd.h index 5d56c78835..96626f8fff 100644 --- a/src/util/virsystemd.h +++ b/src/util/virsystemd.h @@ -50,7 +50,8 @@ int virSystemdCreateMachine(const char *name, bool iscontainer, size_t nnicindexes, int *nicindexes, - const char *partition); + const char *partition, + unsigned int maxthreads); int virSystemdTerminateMachine(const char *name); diff --git a/tests/virsystemdtest.c b/tests/virsystemdtest.c index 7aaa8f97fa..340b038095 100644 --- a/tests/virsystemdtest.c +++ b/tests/virsystemdtest.c @@ -175,7 +175,7 @@ static int testCreateContainer(const void *opaque ATTRIBUTE_UNUSED) 123, true, 0, NULL, - "highpriority.slice") < 0) { + "highpriority.slice", 0) < 0) { fprintf(stderr, "%s", "Failed to create LXC machine\n"); return -1; } @@ -208,7 +208,7 @@ static int testCreateMachine(const void *opaque ATTRIBUTE_UNUSED) 123, false, 0, NULL, - NULL) < 0) { + NULL, 0) < 0) { fprintf(stderr, "%s", "Failed to create KVM machine\n"); return -1; } @@ -245,7 +245,7 @@ static int testCreateNoSystemd(const void *opaque ATTRIBUTE_UNUSED) 123, false, 0, NULL, - NULL)) == 0) { + NULL, 0)) == 0) { unsetenv("FAIL_NO_SERVICE"); fprintf(stderr, "%s", "Unexpected create machine success\n"); return -1; @@ -279,7 +279,7 @@ static int testCreateSystemdNotRunning(const void *opaque ATTRIBUTE_UNUSED) 123, false, 0, NULL, - NULL)) == 0) { + NULL, 0)) == 0) { unsetenv("FAIL_NOT_REGISTERED"); fprintf(stderr, "%s", "Unexpected create machine success\n"); return -1; @@ -313,7 +313,7 @@ static int testCreateBadSystemd(const void *opaque ATTRIBUTE_UNUSED) 123, false, 0, NULL, - NULL)) == 0) { + NULL, 0)) == 0) { unsetenv("FAIL_BAD_SERVICE"); fprintf(stderr, "%s", "Unexpected create machine success\n"); return -1; @@ -348,7 +348,7 @@ static int testCreateNetwork(const void *opaque ATTRIBUTE_UNUSED) 123, true, nnicindexes, nicindexes, - "highpriority.slice") < 0) { + "highpriority.slice", 0) < 0) { fprintf(stderr, "%s", "Failed to create LXC machine\n"); return -1; } -- 2.22.0