4:56 p.m.
Add the external swtpm to the emulator cgroup so that upper limits of CPU
usage can be enforced on the emulated TPM.
To enable this we need to have the swtpm write its pid into a file. We then
take the pid from this file to configure the emulator cgroup.
Signed-off-by: Stefan Berger <stefanb(a)linux.vnet.ibm.com>
---
src/conf/domain_conf.c | 1 +
src/conf/domain_conf.h | 1 +
src/libvirt_private.syms | 1 +
src/qemu/qemu_cgroup.c | 53 ++++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_cgroup.h | 1 +
src/qemu/qemu_extdevice.c | 40 +++++++++++++++++++++++++------
src/qemu/qemu_extdevice.h | 2 +-
src/qemu/qemu_process.c | 8 +++++--
src/util/vircgroup.c | 42 ++++++++++++++++++++++++++++++++
src/util/vircgroup.h | 1 +
src/util/virtpm.c | 61 +++++++++++++++++++++++++++++++++++++++++++++--
src/util/virtpm.h | 6 +++--
12 files changed, 203 insertions(+), 14 deletions(-)
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 5498e2e..1b41584 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -2625,6 +2625,7 @@ void virDomainTPMDefFree(virDomainTPMDefPtr def)
VIR_FREE(def->data.emulator.source.data.nix.path);
VIR_FREE(def->data.emulator.storagepath);
VIR_FREE(def->data.emulator.logfile);
+ VIR_FREE(def->data.emulator.pidfile);
break;
case VIR_DOMAIN_TPM_TYPE_LAST:
break;
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 4aab54b..29ef7ff 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -1310,6 +1310,7 @@ struct _virDomainTPMDef {
/* swtpm storage path and logfile */
char *storagepath;
char *logfile;
+ char *pidfile;
} emulator;
} data;
};
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 488882d..db69b56 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -1468,6 +1468,7 @@ virBufferVasprintf;
# util/vircgroup.h
virCgroupAddMachineTask;
+virCgroupAddProc;
virCgroupAddTask;
virCgroupAddTaskController;
virCgroupAllowAllDevices;
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index bd4859c..859ed55 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -37,6 +37,7 @@
#include "virtypedparam.h"
#include "virnuma.h"
#include "virsystemd.h"
+#include "virpidfile.h"
#define VIR_FROM_THIS VIR_FROM_QEMU
@@ -1106,6 +1107,58 @@ qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup,
int
+qemuSetupCgroupForExtDevices(virDomainObjPtr vm)
+{
+ qemuDomainObjPrivatePtr priv = vm->privateData;
+ virDomainTPMDefPtr tpm = vm->def->tpm;
+ virCgroupPtr cgroup_temp = NULL;
+ pid_t pid;
+ int ret = -1;
+
+ if (priv->cgroup == NULL)
+ return 0; /* Not supported, so claim success */
+
+ /*
+ * If CPU cgroup controller is not initialized here, then we need
+ * neither period nor quota settings. And if CPUSET controller is
+ * not initialized either, then there's nothing to do anyway.
+ */
+ if (!virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
+ !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET))
+ return 0;
+
+ if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
+ false, &cgroup_temp) < 0)
+ goto cleanup;
+
+ if (tpm) {
+ switch (tpm->type) {
+ case VIR_DOMAIN_TPM_TYPE_EMULATOR:
+ if (virPidFileReadPath(tpm->data.emulator.pidfile, &pid) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("Could not read swtpm's pidfile %s"),
+ tpm->data.emulator.pidfile);
+ goto cleanup;
+ }
+ if (virCgroupAddProc(cgroup_temp, pid) < 0)
+ goto cleanup;
+ break;
+ case VIR_DOMAIN_TPM_TYPE_PASSTHROUGH:
+ case VIR_DOMAIN_TPM_TYPE_LAST:
+ break;
+ }
+ }
+
+ ret = 0;
+
+cleanup:
+ virCgroupFree(&cgroup_temp);
+
+ return ret;
+}
+
+
+int
qemuSetupGlobalCpuCgroup(virDomainObjPtr vm)
{
qemuDomainObjPrivatePtr priv = vm->privateData;
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index 3b8ff60..478bf7e 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -69,6 +69,7 @@ int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup,
long long quota);
int qemuSetupCgroupCpusetCpus(virCgroupPtr cgroup, virBitmapPtr cpumask);
int qemuSetupGlobalCpuCgroup(virDomainObjPtr vm);
+int qemuSetupCgroupForExtDevices(virDomainObjPtr vm);
int qemuRemoveCgroup(virDomainObjPtr vm);
typedef struct _qemuCgroupEmulatorAllNodesData qemuCgroupEmulatorAllNodesData;
diff --git a/src/qemu/qemu_extdevice.c b/src/qemu/qemu_extdevice.c
index e685faf..439cb31 100644
--- a/src/qemu/qemu_extdevice.c
+++ b/src/qemu/qemu_extdevice.c
@@ -35,6 +35,7 @@
VIR_LOG_INIT("qemu.qemu_extdevice")
+
static int
qemuExtDeviceLogCommand(qemuDomainLogContextPtr logCtxt,
virCommandPtr cmd,
@@ -93,11 +94,15 @@ qemuExtTPMStartEmulator(virQEMUDriverPtr driver,
virDomainDefPtr def = vm->def;
unsigned char *vmuuid = def->uuid;
virDomainTPMDefPtr tpm = def->tpm;
+ char *pidfiledata = NULL;
+ int timeout;
+ int len;
/* stop any left-over TPM emulator for this VM */
- virTPMStopEmulator(tpm, vmuuid, false);
+ virTPMStopEmulator(tpm, vmuuid, false, cfg->stateDir, def->name);
- if (!(cmd = virTPMEmulatorBuildCommand(tpm, vmuuid, cfg->swtpm_user)))
+ if (!(cmd = virTPMEmulatorBuildCommand(tpm, vmuuid, cfg->swtpm_user,
+ cfg->stateDir, def->name)))
goto cleanup;
if (qemuExtDeviceLogCommand(logCtxt, cmd, "TPM Emulator") < 0)
@@ -147,6 +152,22 @@ qemuExtTPMStartEmulator(virQEMUDriverPtr driver,
goto error;
}
+ /* check that the swtpm has written its pid into the file */
+ timeout = 1000; /* ms */
+ while ((len = virFileReadHeaderQuiet(tpm->data.emulator.pidfile,
+ 10, &pidfiledata)) <= 0) {
+ if (len == 0 && timeout > 0) {
+ timeout -= 50;
+ usleep(50 * 1000);
+ continue;
+ }
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("swtpm did not write pidfile '%s'"),
+ tpm->data.emulator.pidfile);
+ goto error;
+ }
+ VIR_FREE(pidfiledata);
+
ret = 0;
cleanup:
@@ -158,7 +179,7 @@ qemuExtTPMStartEmulator(virQEMUDriverPtr driver,
return ret;
error:
- virTPMStopEmulator(tpm, vmuuid, false);
+ virTPMStopEmulator(tpm, vmuuid, false, cfg->stateDir, def->name);
VIR_FREE(tpm->data.emulator.source.data.nix.path);
goto cleanup;
@@ -186,11 +207,15 @@ qemuExtTPMStart(virQEMUDriverPtr driver,
}
static void
-qemuExtTPMStop(virDomainObjPtr vm)
+qemuExtTPMStop(virQEMUDriverPtr driver,
+ virDomainObjPtr vm)
{
+ virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
+
switch (vm->def->tpm->type) {
case VIR_DOMAIN_TPM_TYPE_EMULATOR:
- virTPMStopEmulator(vm->def->tpm, vm->def->uuid, false);
+ virTPMStopEmulator(vm->def->tpm, vm->def->uuid, false,
+ cfg->stateDir, vm->def->name);
break;
case VIR_DOMAIN_TPM_TYPE_PASSTHROUGH:
case VIR_DOMAIN_TPM_TYPE_LAST:
@@ -212,8 +237,9 @@ qemuExtDevicesStart(virQEMUDriverPtr driver,
}
void
-qemuExtDevicesStop(virDomainObjPtr vm)
+qemuExtDevicesStop(virQEMUDriverPtr driver,
+ virDomainObjPtr vm)
{
if (vm->def->tpm)
- qemuExtTPMStop(vm);
+ qemuExtTPMStop(driver, vm);
}
diff --git a/src/qemu/qemu_extdevice.h b/src/qemu/qemu_extdevice.h
index 4dcaec3..3bf81ad 100644
--- a/src/qemu/qemu_extdevice.h
+++ b/src/qemu/qemu_extdevice.h
@@ -30,7 +30,7 @@ int qemuExtDevicesStart(virQEMUDriverPtr driver,
qemuDomainLogContextPtr logCtxt)
ATTRIBUTE_RETURN_CHECK;
-void qemuExtDevicesStop(virDomainObjPtr vm);
+void qemuExtDevicesStop(virQEMUDriverPtr driver, virDomainObjPtr vm);
#endif /* __QEMU_EXTDEVICE_H__ */
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 26acfab..804412a 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -6068,6 +6068,10 @@ qemuProcessLaunch(virConnectPtr conn,
if (qemuProcessSetupEmulator(vm) < 0)
goto cleanup;
+ VIR_DEBUG("Setting cgroup for external devices (if required)");
+ if (qemuSetupCgroupForExtDevices(vm) < 0)
+ goto cleanup;
+
VIR_DEBUG("Setting up resctrl");
if (qemuProcessResctrlCreate(driver, vm) < 0)
goto cleanup;
@@ -6196,7 +6200,7 @@ qemuProcessLaunch(virConnectPtr conn,
cleanup:
if (ret)
- qemuExtDevicesStop(vm);
+ qemuExtDevicesStop(driver, vm);
qemuDomainSecretDestroy(vm);
virCommandFree(cmd);
virObjectUnref(logCtxt);
@@ -6563,7 +6567,7 @@ void qemuProcessStop(virQEMUDriverPtr driver,
/* Clear network bandwidth */
virDomainClearNetBandwidth(vm);
- qemuExtDevicesStop(vm);
+ qemuExtDevicesStop(driver, vm);
virDomainConfVMNWFilterTeardown(vm);
diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c
index 0a31947..4809f12 100644
--- a/src/util/vircgroup.c
+++ b/src/util/vircgroup.c
@@ -1245,6 +1245,38 @@ virCgroupAddMachineTask(virCgroupPtr group, pid_t pid)
return virCgroupAddTaskInternal(group, pid, true);
}
+/**
+ * virCgroupAddProc:
+ *
+ * @group: The cgroup to add a process to
+ * @pid: The pid of the process to add
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int
+virCgroupAddProc(virCgroupPtr group, pid_t pid)
+{
+ int ret = -1;
+ size_t i;
+
+ for (i = 0; i < VIR_CGROUP_CONTROLLER_LAST; i++) {
+ /* Skip over controllers not mounted */
+ if (!group->controllers[i].mountPoint)
+ continue;
+
+ /* We must never add tasks in systemd's hierarchy */
+ if (i == VIR_CGROUP_CONTROLLER_SYSTEMD)
+ continue;
+
+ if (virCgroupSetValueU64(group, i, "cgroup.procs", pid) < 0)
+ goto cleanup;
+ }
+
+ ret = 0;
+ cleanup:
+ return ret;
+}
+
/**
* virCgroupAddTaskController:
@@ -4298,6 +4330,16 @@ virCgroupAddMachineTask(virCgroupPtr group ATTRIBUTE_UNUSED,
int
+virCgroupAddProc(virCgroupPtr group ATTRIBUTE_UNUSED,
+ pid_t pid ATTRIBUTE_UNUSED)
+{
+ virReportSystemError(ENXIO, "%s",
+ _("Control groups not supported on this platform"));
+ return -1;
+}
+
+
+int
virCgroupAddTaskController(virCgroupPtr group ATTRIBUTE_UNUSED,
pid_t pid ATTRIBUTE_UNUSED,
int controller ATTRIBUTE_UNUSED)
diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h
index d833927..82b3964 100644
--- a/src/util/vircgroup.h
+++ b/src/util/vircgroup.h
@@ -132,6 +132,7 @@ int virCgroupPathOfController(virCgroupPtr group,
int virCgroupAddTask(virCgroupPtr group, pid_t pid);
int virCgroupAddMachineTask(virCgroupPtr group, pid_t pid);
+int virCgroupAddProc(virCgroupPtr group, pid_t pid);
int virCgroupAddTaskController(virCgroupPtr group,
pid_t pid,
diff --git a/src/util/virtpm.c b/src/util/virtpm.c
index 7390895..6c4a024 100644
--- a/src/util/virtpm.c
+++ b/src/util/virtpm.c
@@ -39,6 +39,7 @@
#include "virlog.h"
#include "virtpm.h"
#include "virutil.h"
+#include "virpidfile.h"
#include "configmake.h"
#define VIR_FROM_THIS VIR_FROM_NONE
@@ -420,11 +421,31 @@ virTPMSetupEmulator(const char *storagepath, const unsigned char
*vmuuid,
}
/*
+ * virTPMGetPidfile
+ */
+static char *virTPMGetPidfile(const char *stateDir, const char *vmname)
+{
+ char *pidfile = NULL;
+ char *devname = NULL;
+
+ if (virAsprintf(&devname, "%s-swtpm", vmname) < 0)
+ return NULL;
+
+ pidfile = virPidFileBuildPath(stateDir, devname);
+
+ VIR_FREE(devname);
+
+ return pidfile;
+}
+
+/*
* virTPMBuildEmulatorCommand
*
* @tpm: TPM definition
* @vmuuid: The UUID of the VM
* @swtpm_user: The uid for the swtpm to run as (drop privileges to from root)
+ * @statedir: libvirt's statedir
+ * @vmname: name of the VM
*
* Create the virCommand use for starting the emulator
* Do some initializations on the way, such as creation of storage
@@ -432,12 +453,15 @@ virTPMSetupEmulator(const char *storagepath, const unsigned char
*vmuuid,
*/
virCommandPtr
virTPMEmulatorBuildCommand(virDomainTPMDefPtr tpm, const unsigned char *vmuuid,
- uid_t swtpm_user)
+ uid_t swtpm_user, const char *stateDir,
+ const char *vmname)
{
virCommandPtr cmd = NULL;
char *storagepath = NULL;
char *logfile = NULL;
+ char *pidfile = NULL;
bool created = false;
+ int pidfilefd = -1;
if (virTPMEmulatorInit() < 0)
return NULL;
@@ -497,9 +521,31 @@ virTPMEmulatorBuildCommand(virDomainTPMDefPtr tpm, const unsigned
char *vmuuid,
break;
}
+ if (!(pidfile = virTPMGetPidfile(stateDir, vmname)))
+ goto error;
+
+ /*
+ * We create the pidfile and pass it as a file descriptor so
+ * the swtpm can write its pid after daemonizing.
+ * swtpm_user cannot delete files in stateDir, so we have to
+ * delete it later as well.
+ */
+ pidfilefd = virFileOpenAs(pidfile, O_CREAT|O_WRONLY|O_TRUNC, 0644,
+ 0, 0, VIR_FILE_OPEN_NOFORK);
+ if (pidfilefd < 0) {
+ virReportSystemError(errno,
+ _("Could not open file %s for writing"),
+ pidfile);
+ goto error;
+ }
+ virCommandAddArg(cmd, "--pid");
+ virCommandAddArgFormat(cmd, "fd=%u", pidfilefd);
+ virCommandPassFD(cmd, pidfilefd, VIR_COMMAND_PASS_FD_CLOSE_PARENT);
+
tpm->data.emulator.storagepath = storagepath;
VIR_FREE(tpm->data.emulator.logfile);
tpm->data.emulator.logfile = logfile;
+ tpm->data.emulator.pidfile = pidfile;
return cmd;
@@ -510,6 +556,8 @@ virTPMEmulatorBuildCommand(virDomainTPMDefPtr tpm, const unsigned char
*vmuuid,
VIR_FREE(tpm->data.emulator.source.data.nix.path);
VIR_FREE(storagepath);
VIR_FREE(logfile);
+ VIR_FREE(pidfile);
+ VIR_FORCE_CLOSE(pidfilefd);
virCommandFree(cmd);
@@ -521,17 +569,20 @@ virTPMEmulatorBuildCommand(virDomainTPMDefPtr tpm, const unsigned
char *vmuuid,
* @tpm: TPM definition
* @vmuuid: the UUID of the VM
* @verbose: whether to report errors
+ * @statedir: libvirt's statedir
+ * @vmname: name of the VM
*
* Gracefully stop the swptm
*/
void
virTPMStopEmulator(virDomainTPMDefPtr tpm, const unsigned char *vmuuid,
- bool verbose)
+ bool verbose, const char *stateDir, const char *vmname)
{
virCommandPtr cmd;
int exitstatus;
char *pathname;
char *errbuf = NULL;
+ char *pidfile;
(void)vmuuid;
if (virTPMEmulatorInit() < 0)
@@ -563,6 +614,12 @@ virTPMStopEmulator(virDomainTPMDefPtr tpm, const unsigned char
*vmuuid,
unlink(pathname);
VIR_FREE(pathname);
+ /* clean up the PID file */
+ if ((pidfile = virTPMGetPidfile(stateDir, vmname))) {
+ unlink(pidfile);
+ VIR_FREE(pidfile);
+ }
+
VIR_FREE(tpm->data.emulator.source.data.nix.path);
tpm->data.emulator.source.type = 0;
VIR_FREE(errbuf);
diff --git a/src/util/virtpm.h b/src/util/virtpm.h
index 424718b..9a6eccb 100644
--- a/src/util/virtpm.h
+++ b/src/util/virtpm.h
@@ -30,9 +30,11 @@ typedef virDomainTPMDef *virDomainTPMDefPtr;
char *virTPMCreateCancelPath(const char *devpath) ATTRIBUTE_NOINLINE;
virCommandPtr virTPMEmulatorBuildCommand(virDomainTPMDefPtr tpm,
const unsigned char *vmuuid,
- uid_t swtpm_user) ATTRIBUTE_RETURN_CHECK;
+ uid_t swtpm_user,
+ const char *stateDir,
+ const char *vmname) ATTRIBUTE_RETURN_CHECK;
void virTPMStopEmulator(virDomainTPMDefPtr tpm, const unsigned char *vmuuid,
- bool verbose);
+ bool verbose, const char *stateDir, const char *vmname);
void virTPMDeleteEmulatorStorage(const unsigned char *vmuuid);
int virTPMTryConnect(const char *pathname, unsigned long timeout_ms);
--
2.5.5