This patch sets up the cgroups for QEMU instances. It creates a cgroup when starting a guest, uses an exec hook to place the process into the correct cgroup, and cleans up empty cgroup when the QEMU process shuts down. Daniel diff --git a/src/qemu_driver.c b/src/qemu_driver.c --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -68,6 +68,7 @@ #include "memory.h" #include "uuid.h" #include "domain_conf.h" +#include "cgroup.h" #define VIR_FROM_THIS VIR_FROM_QEMU @@ -1134,6 +1135,79 @@ static int qemudNextFreeVNCPort(struct q static virDomainPtr qemudDomainLookupByName(virConnectPtr conn, const char *name); +static int qemuSetupCgroup(virConnectPtr conn, + struct qemud_driver *driver ATTRIBUTE_UNUSED, + virDomainObjPtr vm) +{ + virCgroupPtr cgroup = NULL; + + if (virCgroupHaveSupport() != 0) + return 0; /* Not supported, so claim success */ + + if (virCgroupForDomain(vm->def, "qemu", &cgroup) !=0 ) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("Unable to create cgroup for %s\n"), + vm->def->name); + goto cleanup; + } + + virCgroupFree(&cgroup); + return 0; + +cleanup: + if (cgroup) { + virCgroupRemove(cgroup); + virCgroupFree(&cgroup); + } + return -1; +} + + +static int qemuRemoveCgroup(virConnectPtr conn, + struct qemud_driver *driver ATTRIBUTE_UNUSED, + virDomainObjPtr vm) +{ + virCgroupPtr cgroup; + + if (virCgroupHaveSupport() != 0) + return 0; /* Not supported, so claim success */ + + if (virCgroupForDomain(vm->def, "qemu", &cgroup) != 0) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("Unable to find cgroup for %s\n"), + vm->def->name); + return -1; + } + + virCgroupRemove(cgroup); + virCgroupFree(&cgroup); + return 0; +} + +static int qemuAddToCgroup(void *data) +{ + virDomainDefPtr def = data; + virCgroupPtr cgroup = NULL; + + if (virCgroupHaveSupport() != 0) + return 0; /* Not supported, so claim success */ + + if (virCgroupForDomain(def, "qemu", &cgroup) != 0) { + qemudReportError(NULL, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("Unable to find cgroup for %s\n"), + def->name); + return -1; + } + + if (virCgroupAddTask(cgroup, getpid()) < 0) { + virCgroupFree(&cgroup); + return -1; + } + + virCgroupFree(&cgroup); + return 0; +} + static int qemudStartVMDaemon(virConnectPtr conn, struct qemud_driver *driver, virDomainObjPtr vm, @@ -1160,6 +1234,8 @@ static int qemudStartVMDaemon(virConnect return -1; } + qemuRemoveCgroup(conn, driver, vm); + if (vm->def->graphics && vm->def->graphics->type == VIR_DOMAIN_GRAPHICS_TYPE_VNC && vm->def->graphics->data.vnc.autoport) { @@ -1185,8 +1261,11 @@ static int qemudStartVMDaemon(virConnect emulator = vm->def->emulator; if (!emulator) emulator = virDomainDefDefaultEmulator(conn, vm->def, driver->caps); - if (!emulator) - return -1; + if (!emulator) { + close(vm->logfile); + vm->logfile = -1; + return -1; + } /* Make sure the binary we are about to try exec'ing exists. * Technically we could catch the exec() failure, but that's @@ -1196,6 +1275,8 @@ static int qemudStartVMDaemon(virConnect virReportSystemError(conn, errno, _("Cannot find QEMU binary %s"), emulator); + close(vm->logfile); + vm->logfile = -1; return -1; } @@ -1205,7 +1286,14 @@ static int qemudStartVMDaemon(virConnect qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("Cannot determine QEMU argv syntax %s"), emulator); - return -1; + close(vm->logfile); + vm->logfile = -1; + return -1; + } + + if (qemuSetupCgroup(conn, driver, vm) < 0) { + close(vm->logfile); + vm->logfile = -1; } vm->def->id = driver->nextvmid++; @@ -1249,9 +1337,10 @@ static int qemudStartVMDaemon(virConnect for (i = 0 ; i < ntapfds ; i++) FD_SET(tapfds[i], &keepfd); - ret = virExec(conn, argv, progenv, &keepfd, &child, - stdin_fd, &vm->logfile, &vm->logfile, - VIR_EXEC_NONBLOCK | VIR_EXEC_DAEMON); + ret = virExecWithHook(conn, argv, progenv, &keepfd, &child, + stdin_fd, &vm->logfile, &vm->logfile, + VIR_EXEC_NONBLOCK | VIR_EXEC_DAEMON, + qemuAddToCgroup, vm->def); /* wait for qemu process to to show up */ if (ret == 0) { @@ -1278,6 +1367,8 @@ static int qemudStartVMDaemon(virConnect "%s", _("Unable to daemonize QEMU process")); ret = -1; } + } else { + qemuRemoveCgroup(conn, driver, vm); } if (ret == 0) { @@ -1315,7 +1406,7 @@ static int qemudStartVMDaemon(virConnect } -static void qemudShutdownVMDaemon(virConnectPtr conn ATTRIBUTE_UNUSED, +static void qemudShutdownVMDaemon(virConnectPtr conn, struct qemud_driver *driver, virDomainObjPtr vm) { if (!virDomainIsActive(vm)) @@ -1351,6 +1442,9 @@ static void qemudShutdownVMDaemon(virCon qemudLog(QEMUD_WARN, _("Failed to remove domain status for %s"), vm->def->name); } + + qemuRemoveCgroup(conn, driver, vm); + vm->pid = -1; vm->def->id = -1; vm->state = VIR_DOMAIN_SHUTOFF; -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|

This patch implements the schedular parameter APIs. This adds a single tunable 'cpu_shares' that is provided by cgroups. This is a slightly more fancy way of doing nice priorities, giving a way to tune relative priority of VMs Daniel diff --git a/src/qemu_driver.c b/src/qemu_driver.c --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -3844,6 +3844,111 @@ cleanup: return ret; } + +static char *qemuGetSchedulerType(virDomainPtr domain ATTRIBUTE_UNUSED, + int *nparams) +{ + if (nparams) + *nparams = 1; + + return strdup("posix"); +} + +static int qemuSetSchedulerParameters(virDomainPtr domain, + virSchedParameterPtr params, + int nparams) +{ + struct qemud_driver *driver = domain->conn->privateData; + int i; + virCgroupPtr group = NULL; + virDomainObjPtr vm = NULL; + int ret = -1; + + if (virCgroupHaveSupport() != 0) + return -1; + + qemuDriverLock(driver); + vm = virDomainFindByUUID(&driver->domains, domain->uuid); + qemuDriverUnlock(driver); + + if (vm == NULL) { + qemudReportError(domain->conn, domain, NULL, VIR_ERR_INTERNAL_ERROR, + _("No such domain %s"), domain->uuid); + goto cleanup; + } + + if (virCgroupForDomain(vm->def, "qemu", &group) != 0) + goto cleanup; + + for (i = 0; i < nparams; i++) { + virSchedParameterPtr param = &params[i]; + + if (STREQ(param->field, "cpu_shares")) { + if (virCgroupSetCpuShares(group, params[i].value.ui) != 0) + goto cleanup; + } else { + qemudReportError(domain->conn, domain, NULL, VIR_ERR_INVALID_ARG, + _("Invalid parameter `%s'"), param->field); + goto cleanup; + } + } + ret = 0; + +cleanup: + virCgroupFree(&group); + if (vm) + virDomainObjUnlock(vm); + return ret; +} + +static int qemuGetSchedulerParameters(virDomainPtr domain, + virSchedParameterPtr params, + int *nparams) +{ + struct qemud_driver *driver = domain->conn->privateData; + virCgroupPtr group = NULL; + virDomainObjPtr vm = NULL; + unsigned long val; + int ret = -1; + + if (virCgroupHaveSupport() != 0) + return -1; + + if ((*nparams) != 1) { + qemudReportError(domain->conn, domain, NULL, VIR_ERR_INVALID_ARG, + "%s", _("Invalid parameter count")); + return -1; + } + + qemuDriverLock(driver); + vm = virDomainFindByUUID(&driver->domains, domain->uuid); + qemuDriverUnlock(driver); + + if (vm == NULL) { + qemudReportError(domain->conn, domain, NULL, VIR_ERR_INTERNAL_ERROR, + _("No such domain %s"), domain->uuid); + goto cleanup; + } + + if (virCgroupForDomain(vm->def, "qemu", &group) != 0) + goto cleanup; + + if (virCgroupGetCpuShares(group, &val) != 0) + goto cleanup; + params[0].value.ul = val; + strncpy(params[0].field, "cpu_shares", sizeof(params[0].field)); + params[0].type = VIR_DOMAIN_SCHED_FIELD_ULLONG; + + ret = 0; + +cleanup: + virCgroupFree(&group); + if (vm) + virDomainObjUnlock(vm); + return ret; +} + + /* This uses the 'info blockstats' monitor command which was * integrated into both qemu & kvm in late 2007. If the command is * not supported we detect this and return the appropriate error. @@ -4650,9 +4755,9 @@ static virDriver qemuDriver = { qemudDomainDetachDevice, /* domainDetachDevice */ qemudDomainGetAutostart, /* domainGetAutostart */ qemudDomainSetAutostart, /* domainSetAutostart */ - NULL, /* domainGetSchedulerType */ - NULL, /* domainGetSchedulerParameters */ - NULL, /* domainSetSchedulerParameters */ + qemuGetSchedulerType, /* domainGetSchedulerType */ + qemuGetSchedulerParameters, /* domainGetSchedulerParameters */ + qemuSetSchedulerParameters, /* domainSetSchedulerParameters */ NULL, /* domainMigratePrepare (v1) */ qemudDomainMigratePerform, /* domainMigratePerform */ NULL, /* domainMigrateFinish */ -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|

This patch is more focused on access control. CGroups has a controller that enforces ACLs on device nodes. This allows us to restrict exactly what block/character devices a guest is allowed to access. So in the absence of something like SELinux sVirt, you can get a degree of isolation between VMs on block device backed disks. This sets up an initial deny-all policy, and then iterates over all the disks defined for a VM, allowing each one in turn. Finally it allows a handy of common nodes like /dev/null, /dev/random, /dev/ptmx and friends, which all processes need to use. Daniel diff --git a/src/cgroup.c b/src/cgroup.c --- a/src/cgroup.c +++ b/src/cgroup.c @@ -788,6 +788,23 @@ int virCgroupAllowDeviceMajor(virCgroupP return rc; } +int virCgroupAllowDevicePath(virCgroupPtr group, + const char *path) +{ + struct stat sb; + + if (stat(path, &sb) < 0) + return -errno; + + if (!S_ISCHR(sb.st_mode) && !S_ISBLK(sb.st_mode)) + return -EINVAL; + + return virCgroupAllowDevice(group, + S_ISCHR(sb.st_mode) ? 'c' : 'b', + major(sb.st_rdev), + minor(sb.st_rdev)); +} + int virCgroupSetCpuShares(virCgroupPtr group, unsigned long shares) { return virCgroupSetValueU64(group, "cpu.shares", (uint64_t)shares); diff --git a/src/cgroup.h b/src/cgroup.h --- a/src/cgroup.h +++ b/src/cgroup.h @@ -38,6 +38,8 @@ int virCgroupAllowDevice(virCgroupPtr gr int virCgroupAllowDeviceMajor(virCgroupPtr group, char type, int major); +int virCgroupAllowDevicePath(virCgroupPtr group, + const char *path); int virCgroupSetCpuShares(virCgroupPtr group, unsigned long shares); int virCgroupGetCpuShares(virCgroupPtr group, unsigned long *shares); diff --git a/src/qemu_driver.c b/src/qemu_driver.c --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -1135,11 +1135,18 @@ static int qemudNextFreeVNCPort(struct q static virDomainPtr qemudDomainLookupByName(virConnectPtr conn, const char *name); +static const char *const devs[] = { + "/dev/null", "/dev/full", "/dev/zero", + "/dev/random", "/dev/urandom", + "/dev/ptmx", "/dev/kvm", "/dev/kqemu", +}; + static int qemuSetupCgroup(virConnectPtr conn, struct qemud_driver *driver ATTRIBUTE_UNUSED, virDomainObjPtr vm) { virCgroupPtr cgroup = NULL; + unsigned int i; if (virCgroupHaveSupport() != 0) return 0; /* Not supported, so claim success */ @@ -1151,6 +1158,41 @@ static int qemuSetupCgroup(virConnectPtr goto cleanup; } + if (virCgroupDenyAllDevices(cgroup) != 0) + goto cleanup; + + for (i = 0; i < vm->def->ndisks ; i++) { + if (vm->def->disks[i]->type != VIR_DOMAIN_DISK_TYPE_BLOCK) + continue; + + if (virCgroupAllowDevicePath(cgroup, + vm->def->disks[i]->src) < 0) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("unable to allow device %s"), + vm->def->disks[i]->src); + goto cleanup; + } + } + + if (virCgroupAllowDeviceMajor(cgroup, 'c', 136) < 0) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("unable to allow device %s"), + devs[i]); + goto cleanup; + } + + for (i = 0; i < ARRAY_CARDINALITY(devs) ; i++) { + int rc; + if ((rc = virCgroupAllowDevicePath(cgroup, + devs[i])) < 0 && + rc != -ENOENT) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("unable to allow device %s"), + devs[i]); + goto cleanup; + } + } + virCgroupFree(&cgroup); return 0; -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|