Given a virPCIDevice, this function returns the path for the device that controls the vfio group the device belongs to, e.g. "/dev/vfio/15". --- src/libvirt_private.syms | 1 + src/util/virpci.c | 34 ++++++++++++++++++++++++++++++++++ src/util/virpci.h | 2 ++ 3 files changed, 37 insertions(+) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 33ec379..2a2c40e 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1609,6 +1609,7 @@ virPCIDeviceGetReprobe; virPCIDeviceGetStubDriver; virPCIDeviceGetUnbindFromStub; virPCIDeviceGetUsedBy; +virPCIDeviceGetVFIOGroupDev; virPCIDeviceIsAssignable; virPCIDeviceListAdd; virPCIDeviceListCount; diff --git a/src/util/virpci.c b/src/util/virpci.c index 73f36d0..673568f 100644 --- a/src/util/virpci.c +++ b/src/util/virpci.c @@ -1726,6 +1726,40 @@ cleanup: return ret; } +/* virPCIDeviceGetVFIOGroupDev - return the name of the device used to + * control this PCI device's group (e.g. "/dev/vfio/15") + */ +char * +virPCIDeviceGetVFIOGroupDev(virPCIDevicePtr dev) +{ + char *devPath = NULL; + char *groupPath = NULL; + char *groupDev = NULL; + + if (virPCIFile(&devPath, dev->name, "iommu_group") < 0) + goto cleanup; + if (virFileIsLink(devPath) != 1) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Invalid device %s iommu_group file %s is not a symlink"), + dev->name, devPath); + goto cleanup; + } + if (virFileResolveLink(devPath, &groupPath) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to resolve device %s iommu_group symlink %s"), + dev->name, devPath); + goto cleanup; + } + if (virAsprintf(&groupDev, "/dev/vfio/%s", basename(groupPath)) < 0) { + virReportOOMError(); + goto cleanup; + } +cleanup: + VIR_FREE(devPath); + VIR_FREE(groupPath); + return groupDev; +} + static int virPCIDeviceDownstreamLacksACS(virPCIDevicePtr dev) { diff --git a/src/util/virpci.h b/src/util/virpci.h index db0be35..3911b72 100644 --- a/src/util/virpci.h +++ b/src/util/virpci.h @@ -111,6 +111,8 @@ typedef int (*virPCIDeviceFileActor)(virPCIDevicePtr dev, int virPCIDeviceFileIterate(virPCIDevicePtr dev, virPCIDeviceFileActor actor, void *opaque); +char * +virPCIDeviceGetVFIOGroupDev(virPCIDevicePtr dev); int virPCIDeviceIsAssignable(virPCIDevicePtr dev, int strict_acs_check); -- 1.7.11.7

On 04/25/2013 11:57 AM, Laine Stump wrote: This matches what we have done for ptys and sound; but it's rather broad in that the access is now ALWAYS allowed. For ptys, always being enabled makes sense (you seldom start a guest without a pty, and even if we blindly grant the privilege to a guest that doesn't use a pty there's not much they could break by a rogue guest opening a pty device); but for vfio, it's tied to something that we don't want to allow rogue access to, when it can be avoided. Do we want to be a bit smarter and only add it to the file at the point when we start a guest with a vfio device and/or hotplug a vfio device, then revoke the cgroup privileges any time we hot-unplug the last vfio device? If so, it would be similar to how we add cgroup device ACL privileges for block disks at hotplug time, except that we'd need some domain tracking to count how many vfio devices are plugged at once, so we aren't revoking access until the last hot-unplug. I can live with this as-is (we at least audit that it happens), but wonder if it is worth respinning this patch to be smarter about when to allow vfio in the guest. -- Eric Blake eblake redhat com +1-919-301-3266 Libvirt virtualization library http://libvirt.org

On 04/25/2013 11:57 AM, Laine Stump wrote: Line break after space... ...line break before space. Looks a bit inconsistent, but the end result is the same. ACK. -- Eric Blake eblake redhat com +1-919-301-3266 Libvirt virtualization library http://libvirt.org

This patch adds two sets of functions: 1) lower level functions that will immediately set the RLIMIT_MEMLOCK. RLIMIT_NPROC, or RLIMIT_NOFILE of either the current process (using setrlimit()) or any other process (using prlimit()). 2) functions for virCommand* that will setup a virCommand object to set those limits at a later time just after it has forked a new process, but before it execs the new program. configure.ac has prlimit and setrlimit added to the list of functions to check for, and the low level functions are hopefully properly "unsupported" error) on all platforms. You'll notice some ugly typecasting around pid_t; this is all an attempt to follow the formula given to me by Eric for getting it to compile without warnings on all platforms. --- configure.ac | 2 +- src/libvirt_private.syms | 6 ++ src/util/vircommand.c | 38 ++++++++++++ src/util/vircommand.h | 4 ++ src/util/virutil.c | 146 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virutil.h | 4 ++ 6 files changed, 199 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 89dae3d..23c24d2 100644 --- a/configure.ac +++ b/configure.ac @@ -194,7 +194,7 @@ dnl Availability of various common functions (non-fatal if missing), dnl and various less common threadsafe functions AC_CHECK_FUNCS_ONCE([cfmakeraw geteuid getgid getgrnam_r getmntent_r \ getpwuid_r getuid initgroups kill mmap newlocale posix_fallocate \ - posix_memalign regexec sched_getaffinity setns symlink]) + posix_memalign prlimit regexec sched_getaffinity setns setrlimit symlink]) dnl Availability of pthread functions (if missing, win32 threading is dnl assumed). Because of $LIB_PTHREAD, we cannot use AC_CHECK_FUNCS_ONCE. diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 2a2c40e..c988c21 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1185,6 +1185,9 @@ virCommandSetErrorFD; virCommandSetGID; virCommandSetInputBuffer; virCommandSetInputFD; +virCommandSetMaxFiles; +virCommandSetMaxMemLock; +virCommandSetMaxProcesses; virCommandSetOutputBuffer; virCommandSetOutputFD; virCommandSetPidFile; @@ -1904,6 +1907,9 @@ virSetBlocking; virSetCloseExec; virSetDeviceUnprivSGIO; virSetInherit; +virSetMaxFiles; +virSetMaxMemLock; +virSetMaxProcesses; virSetNonBlock; virSetUIDGID; virSetUIDGIDWithCaps; diff --git a/src/util/vircommand.c b/src/util/vircommand.c index ac56a63..d0def8c 100644 --- a/src/util/vircommand.c +++ b/src/util/vircommand.c @@ -107,6 +107,10 @@ struct _virCommand { char *pidfile; bool reap; + unsigned long long maxMemLock; + unsigned int maxProcesses; + unsigned int maxFiles; + uid_t uid; gid_t gid; unsigned long long capabilities; @@ -598,6 +602,13 @@ virExec(virCommandPtr cmd) goto fork_error; } + if (virSetMaxMemLock(-1, cmd->maxMemLock) < 0) + goto fork_error; + if (virSetMaxProcesses(-1, cmd->maxProcesses) < 0) + goto fork_error; + if (virSetMaxFiles(-1, cmd->maxFiles) < 0) + goto fork_error; + if (cmd->hook) { VIR_DEBUG("Run hook %p %p", cmd->hook, cmd->opaque); ret = cmd->hook(cmd->opaque); @@ -958,6 +969,33 @@ virCommandSetUID(virCommandPtr cmd, uid_t uid) cmd->uid = uid; } +void +virCommandSetMaxMemLock(virCommandPtr cmd, unsigned long long bytes) +{ + if (!cmd || cmd->has_error) + return; + + cmd->maxMemLock = bytes; +} + +void +virCommandSetMaxProcesses(virCommandPtr cmd, unsigned int procs) +{ + if (!cmd || cmd->has_error) + return; + + cmd->maxProcesses = procs; +} + +void +virCommandSetMaxFiles(virCommandPtr cmd, unsigned int files) +{ + if (!cmd || cmd->has_error) + return; + + cmd->maxFiles = files; +} + /** * virCommandClearCaps: * @cmd: the command to modify diff --git a/src/util/vircommand.h b/src/util/vircommand.h index 6c13795..18568fe 100644 --- a/src/util/vircommand.h +++ b/src/util/vircommand.h @@ -65,6 +65,10 @@ void virCommandSetGID(virCommandPtr cmd, gid_t gid); void virCommandSetUID(virCommandPtr cmd, uid_t uid); +void virCommandSetMaxMemLock(virCommandPtr cmd, unsigned long long bytes); +void virCommandSetMaxProcesses(virCommandPtr cmd, unsigned int procs); +void virCommandSetMaxFiles(virCommandPtr cmd, unsigned int files); + void virCommandClearCaps(virCommandPtr cmd); void virCommandAllowCap(virCommandPtr cmd, diff --git a/src/util/virutil.c b/src/util/virutil.c index b9de33c..058a069 100644 --- a/src/util/virutil.c +++ b/src/util/virutil.c @@ -38,6 +38,10 @@ #include <sys/types.h> #include <sys/ioctl.h> #include <sys/wait.h> +#if HAVE_SETRLIMIT +# include <sys/time.h> +# include <sys/resource.h> +#endif #if HAVE_MMAP # include <sys/mman.h> #endif @@ -2992,6 +2996,148 @@ virGetGroupName(gid_t gid ATTRIBUTE_UNUSED) } #endif /* HAVE_GETPWUID_R */ +#if HAVE_SETRLIMIT + +# if HAVE_PRLIMIT +static int +virPrLimit(pid_t pid, int resource, struct rlimit *rlim) +{ + return prlimit(pid, resource, rlim, NULL); +} +# else +static int +virPrLimit(pid_t pid ATTRIBUTE_UNUSED, + int resource ATTRIBUTE_UNUSED, + struct rlimit *rlim ATTRIBUTE_UNUSED) +{ + virReportSystemError(ENOSYS, "%s", _("Not supported on this platform")); + return -1; +} +# endif + +int +virSetMaxMemLock(pid_t pid, unsigned long long bytes) +{ + struct rlimit rlim; + + if (bytes == 0) + return 0; + + rlim.rlim_cur = rlim.rlim_max = bytes; + if (pid == (pid_t)-1) { + if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) { + virReportSystemError(errno, + _("cannot limit locked memory to %llu"), + bytes); + return -1; + } + } else { + if (virPrLimit(pid, RLIMIT_MEMLOCK, &rlim) < 0) { + virReportSystemError(errno, + _("cannot limit locked memory " + "of process %lld to %llu"), + (long long int)pid, bytes); + return -1; + } + } + return 0; +} + +int +virSetMaxProcesses(pid_t pid, unsigned int procs) +{ + struct rlimit rlim; + + if (procs == 0) + return 0; + + rlim.rlim_cur = rlim.rlim_max = procs; + if (pid == (pid_t)-1) { + if (setrlimit(RLIMIT_NPROC, &rlim) < 0) { + virReportSystemError(errno, + _("cannot limit number of subprocesses to %u"), + procs); + return -1; + } + } else { + if (virPrLimit(pid, RLIMIT_NPROC, &rlim) < 0) { + virReportSystemError(errno, + _("cannot limit number of subprocesses " + "of process %lld to %u"), + (long long int)pid, procs); + return -1; + } + } + return 0; +} + +int +virSetMaxFiles(pid_t pid, unsigned int files) +{ + struct rlimit rlim; + + if (files == 0) + return 0; + + /* Max number of opened files is one greater than actual limit. See + * man setrlimit. + * + * NB: That indicates to me that we would want the following code + * to say "files - 1", but the original of this code in + * qemu_process.c also had files + 1, so this preserves current + * behavior. + */ + rlim.rlim_cur = rlim.rlim_max = files + 1; + if (pid == (pid_t)-1) { + if (setrlimit(RLIMIT_NOFILE, &rlim) < 0) { + virReportSystemError(errno, + _("cannot limit number of open files to %u"), + files); + return -1; + } + } else { + if (virPrLimit(pid, RLIMIT_NOFILE, &rlim) < 0) { + virReportSystemError(errno, + _("cannot limit number of open files " + "of process %lld to %u"), + (long long int)pid, files); + return -1; + } + } + return 0; +} +#else +int +virSetMaxMemLock(pid_t pid ATTRIBUTE_UNUSED, unsigned long long bytes) +{ + if (bytes == 0) + return 0; + + virReportSystemError(ENOSYS, "%s", _("Not supported on this platform")); + return -1; +} + +int +virSetMaxProcesses(pid_t pid ATTRIBUTE_UNUSED, unsigned int procs) +{ + if (procs == 0) + return 0; + + virReportSystemError(ENOSYS, "%s", _("Not supported on this platform")); + return -1; +} + +int +virSetMaxFiles(pid_t pid ATTRIBUTE_UNUSED, unsigned int files) +{ + if (files == 0) + return 0; + + virReportSystemError(ENOSYS, "%s", _("Not supported on this platform")); + return -1; +} +#endif + #if WITH_CAPNG /* Set the real and effective uid and gid to the given values, while * maintaining the capabilities indicated by bits in @capBits. Return diff --git a/src/util/virutil.h b/src/util/virutil.h index 39033db..8b258c9 100644 --- a/src/util/virutil.h +++ b/src/util/virutil.h @@ -53,6 +53,10 @@ int virSetCloseExec(int fd) ATTRIBUTE_RETURN_CHECK; int virPipeReadUntilEOF(int outfd, int errfd, char **outbuf, char **errbuf); +int virSetMaxMemLock(pid_t pid, unsigned long long bytes); +int virSetMaxProcesses(pid_t pid, unsigned int procs); +int virSetMaxFiles(pid_t pid, unsigned int files); + int virSetUIDGID(uid_t uid, gid_t gid); int virSetUIDGIDWithCaps(uid_t uid, gid_t gid, unsigned long long capBits, bool clearExistingCaps); -- 1.7.11.7

These were previously being set in a custom hook function, but now that virCommand directly supports setting them, we can eliminate that part of the hook and call the APIs directly. --- src/qemu/qemu_process.c | 38 ++------------------------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 925939d..f12d7d5 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -25,8 +25,6 @@ #include <unistd.h> #include <signal.h> #include <sys/stat.h> -#include <sys/time.h> -#include <sys/resource.h> #if defined(__linux__) # include <linux/capability.h> #elif defined(__FreeBSD__) @@ -2453,37 +2451,6 @@ qemuProcessPrepareChardevDevice(virDomainDefPtr def ATTRIBUTE_UNUSED, } -static int -qemuProcessLimits(virQEMUDriverConfigPtr cfg) -{ - struct rlimit rlim; - - if (cfg->maxProcesses > 0) { - rlim.rlim_cur = rlim.rlim_max = cfg->maxProcesses; - if (setrlimit(RLIMIT_NPROC, &rlim) < 0) { - virReportSystemError(errno, - _("cannot limit number of processes to %d"), - cfg->maxProcesses); - return -1; - } - } - - if (cfg->maxFiles > 0) { - /* Max number of opened files is one greater than - * actual limit. See man setrlimit */ - rlim.rlim_cur = rlim.rlim_max = cfg->maxFiles + 1; - if (setrlimit(RLIMIT_NOFILE, &rlim) < 0) { - virReportSystemError(errno, - _("cannot set max opened files to %d"), - cfg->maxFiles); - return -1; - } - } - - return 0; -} - - struct qemuProcessHookData { virConnectPtr conn; virDomainObjPtr vm; @@ -2526,9 +2493,6 @@ static int qemuProcessHook(void *data) if (virSecurityManagerClearSocketLabel(h->driver->securityManager, h->vm->def) < 0) goto cleanup; - if (qemuProcessLimits(h->cfg) < 0) - goto cleanup; - /* This must take place before exec(), so that all QEMU * memory allocation is on the correct NUMA node */ @@ -3697,6 +3661,8 @@ int qemuProcessStart(virConnectPtr conn, } virCommandSetPreExecHook(cmd, qemuProcessHook, &hookData); + virCommandSetMaxProcesses(cmd, cfg->maxProcesses); + virCommandSetMaxFiles(cmd, cfg->maxFiles); VIR_DEBUG("Setting up security labelling"); if (virSecurityManagerSetChildProcessLabel(driver->securityManager, -- 1.7.11.7

VFIO requires all of the guest's memory and IO space to be lockable in RAM. The domain's max_balloon is the maximum amount of memory the domain can have (in KiB). We add a generous 1GiB to that for IO space (still much better than KVM device assignment, where the KVM module actually *ignores* the process limits and locks everything anyway), and convert from KiB to bytes. In the case of hotplug, we are changing the limit for the already existing qemu process (prlimit() is used under the hood), and for regular commandline additions of vfio devices, we schedule a call to setrlimit() that will happen after the qemu process is forked. --- src/qemu/qemu_command.c | 22 +++++++++++++++------- src/qemu/qemu_hotplug.c | 24 +++++++++++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 761291c..80a0dec 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7906,13 +7906,21 @@ qemuBuildCommandLine(virConnectPtr conn, if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS && hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) { - if ((hostdev->source.subsys.u.pci.backend - == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_VFIO) && - !virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("VFIO PCI device assignment is not supported by " - "this version of qemu")); - goto error; + if (hostdev->source.subsys.u.pci.backend + == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_VFIO) { + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("VFIO PCI device assignment is not " + "supported by this version of qemu")); + goto error; + } + /* VFIO requires all of the guest's memory to be + * locked resident, plus some amount for IO + * space. Alex Williamson suggested adding 1GiB for IO + * space just to be safe (some finer tuning might be + * nice, though). + */ + virCommandSetMaxMemLock(cmd, ((unsigned long long)def->mem.max_balloon + (1024 * 1024)) * 1024); } if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE)) { diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 30c8bda..af0b8a6 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -999,13 +999,23 @@ int qemuDomainAttachHostPciDevice(virQEMUDriverPtr driver, &hostdev, 1) < 0) return -1; - if ((hostdev->source.subsys.u.pci.backend - == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_VFIO) && - !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI)) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("VFIO PCI device assignment is not supported by " - "this version of qemu")); - goto error; + if (hostdev->source.subsys.u.pci.backend + == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_VFIO) { + if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("VFIO PCI device assignment is not " + "supported by this version of qemu")); + goto error; + } + /* VFIO requires all of the guest's memory to be locked + * resident, plus some amount for IO space. Alex Williamson + * suggested adding 1GiB for IO space just to be safe (some + * finer tuning might be nice, though). + * In this case, the guest's memory may already be locked, but + * it doesn't hurt to "change" the limit to the same value. + */ + virSetMaxMemLock(vm->pid, + ((unsigned long long)vm->def->mem.max_balloon + (1024 * 1024)) * 1024); } if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE)) { -- 1.7.11.7