[libvirt] [PATCH v2 0/2] lxc: properly clean up qemu-nbd (resent)
Just resending the initial patch series as it went into a black hole.
Hi all,
Here is the very same patch, but split in two patches. Well, I also moved two comments around between v1 and v2.
Cédric Bosdonnat (2): Add virProcessGetPids to get all tasks of a process lxc: properly clean up qemu-nbd src/libvirt_private.syms | 1 + src/lxc/lxc_controller.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ src/util/virprocess.c | 47 ++++++++++++++++++++++++++++++++++++++++ src/util/virprocess.h | 2 ++ 4 files changed, 106 insertions(+) -- 2.1.4
This function gets all the PIDs listed in /proc/PID/task. This will be needed at least to move all qmeu-nbd tasks to the container cgroup. --- src/libvirt_private.syms | 1 + src/util/virprocess.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virprocess.h | 2 ++ 3 files changed, 50 insertions(+) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 6a95fb9..780cfbb 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1986,6 +1986,7 @@ virProcessAbort; virProcessExitWithStatus; virProcessGetAffinity; virProcessGetNamespaces; +virProcessGetPids; virProcessGetStartTime; virProcessKill; virProcessKillPainfully; diff --git a/src/util/virprocess.c b/src/util/virprocess.c index 7a79970..ce5e106 100644 --- a/src/util/virprocess.c +++ b/src/util/virprocess.c @@ -608,6 +608,53 @@ int virProcessGetAffinity(pid_t pid ATTRIBUTE_UNUSED, #endif /* HAVE_SCHED_GETAFFINITY */ +int virProcessGetPids(pid_t pid, size_t *npids, pid_t **pids) +{ + int ret = -1; + char *taskPath = NULL; + DIR *dir = NULL; + int value; + struct dirent *ent; + + *npids = 0; + *pids = NULL; + + if (virAsprintf(&taskPath, "/proc/%llu/task", + (unsigned long long)pid) < 0) + goto cleanup; + + if (!(dir = opendir(taskPath))) + goto cleanup; + + while ((value = virDirRead(dir, &ent, taskPath)) > 0) { + pid_t tmp_pid; + + /* Skip . and .. */ + if (STRPREFIX(ent->d_name, ".")) + continue; + + if (virStrToLong_i(ent->d_name, NULL, 10, &tmp_pid) < 0) + goto cleanup; + + if (VIR_APPEND_ELEMENT(*pids, *npids, tmp_pid) < 0) + goto cleanup; + } + + if (value < 0) + goto cleanup; + + ret = 0; + + cleanup: + if (!dir) + closedir(dir); + VIR_FREE(taskPath); + if (ret < 0) + VIR_FREE(*pids); + return ret; +} + + int virProcessGetNamespaces(pid_t pid, size_t *nfdlist, int **fdlist) diff --git a/src/util/virprocess.h b/src/util/virprocess.h index c812882..86a633d 100644 --- a/src/util/virprocess.h +++ b/src/util/virprocess.h @@ -62,6 +62,8 @@ int virProcessGetAffinity(pid_t pid, virBitmapPtr *map, int maxcpu); +int virProcessGetPids(pid_t pid, size_t *npids, pid_t **pids); + int virProcessGetStartTime(pid_t pid, unsigned long long *timestamp); -- 2.1.4
On Tue, Jun 16, 2015 at 11:18:48AM +0200, Cédric Bosdonnat wrote:
This function gets all the PIDs listed in /proc/PID/task. This will be needed at least to move all qmeu-nbd tasks to the container cgroup. --- src/libvirt_private.syms | 1 + src/util/virprocess.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virprocess.h | 2 ++ 3 files changed, 50 insertions(+)
ACK Regards, Daniel -- |: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :| |: http://libvirt.org -o- http://virt-manager.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|
On 06/16/2015 05:18 AM, Cédric Bosdonnat wrote:
This function gets all the PIDs listed in /proc/PID/task. This will be needed at least to move all qmeu-nbd tasks to the container cgroup. --- src/libvirt_private.syms | 1 + src/util/virprocess.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virprocess.h | 2 ++ 3 files changed, 50 insertions(+)
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 6a95fb9..780cfbb 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1986,6 +1986,7 @@ virProcessAbort; virProcessExitWithStatus; virProcessGetAffinity; virProcessGetNamespaces; +virProcessGetPids; virProcessGetStartTime; virProcessKill; virProcessKillPainfully; diff --git a/src/util/virprocess.c b/src/util/virprocess.c index 7a79970..ce5e106 100644 --- a/src/util/virprocess.c +++ b/src/util/virprocess.c @@ -608,6 +608,53 @@ int virProcessGetAffinity(pid_t pid ATTRIBUTE_UNUSED, #endif /* HAVE_SCHED_GETAFFINITY */
+int virProcessGetPids(pid_t pid, size_t *npids, pid_t **pids) +{ + int ret = -1; + char *taskPath = NULL; + DIR *dir = NULL; + int value; + struct dirent *ent; + + *npids = 0; + *pids = NULL; + + if (virAsprintf(&taskPath, "/proc/%llu/task", + (unsigned long long)pid) < 0) + goto cleanup; + + if (!(dir = opendir(taskPath))) + goto cleanup; + + while ((value = virDirRead(dir, &ent, taskPath)) > 0) { + pid_t tmp_pid; + + /* Skip . and .. */ + if (STRPREFIX(ent->d_name, ".")) + continue; + + if (virStrToLong_i(ent->d_name, NULL, 10, &tmp_pid) < 0) + goto cleanup; + + if (VIR_APPEND_ELEMENT(*pids, *npids, tmp_pid) < 0) + goto cleanup; + } + + if (value < 0) + goto cleanup; + + ret = 0; + + cleanup: + if (!dir) + closedir(dir);
^^ Coverity got grumpy about a RESOURCE_LEAK here. I'll fix and push the obvious 1 character less patch John
+ VIR_FREE(taskPath); + if (ret < 0) + VIR_FREE(*pids); + return ret; +} + + int virProcessGetNamespaces(pid_t pid, size_t *nfdlist, int **fdlist) diff --git a/src/util/virprocess.h b/src/util/virprocess.h index c812882..86a633d 100644 --- a/src/util/virprocess.h +++ b/src/util/virprocess.h @@ -62,6 +62,8 @@ int virProcessGetAffinity(pid_t pid, virBitmapPtr *map, int maxcpu);
+int virProcessGetPids(pid_t pid, size_t *npids, pid_t **pids); + int virProcessGetStartTime(pid_t pid, unsigned long long *timestamp);
Add the qemu-nbd tasks to the container cgroup to make sure those will be killed when the container is stopped. In order to reliably get the qemu-nbd tasks PIDs, we use /sys/devices/virtual/block/<DEV>/pid as qemu-nbd is daemonizing itself. --- src/lxc/lxc_controller.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c index efbe71f..9b6f0c8 100644 --- a/src/lxc/lxc_controller.c +++ b/src/lxc/lxc_controller.c @@ -107,6 +107,9 @@ struct _virLXCController { pid_t initpid; + size_t nnbdpids; + pid_t *nbdpids; + size_t nveths; char **veths; @@ -283,6 +286,8 @@ static void virLXCControllerFree(virLXCControllerPtr ctrl) virObjectUnref(ctrl->server); virLXCControllerFreeFuse(ctrl); + VIR_FREE(ctrl->nbdpids); + virCgroupFree(&ctrl->cgroup); /* This must always be the last thing to be closed */ @@ -525,6 +530,38 @@ static int virLXCControllerSetupNBDDeviceDisk(virDomainDiskDefPtr disk) return 0; } +static int virLXCControllerAppendNBDPids(virLXCControllerPtr ctrl, + const char *dev) +{ + char *pidpath = NULL; + pid_t *pids; + size_t npids; + size_t i; + int ret = -1; + pid_t pid; + + if (!STRPREFIX(dev, "/dev/") || + virAsprintf(&pidpath, "/sys/devices/virtual/block/%s/pid", dev + 5) < 0) + goto cleanup; + + if (virPidFileReadPath(pidpath, &pid) < 0) + goto cleanup; + + if (virProcessGetPids(pid, &npids, &pids) < 0) + goto cleanup; + + for (i = 0; i < npids; i++) { + if (VIR_APPEND_ELEMENT(ctrl->nbdpids, ctrl->nnbdpids, pids[i]) < 0) + goto cleanup; + } + + ret = 0; + + cleanup: + VIR_FREE(pids); + VIR_FREE(pidpath); + return ret; +} static int virLXCControllerSetupLoopDevices(virLXCControllerPtr ctrl) { @@ -570,6 +607,12 @@ static int virLXCControllerSetupLoopDevices(virLXCControllerPtr ctrl) } else if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_NBD) { if (virLXCControllerSetupNBDDeviceFS(fs) < 0) goto cleanup; + + /* The NBD device will be cleaned up while the cgroup will end. + * For this we need to remember the qemu-nbd pid and add it to + * the cgroup*/ + if (virLXCControllerAppendNBDPids(ctrl, fs->src) < 0) + goto cleanup; } else { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("fs driver %s is not supported"), @@ -629,6 +672,12 @@ static int virLXCControllerSetupLoopDevices(virLXCControllerPtr ctrl) } if (virLXCControllerSetupNBDDeviceDisk(disk) < 0) goto cleanup; + + /* The NBD device will be cleaned up while the cgroup will end. + * For this we need to remember the qemu-nbd pid and add it to + * the cgroup*/ + if (virLXCControllerAppendNBDPids(ctrl, virDomainDiskGetSource(disk)) < 0) + goto cleanup; } else { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("disk driver %s is not supported"), @@ -781,6 +830,7 @@ static int virLXCControllerSetupCgroupLimits(virLXCControllerPtr ctrl) virBitmapPtr auto_nodeset = NULL; int ret = -1; virBitmapPtr nodeset = NULL; + size_t i; VIR_DEBUG("Setting up cgroup resource limits"); @@ -798,6 +848,12 @@ static int virLXCControllerSetupCgroupLimits(virLXCControllerPtr ctrl) if (virCgroupAddTask(ctrl->cgroup, getpid()) < 0) goto cleanup; + /* Add all qemu-nbd tasks to the cgroup */ + for (i = 0; i < ctrl->nnbdpids; i++) { + if (virCgroupAddTask(ctrl->cgroup, ctrl->nbdpids[i]) < 0) + goto cleanup; + } + if (virLXCCgroupSetup(ctrl->def, ctrl->cgroup, nodeset) < 0) goto cleanup; -- 2.1.4
On Tue, Jun 16, 2015 at 11:18:49AM +0200, Cédric Bosdonnat wrote:
Add the qemu-nbd tasks to the container cgroup to make sure those will be killed when the container is stopped. In order to reliably get the qemu-nbd tasks PIDs, we use /sys/devices/virtual/block/<DEV>/pid as qemu-nbd is daemonizing itself. --- src/lxc/lxc_controller.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+)
ACK Regards, Daniel -- |: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :| |: http://libvirt.org -o- http://virt-manager.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|
participants (3)
-
Cédric Bosdonnat -
Daniel P. Berrange -
John Ferlan