Our virCommand module allows us to set a pidfile for commands we want to spawn. The caller constructs the string of pidfile path and then uses virCommandSetPidFile() to tell the module to write the pidfile once the command is ran. This usually works, but has two flaws: 1) the child process does not hold the pidfile open & locked. Therefore, the caller (or anybody else) can't use our fancy virPidFileForceCleanupPath() function to kill the command afterwards. Also, for everybody else on the system it's needlessly harder to check if the pid from the pidfile is still alive or not. 2) if the caller ever makes a mistake and passes the same pidfile path for two different commands, the start of the second command will overwrite the pidfile even though the first command might still be running. Signed-off-by: Michal Privoznik <mprivozn(a)redhat.com&gt; --- docs/internals/command.html.in | 4 ++- src/util/vircommand.c | 56 +++++++++++++++++++++++++++++----- tests/commanddata/test4.log | 1 + 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/docs/internals/command.html.in b/docs/internals/command.html.in index 8a9061e70f..823d89cc71 100644 --- a/docs/internals/command.html.in +++ b/docs/internals/command.html.in @@ -226,7 +226,9 @@ virCommandSetPidFile(cmd, "/var/run/dnsmasq.pid"); <p> This PID file is guaranteed to be written before - the intermediate process exits. + the intermediate process exits. Moreover, the daemonized + process will inherit the FD of the opened and locked PID + file. </p> <h3><a id="privs">Reducing command privileges</a></h3> diff --git a/src/util/vircommand.c b/src/util/vircommand.c index 9ffa0cf49b..77078d09fb 100644 --- a/src/util/vircommand.c +++ b/src/util/vircommand.c @@ -612,6 +612,7 @@ virExec(virCommandPtr cmd) int null = -1; int pipeout[2] = {-1, -1}; int pipeerr[2] = {-1, -1}; + int pipesync[2] = {-1, -1}; int childin = cmd->infd; int childout = -1; int childerr = -1; @@ -745,9 +746,15 @@ virExec(virCommandPtr cmd) /* Initialize full logging for a while */ virLogSetFromEnv(); + if (cmd->pidfile && + virPipe(pipesync) < 0) + goto fork_error; + /* Daemonize as late as possible, so the parent process can detect * the above errors with wait* */ if (cmd->flags & VIR_EXEC_DAEMON) { + char c; + if (setsid() < 0) { virReportSystemError(errno, "%s", _("cannot become session leader")); @@ -768,12 +775,13 @@ virExec(virCommandPtr cmd) } if (pid > 0) { - if (cmd->pidfile && (virPidFileWritePath(cmd->pidfile, pid) < 0)) { - if (virProcessKillPainfully(pid, true) >= 0) - virReportSystemError(errno, - _("could not write pidfile %s for %d"), - cmd->pidfile, pid); - goto fork_error; + /* The parent expect us to have written the pid file before + * exiting. Wait here for the child to write it and signal us. */ + if (cmd->pidfile && + saferead(pipesync[0], &c, sizeof(c)) != sizeof(c)) { + virReportSystemError(errno, "%s", + _("Unable to wait for child process")); + _exit(EXIT_FAILURE); } _exit(EXIT_SUCCESS); } @@ -788,6 +796,37 @@ virExec(virCommandPtr cmd) if (cmd->setMaxCore && virProcessSetMaxCoreSize(0, cmd->maxCore) < 0) goto fork_error; + if (cmd->pidfile) { + VIR_AUTOCLOSE pidfilefd = -1; + int newpidfilefd = -1; + char c; + + pidfilefd = virPidFileAcquirePath(cmd->pidfile, false, getpid()); + if (pidfilefd < 0) + goto fork_error; + if (virSetInherit(pidfilefd, true) < 0) { + virReportSystemError(errno, "%s", + _("Cannot disable close-on-exec flag")); + goto fork_error; + } + + c = '1'; + if (safewrite(pipesync[1], &c, sizeof(c)) != sizeof(c)) { + virReportSystemError(errno, "%s", _("Unable to notify child process")); + goto fork_error; + } + VIR_FORCE_CLOSE(pipesync[0]); + VIR_FORCE_CLOSE(pipesync[1]); + + /* This is here only to move the pidfilefd + * to the lowest possible number. */ + if ((newpidfilefd = dup(pidfilefd)) < 0) { + virReportSystemError(errno, "%s", _("Unable to dup FD")); + goto fork_error; + } + + /* newpidfilefd is intentionally leaked. */ + } if (cmd->hook) { VIR_DEBUG("Run hook %p %p", cmd->hook, cmd->opaque); @@ -1080,8 +1119,9 @@ virCommandPassFDGetFDIndex(virCommandPtr cmd, int fd) * @cmd: the command to modify * @pidfile: filename to use * - * Save the child PID in a pidfile. The pidfile will be populated - * before the exec of the child. + * Save the child PID in a pidfile. The pidfile will be populated before the + * exec of the child and the child will inherit opened and locked FD to the + * pidfile. */ void virCommandSetPidFile(virCommandPtr cmd, const char *pidfile) diff --git a/tests/commanddata/test4.log b/tests/commanddata/test4.log index f170be204e..5820f28307 100644 --- a/tests/commanddata/test4.log +++ b/tests/commanddata/test4.log @@ -9,6 +9,7 @@ ENV:USER=test FD:0 FD:1 FD:2 +FD:3 DAEMON:yes CWD:/ UMASK:0022 -- 2.24.1

Now, that our virCommandSetPidFile() is more intelligent we don't need to rely on the daemon to create and lock the pidfile and use virCommandSetPidFile() at the same time. Signed-off-by: Michal Privoznik <mprivozn(a)redhat.com&gt; --- src/qemu/qemu_process.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index e8401030a2..0324857913 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -2850,7 +2850,6 @@ qemuProcessStartManagedPRDaemon(virDomainObjPtr vm) g_autoptr(virQEMUDriverConfig) cfg = NULL; int errfd = -1; g_autofree char *pidfile = NULL; - int pidfd = -1; g_autofree char *socketPath = NULL; pid_t cpid = -1; g_autoptr(virCommand) cmd = NULL; @@ -2869,10 +2868,6 @@ qemuProcessStartManagedPRDaemon(virDomainObjPtr vm) if (!(pidfile = qemuProcessBuildPRHelperPidfilePath(vm))) goto cleanup; - /* Just try to acquire. Dummy pid will be replaced later */ - if ((pidfd = virPidFileAcquirePath(pidfile, false, -1)) < 0) - goto cleanup; - if (!(socketPath = qemuDomainGetManagedPRSocketPath(priv))) goto cleanup; @@ -2887,13 +2882,10 @@ qemuProcessStartManagedPRDaemon(virDomainObjPtr vm) if (!(cmd = virCommandNewArgList(cfg->prHelperName, "-k", socketPath, - "-f", pidfile, NULL))) goto cleanup; virCommandDaemonize(cmd); - /* We want our virCommand to write child PID into the pidfile - * so that we can read it even before exec(). */ virCommandSetPidFile(cmd, pidfile); virCommandSetErrorFD(cmd, &errfd); @@ -2956,7 +2948,6 @@ qemuProcessStartManagedPRDaemon(virDomainObjPtr vm) if (pidfile) unlink(pidfile); } - VIR_FORCE_CLOSE(pidfd); VIR_FORCE_CLOSE(errfd); return ret; } -- 2.24.1

Now, that we know that the slirp helper will have the pidfile open and locked we can use virPidFileForceCleanupPath() to kill it and unlink the pidfile. Signed-off-by: Michal Privoznik <mprivozn(a)redhat.com&gt; --- src/qemu/qemu_slirp.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/qemu/qemu_slirp.c b/src/qemu/qemu_slirp.c index 5266b36eaa..be586ade12 100644 --- a/src/qemu/qemu_slirp.c +++ b/src/qemu/qemu_slirp.c @@ -246,8 +246,6 @@ qemuSlirpStop(qemuSlirpPtr slirp, g_autofree char *dbus_path = NULL; g_autofree char *id = qemuSlirpGetDBusVMStateId(net); virErrorPtr orig_err; - pid_t pid; - int rc; if (!(pidfile = qemuSlirpCreatePidFilename(cfg, vm->def, net->info.alias))) { VIR_WARN("Unable to construct slirp pidfile path"); @@ -261,17 +259,11 @@ qemuSlirpStop(qemuSlirpPtr slirp, } virErrorPreserveLast(&orig_err); - rc = virPidFileReadPathIfAlive(pidfile, &pid, cfg->slirpHelperName); - if (rc >= 0 && pid != (pid_t) -1) - virProcessKillPainfully(pid, true); - - if (unlink(pidfile) < 0 && - errno != ENOENT) { - virReportSystemError(errno, - _("Unable to remove stale pidfile %s"), - pidfile); + if (virPidFileForceCleanupPath(pidfile) < 0) { + VIR_WARN("Unable to kill slirp process"); + } else { + slirp->pid = 0; } - slirp->pid = 0; dbus_path = qemuSlirpGetDBusPath(cfg, vm->def, net->info.alias); if (dbus_path) { -- 2.24.1

Now, that we know that the virtiofsd will have the pidfile open and locked we can use virPidFileForceCleanupPath() to kill it and unlink the pidfile. Signed-off-by: Michal Privoznik <mprivozn(a)redhat.com&gt; --- src/qemu/qemu_virtiofs.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/qemu/qemu_virtiofs.c b/src/qemu/qemu_virtiofs.c index d579ce1d33..bbe93e0186 100644 --- a/src/qemu/qemu_virtiofs.c +++ b/src/qemu/qemu_virtiofs.c @@ -36,6 +36,8 @@ #define VIR_FROM_THIS VIR_FROM_QEMU +VIR_LOG_INIT("qemu.virtiofs"); + char * qemuVirtioFSCreatePidFilename(virDomainObjPtr vm, @@ -275,28 +277,19 @@ qemuVirtioFSStop(virQEMUDriverPtr driver G_GNUC_UNUSED, { g_autofree char *pidfile = NULL; virErrorPtr orig_err; - pid_t pid = -1; - int rc; virErrorPreserveLast(&orig_err); if (!(pidfile = qemuVirtioFSCreatePidFilename(vm, fs->info.alias))) goto cleanup; - rc = virPidFileReadPathIfAlive(pidfile, &pid, NULL); - if (rc >= 0 && pid != (pid_t) -1) - virProcessKillPainfully(pid, true); - - if (unlink(pidfile) < 0 && - errno != ENOENT) { - virReportSystemError(errno, - _("Unable to remove stale pidfile %s"), - pidfile); + if (virPidFileForceCleanupPath(pidfile) < 0) { + VIR_WARN("Unable to kill virtiofsd process"); + } else { + if (QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock) + unlink(QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock); } - if (QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock) - unlink(QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock); - cleanup: virErrorRestore(&orig_err); } -- 2.24.1

In the network driver code there's networkKillDaemon() which is the same as virProcessKillPainfully(). Replace the former with the later and drop what becomes unused function. Signed-off-by: Michal Privoznik <mprivozn(a)redhat.com&gt; --- src/network/bridge_driver.c | 106 ++++++------------------------------ 1 file changed, 18 insertions(+), 88 deletions(-) diff --git a/src/network/bridge_driver.c b/src/network/bridge_driver.c index ae52455761..f06099297a 100644 --- a/src/network/bridge_driver.c +++ b/src/network/bridge_driver.c @@ -966,67 +966,6 @@ static int networkConnectIsAlive(virConnectPtr conn G_GNUC_UNUSED) } -/* networkKillDaemon: - * - * kill the specified pid/name, and wait a bit to make sure it's dead. - */ -static int -networkKillDaemon(pid_t pid, - const char *daemonName, - const char *networkName) -{ - size_t i; - int ret = -1; - const char *signame = "TERM"; - - /* send SIGTERM, then wait up to 3 seconds for the process to - * disappear, send SIGKILL, then wait for up to another 2 - * seconds. If that fails, log a warning and continue, hoping - * for the best. - */ - for (i = 0; i < 25; i++) { - int signum = 0; - if (i == 0) { - signum = SIGTERM; - } else if (i == 15) { - signum = SIGKILL; - signame = "KILL"; - } - if (kill(pid, signum) < 0) { - if (errno == ESRCH) { - ret = 0; - } else { - VIR_WARN("Failed to terminate %s process %d " - "for network '%s' with SIG%s: %s", - daemonName, pid, networkName, signame, - g_strerror(errno)); - } - return ret; - } - /* NB: since networks have no reference count like - * domains, there is no safe way to unlock the network - * object temporarily, and so we can't follow the - * procedure used by the qemu driver of 1) unlock driver - * 2) sleep, 3) add ref to object 4) unlock object, 5) - * re-lock driver, 6) re-lock object. We may need to add - * that functionality eventually, but for now this - * function is rarely used and, at worst, leaving the - * network driver locked during this loop of sleeps will - * have the effect of holding up any other thread trying - * to make modifications to a network for up to 5 seconds; - * since modifications to networks are much less common - * than modifications to domains, this seems a reasonable - * tradeoff in exchange for less code disruption. - */ - g_usleep(20 * 1000); - } - VIR_WARN("Timed out waiting after SIG%s to %s process %d " - "(network '%s')", - signame, daemonName, pid, networkName); - return ret; -} - - /* the following does not build a file, it builds a list * which is later saved into a file */ @@ -1832,12 +1771,11 @@ static int networkRestartDhcpDaemon(virNetworkDriverStatePtr driver, virNetworkObjPtr obj) { - virNetworkDefPtr def = virNetworkObjGetDef(obj); pid_t dnsmasqPid = virNetworkObjGetDnsmasqPid(obj); /* if there is a running dnsmasq, kill it */ if (dnsmasqPid > 0) { - networkKillDaemon(dnsmasqPid, "dnsmasq", def->name); + virProcessKillPainfully(dnsmasqPid, false); virNetworkObjSetDnsmasqPid(obj, -1); } /* now start dnsmasq if it should be started */ @@ -2066,23 +2004,19 @@ networkRefreshRadvd(virNetworkDriverStatePtr driver, { virNetworkDefPtr def = virNetworkObjGetDef(obj); dnsmasqCapsPtr dnsmasq_caps = networkGetDnsmasqCaps(driver); - char *radvdpidbase; + g_autofree char *radvdpidbase = NULL; + g_autofree char *pidfile = NULL; pid_t radvdPid; /* Is dnsmasq handling RA? */ if (DNSMASQ_RA_SUPPORT(dnsmasq_caps)) { virObjectUnref(dnsmasq_caps); - radvdPid = virNetworkObjGetRadvdPid(obj); - if (radvdPid <= 0) - return 0; - /* radvd should not be running but in case it is */ - if ((networkKillDaemon(radvdPid, "radvd", def->name) >= 0) && - ((radvdpidbase = networkRadvdPidfileBasename(def->name)) - != NULL)) { - virPidFileDelete(driver->pidDir, radvdpidbase); - VIR_FREE(radvdpidbase); + if ((radvdpidbase = networkRadvdPidfileBasename(def->name)) && + (pidfile = virPidFileBuildPath(driver->pidDir, radvdpidbase))) { + /* radvd should not be running but in case it is */ + virPidFileForceCleanupPath(pidfile); + virNetworkObjSetRadvdPid(obj, -1); } - virNetworkObjSetRadvdPid(obj, -1); return 0; } virObjectUnref(dnsmasq_caps); @@ -2110,23 +2044,19 @@ static int networkRestartRadvd(virNetworkObjPtr obj) { virNetworkDefPtr def = virNetworkObjGetDef(obj); - char *radvdpidbase; - pid_t radvdPid = virNeworkObjGetRadvdPid(obj); + g_autofree char *radvdpidbase = NULL; + g_autofree char *pidfile = NULL; - /* if there is a running radvd, kill it */ - if (radvdPid > 0) { - /* essentially ignore errors from the following two functions, - * since there's really no better recovery to be done than to - * just push ahead (and that may be exactly what's needed). - */ - if ((networkKillDaemon(radvdPid, "radvd", def->name) >= 0) && - ((radvdpidbase = networkRadvdPidfileBasename(def->name)) - != NULL)) { - virPidFileDelete(driver->pidDir, radvdpidbase); - VIR_FREE(radvdpidbase); - } + /* If there is a running radvd, kill it. Essentially ignore errors from the + * following two functions, since there's really no better recovery to be + * done than to just push ahead (and that may be exactly what's needed). + */ + if ((radvdpidbase = networkRadvdPidfileBasename(def->name)) && + (pidfile = virPidFileBuildPath(driver->pidDir, radvdpidbase))) { + virPidFileForceCleanupPath(pidfile); virNetworkObjSetRadvdPid(obj, -1); } + /* now start radvd if it should be started */ return networkStartRadvd(obj); } -- 2.24.1