>From cf7671af462298650343103b5ec386665533d53c Mon Sep 17 00:00:00 2001
From: =?utf-8?q?Guido=20G=C3=BCnther?= <agx@sigxcpu.org>
Date: Sun, 23 Nov 2008 16:22:59 +0100
Subject: [PATCH] let qemu/kvm survive libvirtd restarts

---
 src/domain_conf.c          |   33 +++--
 src/domain_conf.h          |    6 +-
 src/libvirt_sym.version.in |    1 +
 src/qemu_conf.c            |    8 +
 src/qemu_conf.h            |    1 +
 src/qemu_driver.c          |  398 +++++++++++++++++++++++++++++++++++++++++---
 src/test.c                 |    4 +-
 src/util.c                 |   20 ++-
 src/util.h                 |    2 +
 tests/qemuxml2argvtest.c   |    2 +-
 10 files changed, 433 insertions(+), 42 deletions(-)

diff --git a/src/domain_conf.c b/src/domain_conf.c
index 414b7ff..66e58f9 100644
--- a/src/domain_conf.c
+++ b/src/domain_conf.c
@@ -1310,7 +1310,7 @@ error:
 /* Parse the XML definition for a graphics device */
 static virDomainGraphicsDefPtr
 virDomainGraphicsDefParseXML(virConnectPtr conn,
-                             xmlNodePtr node) {
+                             xmlNodePtr node, int flags) {
     virDomainGraphicsDefPtr def;
     char *type = NULL;
 
@@ -1347,7 +1347,8 @@ virDomainGraphicsDefParseXML(virConnectPtr conn,
             VIR_FREE(port);
             /* Legacy compat syntax, used -1 for auto-port */
             if (def->data.vnc.port == -1) {
-                def->data.vnc.port = 0;
+                if (!flags & VIR_DOMAIN_XML_STATE)
+                    def->data.vnc.port = 0;
                 def->data.vnc.autoport = 1;
             }
         } else {
@@ -1357,7 +1358,8 @@ virDomainGraphicsDefParseXML(virConnectPtr conn,
 
         if ((autoport = virXMLPropString(node, "autoport")) != NULL) {
             if (STREQ(autoport, "yes")) {
-                def->data.vnc.port = 0;
+                if (!flags & VIR_DOMAIN_XML_STATE)
+                    def->data.vnc.port = 0;
                 def->data.vnc.autoport = 1;
             }
             VIR_FREE(autoport);
@@ -1691,11 +1693,12 @@ int virDomainDiskQSort(const void *a, const void *b)
 #ifndef PROXY
 static virDomainDefPtr virDomainDefParseXML(virConnectPtr conn,
                                             virCapsPtr caps,
-                                            xmlXPathContextPtr ctxt)
+                                            xmlXPathContextPtr ctxt, int flags)
 {
     xmlNodePtr *nodes = NULL, node = NULL;
     char *tmp = NULL;
     int i, n;
+    long id = -1;
     virDomainDefPtr def;
 
     if (VIR_ALLOC(def) < 0) {
@@ -1703,7 +1706,11 @@ static virDomainDefPtr virDomainDefParseXML(virConnectPtr conn,
                          "%s", _("failed to allocate space for xmlXPathContext"));
         return NULL;
     }
-    def->id = -1;
+
+    if (flags & VIR_DOMAIN_XML_STATE)
+        if((virXPathLong(conn, "string(./@id)", ctxt, &id)) < 0)
+            id = -1;    
+    def->id = (int)id;
 
     /* Find out what type of virtualization to use */
     if (!(tmp = virXPathString(conn, "string(./@type)", ctxt))) {
@@ -2093,7 +2100,8 @@ static virDomainDefPtr virDomainDefParseXML(virConnectPtr conn,
     }
     if (n > 0) {
         virDomainGraphicsDefPtr graphics = virDomainGraphicsDefParseXML(conn,
-                                                                        nodes[0]);
+                                                                        nodes[0],
+                                                                        flags);
         if (!graphics)
             goto error;
 
@@ -2239,7 +2247,7 @@ virDomainDefPtr virDomainDefParseString(virConnectPtr conn,
         goto cleanup;
     }
 
-    def = virDomainDefParseNode(conn, caps, xml, root);
+    def = virDomainDefParseNode(conn, caps, xml, root, 0);
 
 cleanup:
     xmlFreeParserCtxt (pctxt);
@@ -2249,7 +2257,7 @@ cleanup:
 
 virDomainDefPtr virDomainDefParseFile(virConnectPtr conn,
                                       virCapsPtr caps,
-                                      const char *filename)
+                                      const char *filename, int flags)
 {
     xmlParserCtxtPtr pctxt;
     xmlDocPtr xml = NULL;
@@ -2280,7 +2288,7 @@ virDomainDefPtr virDomainDefParseFile(virConnectPtr conn,
         goto cleanup;
     }
 
-    def = virDomainDefParseNode(conn, caps, xml, root);
+    def = virDomainDefParseNode(conn, caps, xml, root, flags);
 
 cleanup:
     xmlFreeParserCtxt (pctxt);
@@ -2292,7 +2300,8 @@ cleanup:
 virDomainDefPtr virDomainDefParseNode(virConnectPtr conn,
                                       virCapsPtr caps,
                                       xmlDocPtr xml,
-                                      xmlNodePtr root)
+                                      xmlNodePtr root,
+                                      int flags)
 {
     xmlXPathContextPtr ctxt = NULL;
     virDomainDefPtr def = NULL;
@@ -2310,7 +2319,7 @@ virDomainDefPtr virDomainDefParseNode(virConnectPtr conn,
     }
 
     ctxt->node = root;
-    def = virDomainDefParseXML(conn, caps, ctxt);
+    def = virDomainDefParseXML(conn, caps, ctxt, flags);
 
 cleanup:
     xmlXPathFreeContext(ctxt);
@@ -3265,7 +3274,7 @@ virDomainObjPtr virDomainLoadConfig(virConnectPtr conn,
     if ((autostart = virFileLinkPointsTo(autostartLink, configFile)) < 0)
         goto error;
 
-    if (!(def = virDomainDefParseFile(conn, caps, configFile)))
+    if (!(def = virDomainDefParseFile(conn, caps, configFile, 0)))
         goto error;
 
     if (virDomainFindByName(doms, def->name))
diff --git a/src/domain_conf.h b/src/domain_conf.h
index 084c448..5a5e7ce 100644
--- a/src/domain_conf.h
+++ b/src/domain_conf.h
@@ -526,11 +526,13 @@ virDomainDefPtr virDomainDefParseString(virConnectPtr conn,
                                         const char *xmlStr);
 virDomainDefPtr virDomainDefParseFile(virConnectPtr conn,
                                       virCapsPtr caps,
-                                      const char *filename);
+                                      const char *filename,
+                                      int flags);
 virDomainDefPtr virDomainDefParseNode(virConnectPtr conn,
                                       virCapsPtr caps,
                                       xmlDocPtr doc,
-                                      xmlNodePtr root);
+                                      xmlNodePtr root,
+                                      int flags);
 #endif
 char *virDomainDefFormat(virConnectPtr conn,
                          virDomainDefPtr def,
diff --git a/src/libvirt_sym.version.in b/src/libvirt_sym.version.in
index ec9ff3d..4f96fd5 100644
--- a/src/libvirt_sym.version.in
+++ b/src/libvirt_sym.version.in
@@ -570,6 +570,7 @@ LIBVIRT_PRIVATE_@VERSION@ {
 	virFileMakePath;
 	virFileOpenTty;
 	virFileReadLimFD;
+	virFilePid;
 	virFileReadPid;
 	virFileLinkPointsTo;
 	virParseNumber;
diff --git a/src/qemu_conf.c b/src/qemu_conf.c
index c66cef9..5735823 100644
--- a/src/qemu_conf.c
+++ b/src/qemu_conf.c
@@ -722,6 +722,7 @@ int qemudBuildCommandLine(virConnectPtr conn,
     const char *emulator;
     char uuid[VIR_UUID_STRING_BUFLEN];
     char domid[50];
+    char* pidfile;
 
     uname(&ut);
 
@@ -814,6 +815,9 @@ int qemudBuildCommandLine(virConnectPtr conn,
     snprintf(memory, sizeof(memory), "%lu", vm->def->memory/1024);
     snprintf(vcpus, sizeof(vcpus), "%lu", vm->def->vcpus);
     snprintf(domid, sizeof(domid), "%d", vm->def->id);
+    pidfile = virFilePid(driver->stateDir, vm->def->name);
+    if (!pidfile)
+        goto error;
 
     ADD_ENV_LIT("LC_ALL=C");
 
@@ -868,6 +872,9 @@ int qemudBuildCommandLine(virConnectPtr conn,
     ADD_ARG_LIT("-monitor");
     ADD_ARG_LIT("pty");
 
+    ADD_ARG_LIT("-pidfile");
+    ADD_ARG(pidfile);
+
     if (vm->def->localtime)
         ADD_ARG_LIT("-localtime");
 
@@ -1315,6 +1322,7 @@ int qemudBuildCommandLine(virConnectPtr conn,
             VIR_FREE((qenv)[i]);
         VIR_FREE(qenv);
     }
+    VIR_FREE(pidfile);
     return -1;
 
 #undef ADD_ARG
diff --git a/src/qemu_conf.h b/src/qemu_conf.h
index 6b2acad..8de35f6 100644
--- a/src/qemu_conf.h
+++ b/src/qemu_conf.h
@@ -60,6 +60,7 @@ struct qemud_driver {
     char *configDir;
     char *autostartDir;
     char *logDir;
+    char *stateDir;
     unsigned int vncTLS : 1;
     unsigned int vncTLSx509verify : 1;
     char *vncTLSx509certdir;
diff --git a/src/qemu_driver.c b/src/qemu_driver.c
index 281ef78..55b99b8 100644
--- a/src/qemu_driver.c
+++ b/src/qemu_driver.c
@@ -156,6 +156,286 @@ qemudAutostartConfigs(struct qemud_driver *driver) {
     }
 }
 
+
+static int
+qemudFileWriteMonitor(const char *dir,
+                      const char *name,
+                      const char *path)
+{
+    int rc;
+    int fd;
+    FILE *file = NULL;
+    char *monitorfile = NULL;
+
+    if ((rc = virFileMakePath(dir)))
+        goto cleanup;
+
+    if (asprintf(&monitorfile, "%s/%s.monitor", dir, name) < 0) {
+        rc = ENOMEM;
+        goto cleanup;
+    }
+
+    if ((fd = open(monitorfile,
+                   O_WRONLY | O_CREAT | O_TRUNC,
+                   S_IRUSR | S_IWUSR)) < 0) {
+        rc = errno;
+        goto cleanup;
+    }
+
+    if (!(file = fdopen(fd, "w"))) {
+        rc = errno;
+        close(fd);
+        goto cleanup;
+    }
+
+    if (fprintf(file, "%s", path) < 0) {
+        rc = errno;
+        goto cleanup;
+    }
+
+    rc = 0;
+
+cleanup:
+    if (file &&
+        fclose(file) < 0) {
+        rc = errno;
+    }
+
+    VIR_FREE(monitorfile);
+    return rc;
+}
+
+static int
+qemudFileReadMonitor(const char *dir,
+                     const char *name,
+                     char **path)
+{
+    int rc = -1;
+    char *statefile = NULL;
+
+    if (asprintf(&statefile, "%s/%s.monitor", dir, name) < 0) {
+        rc = ENOMEM;
+        goto cleanup;
+    }
+
+    if (virFileReadAll(statefile, PATH_MAX, path) < 0)
+        goto cleanup;
+
+    rc = 0;
+ cleanup:
+    VIR_FREE(statefile);
+    return rc;
+}
+
+
+static int
+qemudGetProcFD(pid_t pid, int fdnum)
+{
+    int fd;
+
+#ifdef __linux__
+    char *path = NULL;
+
+    if (!asprintf(&path, "/proc/%d/fd/%d", pid, fdnum)) {
+        qemudReportError(NULL, NULL, NULL, VIR_ERR_NO_MEMORY, NULL);
+        return -1;
+    }
+
+    if((fd = open(path, O_RDONLY)) < 0) {
+        qemudReportError(NULL, NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                         _("Unable to open %s"), path);
+        return -1;
+    }
+    if (qemudSetCloseExec(fd) < 0) {
+        qemudReportError(NULL, NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                         _("Unable to set close-on-exec flag on %s"), path);
+        goto error;
+    }
+
+    if (qemudSetNonBlock(fd) < 0) {
+        qemudReportError(NULL, NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                          _("Unable to put %s into non-blocking mode"), path);
+        goto error;
+    }
+
+    VIR_FREE(path);
+    return fd;
+error:
+   VIR_FREE(path);
+   close(fd);
+#endif
+   return -1;
+}
+
+
+static int
+qemudGetProcMonitorPath(int fdnum, char* monitor)
+{
+    int ret = -1;
+#ifdef __linux__
+    char *path = NULL;
+    pid_t pid = getpid();
+
+    if (!asprintf(&path, "/proc/%d/fd/%d", pid, fdnum)) {
+        qemudReportError(NULL, NULL, NULL, VIR_ERR_NO_MEMORY, NULL);
+        goto error;
+    }
+
+    monitor[PATH_MAX-1] = '\0';
+    if (readlink(path, monitor, PATH_MAX-1) < 0)
+        goto error;
+    ret = 0;
+
+error:    
+    VIR_FREE(path);
+#endif
+    return ret;
+}
+
+
+static int 
+qemudUnlinkStateFile(const char* dir, const char* name, const char* ext)
+{
+    int rc;
+    char *file = NULL;
+
+    if ((rc = virFileMakePath(dir)))
+        goto cleanup;
+
+    if (asprintf(&file, "%s/%s.%s", dir, name, ext) < 0) {
+        rc = ENOMEM;
+        goto cleanup;
+    }
+
+    if (unlink(file) < 0 && errno != ENOENT && errno != ENOTDIR) {
+        rc = errno;
+        goto cleanup;
+    }
+
+    rc = 0;
+cleanup:
+    VIR_FREE(file);
+    return rc;
+}
+
+
+/**
+ * qemudRemoveDomainState
+ *
+ * remove all state files of a domain from statedir
+ *
+ * Returns 0 on success
+ */
+static int
+qemudRemoveDomainState(struct qemud_driver *driver, virDomainObjPtr vm) {
+    int rc;
+
+    rc = virFileDeletePid(driver->stateDir, vm->def->name);
+    if (qemudUnlinkStateFile(driver->stateDir, vm->def->name, "monitor"))
+        rc = -1;
+    if (qemudUnlinkStateFile(driver->stateDir, vm->def->name, "xml"))
+        rc = -1;
+    return rc;
+}
+
+
+static int qemudOpenMonitor(virConnectPtr conn,
+                            struct qemud_driver *driver,
+                            virDomainObjPtr vm,
+                            const char *monitor,
+                            int reconnect);
+/**
+ * qemudReconnectVMs
+ *
+ * Reconnect running vms to the daemon process
+ */
+static int
+qemudReconnectVMs(struct qemud_driver *driver)
+{
+    int i;
+
+    for (i = 0 ; i < qemu_driver->domains.count ; i++) {
+        virDomainObjPtr vm = qemu_driver->domains.objs[i];
+        virDomainDefPtr tmp;
+        char *config = NULL;
+        char *monitor = NULL;
+        int rc;
+
+        /* Read pid */
+        if ((rc = virFileReadPid(driver->stateDir, vm->def->name, &vm->pid)) == 0)
+            DEBUG("Found pid %d for '%s'", vm->pid, vm->def->name);
+        else
+            goto next;
+
+        if ((config = virDomainConfigFile(NULL,
+                                          driver->stateDir,
+                                          vm->def->name)) == NULL) {
+            qemudLog(QEMUD_ERR, _("Failed to read back domain config for %s\n"),
+                     vm->def->name);
+            goto next_error;
+        }
+
+        tmp = virDomainDefParseFile(NULL, driver->caps, config, VIR_DOMAIN_XML_STATE);
+        if (tmp) {
+            vm->newDef = vm->def;
+            vm->def = tmp;
+        }
+
+        /* read back monitor path */
+        if ((rc = qemudFileReadMonitor(driver->stateDir, vm->def->name, &monitor)) != 0) {
+            qemudLog(QEMUD_ERR, _("Failed to read back monitor path for %s: %s\n"),
+                     vm->def->name, strerror(rc));
+            goto next_error;
+        }
+
+        if ((rc = qemudOpenMonitor(NULL, driver, vm, monitor, 1)) != 0) {
+            qemudLog(QEMUD_ERR, _("Failed to reconnect monitor for %s: %d\n"),
+                     vm->def->name, rc);
+            goto next_error;
+        }
+
+        vm->stdin_fd  = qemudGetProcFD(vm->pid, 0);
+        vm->stdout_fd = qemudGetProcFD(vm->pid, 1);
+        vm->stderr_fd = qemudGetProcFD(vm->pid, 2);
+
+        if (((vm->stdout_watch = virEventAddHandle(vm->stdout_fd,
+                                                   VIR_EVENT_HANDLE_READABLE |
+                                                   VIR_EVENT_HANDLE_ERROR |
+                                                   VIR_EVENT_HANDLE_HANGUP,
+                                                   qemudDispatchVMEvent,
+                                                   driver, NULL)) < 0) ||
+            ((vm->stderr_watch = virEventAddHandle(vm->stderr_fd,
+                                                   VIR_EVENT_HANDLE_READABLE |
+                                                   VIR_EVENT_HANDLE_ERROR |
+                                                   VIR_EVENT_HANDLE_HANGUP,
+                                                   qemudDispatchVMEvent,
+                                                   driver, NULL)) < 0)) {
+    
+            qemudLog(QEMUD_ERR, _("Failed to reconnect to stdout/stderr\n"));
+            goto next_error;
+        }
+
+        if (vm->def->id >= driver->nextvmid)
+            driver->nextvmid = vm->def->id + 1;
+
+        /* FIXME: domain could be suspended or in migration */
+        vm->state = VIR_DOMAIN_RUNNING;
+        goto next;
+
+next_error:
+        /* we failed to reconnect the vm so remove it's traces */
+        vm->def->id = -1;
+        qemudRemoveDomainState(driver, vm);
+        virDomainDefFree(vm->def);
+        vm->def = vm->newDef;
+        vm->newDef = NULL;
+next:
+        VIR_FREE(config);
+        VIR_FREE(monitor);
+    }
+    return 0;
+}
+
 /**
  * qemudStartup:
  *
@@ -185,11 +465,15 @@ qemudStartup(void) {
 
         if ((base = strdup (SYSCONF_DIR "/libvirt")) == NULL)
             goto out_of_memory;
+
+        if (asprintf (&qemu_driver->stateDir,
+                      "%s/run/libvirt/qemu/", LOCAL_STATE_DIR) == -1)
+            goto out_of_memory;
     } else {
         if (!(pw = getpwuid(uid))) {
             qemudLog(QEMUD_ERR, _("Failed to find user record for uid '%d': %s\n"),
                      uid, strerror(errno));
-            goto out_nouid;
+            goto error;
         }
 
         if (asprintf(&qemu_driver->logDir,
@@ -198,6 +482,16 @@ qemudStartup(void) {
 
         if (asprintf (&base, "%s/.libvirt", pw->pw_dir) == -1)
             goto out_of_memory;
+
+        if (asprintf (&qemu_driver->stateDir,
+                      "%s/run/libvirt/qemu/", base) == -1)
+            goto out_of_memory;
+    }
+
+    if (virFileMakePath(qemu_driver->stateDir) < 0) {
+            qemudLog(QEMUD_ERR, _("Failed to create state dir '%s': %s\n"),
+                     qemu_driver->stateDir, strerror(errno));
+            goto error;
     }
 
     /* Configuration paths are either ~/.libvirt/qemu/... (session) or
@@ -232,6 +526,7 @@ qemudStartup(void) {
         qemudShutdown();
         return -1;
     }
+    qemudReconnectVMs(qemu_driver);
     qemudAutostartConfigs(qemu_driver);
 
     return 0;
@@ -239,7 +534,7 @@ qemudStartup(void) {
  out_of_memory:
     qemudLog (QEMUD_ERR,
               "%s", _("qemudStartup: out of memory\n"));
- out_nouid:
+ error:
     VIR_FREE(base);
     VIR_FREE(qemu_driver);
     return -1;
@@ -302,6 +597,40 @@ qemudActive(void) {
 }
 
 /**
+ * qemudSaveDomainState
+ *
+ * Save the full state of a running domain to statedir
+ *
+ * Returns 0 on success
+ */
+static int
+qemudSaveDomainState(virConnectPtr conn, 
+                     struct qemud_driver *driver,
+                     virDomainObjPtr vm) {
+    int ret;
+    char monitor[PATH_MAX];
+
+    if ((ret = virDomainSaveConfig(NULL, driver->stateDir, vm->def))) {
+        qemudReportError(conn, vm, NULL, VIR_ERR_INTERNAL_ERROR,
+                         _("Unable to save domain state\n"));
+        return ret;
+    }
+
+    if (qemudGetProcMonitorPath(vm->monitor, monitor) < 0) {
+        qemudReportError(conn, vm, NULL, VIR_ERR_INTERNAL_ERROR,
+                         _("Unable to read monitor file\n"));
+    }
+    if ((ret = qemudFileWriteMonitor(driver->stateDir, vm->def->name, monitor)) != 0) {
+        qemudReportError(conn, vm, NULL, VIR_ERR_INTERNAL_ERROR,
+                         _("Unable to save monitor file: %s\n"), 
+                         strerror(ret));
+        return ret;
+    }
+    return 0;
+}
+
+
+/**
  * qemudShutdown:
  *
  * Shutdown the QEmu daemon, it will stop all active domains and networks
@@ -317,12 +646,15 @@ qemudShutdown(void) {
 
     /* shutdown active VMs */
     for (i = 0 ; i < qemu_driver->domains.count ; i++) {
+    /* FIXME: don't shutdown VMs on daemon shutdown for now */
+#if 0
         virDomainObjPtr dom = qemu_driver->domains.objs[i];
         if (virDomainIsActive(dom))
             qemudShutdownVMDaemon(NULL, qemu_driver, dom);
         if (!dom->persistent)
             virDomainRemoveInactive(&qemu_driver->domains,
                                     dom);
+#endif
     }
 
     virDomainObjListFree(&qemu_driver->domains);
@@ -335,6 +667,7 @@ qemudShutdown(void) {
 
     /* Free domain callback list */
     virDomainEventCallbackListFree(qemu_driver->domainEventCallbacks);
+    VIR_FREE(qemu_driver->stateDir);
 
     if (qemu_driver->brctl)
         brShutdown(qemu_driver->brctl);
@@ -442,7 +775,7 @@ qemudCheckMonitorPrompt(virConnectPtr conn ATTRIBUTE_UNUSED,
 static int qemudOpenMonitor(virConnectPtr conn,
                             struct qemud_driver *driver,
                             virDomainObjPtr vm,
-                            const char *monitor) {
+                            const char *monitor, int reconnect) {
     int monfd;
     char buf[1024];
     int ret = -1;
@@ -463,11 +796,20 @@ static int qemudOpenMonitor(virConnectPtr conn,
         goto error;
     }
 
-    ret = qemudReadMonitorOutput(conn,
-                                 driver, vm, monfd,
-                                 buf, sizeof(buf),
-                                 qemudCheckMonitorPrompt,
-                                 "monitor");
+    if (!reconnect) {
+        ret = qemudReadMonitorOutput(conn,
+                                     driver, vm, monfd,
+                                     buf, sizeof(buf),
+                                     qemudCheckMonitorPrompt,
+                                     "monitor");
+
+    } else {
+        vm->monitor = monfd;
+        ret = 0;
+    }
+
+    if (ret != 0)
+         goto error;
 
     /* Keep monitor open upon success */
     if (ret == 0)
@@ -563,7 +905,7 @@ qemudFindCharDevicePTYs(virConnectPtr conn,
     }
 
     /* Got them all, so now open the monitor console */
-    ret = qemudOpenMonitor(conn, driver, vm, monitor);
+    ret = qemudOpenMonitor(conn, driver, vm, monitor, 0);
 
 cleanup:
     VIR_FREE(monitor);
@@ -797,6 +1139,7 @@ static int qemudStartVMDaemon(virConnectPtr conn,
     unsigned int qemuCmdFlags;
     fd_set keepfd;
     const char *emulator;
+    pid_t child;
 
     FD_ZERO(&keepfd);
 
@@ -923,12 +1266,26 @@ static int qemudStartVMDaemon(virConnectPtr conn,
     for (i = 0 ; i < ntapfds ; i++)
         FD_SET(tapfds[i], &keepfd);
 
-    ret = virExec(conn, argv, progenv, &keepfd, &vm->pid,
+    ret = virExec(conn, argv, progenv, &keepfd, &child,
                   vm->stdin_fd, &vm->stdout_fd, &vm->stderr_fd,
-                  VIR_EXEC_NONBLOCK);
-    if (ret == 0)
+                  VIR_EXEC_NONBLOCK | VIR_EXEC_DAEMON);
+
+    /* wait for qemu process to to show up */
+    if (ret == 0) {
+        int retries = 100;
+        while (retries) {
+            if ((ret = virFileReadPid(driver->stateDir, vm->def->name, &vm->pid)) == 0)
+                break;
+            usleep(10*1000);
+            retries--;
+        }
+        if (ret)
+            qemudLog(QEMUD_WARN, _("Domain %s didn't show up\n"), vm->def->name);
+    }
+
+    if (ret == 0) {
         vm->state = migrateFrom ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_RUNNING;
-    else
+    } else
         vm->def->id = -1;
 
     for (i = 0 ; argv[i] ; i++)
@@ -966,6 +1323,7 @@ static int qemudStartVMDaemon(virConnectPtr conn,
             return -1;
         }
     }
+    qemudSaveDomainState(conn, qemu_driver, vm);
 
     return ret;
 }
@@ -1004,7 +1362,9 @@ static void qemudShutdownVMDaemon(virConnectPtr conn ATTRIBUTE_UNUSED,
 
     qemudLog(QEMUD_INFO, _("Shutting down VM '%s'\n"), vm->def->name);
 
-    kill(vm->pid, SIGTERM);
+    if (kill(vm->pid, SIGTERM) < 0)
+        qemudLog(QEMUD_ERROR, _("Failed to send SIGTERM to %s (%d): %s\n"),
+                 vm->def->name, vm->pid, strerror(errno));
 
     qemudVMData(driver, vm, vm->stdout_fd);
     qemudVMData(driver, vm, vm->stderr_fd);
@@ -1024,13 +1384,9 @@ static void qemudShutdownVMDaemon(virConnectPtr conn ATTRIBUTE_UNUSED,
     vm->stderr_fd = -1;
     vm->monitor = -1;
 
-    if (waitpid(vm->pid, NULL, WNOHANG) != vm->pid) {
-        kill(vm->pid, SIGKILL);
-        if (waitpid(vm->pid, NULL, 0) != vm->pid) {
-            qemudLog(QEMUD_WARN,
-                     "%s", _("Got unexpected pid, damn\n"));
-        }
-    }
+    /* shut if off for sure */
+    kill(vm->pid, SIGKILL);
+    qemudRemoveDomainState(driver, vm);
 
     vm->pid = -1;
     vm->def->id = -1;
diff --git a/src/test.c b/src/test.c
index 3648c05..2313746 100644
--- a/src/test.c
+++ b/src/test.c
@@ -504,12 +504,12 @@ static int testOpenFromFile(virConnectPtr conn,
                 testError(NULL, VIR_ERR_INTERNAL_ERROR, "%s", _("resolving domain filename"));
                 goto error;
             }
-            def = virDomainDefParseFile(conn, privconn->caps, absFile);
+            def = virDomainDefParseFile(conn, privconn->caps, absFile, 0);
             VIR_FREE(absFile);
             if (!def)
                 goto error;
         } else {
-            if ((def = virDomainDefParseNode(conn, privconn->caps, xml, domains[i])) == NULL)
+            if ((def = virDomainDefParseNode(conn, privconn->caps, xml, domains[i], 0)) == NULL)
                 goto error;
         }
 
diff --git a/src/util.c b/src/util.c
index 25eec54..1bfdde1 100644
--- a/src/util.c
+++ b/src/util.c
@@ -914,6 +914,17 @@ int virFileOpenTty(int *ttymaster ATTRIBUTE_UNUSED,
 #endif
 
 
+char* virFilePid(const char *dir, const char* name)
+{
+    char* pidfile;
+
+    if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) {
+        pidfile = NULL;
+    }
+    return pidfile;
+}
+
+
 int virFileWritePid(const char *dir,
                     const char *name,
                     pid_t pid)
@@ -926,7 +937,7 @@ int virFileWritePid(const char *dir,
     if ((rc = virFileMakePath(dir)))
         goto cleanup;
 
-    if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) {
+    if (!(pidfile = virFilePid(dir, name))) {
         rc = ENOMEM;
         goto cleanup;
     }
@@ -969,7 +980,8 @@ int virFileReadPid(const char *dir,
     FILE *file;
     char *pidfile = NULL;
     *pid = 0;
-    if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) {
+
+    if (!(pidfile = virFilePid(dir, name))) {
         rc = ENOMEM;
         goto cleanup;
     }
@@ -1002,8 +1014,8 @@ int virFileDeletePid(const char *dir,
     int rc = 0;
     char *pidfile = NULL;
 
-    if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) {
-        rc = errno;
+    if (!(pidfile = virFilePid(dir, name))) {
+        rc = ENOMEM;
         goto cleanup;
     }
 
diff --git a/src/util.h b/src/util.h
index 0748cbf..58488ae 100644
--- a/src/util.h
+++ b/src/util.h
@@ -79,6 +79,8 @@ int virFileOpenTty(int *ttymaster,
                    char **ttyName,
                    int rawmode);
 
+char* virFilePid(const char *dir,
+                 const char *name);
 int virFileWritePid(const char *dir,
                     const char *name,
                     pid_t pid);
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index b6a7f68..e437d3d 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -38,7 +38,7 @@ static int testCompareXMLToArgvFiles(const char *xml,
     if (virtTestLoadFile(cmd, &expectargv, MAX_FILE) < 0)
         goto fail;
 
-    if (!(vmdef = virDomainDefParseFile(NULL, driver.caps, xml)))
+    if (!(vmdef = virDomainDefParseFile(NULL, driver.caps, xml, 0)))
         goto fail;
 
     memset(&vm, 0, sizeof vm);
-- 
1.6.0.2

