Hi, I've implemented save/restore support for KVM using the live migration feature. First, a few patches to KVM to fix bugs: Subject: [PATCH 1/3] qemu: fix freed pointer dereference http://article.gmane.org/gmane.comp.emulators.kvm.devel/5572 Subject: [PATCH 2/3] qemu: don't start a new migration if one is already in progress http://article.gmane.org/gmane.comp.emulators.kvm.devel/5575 Subject: [PATCH 3/3] qemu: reset buffer pointers after CR/LF http://article.gmane.org/gmane.comp.emulators.kvm.devel/5573 (If compatibility with old KVM is wanted, it might be possible to work around the kvm bugs in other ways, but I'm not sure). Then, another patch to KVM to support inbound migration from a filename. It already supports migration from stdin, but adding this seemed easier from a libvirt perspective. Subject: [PATCH] qemu: accept filename for incoming migration http://article.gmane.org/gmane.comp.emulators.kvm.devel/5590 Finally, the libvirt side. Some notes: - Arbitrary decisions about VM status: I pause the VM before starting migration, and destroy it once it's finished. Neither is required by KVM but I'd be concerned about the disk image state otherwise. Also, after resuming, the VM is still paused. I don't know how Xen behaves in this regard but the behavior here is easy enough to change. - I append the domain's UUID at the end of the migration image. This doesn't affect KVM at all (it ignores the extra data). Does that seem reasonable? It's unclear how the saved image is supposed to get associated with a particular VM configuration without doing something like this. - I added the migrateFrom field to the qemu_vm struct. Dunno if that's the best place. Could also add put it in qemu_vm_def, or add parameters to qemudStartVMDaemon, etc.. Patch against current CVS is below. Signed-off-by: Jim Paris <jim(a)jtan.com&gt; -jim Index: src/qemu_conf.c =================================================================== RCS file: /data/cvs/libvirt/src/qemu_conf.c,v retrieving revision 1.10 diff -u -r1.10 qemu_conf.c --- src/qemu_conf.c 7 Aug 2007 13:02:35 -0000 1.10 +++ src/qemu_conf.c 9 Aug 2007 21:15:10 -0000 @@ -1518,7 +1518,8 @@ (vm->def->os.initrd[0] ? 2 : 0) + /* initrd */ (vm->def->os.cmdline[0] ? 2 : 0) + /* cmdline */ (vm->def->graphicsType == QEMUD_GRAPHICS_VNC ? 2 : - (vm->def->graphicsType == QEMUD_GRAPHICS_SDL ? 0 : 1)); /* graphics */ + (vm->def->graphicsType == QEMUD_GRAPHICS_SDL ? 0 : 1)) + /* graphics */ + (vm->migrateFrom[0] ? 3 : 0); /* migrateFrom */ snprintf(memory, sizeof(memory), "%d", vm->def->memory/1024); snprintf(vcpus, sizeof(vcpus), "%d", vm->def->vcpus); @@ -1767,6 +1768,15 @@ /* SDL is the default. no args needed */ } + if (vm->migrateFrom[0]) { + if (!((*argv)[++n] = strdup("-S"))) + goto no_memory; + if (!((*argv)[++n] = strdup("-incoming"))) + goto no_memory; + if (!((*argv)[++n] = strdup(vm->migrateFrom))) + goto no_memory; + } + (*argv)[++n] = NULL; return 0; Index: src/qemu_conf.h =================================================================== RCS file: /data/cvs/libvirt/src/qemu_conf.h,v retrieving revision 1.6 diff -u -r1.6 qemu_conf.h --- src/qemu_conf.h 30 Jul 2007 09:59:06 -0000 1.6 +++ src/qemu_conf.h 9 Aug 2007 21:15:10 -0000 @@ -213,6 +213,7 @@ char configFile[PATH_MAX]; char autostartLink[PATH_MAX]; + char migrateFrom[PATH_MAX]; struct qemud_vm_def *def; /* The current definition */ struct qemud_vm_def *newDef; /* New definition to activate at shutdown */ Index: src/qemu_driver.c =================================================================== RCS file: /data/cvs/libvirt/src/qemu_driver.c,v retrieving revision 1.14 diff -u -r1.14 qemu_driver.c --- src/qemu_driver.c 30 Jul 2007 09:59:06 -0000 1.14 +++ src/qemu_driver.c 9 Aug 2007 21:15:10 -0000 @@ -656,7 +656,7 @@ if (virExecNonBlock(conn, argv, &vm->pid, &vm->stdout, &vm->stderr) == 0) { vm->id = driver->nextvmid++; - vm->state = VIR_DOMAIN_RUNNING; + vm->state = vm->migrateFrom[0] ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_RUNNING; driver->ninactivevms--; driver->nactivevms++; @@ -1852,29 +1852,151 @@ return 0; } +#define QEMUD_SAVE_MAGIC "LibvirtQemudUUID" -static int qemudDomainSave(virDomainPtr dom, - const char *path ATTRIBUTE_UNUSED) { +static int qemudDomainSave(virDomainPtr dom, + const char *path) { struct qemud_driver *driver = (struct qemud_driver *)dom->conn->privateData; struct qemud_vm *vm = qemudFindVMByID(driver, dom->id); + char *command, *info; + int fd; + if (!vm) { - qemudReportError(dom->conn, dom, NULL, VIR_ERR_INVALID_DOMAIN, "no domain with matching id %d", dom->id); + qemudReportError(dom->conn, dom, NULL, VIR_ERR_INVALID_DOMAIN, + "no domain with matching id %d", dom->id); return -1; } + if (!qemudIsActiveVM(vm)) { - qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, "domain is not running"); + qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, + "domain is not running"); return -1; } - qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, "save is not supported"); - return -1; + + if (strchr(path, '\'') || strchr(path, '\\') ) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, + "invalid filename"); + return -1; + } + + /* Pause */ + if (qemudDomainSuspend(dom) != 0) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, + "failed to pause domain"); + return -1; + } + + /* Migrate to file. */ + if (asprintf (&command, "migrate \"exec:dd of='%s' 2>/dev/null\"\n", + path) == -1) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, + "out of memory"); + return -1; + } + + if (qemudMonitorCommand(driver, vm, command, &info) < 0) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, + "migrate operation failed"); + free(command); + return -1; + } + + free(info); + free(command); + + /* Append the UUID, which we'll need in order to restore the domain. */ + if ((fd = open(path, O_APPEND|O_WRONLY)) < 0) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, + "failed to open '%s'", path); + return -1; + } + + if (write(fd, QEMUD_SAVE_MAGIC, sizeof(QEMUD_SAVE_MAGIC)) != + sizeof(QEMUD_SAVE_MAGIC)) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, + "failed to write magic"); + close(fd); + return -1; + } + + if (write(fd, dom->uuid, VIR_UUID_BUFLEN) != VIR_UUID_BUFLEN) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED, + "failed to write uuid"); + close(fd); + return -1; + } + close(fd); + + /* Shut it down */ + qemudShutdownVMDaemon(dom->conn, driver, vm); + if (!vm->configFile[0]) + qemudRemoveInactiveVM(driver, vm); + + return 0; } static int qemudDomainRestore(virConnectPtr conn, - const char *path ATTRIBUTE_UNUSED) { - /*struct qemud_driver *driver = (struct qemud_driver *)conn->privateData;*/ - qemudReportError(conn, NULL, NULL, VIR_ERR_OPERATION_FAILED, "restore is not supported"); - return -1; + const char *path) { + struct qemud_driver *driver = (struct qemud_driver *)conn->privateData; + unsigned char uuid[VIR_UUID_BUFLEN]; + unsigned char magic[sizeof(QEMUD_SAVE_MAGIC)]; + virDomainPtr dom; + struct qemud_vm *vm; + int fd; + + /* Find the UUID */ + if ((fd = open(path, O_RDONLY)) < 0) { + qemudReportError(conn, NULL, NULL, VIR_ERR_OPERATION_FAILED, + "cannot read domain image"); + return -1; + } + if (lseek(fd, 0 - (sizeof(magic) + sizeof(uuid)), SEEK_END) == -1) { + qemudReportError(conn, NULL, NULL, VIR_ERR_OPERATION_FAILED, + "failed to seek to end of domain image"); + close(fd); + return -1; + } + if (read(fd, magic, sizeof(magic)) != sizeof(magic)) { + qemudReportError(conn, NULL, NULL, VIR_ERR_OPERATION_FAILED, + "failed to read domain image"); + close(fd); + return -1; + } + if (memcmp(magic, QEMUD_SAVE_MAGIC, sizeof(magic))) { + qemudReportError(conn, NULL, NULL, VIR_ERR_OPERATION_FAILED, + "image magic is not correct"); + close(fd); + return -1; + } + if (read(fd, uuid, sizeof(uuid)) != sizeof(uuid)) { + qemudReportError(conn, NULL, NULL, VIR_ERR_OPERATION_FAILED, + "failed to read domain UUID from image"); + close(fd); + return -1; + } + close(fd); + + /* Find the domain & vm */ + dom = qemudDomainLookupByUUID(conn, uuid); + if (!dom) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INVALID_DOMAIN, + "no domain with matching uuid"); + return -1; + } + + vm = qemudFindVMByUUID(driver, dom->uuid); + if (!vm) { + qemudReportError(conn, dom, NULL, VIR_ERR_INVALID_DOMAIN, + "no vm with matching uuid"); + return -1; + } + + /* Set the migration source and start it up. */ + snprintf(vm->migrateFrom, sizeof(vm->migrateFrom), "file://%s", path); + vm->migrateFrom[sizeof(vm->migrateFrom)-1] = '\0'; + + return qemudStartVMDaemon(dom->conn, driver, vm); } @@ -1931,6 +2053,7 @@ return -1; } + vm->migrateFrom[0] = '\0'; return qemudStartVMDaemon(dom->conn, driver, vm); }