Hi, Doug for your comment, I changed to use virDomainGetBlockInfo() to prepare disk images for non-shared migration, but, it really looks ugly and doesn't make much sense. what's your opinion? 在 2012-11-23五的 13:28 +0800，liguang写道：

try to do non-shared migration without bothering to create disk images at target by hand.

consider this situation: 1. non-shared migration virsh migrate --copy-storage-all ... 2. migration fails 3. create disk images required qemu-img create ... 4 migration run smoothly so, try do remove step 2, 3, 4

this kind of usage had been discussed before, http://www.redhat.com/archives/libvir-list/2011-December/msg00451.html

this patch depends on my support offline migration patch: https://www.redhat.com/archives/libvir-list/2012-November/msg00512.html

Signed-off-by: liguang <lig.fnst@cn.fujitsu.com> --- src/qemu/qemu_conf.h | 2 + src/qemu/qemu_driver.c | 1 + src/qemu/qemu_migration.c | 275 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 277 insertions(+), 1 deletions(-)

diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index 47f349c..ad789e9 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -102,6 +102,8 @@ struct qemud_driver { char *hugetlbfs_mount; char *hugepage_path;

+ void *private; + unsigned int macFilter : 1; ebtablesContext *ebtables;

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index b23056b..db11629 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -1133,6 +1133,7 @@ static virDrvOpenStatus qemudOpen(virConnectPtr conn, } } } + qemu_driver->private = conn; conn->privateData = qemu_driver;

return VIR_DRV_OPEN_SUCCESS; diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 53171df..8b4e563 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -50,6 +50,7 @@ #include "storage_file.h" #include "viruri.h" #include "hooks.h" +#include "dirname.h"

#define VIR_FROM_THIS VIR_FROM_QEMU @@ -72,6 +73,7 @@ enum qemuMigrationCookieFlags { QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE, QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT, QEMU_MIGRATION_COOKIE_FLAG_NETWORK, + QEMU_MIGRATION_COOKIE_FLAG_COPYSTORAGE,

QEMU_MIGRATION_COOKIE_FLAG_LAST }; @@ -79,13 +81,14 @@ enum qemuMigrationCookieFlags { VIR_ENUM_DECL(qemuMigrationCookieFlag); VIR_ENUM_IMPL(qemuMigrationCookieFlag, QEMU_MIGRATION_COOKIE_FLAG_LAST, - "graphics", "lockstate", "persistent", "network"); + "graphics", "lockstate", "persistent", "network", "storage");

enum qemuMigrationCookieFeatures { QEMU_MIGRATION_COOKIE_GRAPHICS = (1 << QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS), QEMU_MIGRATION_COOKIE_LOCKSTATE = (1 << QEMU_MIGRATION_COOKIE_FLAG_LOCKSTATE), QEMU_MIGRATION_COOKIE_PERSISTENT = (1 << QEMU_MIGRATION_COOKIE_FLAG_PERSISTENT), QEMU_MIGRATION_COOKIE_NETWORK = (1 << QEMU_MIGRATION_COOKIE_FLAG_NETWORK), + QEMU_MIGRATION_COOKIE_COPYSTORAGE = (1 << QEMU_MIGRATION_COOKIE_FLAG_COPYSTORAGE), };

typedef struct _qemuMigrationCookieGraphics qemuMigrationCookieGraphics; @@ -119,6 +122,19 @@ struct _qemuMigrationCookieNetwork { qemuMigrationCookieNetDataPtr net; };

+typedef struct _qemuMigrationCookieStorageData qemuMigrationCookieStorageData; +typedef qemuMigrationCookieStorageData *qemuMigrationCookieStorageDataPtr; +struct _qemuMigrationCookieStorageData { + char *dsize; +}; + +typedef struct _qemuMigrationCookieStorage qemuMigrationCookieStorage; +typedef qemuMigrationCookieStorage *qemuMigrationCookieStoragePtr; +struct _qemuMigrationCookieStorage { + int ndisks; + qemuMigrationCookieStorageDataPtr disk; +}; + typedef struct _qemuMigrationCookie qemuMigrationCookie; typedef qemuMigrationCookie *qemuMigrationCookiePtr; struct _qemuMigrationCookie { @@ -147,6 +163,9 @@ struct _qemuMigrationCookie {

/* If (flags & QEMU_MIGRATION_COOKIE_NETWORK) */ qemuMigrationCookieNetworkPtr network; + + /* If (flags & QEMU_MIGRATION_COOKIE_COPYSTORAGE) */ + qemuMigrationCookieStoragePtr storage; };

static void qemuMigrationCookieGraphicsFree(qemuMigrationCookieGraphicsPtr grap) @@ -175,6 +194,21 @@ qemuMigrationCookieNetworkFree(qemuMigrationCookieNetworkPtr network) VIR_FREE(network); }

+static void +qemuMigrationCookieStorageFree(qemuMigrationCookieStoragePtr storage) +{ + int i; + + if (!storage) + return; + + if (storage->disk) { + for (i = 0; i < storage->ndisks; i++) + VIR_FREE(storage->disk[i].dsize); + } + VIR_FREE(storage->disk); + VIR_FREE(storage); +}

static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig) { @@ -187,6 +221,9 @@ static void qemuMigrationCookieFree(qemuMigrationCookiePtr mig) if (mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) qemuMigrationCookieNetworkFree(mig->network);

+ if (mig->flags & QEMU_MIGRATION_COOKIE_COPYSTORAGE) + qemuMigrationCookieStorageFree(mig->storage); + VIR_FREE(mig->localHostname); VIR_FREE(mig->remoteHostname); VIR_FREE(mig->name); @@ -356,6 +393,45 @@ error: return NULL; }

+static qemuMigrationCookieStoragePtr +qemuMigrationCookieStorageAlloc(struct qemud_driver *driver, + virDomainDefPtr def) +{ + int i; + qemuMigrationCookieStoragePtr mig; + virDomainBlockInfo info; + virDomainPtr dom; + virConnectPtr conn = driver->private; + + dom = virGetDomain(conn, def->name, def->uuid); + if (!dom) + goto error; + dom->id = def->id; + + if (VIR_ALLOC(mig) < 0) + goto no_memory; + if (VIR_ALLOC_N(mig->disk, def->ndisks) < 0) + goto no_memory; + + mig->ndisks = def->ndisks; + + for (i = 0; i < mig->ndisks; i++) { + if (virDomainGetBlockInfo(dom, def->disks[i]->src, + &info, 0) < 0) + goto error; + if (virAsprintf(&(mig->disk[i].dsize), "%llu", info.capacity) < 0) + goto no_memory; + } + + return mig; + +no_memory: + virReportOOMError(); +error: + qemuMigrationCookieStorageFree(mig); + return NULL; +} + static qemuMigrationCookiePtr qemuMigrationCookieNew(virDomainObjPtr dom) { @@ -491,6 +567,31 @@ qemuMigrationCookieAddNetwork(qemuMigrationCookiePtr mig, return 0; }

+static int qemuMigrationCookieAddStorage(qemuMigrationCookiePtr mig, + struct qemud_driver *driver, + virDomainObjPtr dom) +{ + if (mig->flags & QEMU_MIGRATION_COOKIE_COPYSTORAGE) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("migration storage data already present")); + return -1; + } + + virDomainObjUnlock(dom); + qemuDriverUnlock(driver); + + if (dom->def->ndisks > 0) { + mig->storage = qemuMigrationCookieStorageAlloc(driver, dom->def); + if (!mig->storage) + return -1; + mig->flags |= QEMU_MIGRATION_COOKIE_COPYSTORAGE; + } + + qemuDriverLock(driver); + virDomainObjLock(dom); + + return 0; +}

static void qemuMigrationCookieGraphicsXMLFormat(virBufferPtr buf, qemuMigrationCookieGraphicsPtr grap) @@ -540,6 +641,19 @@ qemuMigrationCookieNetworkXMLFormat(virBufferPtr buf, virBufferAddLit(buf, " </network>\n"); }

+static void +qemuMigrationCookieStorageXMLFormat(virBufferPtr buf, + qemuMigrationCookieStoragePtr dsz) +{ + int i = 0; + + for (i = 0; i < dsz->ndisks; i++) { + char *dsize = dsz->disk[i].dsize; + virBufferAsprintf(buf, " <copystorage>\n"); + virBufferAsprintf(buf, " <disksize>%s</disksize>\n", dsize); + virBufferAddLit(buf, " </copystorage>\n"); + } +}

static int qemuMigrationCookieXMLFormat(struct qemud_driver *driver, @@ -594,6 +708,9 @@ qemuMigrationCookieXMLFormat(struct qemud_driver *driver, if ((mig->flags & QEMU_MIGRATION_COOKIE_NETWORK) && mig->network) qemuMigrationCookieNetworkXMLFormat(buf, mig->network);

+ if ((mig->flags & QEMU_MIGRATION_COOKIE_COPYSTORAGE) && mig->storage) + qemuMigrationCookieStorageXMLFormat(buf, mig->storage); + virBufferAddLit(buf, "</qemu-migration>\n"); return 0; } @@ -721,6 +838,44 @@ error: goto cleanup; }

+static qemuMigrationCookieStoragePtr +qemuMigrationCookieStorageXMLParse(xmlXPathContextPtr ctxt) +{ + qemuMigrationCookieStoragePtr dsz; + int i, n; + xmlNodePtr *storage = NULL; + + if (VIR_ALLOC(dsz) < 0) + goto no_memory; + + if ((n = virXPathNodeSet("./copystorage", ctxt, &storage)) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing storage information")); + goto error; + } + + dsz->ndisks = n; + if (VIR_ALLOC_N(dsz->disk, dsz->ndisks) < 0) + goto no_memory; + + for (i = 0; i < dsz->ndisks; i++) { + ctxt->node = storage[i]; + if (!(dsz->disk[i].dsize = virXPathString("string(./disksize[1])", ctxt))) + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("missing tlsPort attribute in migration data")); + } + +cleanup: + VIR_FREE(storage); + return dsz; + +no_memory: + virReportOOMError(); +error: + qemuMigrationCookieStorageFree(dsz); + dsz = NULL; + goto cleanup; +}

static int qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, @@ -873,6 +1028,11 @@ qemuMigrationCookieXMLParse(qemuMigrationCookiePtr mig, (!(mig->network = qemuMigrationCookieNetworkXMLParse(ctxt)))) goto error;

+ if ((flags & QEMU_MIGRATION_COOKIE_COPYSTORAGE) && + virXPathBoolean("count(./copystorage) > 0", ctxt) && + (!(mig->storage = qemuMigrationCookieStorageXMLParse(ctxt)))) + goto error; + return 0;

error: @@ -937,6 +1097,11 @@ qemuMigrationBakeCookie(qemuMigrationCookiePtr mig, return -1; }

+ if (flags & QEMU_MIGRATION_COOKIE_COPYSTORAGE && + qemuMigrationCookieAddStorage(mig, driver, dom) < 0) { + return -1; + } + if (!(*cookieout = qemuMigrationCookieXMLFormatStr(driver, mig))) return -1;

@@ -1442,6 +1607,14 @@ char *qemuMigrationBegin(struct qemud_driver *driver, QEMU_MIGRATION_COOKIE_LOCKSTATE) < 0) goto cleanup;

+ if (flags & (VIR_MIGRATE_NON_SHARED_DISK | + VIR_MIGRATE_NON_SHARED_INC)) { + if (qemuMigrationBakeCookie(mig, driver, vm, + cookieout, cookieoutlen, + QEMU_MIGRATION_COOKIE_COPYSTORAGE) < 0) + goto cleanup; + } + if (flags & VIR_MIGRATE_OFFLINE) { if (flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC)) { @@ -1487,6 +1660,91 @@ cleanup: }

+/* + if gen is true, find out disk images migration required, + so try to generate them at target, + if gen is false, delete disk images generated before. +*/ +static int qemuMigrationHandleDiskFiles(struct qemud_driver *driver, + virDomainDefPtr def, bool gen, + qemuMigrationCookiePtr mig) +{ + char *tmp_dir = NULL, *outbuf = NULL; + const char *img_tool = driver->qemuImgTool; + const char *disk_format[] = {"none", "raw", "none", "none", "none", + "cow", "none", "none", "qcow", "qcow2", + "qed", "vmdk", "vpc","none", "none" + }; + virCommandPtr cmd = NULL; + int i, ret = -1; + + if (!def->ndisks) + return 0; + + if (img_tool == NULL) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("unable to find kvm-img or qemu-img")); + goto error; + } + + for (i = 0; i < def->ndisks; i++) { + if (STRNEQ(def->disks[i]->driverName, "qemu")) + continue; + if (def->disks[i]->src == NULL) + continue; + if (virFileExists(def->disks[i]->src) && gen) + continue; + if (!gen && !virFileExists(def->disks[i]->src)) + continue; + if ((tmp_dir = mdir_name(def->disks[i]->src)) == NULL) + continue; + if (!virFileExists(tmp_dir)) + if (virFileMakePath(tmp_dir) < 0) + continue; + if (STREQ(disk_format[def->disks[i]->format], "none")) + continue; + if (def->disks[i]->format < VIR_STORAGE_FILE_RAW) + goto error; + if (def->disks[i]->format >= VIR_STORAGE_FILE_LAST) + goto error; + + if (gen) { + char *dsize = mig->storage->disk[i].dsize; + cmd = virCommandNewArgList(img_tool, "create", "-f", + disk_format[def->disks[i]->format], + def->disks[i]->src, NULL); + virCommandAddArgFormat(cmd, "%s", dsize); + if (def->disks[i]->encryption) + virCommandAddArgList(cmd, "-o", "encryption=on", NULL); + virCommandSetOutputBuffer(cmd, &outbuf); + if (virCommandRun(cmd, NULL) < 0) { + virReportSystemError(errno, "%s", outbuf); + goto cleanup; + } + } else { + if (unlink(def->disks[i]->src) < 0) { + virReportError(errno, "%s", _("fail to unlink disk image file")); + goto cleanup; + } + } + virCommandFree(cmd); + VIR_FREE(tmp_dir); + VIR_FREE(outbuf); + } + + ret = 0; + +cleanup: + if (ret < 0) { + virCommandFree(cmd); + VIR_FREE(tmp_dir); + VIR_FREE(outbuf); + } +error: + return ret; +} + + /* Prepare is the first step, and it runs on the destination host. */

@@ -1602,6 +1860,15 @@ qemuMigrationPrepareAny(struct qemud_driver *driver, /* virDomainAssignDef already set the error */ goto cleanup; } + + if (!(mig = qemuMigrationEatCookie(driver, vm, cookiein, cookieinlen, + QEMU_MIGRATION_COOKIE_COPYSTORAGE))) + goto cleanup; + + if (flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC)) + if (qemuMigrationHandleDiskFiles(driver, def, true, mig) < 0) + goto endjob; + def = NULL; priv = vm->privateData; priv->origname = origname; @@ -3247,6 +3514,7 @@ qemuMigrationFinish(struct qemud_driver *driver, virErrorPtr orig_err = NULL; int cookie_flags = 0; qemuDomainObjPrivatePtr priv = vm->privateData; + bool migration_status = false;

VIR_DEBUG("driver=%p, dconn=%p, vm=%p, cookiein=%s, cookieinlen=%d, " "cookieout=%p, cookieoutlen=%p, flags=%lx, retcode=%d", @@ -3416,7 +3684,12 @@ qemuMigrationFinish(struct qemud_driver *driver, if (qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, 0) < 0) VIR_WARN("Unable to encode migration cookie");

+ migration_status = true; + endjob: + if (!migration_status && flags & + (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC)) + qemuMigrationHandleDiskFiles(driver, vm->def, false, NULL); if (qemuMigrationJobFinish(driver, vm) == 0) { vm = NULL; } else if (!vm->persistent && !virDomainObjIsActive(vm)) {

-- li guang lig.fnst@cn.fujitsu.com linux kernel team at FNST, china