From: Cristian Klein <cristiklein@gmail.com> To use post-copy one has to start the migration with VIR_MIGRATE_POSTCOPY flag and, while migration is in progress, call virDomainMigrateStartPostCopy() to switch from pre-copy to post-copy. Signed-off-by: Cristian Klein <cristiklein@gmail.com> Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 2: - POSTCOPY_AFTER_PRECOPY flag removed include/libvirt/libvirt-domain.h | 4 ++ src/driver-hypervisor.h | 5 ++ src/libvirt-domain.c | 114 +++++++++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 4 ++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 ++++- src/remote_protocol-structs | 5 ++ 7 files changed, 145 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 50342d2..ccbf6a7 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -662,6 +662,7 @@ typedef enum { VIR_MIGRATE_ABORT_ON_ERROR = (1 << 12), /* abort migration on I/O errors happened during migration */ VIR_MIGRATE_AUTO_CONVERGE = (1 << 13), /* force convergence */ VIR_MIGRATE_RDMA_PIN_ALL = (1 << 14), /* RDMA memory pinning */ + VIR_MIGRATE_POSTCOPY = (1 << 15), /* enable (but do not start) post-copy migration */ } virDomainMigrateFlags; @@ -810,6 +811,9 @@ int virDomainMigrateGetMaxSpeed(virDomainPtr domain, unsigned long *bandwidth, unsigned int flags); +int virDomainMigrateStartPostCopy(virDomainPtr domain, + unsigned int flags); + char * virConnectGetDomainCapabilities(virConnectPtr conn, const char *emulatorbin, const char *arch, diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index ae2ec4d..68a7730 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -638,6 +638,10 @@ typedef int const char *dom_xml); typedef int +(*virDrvDomainMigrateStartPostCopy)(virDomainPtr domain, + unsigned int flags); + +typedef int (*virDrvConnectIsEncrypted)(virConnectPtr conn); typedef int @@ -1443,6 +1447,7 @@ struct _virHypervisorDriver { virDrvDomainGetFSInfo domainGetFSInfo; virDrvDomainInterfaceAddresses domainInterfaceAddresses; virDrvDomainSetUserPassword domainSetUserPassword; + virDrvDomainMigrateStartPostCopy domainMigrateStartPostCopy; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 9491845..676f0f7 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -3537,6 +3537,7 @@ virDomainMigrateUnmanaged(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set. * Applications using the VIR_MIGRATE_PEER2PEER flag will probably @@ -3573,6 +3574,11 @@ virDomainMigrateUnmanaged(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -3748,6 +3754,7 @@ virDomainMigrate(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set. * Applications using the VIR_MIGRATE_PEER2PEER flag will probably @@ -3784,6 +3791,11 @@ virDomainMigrate(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -3968,6 +3980,11 @@ virDomainMigrate2(virDomainPtr domain, * can use either VIR_MIGRATE_NON_SHARED_DISK or * VIR_MIGRATE_NON_SHARED_INC as they are mutually exclusive. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * There are many limitations on migration imposed by the underlying * technology - for example it may not be possible to migrate between * different processors even with the same architecture, or between @@ -4208,6 +4225,7 @@ int virDomainMigrateUnmanagedCheckCompat(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * The operation of this API hinges on the VIR_MIGRATE_PEER2PEER flag. * If the VIR_MIGRATE_PEER2PEER flag is NOT set, the duri parameter @@ -4240,6 +4258,11 @@ int virDomainMigrateUnmanagedCheckCompat(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -4321,6 +4344,7 @@ virDomainMigrateToURI(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * The operation of this API hinges on the VIR_MIGRATE_PEER2PEER flag. * @@ -4366,6 +4390,11 @@ virDomainMigrateToURI(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -4446,6 +4475,11 @@ virDomainMigrateToURI2(virDomainPtr domain, * can use either VIR_MIGRATE_NON_SHARED_DISK or * VIR_MIGRATE_NON_SHARED_INC as they are mutually exclusive. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * There are many limitations on migration imposed by the underlying * technology - for example it may not be possible to migrate between * different processors even with the same architecture, or between @@ -9163,6 +9197,86 @@ virDomainMigrateGetMaxSpeed(virDomainPtr domain, /** + * virDomainMigrateStartPostCopy: + * @domain: a domain object + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Starts post-copy migration. This function has to be called while + * migration (initiated with VIR_MIGRATE_POSTCOPY flag) is in progress. + * + * Traditional post-copy migration iteratively walks through guest memory + * pages and migrates those that changed since the previous iteration. The + * iterative phase stops when the number of dirty pages is low enough so that + * the virtual CPUs can be paused, all dirty pages transferred to the + * destination, where the virtual CPUs are unpaused, and all this can happen + * within a predefined downtime period. It's clear that this process may never + * converge if downtime is too short and/or the guest keeps changing a lot of + * memory pages. + * + * When migration is switched to post-copy mode, the virtual CPUs are paused + * immediately, only a minimum set of pages is transferred, and the CPUs are + * unpaused on destination. The source keeps sending all remaining memory pages + * to the destination while the guest is already running there. Whenever the + * guest tries to read a memory page which has not been migrated yet, the + * hypervisor has to tell the source to transfer that page in a priority + * channel. To minimize such page faults, it is a good idea to run at least one + * iteration of pre-copy migration before switching to post-copy. + * + * Post-copy migration is guaranteed to converge since each page is transferred + * at most once no matter how fast it changes. On the other hand once the + * guest is running on the destination host, the migration can no longer be + * rolled back because none of the hosts has complete state. If this happens, + * libvirt will leave the domain paused on both hosts with + * VIR_DOMAIN_PAUSED_POSTCOPY_FAILED reason. It's up to the upper layer to + * decide what to do in such case. + * + * The following domain life cycle events are emitted during post-copy + * migration: + * VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY (on the source) -- migration entered + * post-copy mode. + * VIR_DOMAIN_EVENT_RESUMED_POSTCOPY (on the destination) -- the guest is + * running on the destination host while some of its memory pages still + * remain on the source host; neither the source nor the destination host + * contain a complete guest state from this point until migration + * finishes. + * VIR_DOMAIN_EVENT_RESUMED_MIGRATED (on the destination), + * VIR_DOMAIN_EVENT_STOPPED_MIGRATED (on the source) -- migration finished + * successfully and the destination host holds a complete guest state. + * VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED (on either side) -- emitted + * when migration fails in post-copy mode and it's unclear whether any + * of the hosts has a complete guest state. + * + * Returns 0 in case of success, -1 otherwise. + */ +int +virDomainMigrateStartPostCopy(virDomainPtr domain, + unsigned int flags) +{ + virConnectPtr conn; + + VIR_DOMAIN_DEBUG(domain); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckReadOnlyGoto(conn->flags, error); + + if (conn->driver->domainMigrateStartPostCopy) { + if (conn->driver->domainMigrateStartPostCopy(domain, flags) < 0) + goto error; + return 0; + } + + virReportUnsupportedError(); + error: + virDispatchError(conn); + return -1; +} + + +/** * virConnectDomainEventRegisterAny: * @conn: pointer to the connection * @dom: pointer to the domain diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index dd94191..d98414d 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -725,4 +725,8 @@ LIBVIRT_1.2.19 { virDomainRename; } LIBVIRT_1.2.17; +LIBVIRT_1.3.2 { + virDomainMigrateStartPostCopy; +} LIBVIRT_1.2.19; + # .... define new API here using predicted next version number .... diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index d9d7ec8..f0a81a6 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8462,6 +8462,7 @@ static virHypervisorDriver hypervisor_driver = { .domainInterfaceAddresses = remoteDomainInterfaceAddresses, /* 1.2.14 */ .domainSetUserPassword = remoteDomainSetUserPassword, /* 1.2.16 */ .domainRename = remoteDomainRename, /* 1.2.19 */ + .domainMigrateStartPostCopy = remoteDomainMigrateStartPostCopy, /* 1.3.2 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index bfdbce7..1e1b301 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -3228,6 +3228,11 @@ struct remote_domain_event_callback_migration_iteration_msg { int iteration; }; +struct remote_domain_migrate_start_post_copy_args { + remote_nonnull_domain dom; + unsigned int flags; +}; + /*----- Protocol. -----*/ /* Define the program number, protocol version and procedure numbers here. */ @@ -5706,5 +5711,11 @@ enum remote_procedure { * @generate: both * @acl: none */ - REMOTE_PROC_DOMAIN_EVENT_CALLBACK_MIGRATION_ITERATION = 359 + REMOTE_PROC_DOMAIN_EVENT_CALLBACK_MIGRATION_ITERATION = 359, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_START_POST_COPY = 360 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index dff54e8..b372f49 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -2697,6 +2697,10 @@ struct remote_domain_event_callback_migration_iteration_msg { remote_nonnull_domain dom; int iteration; }; +struct remote_domain_migrate_start_post_copy_args { + remote_nonnull_domain dom; + u_int flags; +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -3057,4 +3061,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_SET_USER_PASSWORD = 357, REMOTE_PROC_DOMAIN_RENAME = 358, REMOTE_PROC_DOMAIN_EVENT_CALLBACK_MIGRATION_ITERATION = 359, + REMOTE_PROC_DOMAIN_MIGRATE_START_POST_COPY = 360, }; -- 2.7.0

Migration enters "postcopy-active" state after QEMU switches to post-copy and pauses guest CPUs. From libvirt's point of view this state is similar to "completed" because we need to transfer guest execution to the destination host. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 2: - no change src/qemu/qemu_migration.c | 84 +++++++++++++++++++++++++++++++++++++------- src/qemu/qemu_monitor.c | 2 +- src/qemu/qemu_monitor.h | 1 + src/qemu/qemu_monitor_json.c | 1 + src/qemu/qemu_process.c | 10 ++++-- 5 files changed, 82 insertions(+), 16 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index c927888..6c98c5c 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2549,6 +2549,7 @@ qemuMigrationUpdateJobType(qemuDomainJobInfoPtr jobInfo) case QEMU_MONITOR_MIGRATION_STATUS_SETUP: case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: case QEMU_MONITOR_MIGRATION_STATUS_LAST: break; @@ -2666,6 +2667,7 @@ enum qemuMigrationCompletedFlags { QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR = (1 << 0), QEMU_MIGRATION_COMPLETED_CHECK_STORAGE = (1 << 1), QEMU_MIGRATION_COMPLETED_UPDATE_STATS = (1 << 2), + QEMU_MIGRATION_COMPLETED_POSTCOPY = (1 << 3), }; /** @@ -2707,6 +2709,20 @@ qemuMigrationCompleted(virQEMUDriverPtr driver, goto error; } + /* In case of postcopy the source considers migration completed at the + * moment it switched from active to postcopy-active state. The destination + * will continue waiting until the migrate state changes to completed. + */ + if (flags & QEMU_MIGRATION_COMPLETED_POSTCOPY && + jobInfo->type == VIR_DOMAIN_JOB_UNBOUNDED && + jobInfo->stats.status == QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY) { + VIR_DEBUG("Migration switched to post-copy"); + if (updateStats && + qemuMigrationUpdateJobStatus(driver, vm, asyncJob) < 0) + goto error; + return 1; + } + if (jobInfo->type == VIR_DOMAIN_JOB_COMPLETED) return 1; else @@ -2740,9 +2756,11 @@ qemuMigrationWaitForCompletion(virQEMUDriverPtr driver, qemuDomainObjPrivatePtr priv = vm->privateData; qemuDomainJobInfoPtr jobInfo = priv->job.current; bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT); - unsigned int flags = QEMU_MIGRATION_COMPLETED_UPDATE_STATS; + unsigned int flags; int rv; + flags = QEMU_MIGRATION_COMPLETED_UPDATE_STATS | + QEMU_MIGRATION_COMPLETED_POSTCOPY; if (abort_on_error) flags |= QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR; if (storage) @@ -2781,9 +2799,11 @@ qemuMigrationWaitForCompletion(virQEMUDriverPtr driver, static int qemuMigrationWaitForDestCompletion(virQEMUDriverPtr driver, virDomainObjPtr vm, - qemuDomainAsyncJob asyncJob) + qemuDomainAsyncJob asyncJob, + bool postcopy) { qemuDomainObjPrivatePtr priv = vm->privateData; + unsigned int flags = 0; int rv; if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT)) @@ -2791,7 +2811,11 @@ qemuMigrationWaitForDestCompletion(virQEMUDriverPtr driver, VIR_DEBUG("Waiting for incoming migration to complete"); - while ((rv = qemuMigrationCompleted(driver, vm, asyncJob, NULL, 0)) != 1) { + if (postcopy) + flags = QEMU_MIGRATION_COMPLETED_POSTCOPY; + + while ((rv = qemuMigrationCompleted(driver, vm, asyncJob, + NULL, flags)) != 1) { if (rv < 0 || virDomainObjWait(vm) < 0) return -1; } @@ -2992,7 +3016,7 @@ qemuMigrationRunIncoming(virQEMUDriverPtr driver, goto cleanup; } - if (qemuMigrationWaitForDestCompletion(driver, vm, asyncJob) < 0) + if (qemuMigrationWaitForDestCompletion(driver, vm, asyncJob, false) < 0) goto cleanup; ret = 0; @@ -4378,6 +4402,7 @@ qemuMigrationRun(virQEMUDriverPtr driver, unsigned int cookieFlags = 0; bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR); bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT); + bool inPostCopy = false; int rc; VIR_DEBUG("driver=%p, vm=%p, cookiein=%s, cookieinlen=%d, " @@ -4563,6 +4588,9 @@ qemuMigrationRun(virQEMUDriverPtr driver, else if (rc == -1) goto cleanup; + if (priv->job.current->stats.status == QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY) + inPostCopy = true; + /* When migration completed, QEMU will have paused the CPUs for us. * Wait for the STOP event to be processed or explicitly stop CPUs * (for old QEMU which does not send events) to release the lock state. @@ -4572,15 +4600,12 @@ qemuMigrationRun(virQEMUDriverPtr driver, priv->signalStop = true; rc = virDomainObjWait(vm); priv->signalStop = false; - if (rc < 0) { - priv->job.current->type = VIR_DOMAIN_JOB_FAILED; - goto cleanup; - } + if (rc < 0) + goto cancelPostCopy; } } else if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING && qemuMigrationSetOffline(driver, vm) < 0) { - priv->job.current->type = VIR_DOMAIN_JOB_FAILED; - goto cleanup; + goto cancelPostCopy; } ret = 0; @@ -4609,7 +4634,7 @@ qemuMigrationRun(virQEMUDriverPtr driver, ignore_value(virTimeMillisNow(&priv->job.completed->sent)); } - if (priv->job.current->type == VIR_DOMAIN_JOB_UNBOUNDED) + if (priv->job.current->type == VIR_DOMAIN_JOB_UNBOUNDED && !inPostCopy) priv->job.current->type = VIR_DOMAIN_JOB_FAILED; cookieFlags |= QEMU_MIGRATION_COOKIE_NETWORK | @@ -4649,6 +4674,13 @@ qemuMigrationRun(virQEMUDriverPtr driver, } } goto cleanup; + + cancelPostCopy: + priv->job.current->type = VIR_DOMAIN_JOB_FAILED; + if (inPostCopy) + goto cancel; + else + goto cleanup; } /* Perform migration using QEMU's native migrate support, @@ -5800,6 +5832,7 @@ qemuMigrationFinish(virQEMUDriverPtr driver, unsigned short port; unsigned long long timeReceived = 0; virObjectEventPtr event; + bool inPostCopy = false; VIR_DEBUG("driver=%p, dconn=%p, vm=%p, cookiein=%s, cookieinlen=%d, " "cookieout=%p, cookieoutlen=%p, flags=%lx, retcode=%d", @@ -5891,7 +5924,8 @@ qemuMigrationFinish(virQEMUDriverPtr driver, * before starting guest CPUs. */ if (qemuMigrationWaitForDestCompletion(driver, vm, - QEMU_ASYNC_JOB_MIGRATION_IN) < 0) { + QEMU_ASYNC_JOB_MIGRATION_IN, + !!(flags & VIR_MIGRATE_POSTCOPY)) < 0) { /* There's not much we can do for v2 protocol since the * original domain on the source host is already gone. */ @@ -5899,13 +5933,17 @@ qemuMigrationFinish(virQEMUDriverPtr driver, goto endjob; } + if (priv->job.current->stats.status == QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY) + inPostCopy = true; + if (!(flags & VIR_MIGRATE_PAUSED)) { /* run 'cont' on the destination, which allows migration on qemu * >= 0.10.6 to work properly. This isn't strictly necessary on * older qemu's, but it also doesn't hurt anything there */ if (qemuProcessStartCPUs(driver, vm, dconn, - VIR_DOMAIN_RUNNING_MIGRATED, + inPostCopy ? VIR_DOMAIN_RUNNING_POSTCOPY + : VIR_DOMAIN_RUNNING_MIGRATED, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) { if (virGetLastError() == NULL) virReportError(VIR_ERR_INTERNAL_ERROR, @@ -5926,6 +5964,13 @@ qemuMigrationFinish(virQEMUDriverPtr driver, if (v3proto) goto endjob; } + + if (inPostCopy) { + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_RESUMED, + VIR_DOMAIN_EVENT_RESUMED_POSTCOPY); + qemuDomainEventQueue(driver, event); + } } if (mig->jobInfo) { @@ -5942,6 +5987,19 @@ qemuMigrationFinish(virQEMUDriverPtr driver, qemuDomainJobInfoUpdateDowntime(priv->job.completed); } + if (inPostCopy) { + if (qemuMigrationWaitForDestCompletion(driver, vm, + QEMU_ASYNC_JOB_MIGRATION_IN, + false) < 0) { + goto endjob; + } + if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING) { + virDomainObjSetState(vm, + VIR_DOMAIN_RUNNING, + VIR_DOMAIN_RUNNING_MIGRATED); + } + } + dom = virGetDomain(dconn, vm->def->name, vm->def->uuid); event = virDomainEventLifecycleNewFromObj(vm, diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index 6b23e88..edb3310 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -161,7 +161,7 @@ VIR_ONCE_GLOBAL_INIT(qemuMonitor) VIR_ENUM_IMPL(qemuMonitorMigrationStatus, QEMU_MONITOR_MIGRATION_STATUS_LAST, "inactive", "setup", - "active", + "active", "postcopy-active", "completed", "failed", "cancelling", "cancelled") diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 9d7d5f3..420d82c 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -467,6 +467,7 @@ typedef enum { QEMU_MONITOR_MIGRATION_STATUS_INACTIVE, QEMU_MONITOR_MIGRATION_STATUS_SETUP, QEMU_MONITOR_MIGRATION_STATUS_ACTIVE, + QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY, QEMU_MONITOR_MIGRATION_STATUS_COMPLETED, QEMU_MONITOR_MIGRATION_STATUS_ERROR, QEMU_MONITOR_MIGRATION_STATUS_CANCELLING, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index 24a8865..8d71dd4 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -2493,6 +2493,7 @@ qemuMonitorJSONGetMigrationStatsReply(virJSONValuePtr reply, break; case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED: case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: ram = virJSONValueObjectGetObject(ret, "ram"); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index cab1aee..5ced279 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -715,8 +715,14 @@ qemuProcessHandleStop(qemuMonitorPtr mon ATTRIBUTE_UNUSED, } if (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT) { - reason = VIR_DOMAIN_PAUSED_MIGRATION; - detail = VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED; + if (priv->job.current->stats.status == + QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY) { + reason = VIR_DOMAIN_PAUSED_POSTCOPY; + detail = VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY; + } else { + reason = VIR_DOMAIN_PAUSED_MIGRATION; + detail = VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED; + } } VIR_DEBUG("Transitioned guest %s to paused state, reason %s", -- 2.7.0

From: Cristian Klein <cristiklein@gmail.com> Signed-off-by: Cristian Klein <cristiklein@gmail.com> Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 2: - no change src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_driver.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_migration.c | 2 ++ 4 files changed, 62 insertions(+) diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 0fa2dbe..15a9aa8 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -188,6 +188,7 @@ qemuDomainObjResetAsyncJob(qemuDomainObjPrivatePtr priv) job->dump_memory_only = false; job->abortJob = false; job->spiceMigrated = false; + job->postcopyEnabled = false; VIR_FREE(job->current); } diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 9fd9076..2bea482 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -138,6 +138,7 @@ struct qemuDomainJobObj { qemuDomainJobInfoPtr completed; /* statistics data of a recently completed job */ bool abortJob; /* abort of the job requested */ bool spiceMigrated; /* spice migration completed */ + bool postcopyEnabled; /* post-copy migration was enabled */ }; typedef void (*qemuDomainCleanupCallback)(virQEMUDriverPtr driver, diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 8ccf68b..c62561b 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -13421,6 +13421,63 @@ qemuDomainMigrateGetMaxSpeed(virDomainPtr dom, } +static int +qemuDomainMigrateStartPostCopy(virDomainPtr dom, + unsigned int flags) +{ + virQEMUDriverPtr driver = dom->conn->privateData; + virDomainObjPtr vm; + qemuDomainObjPrivatePtr priv; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = qemuDomObjFromDomain(dom))) + goto cleanup; + + if (virDomainMigrateStartPostCopyEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MIGRATION_OP) < 0) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("domain is not running")); + goto endjob; + } + + priv = vm->privateData; + + if (priv->job.asyncJob != QEMU_ASYNC_JOB_MIGRATION_OUT) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("post-copy can only be started while " + "outgoing migration is in progress")); + goto endjob; + } + + if (!priv->job.postcopyEnabled) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("switching to post-copy requires migration to be " + "started with VIR_MIGRATE_POSTCOPY flag")); + goto endjob; + } + + VIR_DEBUG("Starting post-copy"); + qemuDomainObjEnterMonitor(driver, vm); + ret = qemuMonitorMigrateStartPostCopy(priv->mon); + if (qemuDomainObjExitMonitor(driver, vm) < 0) + ret = -1; + + endjob: + qemuDomainObjEndJob(driver, vm); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + + typedef enum { VIR_DISK_CHAIN_NO_ACCESS, VIR_DISK_CHAIN_READ_ONLY, @@ -20222,6 +20279,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainInterfaceAddresses = qemuDomainInterfaceAddresses, /* 1.2.14 */ .domainSetUserPassword = qemuDomainSetUserPassword, /* 1.2.16 */ .domainRename = qemuDomainRename, /* 1.2.19 */ + .domainMigrateStartPostCopy = qemuDomainMigrateStartPostCopy, /* 1.3.2 */ }; diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 64c1eb8..c5bd2fc 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2542,6 +2542,8 @@ qemuMigrationSetPostCopy(virQEMUDriverPtr driver, cleanup: if (qemuDomainObjExitMonitor(driver, vm) < 0) ret = -1; + if (ret == 0) + priv->job.postcopyEnabled = state; return ret; } -- 2.7.0

Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 2: - no change tools/virsh-domain.c | 63 +++++++++++++++++++++++++++++++++++++++++++++------- tools/virsh.pod | 15 ++++++++----- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index bcb6809..96125bb 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -9645,7 +9645,16 @@ static const vshCmdOptDef opts_migrate[] = { }, {.name = "timeout", .type = VSH_OT_INT, - .help = N_("force guest to suspend if live migration exceeds timeout (in seconds)") + .help = N_("run action specified by --timeout-* option (suspend by " + "default) if live migration exceeds timeout (in seconds)") + }, + {.name = "timeout-suspend", + .type = VSH_OT_BOOL, + .help = N_("suspend the guest after timeout") + }, + {.name = "timeout-postcopy", + .type = VSH_OT_BOOL, + .help = N_("switch to post-copy after timeout") }, {.name = "xml", .type = VSH_OT_STRING, @@ -9823,14 +9832,35 @@ doMigrate(void *opaque) goto out; } +typedef enum { + VIRSH_MIGRATE_TIMEOUT_DEFAULT, + VIRSH_MIGRATE_TIMEOUT_SUSPEND, + VIRSH_MIGRATE_TIMEOUT_POSTCOPY, +} virshMigrateTimeoutAction; + static void -virshMigrationTimeout(vshControl *ctl, - virDomainPtr dom, - void *opaque ATTRIBUTE_UNUSED) +virshMigrateTimeout(vshControl *ctl, + virDomainPtr dom, + void *opaque) { - vshDebug(ctl, VSH_ERR_DEBUG, "suspending the domain, " - "since migration timed out\n"); - virDomainSuspend(dom); + virshMigrateTimeoutAction action = *(virshMigrateTimeoutAction *) opaque; + + switch (action) { + case VIRSH_MIGRATE_TIMEOUT_DEFAULT: /* unreachable */ + case VIRSH_MIGRATE_TIMEOUT_SUSPEND: + vshDebug(ctl, VSH_ERR_DEBUG, + "migration timed out; suspending domain\n"); + if (virDomainSuspend(dom) < 0) + vshDebug(ctl, VSH_ERR_INFO, "suspending domain failed\n"); + break; + + case VIRSH_MIGRATE_TIMEOUT_POSTCOPY: + vshDebug(ctl, VSH_ERR_DEBUG, + "migration timed out; switching to post-copy\n"); + if (virDomainMigrateStartPostCopy(dom, 0) < 0) + vshDebug(ctl, VSH_ERR_INFO, "switching to post-copy failed\n"); + break; + } } static bool @@ -9842,9 +9872,12 @@ cmdMigrate(vshControl *ctl, const vshCmd *cmd) bool verbose = false; bool functionReturn = false; int timeout = 0; + virshMigrateTimeoutAction timeoutAction = VIRSH_MIGRATE_TIMEOUT_DEFAULT; bool live_flag = false; virshCtrlData data = { .dconn = NULL }; + VSH_EXCLUSIVE_OPTIONS("timeout-suspend", "timeout-postcopy"); + if (!(dom = virshCommandOptDomain(ctl, cmd, NULL))) return false; @@ -9861,6 +9894,19 @@ cmdMigrate(vshControl *ctl, const vshCmd *cmd) goto cleanup; } + if (vshCommandOptBool(cmd, "timeout-suspend")) + timeoutAction = VIRSH_MIGRATE_TIMEOUT_SUSPEND; + if (vshCommandOptBool(cmd, "timeout-postcopy")) + timeoutAction = VIRSH_MIGRATE_TIMEOUT_POSTCOPY; + if (timeout > 0) { + if (timeoutAction == VIRSH_MIGRATE_TIMEOUT_DEFAULT) + timeoutAction = VIRSH_MIGRATE_TIMEOUT_SUSPEND; + } else if (timeoutAction) { + vshError(ctl, "%s", + _("migrate: Unexpected --timeout-* option without --timeout")); + goto cleanup; + } + if (pipe(p) < 0) goto cleanup; @@ -9891,7 +9937,8 @@ cmdMigrate(vshControl *ctl, const vshCmd *cmd) &data) < 0) goto cleanup; functionReturn = virshWatchJob(ctl, dom, verbose, p[0], timeout, - virshMigrationTimeout, NULL, _("Migration")); + virshMigrateTimeout, + &timeoutAction, _("Migration")); virThreadJoin(&workerThread); diff --git a/tools/virsh.pod b/tools/virsh.pod index 109c608..64b4287 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -1531,8 +1531,8 @@ to the I<uri> namespace is displayed instead of being modified. [I<--copy-storage-inc>] [I<--change-protection>] [I<--unsafe>] [I<--verbose>] [I<--compressed>] [I<--abort-on-error>] [I<--auto-converge>] [I<--postcopy>] I<domain> I<desturi> [I<migrateuri>] [I<graphicsuri>] [I<listen-address>] -[I<dname>] [I<--timeout> B<seconds>] [I<--xml> B<file>] -[I<--migrate-disks> B<disk-list>] +[I<dname>] [I<--timeout> B<seconds> [I<--timeout-suspend> | I<--timeout-postcopy>]] +[I<--xml> B<file>] [I<--migrate-disks> B<disk-list>] Migrate domain to another host. Add I<--live> for live migration; <--p2p> for peer-2-peer migration; I<--direct> for direct migration; or I<--tunnelled> @@ -1582,9 +1582,14 @@ the destination to supply a larger set of changes to any host-specific portions of the domain XML, such as accounting for naming differences between source and destination in accessing underlying storage. -I<--timeout> B<seconds> forces guest to suspend when live migration exceeds -that many seconds, and -then the migration will complete offline. It can only be used with I<--live>. +I<--timeout> B<seconds> tells virsh to run a specified action when live +migration exceeds that many seconds. It can only be used with I<--live>. +If I<--timeout-suspend> is specified, the domain will be suspended after +the timeout and the migration will complete offline; this is the default +if no I<--timeout-*> option is specified on the command line. When +I<--timeout-postcopy> is used, virsh will switch migration from pre-copy +to post-copy upon timeout; migration has to be started with I<--postcopy> +option for this to work. Running migration can be canceled by interrupting virsh (usually using C<Ctrl-C>) or by B<domjobabort> command sent from another virsh instance. -- 2.7.0

When destination libvirtd is restarted during migration in Finish phase just after the point we started guest CPUs, we should not kill the domain. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 2: - no change src/qemu/qemu_process.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 5ced279..879cd12 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3304,9 +3304,13 @@ qemuProcessRecoverMigration(virQEMUDriverPtr driver, case QEMU_MIGRATION_PHASE_FINISH3: /* migration finished, we started resuming the domain but didn't - * confirm success or failure yet; killing it seems safest */ - VIR_DEBUG("Killing migrated domain %s", vm->def->name); - return -1; + * confirm success or failure yet; killing it seems safest unless + * we already started guest CPUs */ + if (state != VIR_DOMAIN_RUNNING) { + VIR_DEBUG("Killing migrated domain %s", vm->def->name); + return -1; + } + break; } } else if (job == QEMU_ASYNC_JOB_MIGRATION_OUT) { switch (phase) { -- 2.7.0

When migration fails in the post-copy mode, it's impossible to just kill the destination domain and resume the source since the source no longer contains current guest state. Let's mark domains on both sides as VIR_DOMAIN_PAUSED_POSTCOPY_FAILED to let the upper layer decide what to do with them. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 2: - no change src/qemu/qemu_migration.c | 87 +++++++++++++++++++++++++++++++++++++++-------- src/qemu/qemu_migration.h | 3 ++ src/qemu/qemu_process.c | 59 ++++++++++++++++++++++++-------- 3 files changed, 119 insertions(+), 30 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index c5bd2fc..2a34b4a 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -1475,14 +1475,21 @@ qemuMigrationRestoreDomainState(virConnectPtr conn, virDomainObjPtr vm) { virQEMUDriverPtr driver = conn->privateData; qemuDomainObjPrivatePtr priv = vm->privateData; - int state = virDomainObjGetState(vm, NULL); + int reason; + virDomainState state = virDomainObjGetState(vm, &reason); bool ret = false; - VIR_DEBUG("driver=%p, vm=%p, pre-mig-state=%d, state=%d", - driver, vm, priv->preMigrationState, state); + VIR_DEBUG("driver=%p, vm=%p, pre-mig-state=%s, state=%s, reason=%s", + driver, vm, + virDomainStateTypeToString(priv->preMigrationState), + virDomainStateTypeToString(state), + virDomainStateReasonToString(state, reason)); - if (state == VIR_DOMAIN_PAUSED && - priv->preMigrationState == VIR_DOMAIN_RUNNING) { + if (state != VIR_DOMAIN_PAUSED || + reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED) + goto cleanup; + + if (priv->preMigrationState == VIR_DOMAIN_RUNNING) { /* This is basically the only restore possibility that's safe * and we should attempt to do */ @@ -2365,6 +2372,46 @@ qemuMigrationSetOffline(virQEMUDriverPtr driver, } +void +qemuMigrationPostcopyFailed(virQEMUDriverPtr driver, + virDomainObjPtr vm) +{ + virObjectEventPtr event; + virDomainState state; + int reason; + + state = virDomainObjGetState(vm, &reason); + + if (state != VIR_DOMAIN_PAUSED && + state != VIR_DOMAIN_RUNNING) + return; + + if (state == VIR_DOMAIN_PAUSED && + reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED) + return; + + VIR_WARN("Migration of domain %s failed during post-copy; " + "leaving the domain paused", vm->def->name); + + if (state == VIR_DOMAIN_RUNNING) { + if (qemuProcessStopCPUs(driver, vm, + VIR_DOMAIN_PAUSED_POSTCOPY_FAILED, + QEMU_ASYNC_JOB_MIGRATION_IN) < 0) { + VIR_WARN("Unable to pause guest CPUs for %s", vm->def->name); + return; + } + } else { + virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, + VIR_DOMAIN_PAUSED_POSTCOPY_FAILED); + } + + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED); + qemuDomainEventQueue(driver, event); +} + + static int qemuMigrationSetCompression(virQEMUDriverPtr driver, virDomainObjPtr vm, @@ -4078,8 +4125,8 @@ qemuMigrationConfirmPhase(virQEMUDriverPtr driver, if (flags & VIR_MIGRATE_OFFLINE) goto done; - /* Did the migration go as planned? If yes, kill off the - * domain object, but if no, resume CPUs + /* Did the migration go as planned? If yes, kill off the domain object. + * If something failed, resume CPUs, but only if we didn't use post-copy. */ if (retcode == 0) { /* If guest uses SPICE and supports seamless migration we have to hold @@ -4095,6 +4142,7 @@ qemuMigrationConfirmPhase(virQEMUDriverPtr driver, VIR_DOMAIN_EVENT_STOPPED_MIGRATED); } else { virErrorPtr orig_err = virSaveLastError(); + int reason; /* cancel any outstanding NBD jobs */ qemuMigrationCancelDriveMirror(driver, vm, false, @@ -4103,7 +4151,10 @@ qemuMigrationConfirmPhase(virQEMUDriverPtr driver, virSetError(orig_err); virFreeError(orig_err); - if (qemuMigrationRestoreDomainState(conn, vm)) { + if (virDomainObjGetState(vm, &reason) == VIR_DOMAIN_PAUSED && + reason == VIR_DOMAIN_PAUSED_POSTCOPY) { + qemuMigrationPostcopyFailed(driver, vm); + } else if (qemuMigrationRestoreDomainState(conn, vm)) { event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_RESUMED, VIR_DOMAIN_EVENT_RESUMED_MIGRATED); @@ -5911,6 +5962,7 @@ qemuMigrationFinish(virQEMUDriverPtr driver, unsigned long long timeReceived = 0; virObjectEventPtr event; bool inPostCopy = false; + bool kill = true; VIR_DEBUG("driver=%p, dconn=%p, vm=%p, cookiein=%s, cookieinlen=%d, " "cookieout=%p, cookieoutlen=%p, flags=%lx, retcode=%d", @@ -6044,6 +6096,7 @@ qemuMigrationFinish(virQEMUDriverPtr driver, } if (inPostCopy) { + kill = false; event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_RESUMED, VIR_DOMAIN_EVENT_RESUMED_POSTCOPY); @@ -6104,13 +6157,17 @@ qemuMigrationFinish(virQEMUDriverPtr driver, if (!dom && !(flags & VIR_MIGRATE_OFFLINE) && virDomainObjIsActive(vm)) { - qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, - VIR_QEMU_PROCESS_STOP_MIGRATED); - virDomainAuditStop(vm, "failed"); - event = virDomainEventLifecycleNewFromObj(vm, - VIR_DOMAIN_EVENT_STOPPED, - VIR_DOMAIN_EVENT_STOPPED_FAILED); - qemuDomainEventQueue(driver, event); + if (kill) { + qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, + VIR_QEMU_PROCESS_STOP_MIGRATED); + virDomainAuditStop(vm, "failed"); + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_FAILED); + qemuDomainEventQueue(driver, event); + } else { + qemuMigrationPostcopyFailed(driver, vm); + } } if (dom && diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index e30b2ec..46b37a3 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -211,4 +211,7 @@ int qemuMigrationRunIncoming(virQEMUDriverPtr driver, const char *uri, qemuDomainAsyncJob asyncJob); +void qemuMigrationPostcopyFailed(virQEMUDriverPtr driver, + virDomainObjPtr vm); + #endif /* __QEMU_MIGRATION_H__ */ diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index a0c93c9..fba9a65 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3270,8 +3270,13 @@ qemuProcessRecoverMigrationIn(virQEMUDriverPtr driver, virConnectPtr conn, qemuMigrationJobPhase phase, virDomainState state, - int reason ATTRIBUTE_UNUSED) + int reason) { + bool postcopy = (state == VIR_DOMAIN_PAUSED && + reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED) || + (state == VIR_DOMAIN_RUNNING && + reason == VIR_DOMAIN_RUNNING_POSTCOPY); + switch (phase) { case QEMU_MIGRATION_PHASE_NONE: case QEMU_MIGRATION_PHASE_PERFORM2: @@ -3304,8 +3309,10 @@ qemuProcessRecoverMigrationIn(virQEMUDriverPtr driver, case QEMU_MIGRATION_PHASE_FINISH3: /* migration finished, we started resuming the domain but didn't * confirm success or failure yet; killing it seems safest unless - * we already started guest CPUs */ - if (state != VIR_DOMAIN_RUNNING) { + * we already started guest CPUs or we were in post-copy mode */ + if (postcopy) { + qemuMigrationPostcopyFailed(driver, vm); + } else if (state != VIR_DOMAIN_RUNNING) { VIR_DEBUG("Killing migrated domain %s", vm->def->name); return -1; } @@ -3323,6 +3330,10 @@ qemuProcessRecoverMigrationOut(virQEMUDriverPtr driver, virDomainState state, int reason) { + bool postcopy = state == VIR_DOMAIN_PAUSED && + (reason == VIR_DOMAIN_PAUSED_POSTCOPY || + reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED); + switch (phase) { case QEMU_MIGRATION_PHASE_NONE: case QEMU_MIGRATION_PHASE_PREPARE: @@ -3340,26 +3351,44 @@ qemuProcessRecoverMigrationOut(virQEMUDriverPtr driver, case QEMU_MIGRATION_PHASE_PERFORM2: case QEMU_MIGRATION_PHASE_PERFORM3: /* migration is still in progress, let's cancel it and resume the - * domain */ - VIR_DEBUG("Cancelling unfinished migration of domain %s", - vm->def->name); - if (qemuMigrationCancel(driver, vm) < 0) { - VIR_WARN("Could not cancel ongoing migration of domain %s", - vm->def->name); + * domain; however we can only do that before migration enters + * post-copy mode + */ + if (postcopy) { + qemuMigrationPostcopyFailed(driver, vm); + } else { + VIR_DEBUG("Cancelling unfinished migration of domain %s", + vm->def->name); + if (qemuMigrationCancel(driver, vm) < 0) { + VIR_WARN("Could not cancel ongoing migration of domain %s", + vm->def->name); + } + goto resume; } - goto resume; + break; case QEMU_MIGRATION_PHASE_PERFORM3_DONE: /* migration finished but we didn't have a chance to get the result - * of Finish3 step; third party needs to check what to do next + * of Finish3 step; third party needs to check what to do next; in + * post-copy mode we can use PAUSED_POSTCOPY_FAILED state for this */ + if (postcopy) + qemuMigrationPostcopyFailed(driver, vm); break; case QEMU_MIGRATION_PHASE_CONFIRM3_CANCELLED: - /* Finish3 failed, we need to resume the domain */ - VIR_DEBUG("Resuming domain %s after failed migration", - vm->def->name); - goto resume; + /* Finish3 failed, we need to resume the domain, but once we enter + * post-copy mode there's no way back, so let's just mark the domain + * as broken in that case + */ + if (postcopy) { + qemuMigrationPostcopyFailed(driver, vm); + } else { + VIR_DEBUG("Resuming domain %s after failed migration", + vm->def->name); + goto resume; + } + break; case QEMU_MIGRATION_PHASE_CONFIRM3: /* migration completed, we need to kill the domain here */ -- 2.7.0

The function already takes two bool arguments, switching to flags makes it a lot easier to read. Especially in case we need to add another boolean in the future. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 2: - moved to the end of the series since it is not required by another patch in this series anymore src/qemu/qemu_migration.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 2a34b4a..55a8be0 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2847,21 +2847,14 @@ qemuMigrationWaitForCompletion(virQEMUDriverPtr driver, virDomainObjPtr vm, qemuDomainAsyncJob asyncJob, virConnectPtr dconn, - bool abort_on_error, - bool storage) + unsigned int flags) { qemuDomainObjPrivatePtr priv = vm->privateData; qemuDomainJobInfoPtr jobInfo = priv->job.current; bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT); - unsigned int flags; int rv; - flags = QEMU_MIGRATION_COMPLETED_UPDATE_STATS | - QEMU_MIGRATION_COMPLETED_POSTCOPY; - if (abort_on_error) - flags |= QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR; - if (storage) - flags |= QEMU_MIGRATION_COMPLETED_CHECK_STORAGE; + flags |= QEMU_MIGRATION_COMPLETED_UPDATE_STATS; jobInfo->type = VIR_DOMAIN_JOB_UNBOUNDED; while ((rv = qemuMigrationCompleted(driver, vm, asyncJob, @@ -4527,6 +4520,7 @@ qemuMigrationRun(virQEMUDriverPtr driver, bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR); bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT); bool inPostCopy = false; + unsigned int waitFlags; int rc; VIR_DEBUG("driver=%p, vm=%p, cookiein=%s, cookieinlen=%d, " @@ -4709,9 +4703,17 @@ qemuMigrationRun(virQEMUDriverPtr driver, fd = -1; } + waitFlags = 0; + if (abort_on_error) + waitFlags |= QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR; + if (mig->nbd) + waitFlags |= QEMU_MIGRATION_COMPLETED_CHECK_STORAGE; + if (flags & VIR_MIGRATE_POSTCOPY) + waitFlags |= QEMU_MIGRATION_COMPLETED_POSTCOPY; + rc = qemuMigrationWaitForCompletion(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT, - dconn, abort_on_error, !!mig->nbd); + dconn, waitFlags); if (rc == -2) goto cancel; else if (rc == -1) @@ -6327,8 +6329,7 @@ qemuMigrationToFile(virQEMUDriverPtr driver, virDomainObjPtr vm, if (rc < 0) goto cleanup; - rc = qemuMigrationWaitForCompletion(driver, vm, asyncJob, - NULL, false, false); + rc = qemuMigrationWaitForCompletion(driver, vm, asyncJob, NULL, 0); if (rc < 0) { if (rc == -2) { -- 2.7.0