From: Cristian Klein <cristiklein@gmail.com> To use post-copy one has to start the migration with VIR_MIGRATE_POSTCOPY flag and, while migration is in progress, call virDomainMigrateStartPostCopy() to switch from pre-copy to post-copy. Signed-off-by: Cristian Klein <cristiklein@gmail.com> Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 3: - VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED will only be emitted on the destination host Version 2: - POSTCOPY_AFTER_PRECOPY flag removed include/libvirt/libvirt-domain.h | 4 ++ src/driver-hypervisor.h | 5 ++ src/libvirt-domain.c | 141 ++++++++++++++++++++++++++++++++++++--- src/libvirt_public.syms | 4 ++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 +++- src/remote_protocol-structs | 5 ++ 7 files changed, 164 insertions(+), 9 deletions(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 3846ab9..dede3da 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -678,6 +678,7 @@ typedef enum { VIR_MIGRATE_ABORT_ON_ERROR = (1 << 12), /* abort migration on I/O errors happened during migration */ VIR_MIGRATE_AUTO_CONVERGE = (1 << 13), /* force convergence */ VIR_MIGRATE_RDMA_PIN_ALL = (1 << 14), /* RDMA memory pinning */ + VIR_MIGRATE_POSTCOPY = (1 << 15), /* enable (but do not start) post-copy migration */ } virDomainMigrateFlags; @@ -826,6 +827,9 @@ int virDomainMigrateGetMaxSpeed(virDomainPtr domain, unsigned long *bandwidth, unsigned int flags); +int virDomainMigrateStartPostCopy(virDomainPtr domain, + unsigned int flags); + char * virConnectGetDomainCapabilities(virConnectPtr conn, const char *emulatorbin, const char *arch, diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 2cbd01b..9bc3211 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -638,6 +638,10 @@ typedef int const char *dom_xml); typedef int +(*virDrvDomainMigrateStartPostCopy)(virDomainPtr domain, + unsigned int flags); + +typedef int (*virDrvConnectIsEncrypted)(virConnectPtr conn); typedef int @@ -1455,6 +1459,7 @@ struct _virHypervisorDriver { virDrvDomainSetUserPassword domainSetUserPassword; virDrvConnectRegisterCloseCallback connectRegisterCloseCallback; virDrvConnectUnregisterCloseCallback connectUnregisterCloseCallback; + virDrvDomainMigrateStartPostCopy domainMigrateStartPostCopy; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index ca32dc1..1b95664 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -3537,6 +3537,7 @@ virDomainMigrateUnmanaged(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set. * Applications using the VIR_MIGRATE_PEER2PEER flag will probably @@ -3573,6 +3574,11 @@ virDomainMigrateUnmanaged(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -3748,6 +3754,7 @@ virDomainMigrate(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set. * Applications using the VIR_MIGRATE_PEER2PEER flag will probably @@ -3784,6 +3791,11 @@ virDomainMigrate(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -3968,6 +3980,11 @@ virDomainMigrate2(virDomainPtr domain, * can use either VIR_MIGRATE_NON_SHARED_DISK or * VIR_MIGRATE_NON_SHARED_INC as they are mutually exclusive. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * There are many limitations on migration imposed by the underlying * technology - for example it may not be possible to migrate between * different processors even with the same architecture, or between @@ -4208,6 +4225,7 @@ int virDomainMigrateUnmanagedCheckCompat(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * The operation of this API hinges on the VIR_MIGRATE_PEER2PEER flag. * If the VIR_MIGRATE_PEER2PEER flag is NOT set, the duri parameter @@ -4240,6 +4258,11 @@ int virDomainMigrateUnmanagedCheckCompat(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -4321,6 +4344,7 @@ virDomainMigrateToURI(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * The operation of this API hinges on the VIR_MIGRATE_PEER2PEER flag. * @@ -4366,6 +4390,11 @@ virDomainMigrateToURI(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -4446,6 +4475,11 @@ virDomainMigrateToURI2(virDomainPtr domain, * can use either VIR_MIGRATE_NON_SHARED_DISK or * VIR_MIGRATE_NON_SHARED_INC as they are mutually exclusive. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * There are many limitations on migration imposed by the underlying * technology - for example it may not be possible to migrate between * different processors even with the same architecture, or between @@ -8868,14 +8902,15 @@ virDomainGetJobInfo(virDomainPtr domain, virDomainJobInfoPtr info) * * When @flags contains VIR_DOMAIN_JOB_STATS_COMPLETED, the function will * return statistics about a recently completed job. Specifically, this - * flag may be used to query statistics of a completed incoming migration. - * Statistics of a completed job are automatically destroyed once read or - * when libvirtd is restarted. Note that time information returned for - * completed migrations may be completely irrelevant unless both source and - * destination hosts have synchronized time (i.e., NTP daemon is running on - * both of them). The statistics of a completed job can also be obtained by - * listening to a VIR_DOMAIN_EVENT_ID_JOB_COMPLETED event (on the source host - * in case of a migration job). + * flag may be used to query statistics of a completed incoming pre-copy + * migration (statistics for post-copy migration are only available on the + * source hsot). Statistics of a completed job are automatically destroyed + * once read or when libvirtd is restarted. Note that time information + * returned for completed migrations may be completely irrelevant unless both + * source and destination hosts have synchronized time (i.e., NTP daemon is + * running on both of them). The statistics of a completed job can also be + * obtained by listening to a VIR_DOMAIN_EVENT_ID_JOB_COMPLETED event (on the + * source host in case of a migration job). * * Returns 0 in case of success and -1 in case of failure. */ @@ -9165,6 +9200,96 @@ virDomainMigrateGetMaxSpeed(virDomainPtr domain, /** + * virDomainMigrateStartPostCopy: + * @domain: a domain object + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Starts post-copy migration. This function has to be called while + * migration (initiated with VIR_MIGRATE_POSTCOPY flag) is in progress. + * + * Traditional post-copy migration iteratively walks through guest memory + * pages and migrates those that changed since the previous iteration. The + * iterative phase stops when the number of dirty pages is low enough so that + * the virtual CPUs can be paused, all dirty pages transferred to the + * destination, where the virtual CPUs are unpaused, and all this can happen + * within a predefined downtime period. It's clear that this process may never + * converge if downtime is too short and/or the guest keeps changing a lot of + * memory pages. + * + * When migration is switched to post-copy mode, the virtual CPUs are paused + * immediately, only a minimum set of pages is transferred, and the CPUs are + * unpaused on destination. The source keeps sending all remaining memory pages + * to the destination while the guest is already running there. Whenever the + * guest tries to read a memory page which has not been migrated yet, the + * hypervisor has to tell the source to transfer that page in a priority + * channel. To minimize such page faults, it is a good idea to run at least one + * iteration of pre-copy migration before switching to post-copy. + * + * Post-copy migration is guaranteed to converge since each page is transferred + * at most once no matter how fast it changes. On the other hand once the + * guest is running on the destination host, the migration can no longer be + * rolled back because none of the hosts has complete state. If this happens, + * libvirt will leave the domain paused on both hosts with + * VIR_DOMAIN_PAUSED_POSTCOPY_FAILED reason. It's up to the upper layer to + * decide what to do in such case. + * + * The following domain life cycle events are emitted during post-copy + * migration: + * VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY (on the source) -- migration entered + * post-copy mode. + * VIR_DOMAIN_EVENT_RESUMED_POSTCOPY (on the destination) -- the guest is + * running on the destination host while some of its memory pages still + * remain on the source host; neither the source nor the destination host + * contain a complete guest state from this point until migration + * finishes. + * VIR_DOMAIN_EVENT_RESUMED_MIGRATED (on the destination), + * VIR_DOMAIN_EVENT_STOPPED_MIGRATED (on the source) -- migration finished + * successfully and the destination host holds a complete guest state. + * VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED (on the destination) -- emitted + * when migration fails in post-copy mode and it's unclear whether any + * of the hosts has a complete guest state. + * + * The progress of a post-copy migration can be monitored normally using + * virDomainGetJobStats on the source host. Fetching statistics of a completed + * post-copy migration can also be done on the source host (by calling + * virDomainGetJobStats or listening to VIR_DOMAIN_EVENT_ID_JOB_COMPLETED + * event, but (in contrast to pre-copy migration) the statistics are not + * available on the destination host. Thus, VIR_DOMAIN_EVENT_ID_JOB_COMPLETED + * event is the only way of getting statistics of a completed post-copy + * migration of a transient domain (because the domain is removed after + * migration and there's no domain to run virDomainGetJobStats on). + * + * Returns 0 in case of success, -1 otherwise. + */ +int +virDomainMigrateStartPostCopy(virDomainPtr domain, + unsigned int flags) +{ + virConnectPtr conn; + + VIR_DOMAIN_DEBUG(domain); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckReadOnlyGoto(conn->flags, error); + + if (conn->driver->domainMigrateStartPostCopy) { + if (conn->driver->domainMigrateStartPostCopy(domain, flags) < 0) + goto error; + return 0; + } + + virReportUnsupportedError(); + error: + virDispatchError(conn); + return -1; +} + + +/** * virConnectDomainEventRegisterAny: * @conn: pointer to the connection * @dom: pointer to the domain diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index dd94191..d98414d 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -725,4 +725,8 @@ LIBVIRT_1.2.19 { virDomainRename; } LIBVIRT_1.2.17; +LIBVIRT_1.3.2 { + virDomainMigrateStartPostCopy; +} LIBVIRT_1.2.19; + # .... define new API here using predicted next version number .... diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 2daa507..09de5f8 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8417,6 +8417,7 @@ static virHypervisorDriver hypervisor_driver = { .domainRename = remoteDomainRename, /* 1.2.19 */ .connectRegisterCloseCallback = remoteConnectRegisterCloseCallback, /* 1.3.2 */ .connectUnregisterCloseCallback = remoteConnectUnregisterCloseCallback, /* 1.3.2 */ + .domainMigrateStartPostCopy = remoteDomainMigrateStartPostCopy, /* 1.3.3 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 952686c..6adad7a 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -3238,6 +3238,11 @@ struct remote_domain_event_callback_job_completed_msg { remote_typed_param params<REMOTE_DOMAIN_JOB_STATS_MAX>; }; +struct remote_domain_migrate_start_post_copy_args { + remote_nonnull_domain dom; + unsigned int flags; +}; + /*----- Protocol. -----*/ /* Define the program number, protocol version and procedure numbers here. */ @@ -5740,5 +5745,11 @@ enum remote_procedure { * @generate: both * @acl: none */ - REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363 + REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_START_POST_COPY = 364 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index 070338c..9d59e0e 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -2708,6 +2708,10 @@ struct remote_domain_event_callback_job_completed_msg { remote_typed_param * params_val; } params; }; +struct remote_domain_migrate_start_post_copy_args { + remote_nonnull_domain dom; + u_int flags; +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -3072,4 +3076,5 @@ enum remote_procedure { REMOTE_PROC_CONNECT_CLOSE_CALLBACK_UNREGISTER = 361, REMOTE_PROC_CONNECT_EVENT_CONNECTION_CLOSED = 362, REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363, + REMOTE_PROC_DOMAIN_MIGRATE_START_POST_COPY = 364, }; -- 2.7.2

Migration enters "postcopy-active" state after QEMU switches to post-copy and pauses guest CPUs. From libvirt's point of view this state is similar to "completed" because we need to transfer guest execution to the destination host. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 3: - remove completed stats for incoming post-copy migration Version 2: - no change src/qemu/qemu_migration.c | 103 +++++++++++++++++++++++++++++++++++++------ src/qemu/qemu_monitor.c | 2 +- src/qemu/qemu_monitor.h | 1 + src/qemu/qemu_monitor_json.c | 1 + src/qemu/qemu_process.c | 10 ++++- 5 files changed, 101 insertions(+), 16 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 38fa81c..fdd95c5 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2460,6 +2460,7 @@ qemuMigrationUpdateJobType(qemuDomainJobInfoPtr jobInfo) case QEMU_MONITOR_MIGRATION_STATUS_SETUP: case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: case QEMU_MONITOR_MIGRATION_STATUS_LAST: break; @@ -2577,6 +2578,7 @@ enum qemuMigrationCompletedFlags { QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR = (1 << 0), QEMU_MIGRATION_COMPLETED_CHECK_STORAGE = (1 << 1), QEMU_MIGRATION_COMPLETED_UPDATE_STATS = (1 << 2), + QEMU_MIGRATION_COMPLETED_POSTCOPY = (1 << 3), }; /** @@ -2618,6 +2620,20 @@ qemuMigrationCompleted(virQEMUDriverPtr driver, goto error; } + /* In case of postcopy the source considers migration completed at the + * moment it switched from active to postcopy-active state. The destination + * will continue waiting until the migrate state changes to completed. + */ + if (flags & QEMU_MIGRATION_COMPLETED_POSTCOPY && + jobInfo->type == VIR_DOMAIN_JOB_UNBOUNDED && + jobInfo->stats.status == QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY) { + VIR_DEBUG("Migration switched to post-copy"); + if (updateStats && + qemuMigrationUpdateJobStatus(driver, vm, asyncJob) < 0) + goto error; + return 1; + } + if (jobInfo->type == VIR_DOMAIN_JOB_COMPLETED) return 1; else @@ -2651,9 +2667,11 @@ qemuMigrationWaitForCompletion(virQEMUDriverPtr driver, qemuDomainObjPrivatePtr priv = vm->privateData; qemuDomainJobInfoPtr jobInfo = priv->job.current; bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT); - unsigned int flags = QEMU_MIGRATION_COMPLETED_UPDATE_STATS; + unsigned int flags; int rv; + flags = QEMU_MIGRATION_COMPLETED_UPDATE_STATS | + QEMU_MIGRATION_COMPLETED_POSTCOPY; if (abort_on_error) flags |= QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR; if (storage) @@ -2692,9 +2710,11 @@ qemuMigrationWaitForCompletion(virQEMUDriverPtr driver, static int qemuMigrationWaitForDestCompletion(virQEMUDriverPtr driver, virDomainObjPtr vm, - qemuDomainAsyncJob asyncJob) + qemuDomainAsyncJob asyncJob, + bool postcopy) { qemuDomainObjPrivatePtr priv = vm->privateData; + unsigned int flags = 0; int rv; if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT)) @@ -2702,7 +2722,11 @@ qemuMigrationWaitForDestCompletion(virQEMUDriverPtr driver, VIR_DEBUG("Waiting for incoming migration to complete"); - while ((rv = qemuMigrationCompleted(driver, vm, asyncJob, NULL, 0)) != 1) { + if (postcopy) + flags = QEMU_MIGRATION_COMPLETED_POSTCOPY; + + while ((rv = qemuMigrationCompleted(driver, vm, asyncJob, + NULL, flags)) != 1) { if (rv < 0 || virDomainObjWait(vm) < 0) return -1; } @@ -2904,7 +2928,7 @@ qemuMigrationRunIncoming(virQEMUDriverPtr driver, goto cleanup; } - if (qemuMigrationWaitForDestCompletion(driver, vm, asyncJob) < 0) + if (qemuMigrationWaitForDestCompletion(driver, vm, asyncJob, false) < 0) goto cleanup; ret = 0; @@ -3895,6 +3919,19 @@ qemuMigrationConfirmPhase(virQEMUDriverPtr driver, /* Update times with the values sent by the destination daemon */ if (mig->jobInfo && jobInfo) { + int reason; + + /* We need to refresh migration statistics after a completed post-copy + * migration since priv->job.completed contains obsolete data from the + * time we switched to post-copy mode. + */ + if (virDomainObjGetState(vm, &reason) == VIR_DOMAIN_PAUSED && + reason == VIR_DOMAIN_PAUSED_POSTCOPY && + qemuMigrationFetchJobStatus(driver, vm, + QEMU_ASYNC_JOB_MIGRATION_OUT, + jobInfo) < 0) + VIR_WARN("Could not refresh migration statistics"); + qemuDomainJobInfoUpdateTime(jobInfo); jobInfo->timeDeltaSet = mig->jobInfo->timeDeltaSet; jobInfo->timeDelta = mig->jobInfo->timeDelta; @@ -4305,6 +4342,7 @@ qemuMigrationRun(virQEMUDriverPtr driver, unsigned int cookieFlags = 0; bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR); bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT); + bool inPostCopy = false; int rc; VIR_DEBUG("driver=%p, vm=%p, cookiein=%s, cookieinlen=%d, " @@ -4493,6 +4531,9 @@ qemuMigrationRun(virQEMUDriverPtr driver, else if (rc == -1) goto cleanup; + if (priv->job.current->stats.status == QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY) + inPostCopy = true; + /* When migration completed, QEMU will have paused the CPUs for us. * Wait for the STOP event to be processed or explicitly stop CPUs * (for old QEMU which does not send events) to release the lock state. @@ -4502,15 +4543,12 @@ qemuMigrationRun(virQEMUDriverPtr driver, priv->signalStop = true; rc = virDomainObjWait(vm); priv->signalStop = false; - if (rc < 0) { - priv->job.current->type = VIR_DOMAIN_JOB_FAILED; - goto cleanup; - } + if (rc < 0) + goto cancelPostCopy; } } else if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING && qemuMigrationSetOffline(driver, vm) < 0) { - priv->job.current->type = VIR_DOMAIN_JOB_FAILED; - goto cleanup; + goto cancelPostCopy; } if (priv->job.completed) priv->job.completed->stopped = priv->job.current->stopped; @@ -4541,7 +4579,7 @@ qemuMigrationRun(virQEMUDriverPtr driver, ignore_value(virTimeMillisNow(&priv->job.completed->sent)); } - if (priv->job.current->type == VIR_DOMAIN_JOB_UNBOUNDED) + if (priv->job.current->type == VIR_DOMAIN_JOB_UNBOUNDED && !inPostCopy) priv->job.current->type = VIR_DOMAIN_JOB_FAILED; cookieFlags |= QEMU_MIGRATION_COOKIE_NETWORK | @@ -4581,6 +4619,13 @@ qemuMigrationRun(virQEMUDriverPtr driver, } } goto cleanup; + + cancelPostCopy: + priv->job.current->type = VIR_DOMAIN_JOB_FAILED; + if (inPostCopy) + goto cancel; + else + goto cleanup; } /* Perform migration using QEMU's native migrate support, @@ -5736,6 +5781,7 @@ qemuMigrationFinish(virQEMUDriverPtr driver, virObjectEventPtr event; int rc; qemuDomainJobInfoPtr jobInfo = NULL; + bool inPostCopy = false; VIR_DEBUG("driver=%p, dconn=%p, vm=%p, cookiein=%s, cookieinlen=%d, " "cookieout=%p, cookieoutlen=%p, flags=%lx, retcode=%d", @@ -5841,7 +5887,8 @@ qemuMigrationFinish(virQEMUDriverPtr driver, * before starting guest CPUs. */ if (qemuMigrationWaitForDestCompletion(driver, vm, - QEMU_ASYNC_JOB_MIGRATION_IN) < 0) { + QEMU_ASYNC_JOB_MIGRATION_IN, + !!(flags & VIR_MIGRATE_POSTCOPY)) < 0) { /* There's not much we can do for v2 protocol since the * original domain on the source host is already gone. */ @@ -5849,13 +5896,17 @@ qemuMigrationFinish(virQEMUDriverPtr driver, goto endjob; } + if (priv->job.current->stats.status == QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY) + inPostCopy = true; + if (!(flags & VIR_MIGRATE_PAUSED)) { /* run 'cont' on the destination, which allows migration on qemu * >= 0.10.6 to work properly. This isn't strictly necessary on * older qemu's, but it also doesn't hurt anything there */ if (qemuProcessStartCPUs(driver, vm, dconn, - VIR_DOMAIN_RUNNING_MIGRATED, + inPostCopy ? VIR_DOMAIN_RUNNING_POSTCOPY + : VIR_DOMAIN_RUNNING_MIGRATED, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) { if (virGetLastError() == NULL) virReportError(VIR_ERR_INTERNAL_ERROR, @@ -5876,6 +5927,13 @@ qemuMigrationFinish(virQEMUDriverPtr driver, if (v3proto) goto endjob; } + + if (inPostCopy) { + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_RESUMED, + VIR_DOMAIN_EVENT_RESUMED_POSTCOPY); + qemuDomainEventQueue(driver, event); + } } if (mig->jobInfo) { @@ -5891,6 +5949,19 @@ qemuMigrationFinish(virQEMUDriverPtr driver, qemuDomainJobInfoUpdateDowntime(jobInfo); } + if (inPostCopy) { + if (qemuMigrationWaitForDestCompletion(driver, vm, + QEMU_ASYNC_JOB_MIGRATION_IN, + false) < 0) { + goto endjob; + } + if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_RUNNING) { + virDomainObjSetState(vm, + VIR_DOMAIN_RUNNING, + VIR_DOMAIN_RUNNING_MIGRATED); + } + } + dom = virGetDomain(dconn, vm->def->name, vm->def->uuid); event = virDomainEventLifecycleNewFromObj(vm, @@ -5933,6 +6004,12 @@ qemuMigrationFinish(virQEMUDriverPtr driver, if (qemuMigrationBakeCookie(mig, driver, vm, cookieout, cookieoutlen, QEMU_MIGRATION_COOKIE_STATS) < 0) VIR_WARN("Unable to encode migration cookie"); + + /* Remove completed stats for post-copy, everything but timing fields + * is obsolete anyway. + */ + if (inPostCopy) + VIR_FREE(priv->job.completed); } qemuMigrationJobFinish(driver, vm); diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index ace3bb4..e831834 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -161,7 +161,7 @@ VIR_ONCE_GLOBAL_INIT(qemuMonitor) VIR_ENUM_IMPL(qemuMonitorMigrationStatus, QEMU_MONITOR_MIGRATION_STATUS_LAST, "inactive", "setup", - "active", + "active", "postcopy-active", "completed", "failed", "cancelling", "cancelled") diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 4467a41..9331dab 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -473,6 +473,7 @@ typedef enum { QEMU_MONITOR_MIGRATION_STATUS_INACTIVE, QEMU_MONITOR_MIGRATION_STATUS_SETUP, QEMU_MONITOR_MIGRATION_STATUS_ACTIVE, + QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY, QEMU_MONITOR_MIGRATION_STATUS_COMPLETED, QEMU_MONITOR_MIGRATION_STATUS_ERROR, QEMU_MONITOR_MIGRATION_STATUS_CANCELLING, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index 8352e53..8a2aed7 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -2548,6 +2548,7 @@ qemuMonitorJSONGetMigrationStatsReply(virJSONValuePtr reply, break; case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED: case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: ram = virJSONValueObjectGetObject(ret, "ram"); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index cd7861e..3757a53 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -711,8 +711,14 @@ qemuProcessHandleStop(qemuMonitorPtr mon ATTRIBUTE_UNUSED, } if (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT) { - reason = VIR_DOMAIN_PAUSED_MIGRATION; - detail = VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED; + if (priv->job.current->stats.status == + QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY) { + reason = VIR_DOMAIN_PAUSED_POSTCOPY; + detail = VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY; + } else { + reason = VIR_DOMAIN_PAUSED_MIGRATION; + detail = VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED; + } } VIR_DEBUG("Transitioned guest %s to paused state, reason %s", -- 2.7.2

From: Cristian Klein <cristiklein@gmail.com> Signed-off-by: Cristian Klein <cristiklein@gmail.com> Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 3: - no change Version 2: - no change src/qemu/qemu_migration.c | 76 +++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_migration.h | 3 +- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index fdd95c5..e9f44a5 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2408,6 +2408,54 @@ qemuMigrationSetOption(virQEMUDriverPtr driver, return ret; } + +static int +qemuMigrationSetPostCopy(virQEMUDriverPtr driver, + virDomainObjPtr vm, + bool state, + qemuDomainAsyncJob job) +{ + qemuDomainObjPrivatePtr priv = vm->privateData; + int ret; + + if (qemuDomainObjEnterMonitorAsync(driver, vm, job) < 0) + return -1; + + ret = qemuMonitorGetMigrationCapability( + priv->mon, + QEMU_MONITOR_MIGRATION_CAPS_POSTCOPY); + + if (ret < 0) { + goto cleanup; + } else if (ret == 0 && !state) { + /* Unsupported but we want it off anyway */ + goto cleanup; + } else if (ret == 0) { + if (job == QEMU_ASYNC_JOB_MIGRATION_IN) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("Post-copy migration is not supported by " + "target QEMU binary")); + } else { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("Post-copy migration is not supported by " + "source QEMU binary")); + } + ret = -1; + goto cleanup; + } + + ret = qemuMonitorSetMigrationCapability( + priv->mon, + QEMU_MONITOR_MIGRATION_CAPS_POSTCOPY, + state); + + cleanup: + if (qemuDomainObjExitMonitor(driver, vm) < 0) + ret = -1; + return ret; +} + + static int qemuMigrationWaitForSpice(virDomainObjPtr vm) { @@ -3044,6 +3092,15 @@ qemuMigrationBeginPhase(virQEMUDriverPtr driver, !qemuMigrationIsSafe(vm->def, nmigrate_disks, migrate_disks)) goto cleanup; + if (flags & VIR_MIGRATE_POSTCOPY && + (!(flags & VIR_MIGRATE_LIVE) || + flags & VIR_MIGRATE_PAUSED)) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("post-copy migration is not supported with non-live " + "or paused migration")); + goto cleanup; + } + if (flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC)) { bool has_drive_mirror = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DRIVE_MIRROR); @@ -3383,6 +3440,15 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, cookieFlags = QEMU_MIGRATION_COOKIE_GRAPHICS; } + if (flags & VIR_MIGRATE_POSTCOPY && + (!(flags & VIR_MIGRATE_LIVE) || + flags & VIR_MIGRATE_PAUSED)) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("post-copy migration is not supported with non-live " + "or paused migration")); + goto cleanup; + } + if (!(caps = virQEMUDriverGetCapabilities(driver, false))) goto cleanup; @@ -3536,6 +3602,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) goto stopjob; + if (qemuMigrationSetPostCopy(driver, vm, + flags & VIR_MIGRATE_POSTCOPY, + QEMU_ASYNC_JOB_MIGRATION_IN) < 0) + goto stopjob; + if (mig->nbd && flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC) && virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_NBD_SERVER)) { @@ -4430,6 +4501,11 @@ qemuMigrationRun(virQEMUDriverPtr driver, QEMU_ASYNC_JOB_MIGRATION_OUT) < 0) goto cleanup; + if (qemuMigrationSetPostCopy(driver, vm, + flags & VIR_MIGRATE_POSTCOPY, + QEMU_ASYNC_JOB_MIGRATION_OUT) < 0) + goto cleanup; + if (qemuDomainObjEnterMonitorAsync(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT) < 0) goto cleanup; diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 2c67a02..953f96b 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -41,7 +41,8 @@ VIR_MIGRATE_COMPRESSED | \ VIR_MIGRATE_ABORT_ON_ERROR | \ VIR_MIGRATE_AUTO_CONVERGE | \ - VIR_MIGRATE_RDMA_PIN_ALL) + VIR_MIGRATE_RDMA_PIN_ALL | \ + VIR_MIGRATE_POSTCOPY) /* All supported migration parameters and their types. */ # define QEMU_MIGRATION_PARAMETERS \ -- 2.7.2

From: Cristian Klein <cristiklein@gmail.com> Signed-off-by: Cristian Klein <cristiklein@gmail.com> Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 3: - no change Version 2: - no change src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_driver.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_migration.c | 2 ++ 4 files changed, 62 insertions(+) diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 5165c97..c663eb5 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -214,6 +214,7 @@ qemuDomainObjResetAsyncJob(qemuDomainObjPrivatePtr priv) job->dump_memory_only = false; job->abortJob = false; job->spiceMigrated = false; + job->postcopyEnabled = false; VIR_FREE(job->current); } diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index b6f493e..e8c9254 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -140,6 +140,7 @@ struct qemuDomainJobObj { bool spiceMigration; /* we asked for spice migration and we * should wait for it to finish */ bool spiceMigrated; /* spice migration completed */ + bool postcopyEnabled; /* post-copy migration was enabled */ }; typedef void (*qemuDomainCleanupCallback)(virQEMUDriverPtr driver, diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 460232a..83d7dd7 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -13409,6 +13409,63 @@ qemuDomainMigrateGetMaxSpeed(virDomainPtr dom, } +static int +qemuDomainMigrateStartPostCopy(virDomainPtr dom, + unsigned int flags) +{ + virQEMUDriverPtr driver = dom->conn->privateData; + virDomainObjPtr vm; + qemuDomainObjPrivatePtr priv; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = qemuDomObjFromDomain(dom))) + goto cleanup; + + if (virDomainMigrateStartPostCopyEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MIGRATION_OP) < 0) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("domain is not running")); + goto endjob; + } + + priv = vm->privateData; + + if (priv->job.asyncJob != QEMU_ASYNC_JOB_MIGRATION_OUT) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("post-copy can only be started while " + "outgoing migration is in progress")); + goto endjob; + } + + if (!priv->job.postcopyEnabled) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("switching to post-copy requires migration to be " + "started with VIR_MIGRATE_POSTCOPY flag")); + goto endjob; + } + + VIR_DEBUG("Starting post-copy"); + qemuDomainObjEnterMonitor(driver, vm); + ret = qemuMonitorMigrateStartPostCopy(priv->mon); + if (qemuDomainObjExitMonitor(driver, vm) < 0) + ret = -1; + + endjob: + qemuDomainObjEndJob(driver, vm); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + + typedef enum { VIR_DISK_CHAIN_NO_ACCESS, VIR_DISK_CHAIN_READ_ONLY, @@ -20225,6 +20282,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainInterfaceAddresses = qemuDomainInterfaceAddresses, /* 1.2.14 */ .domainSetUserPassword = qemuDomainSetUserPassword, /* 1.2.16 */ .domainRename = qemuDomainRename, /* 1.2.19 */ + .domainMigrateStartPostCopy = qemuDomainMigrateStartPostCopy, /* 1.3.2 */ }; diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index e9f44a5..202e955 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2452,6 +2452,8 @@ qemuMigrationSetPostCopy(virQEMUDriverPtr driver, cleanup: if (qemuDomainObjExitMonitor(driver, vm) < 0) ret = -1; + if (ret == 0) + priv->job.postcopyEnabled = state; return ret; } -- 2.7.2

Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 3: - no change Version 2: - no change tools/virsh-domain.c | 62 +++++++++++++++++++++++++++++++++++++++++++++------- tools/virsh.pod | 15 ++++++++----- 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 25620db..83fec37 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -9660,7 +9660,16 @@ static const vshCmdOptDef opts_migrate[] = { }, {.name = "timeout", .type = VSH_OT_INT, - .help = N_("force guest to suspend if live migration exceeds timeout (in seconds)") + .help = N_("run action specified by --timeout-* option (suspend by " + "default) if live migration exceeds timeout (in seconds)") + }, + {.name = "timeout-suspend", + .type = VSH_OT_BOOL, + .help = N_("suspend the guest after timeout") + }, + {.name = "timeout-postcopy", + .type = VSH_OT_BOOL, + .help = N_("switch to post-copy after timeout") }, {.name = "xml", .type = VSH_OT_STRING, @@ -9838,14 +9847,35 @@ doMigrate(void *opaque) goto out; } +typedef enum { + VIRSH_MIGRATE_TIMEOUT_DEFAULT, + VIRSH_MIGRATE_TIMEOUT_SUSPEND, + VIRSH_MIGRATE_TIMEOUT_POSTCOPY, +} virshMigrateTimeoutAction; + static void -virshMigrationTimeout(vshControl *ctl, - virDomainPtr dom, - void *opaque ATTRIBUTE_UNUSED) +virshMigrateTimeout(vshControl *ctl, + virDomainPtr dom, + void *opaque) { - vshDebug(ctl, VSH_ERR_DEBUG, "suspending the domain, " - "since migration timed out\n"); - virDomainSuspend(dom); + virshMigrateTimeoutAction action = *(virshMigrateTimeoutAction *) opaque; + + switch (action) { + case VIRSH_MIGRATE_TIMEOUT_DEFAULT: /* unreachable */ + case VIRSH_MIGRATE_TIMEOUT_SUSPEND: + vshDebug(ctl, VSH_ERR_DEBUG, + "migration timed out; suspending domain\n"); + if (virDomainSuspend(dom) < 0) + vshDebug(ctl, VSH_ERR_INFO, "suspending domain failed\n"); + break; + + case VIRSH_MIGRATE_TIMEOUT_POSTCOPY: + vshDebug(ctl, VSH_ERR_DEBUG, + "migration timed out; switching to post-copy\n"); + if (virDomainMigrateStartPostCopy(dom, 0) < 0) + vshDebug(ctl, VSH_ERR_INFO, "switching to post-copy failed\n"); + break; + } } static bool @@ -9857,10 +9887,12 @@ cmdMigrate(vshControl *ctl, const vshCmd *cmd) bool verbose = false; bool functionReturn = false; int timeout = 0; + virshMigrateTimeoutAction timeoutAction = VIRSH_MIGRATE_TIMEOUT_DEFAULT; bool live_flag = false; virshCtrlData data = { .dconn = NULL }; VSH_EXCLUSIVE_OPTIONS("live", "offline"); + VSH_EXCLUSIVE_OPTIONS("timeout-suspend", "timeout-postcopy"); if (!(dom = virshCommandOptDomain(ctl, cmd, NULL))) return false; @@ -9878,6 +9910,19 @@ cmdMigrate(vshControl *ctl, const vshCmd *cmd) goto cleanup; } + if (vshCommandOptBool(cmd, "timeout-suspend")) + timeoutAction = VIRSH_MIGRATE_TIMEOUT_SUSPEND; + if (vshCommandOptBool(cmd, "timeout-postcopy")) + timeoutAction = VIRSH_MIGRATE_TIMEOUT_POSTCOPY; + if (timeout > 0) { + if (timeoutAction == VIRSH_MIGRATE_TIMEOUT_DEFAULT) + timeoutAction = VIRSH_MIGRATE_TIMEOUT_SUSPEND; + } else if (timeoutAction) { + vshError(ctl, "%s", + _("migrate: Unexpected --timeout-* option without --timeout")); + goto cleanup; + } + if (pipe(p) < 0) goto cleanup; @@ -9908,7 +9953,8 @@ cmdMigrate(vshControl *ctl, const vshCmd *cmd) &data) < 0) goto cleanup; functionReturn = virshWatchJob(ctl, dom, verbose, p[0], timeout, - virshMigrationTimeout, NULL, _("Migration")); + virshMigrateTimeout, + &timeoutAction, _("Migration")); virThreadJoin(&workerThread); diff --git a/tools/virsh.pod b/tools/virsh.pod index e232adf..7dab4cd 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -1531,8 +1531,8 @@ to the I<uri> namespace is displayed instead of being modified. [I<--copy-storage-inc>] [I<--change-protection>] [I<--unsafe>] [I<--verbose>] [I<--compressed>] [I<--abort-on-error>] [I<--auto-converge>] [I<--postcopy>] I<domain> I<desturi> [I<migrateuri>] [I<graphicsuri>] [I<listen-address>] -[I<dname>] [I<--timeout> B<seconds>] [I<--xml> B<file>] -[I<--migrate-disks> B<disk-list>] +[I<dname>] [I<--timeout> B<seconds> [I<--timeout-suspend> | I<--timeout-postcopy>]] +[I<--xml> B<file>] [I<--migrate-disks> B<disk-list>] Migrate domain to another host. Add I<--live> for live migration; <--p2p> for peer-2-peer migration; I<--direct> for direct migration; or I<--tunnelled> @@ -1582,9 +1582,14 @@ the destination to supply a larger set of changes to any host-specific portions of the domain XML, such as accounting for naming differences between source and destination in accessing underlying storage. -I<--timeout> B<seconds> forces guest to suspend when live migration exceeds -that many seconds, and -then the migration will complete offline. It can only be used with I<--live>. +I<--timeout> B<seconds> tells virsh to run a specified action when live +migration exceeds that many seconds. It can only be used with I<--live>. +If I<--timeout-suspend> is specified, the domain will be suspended after +the timeout and the migration will complete offline; this is the default +if no I<--timeout-*> option is specified on the command line. When +I<--timeout-postcopy> is used, virsh will switch migration from pre-copy +to post-copy upon timeout; migration has to be started with I<--postcopy> +option for this to work. Running migration can be canceled by interrupting virsh (usually using C<Ctrl-C>) or by B<domjobabort> command sent from another virsh instance. -- 2.7.2

When destination libvirtd is restarted during migration in Finish phase just after the point we started guest CPUs, we should not kill the domain. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 3: - no change Version 2: - no change src/qemu/qemu_process.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 3757a53..490e148 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3173,9 +3173,13 @@ qemuProcessRecoverMigration(virQEMUDriverPtr driver, case QEMU_MIGRATION_PHASE_FINISH3: /* migration finished, we started resuming the domain but didn't - * confirm success or failure yet; killing it seems safest */ - VIR_DEBUG("Killing migrated domain %s", vm->def->name); - return -1; + * confirm success or failure yet; killing it seems safest unless + * we already started guest CPUs */ + if (state != VIR_DOMAIN_RUNNING) { + VIR_DEBUG("Killing migrated domain %s", vm->def->name); + return -1; + } + break; } } else if (job == QEMU_ASYNC_JOB_MIGRATION_OUT) { switch (phase) { -- 2.7.2

When migration fails in the post-copy mode, it's impossible to just kill the destination domain and resume the source since the source no longer contains current guest state. Let's mark domains on both sides as VIR_DOMAIN_PAUSED_POSTCOPY_FAILED to let the upper layer decide what to do with them. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 3: - send VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED event only on the destination host Version 2: - no change src/qemu/qemu_migration.c | 91 ++++++++++++++++++++++++++++++++++++++--------- src/qemu/qemu_migration.h | 3 ++ src/qemu/qemu_process.c | 59 ++++++++++++++++++++++-------- 3 files changed, 122 insertions(+), 31 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 202e955..d67eca8 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -1475,14 +1475,21 @@ qemuMigrationRestoreDomainState(virConnectPtr conn, virDomainObjPtr vm) { virQEMUDriverPtr driver = conn->privateData; qemuDomainObjPrivatePtr priv = vm->privateData; - int state = virDomainObjGetState(vm, NULL); + int reason; + virDomainState state = virDomainObjGetState(vm, &reason); bool ret = false; - VIR_DEBUG("driver=%p, vm=%p, pre-mig-state=%d, state=%d", - driver, vm, priv->preMigrationState, state); + VIR_DEBUG("driver=%p, vm=%p, pre-mig-state=%s, state=%s, reason=%s", + driver, vm, + virDomainStateTypeToString(priv->preMigrationState), + virDomainStateTypeToString(state), + virDomainStateReasonToString(state, reason)); - if (state == VIR_DOMAIN_PAUSED && - priv->preMigrationState == VIR_DOMAIN_RUNNING) { + if (state != VIR_DOMAIN_PAUSED || + reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED) + goto cleanup; + + if (priv->preMigrationState == VIR_DOMAIN_RUNNING) { /* This is basically the only restore possibility that's safe * and we should attempt to do */ @@ -2364,6 +2371,48 @@ qemuMigrationSetOffline(virQEMUDriverPtr driver, return ret; } + +void +qemuMigrationPostcopyFailed(virQEMUDriverPtr driver, + virDomainObjPtr vm) +{ + virDomainState state; + int reason; + + state = virDomainObjGetState(vm, &reason); + + if (state != VIR_DOMAIN_PAUSED && + state != VIR_DOMAIN_RUNNING) + return; + + if (state == VIR_DOMAIN_PAUSED && + reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED) + return; + + VIR_WARN("Migration of domain %s failed during post-copy; " + "leaving the domain paused", vm->def->name); + + if (state == VIR_DOMAIN_RUNNING) { + virObjectEventPtr event; + + if (qemuProcessStopCPUs(driver, vm, + VIR_DOMAIN_PAUSED_POSTCOPY_FAILED, + QEMU_ASYNC_JOB_MIGRATION_IN) < 0) { + VIR_WARN("Unable to pause guest CPUs for %s", vm->def->name); + return; + } + + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED); + qemuDomainEventQueue(driver, event); + } else { + virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, + VIR_DOMAIN_PAUSED_POSTCOPY_FAILED); + } +} + + static int qemuMigrationSetOption(virQEMUDriverPtr driver, virDomainObjPtr vm, @@ -4015,8 +4064,8 @@ qemuMigrationConfirmPhase(virQEMUDriverPtr driver, if (flags & VIR_MIGRATE_OFFLINE) goto done; - /* Did the migration go as planned? If yes, kill off the - * domain object, but if no, resume CPUs + /* Did the migration go as planned? If yes, kill off the domain object. + * If something failed, resume CPUs, but only if we didn't use post-copy. */ if (retcode == 0) { /* If guest uses SPICE and supports seamless migration we have to hold @@ -4035,6 +4084,7 @@ qemuMigrationConfirmPhase(virQEMUDriverPtr driver, qemuDomainEventEmitJobCompleted(driver, vm); } else { virErrorPtr orig_err = virSaveLastError(); + int reason; /* cancel any outstanding NBD jobs */ qemuMigrationCancelDriveMirror(driver, vm, false, @@ -4043,7 +4093,10 @@ qemuMigrationConfirmPhase(virQEMUDriverPtr driver, virSetError(orig_err); virFreeError(orig_err); - if (qemuMigrationRestoreDomainState(conn, vm)) { + if (virDomainObjGetState(vm, &reason) == VIR_DOMAIN_PAUSED && + reason == VIR_DOMAIN_PAUSED_POSTCOPY) { + qemuMigrationPostcopyFailed(driver, vm); + } else if (qemuMigrationRestoreDomainState(conn, vm)) { event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_RESUMED, VIR_DOMAIN_EVENT_RESUMED_MIGRATED); @@ -5860,6 +5913,7 @@ qemuMigrationFinish(virQEMUDriverPtr driver, int rc; qemuDomainJobInfoPtr jobInfo = NULL; bool inPostCopy = false; + bool kill = true; VIR_DEBUG("driver=%p, dconn=%p, vm=%p, cookiein=%s, cookieinlen=%d, " "cookieout=%p, cookieoutlen=%p, flags=%lx, retcode=%d", @@ -6007,6 +6061,7 @@ qemuMigrationFinish(virQEMUDriverPtr driver, } if (inPostCopy) { + kill = false; event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_RESUMED, VIR_DOMAIN_EVENT_RESUMED_POSTCOPY); @@ -6066,14 +6121,18 @@ qemuMigrationFinish(virQEMUDriverPtr driver, if (!dom && !(flags & VIR_MIGRATE_OFFLINE) && virDomainObjIsActive(vm)) { - qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, - QEMU_ASYNC_JOB_MIGRATION_IN, - VIR_QEMU_PROCESS_STOP_MIGRATED); - virDomainAuditStop(vm, "failed"); - event = virDomainEventLifecycleNewFromObj(vm, - VIR_DOMAIN_EVENT_STOPPED, - VIR_DOMAIN_EVENT_STOPPED_FAILED); - qemuDomainEventQueue(driver, event); + if (kill) { + qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED, + QEMU_ASYNC_JOB_MIGRATION_IN, + VIR_QEMU_PROCESS_STOP_MIGRATED); + virDomainAuditStop(vm, "failed"); + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_FAILED); + qemuDomainEventQueue(driver, event); + } else { + qemuMigrationPostcopyFailed(driver, vm); + } } if (dom) { diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 953f96b..6ea4424 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -210,4 +210,7 @@ int qemuMigrationRunIncoming(virQEMUDriverPtr driver, const char *uri, qemuDomainAsyncJob asyncJob); +void qemuMigrationPostcopyFailed(virQEMUDriverPtr driver, + virDomainObjPtr vm); + #endif /* __QEMU_MIGRATION_H__ */ diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index f19a20c..1854132 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3139,8 +3139,13 @@ qemuProcessRecoverMigrationIn(virQEMUDriverPtr driver, virConnectPtr conn, qemuMigrationJobPhase phase, virDomainState state, - int reason ATTRIBUTE_UNUSED) + int reason) { + bool postcopy = (state == VIR_DOMAIN_PAUSED && + reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED) || + (state == VIR_DOMAIN_RUNNING && + reason == VIR_DOMAIN_RUNNING_POSTCOPY); + switch (phase) { case QEMU_MIGRATION_PHASE_NONE: case QEMU_MIGRATION_PHASE_PERFORM2: @@ -3173,8 +3178,10 @@ qemuProcessRecoverMigrationIn(virQEMUDriverPtr driver, case QEMU_MIGRATION_PHASE_FINISH3: /* migration finished, we started resuming the domain but didn't * confirm success or failure yet; killing it seems safest unless - * we already started guest CPUs */ - if (state != VIR_DOMAIN_RUNNING) { + * we already started guest CPUs or we were in post-copy mode */ + if (postcopy) { + qemuMigrationPostcopyFailed(driver, vm); + } else if (state != VIR_DOMAIN_RUNNING) { VIR_DEBUG("Killing migrated domain %s", vm->def->name); return -1; } @@ -3192,6 +3199,10 @@ qemuProcessRecoverMigrationOut(virQEMUDriverPtr driver, virDomainState state, int reason) { + bool postcopy = state == VIR_DOMAIN_PAUSED && + (reason == VIR_DOMAIN_PAUSED_POSTCOPY || + reason == VIR_DOMAIN_PAUSED_POSTCOPY_FAILED); + switch (phase) { case QEMU_MIGRATION_PHASE_NONE: case QEMU_MIGRATION_PHASE_PREPARE: @@ -3209,26 +3220,44 @@ qemuProcessRecoverMigrationOut(virQEMUDriverPtr driver, case QEMU_MIGRATION_PHASE_PERFORM2: case QEMU_MIGRATION_PHASE_PERFORM3: /* migration is still in progress, let's cancel it and resume the - * domain */ - VIR_DEBUG("Cancelling unfinished migration of domain %s", - vm->def->name); - if (qemuMigrationCancel(driver, vm) < 0) { - VIR_WARN("Could not cancel ongoing migration of domain %s", - vm->def->name); + * domain; however we can only do that before migration enters + * post-copy mode + */ + if (postcopy) { + qemuMigrationPostcopyFailed(driver, vm); + } else { + VIR_DEBUG("Cancelling unfinished migration of domain %s", + vm->def->name); + if (qemuMigrationCancel(driver, vm) < 0) { + VIR_WARN("Could not cancel ongoing migration of domain %s", + vm->def->name); + } + goto resume; } - goto resume; + break; case QEMU_MIGRATION_PHASE_PERFORM3_DONE: /* migration finished but we didn't have a chance to get the result - * of Finish3 step; third party needs to check what to do next + * of Finish3 step; third party needs to check what to do next; in + * post-copy mode we can use PAUSED_POSTCOPY_FAILED state for this */ + if (postcopy) + qemuMigrationPostcopyFailed(driver, vm); break; case QEMU_MIGRATION_PHASE_CONFIRM3_CANCELLED: - /* Finish3 failed, we need to resume the domain */ - VIR_DEBUG("Resuming domain %s after failed migration", - vm->def->name); - goto resume; + /* Finish3 failed, we need to resume the domain, but once we enter + * post-copy mode there's no way back, so let's just mark the domain + * as broken in that case + */ + if (postcopy) { + qemuMigrationPostcopyFailed(driver, vm); + } else { + VIR_DEBUG("Resuming domain %s after failed migration", + vm->def->name); + goto resume; + } + break; case QEMU_MIGRATION_PHASE_CONFIRM3: /* migration completed, we need to kill the domain here */ -- 2.7.2

The function already takes two bool arguments, switching to flags makes it a lot easier to read. Especially in case we need to add another boolean in the future. Signed-off-by: Jiri Denemark <jdenemar@redhat.com> --- Notes: Version 3: - no change Version 2: - moved to the end of the series since it is not required by another patch in this series anymore src/qemu/qemu_migration.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index d67eca8..5bfd304 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2760,21 +2760,14 @@ qemuMigrationWaitForCompletion(virQEMUDriverPtr driver, virDomainObjPtr vm, qemuDomainAsyncJob asyncJob, virConnectPtr dconn, - bool abort_on_error, - bool storage) + unsigned int flags) { qemuDomainObjPrivatePtr priv = vm->privateData; qemuDomainJobInfoPtr jobInfo = priv->job.current; bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT); - unsigned int flags; int rv; - flags = QEMU_MIGRATION_COMPLETED_UPDATE_STATS | - QEMU_MIGRATION_COMPLETED_POSTCOPY; - if (abort_on_error) - flags |= QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR; - if (storage) - flags |= QEMU_MIGRATION_COMPLETED_CHECK_STORAGE; + flags |= QEMU_MIGRATION_COMPLETED_UPDATE_STATS; jobInfo->type = VIR_DOMAIN_JOB_UNBOUNDED; while ((rv = qemuMigrationCompleted(driver, vm, asyncJob, @@ -4469,6 +4462,7 @@ qemuMigrationRun(virQEMUDriverPtr driver, bool abort_on_error = !!(flags & VIR_MIGRATE_ABORT_ON_ERROR); bool events = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_EVENT); bool inPostCopy = false; + unsigned int waitFlags; int rc; VIR_DEBUG("driver=%p, vm=%p, cookiein=%s, cookieinlen=%d, " @@ -4654,9 +4648,17 @@ qemuMigrationRun(virQEMUDriverPtr driver, fd = -1; } + waitFlags = 0; + if (abort_on_error) + waitFlags |= QEMU_MIGRATION_COMPLETED_ABORT_ON_ERROR; + if (mig->nbd) + waitFlags |= QEMU_MIGRATION_COMPLETED_CHECK_STORAGE; + if (flags & VIR_MIGRATE_POSTCOPY) + waitFlags |= QEMU_MIGRATION_COMPLETED_POSTCOPY; + rc = qemuMigrationWaitForCompletion(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT, - dconn, abort_on_error, !!mig->nbd); + dconn, waitFlags); if (rc == -2) goto cancel; else if (rc == -1) @@ -6268,8 +6270,7 @@ qemuMigrationToFile(virQEMUDriverPtr driver, virDomainObjPtr vm, if (rc < 0) goto cleanup; - rc = qemuMigrationWaitForCompletion(driver, vm, asyncJob, - NULL, false, false); + rc = qemuMigrationWaitForCompletion(driver, vm, asyncJob, NULL, 0); if (rc < 0) { if (rc == -2) { -- 2.7.2