[libvirt] [PATCH v3 02/14] Add public APIs for post-copy migration

From: Cristian Klein <cristiklein(a)gmail.com&gt; To use post-copy one has to start the migration with VIR_MIGRATE_POSTCOPY flag and, while migration is in progress, call virDomainMigrateStartPostCopy() to switch from pre-copy to post-copy. Signed-off-by: Cristian Klein <cristiklein(a)gmail.com&gt; Signed-off-by: Jiri Denemark <jdenemar(a)redhat.com&gt; --- Notes: Version 3: - VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED will only be emitted on the destination host Version 2: - POSTCOPY_AFTER_PRECOPY flag removed include/libvirt/libvirt-domain.h | 4 ++ src/driver-hypervisor.h | 5 ++ src/libvirt-domain.c | 141 ++++++++++++++++++++++++++++++++++++--- src/libvirt_public.syms | 4 ++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 +++- src/remote_protocol-structs | 5 ++ 7 files changed, 164 insertions(+), 9 deletions(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 3846ab9..dede3da 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -678,6 +678,7 @@ typedef enum { VIR_MIGRATE_ABORT_ON_ERROR = (1 << 12), /* abort migration on I/O errors happened during migration */ VIR_MIGRATE_AUTO_CONVERGE = (1 << 13), /* force convergence */ VIR_MIGRATE_RDMA_PIN_ALL = (1 << 14), /* RDMA memory pinning */ + VIR_MIGRATE_POSTCOPY = (1 << 15), /* enable (but do not start) post-copy migration */ } virDomainMigrateFlags; @@ -826,6 +827,9 @@ int virDomainMigrateGetMaxSpeed(virDomainPtr domain, unsigned long *bandwidth, unsigned int flags); +int virDomainMigrateStartPostCopy(virDomainPtr domain, + unsigned int flags); + char * virConnectGetDomainCapabilities(virConnectPtr conn, const char *emulatorbin, const char *arch, diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 2cbd01b..9bc3211 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -638,6 +638,10 @@ typedef int const char *dom_xml); typedef int +(*virDrvDomainMigrateStartPostCopy)(virDomainPtr domain, + unsigned int flags); + +typedef int (*virDrvConnectIsEncrypted)(virConnectPtr conn); typedef int @@ -1455,6 +1459,7 @@ struct _virHypervisorDriver { virDrvDomainSetUserPassword domainSetUserPassword; virDrvConnectRegisterCloseCallback connectRegisterCloseCallback; virDrvConnectUnregisterCloseCallback connectUnregisterCloseCallback; + virDrvDomainMigrateStartPostCopy domainMigrateStartPostCopy; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index ca32dc1..1b95664 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -3537,6 +3537,7 @@ virDomainMigrateUnmanaged(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set. * Applications using the VIR_MIGRATE_PEER2PEER flag will probably @@ -3573,6 +3574,11 @@ virDomainMigrateUnmanaged(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -3748,6 +3754,7 @@ virDomainMigrate(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set. * Applications using the VIR_MIGRATE_PEER2PEER flag will probably @@ -3784,6 +3791,11 @@ virDomainMigrate(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -3968,6 +3980,11 @@ virDomainMigrate2(virDomainPtr domain, * can use either VIR_MIGRATE_NON_SHARED_DISK or * VIR_MIGRATE_NON_SHARED_INC as they are mutually exclusive. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * There are many limitations on migration imposed by the underlying * technology - for example it may not be possible to migrate between * different processors even with the same architecture, or between @@ -4208,6 +4225,7 @@ int virDomainMigrateUnmanagedCheckCompat(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * The operation of this API hinges on the VIR_MIGRATE_PEER2PEER flag. * If the VIR_MIGRATE_PEER2PEER flag is NOT set, the duri parameter @@ -4240,6 +4258,11 @@ int virDomainMigrateUnmanagedCheckCompat(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -4321,6 +4344,7 @@ virDomainMigrateToURI(virDomainPtr domain, * automatically when supported). * VIR_MIGRATE_UNSAFE Force migration even if it is considered unsafe. * VIR_MIGRATE_OFFLINE Migrate offline + * VIR_MIGRATE_POSTCOPY Enable (but do not start) post-copy * * The operation of this API hinges on the VIR_MIGRATE_PEER2PEER flag. * @@ -4366,6 +4390,11 @@ virDomainMigrateToURI(virDomainPtr domain, * not support this feature and will return an error if bandwidth * is not 0. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * To see which features are supported by the current hypervisor, * see virConnectGetCapabilities, /capabilities/host/migration_features. * @@ -4446,6 +4475,11 @@ virDomainMigrateToURI2(virDomainPtr domain, * can use either VIR_MIGRATE_NON_SHARED_DISK or * VIR_MIGRATE_NON_SHARED_INC as they are mutually exclusive. * + * Enabling the VIR_MIGRATE_POSTCOPY flag tells libvirt to enable post-copy + * migration. Use virDomainMigrateStartPostCopy to switch migration into + * the post-copy mode. See virDomainMigrateStartPostCopy for more details + * about post-copy. + * * There are many limitations on migration imposed by the underlying * technology - for example it may not be possible to migrate between * different processors even with the same architecture, or between @@ -8868,14 +8902,15 @@ virDomainGetJobInfo(virDomainPtr domain, virDomainJobInfoPtr info) * * When @flags contains VIR_DOMAIN_JOB_STATS_COMPLETED, the function will * return statistics about a recently completed job. Specifically, this - * flag may be used to query statistics of a completed incoming migration. - * Statistics of a completed job are automatically destroyed once read or - * when libvirtd is restarted. Note that time information returned for - * completed migrations may be completely irrelevant unless both source and - * destination hosts have synchronized time (i.e., NTP daemon is running on - * both of them). The statistics of a completed job can also be obtained by - * listening to a VIR_DOMAIN_EVENT_ID_JOB_COMPLETED event (on the source host - * in case of a migration job). + * flag may be used to query statistics of a completed incoming pre-copy + * migration (statistics for post-copy migration are only available on the + * source hsot). Statistics of a completed job are automatically destroyed + * once read or when libvirtd is restarted. Note that time information + * returned for completed migrations may be completely irrelevant unless both + * source and destination hosts have synchronized time (i.e., NTP daemon is + * running on both of them). The statistics of a completed job can also be + * obtained by listening to a VIR_DOMAIN_EVENT_ID_JOB_COMPLETED event (on the + * source host in case of a migration job). * * Returns 0 in case of success and -1 in case of failure. */ @@ -9165,6 +9200,96 @@ virDomainMigrateGetMaxSpeed(virDomainPtr domain, /** + * virDomainMigrateStartPostCopy: + * @domain: a domain object + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Starts post-copy migration. This function has to be called while + * migration (initiated with VIR_MIGRATE_POSTCOPY flag) is in progress. + * + * Traditional post-copy migration iteratively walks through guest memory + * pages and migrates those that changed since the previous iteration. The + * iterative phase stops when the number of dirty pages is low enough so that + * the virtual CPUs can be paused, all dirty pages transferred to the + * destination, where the virtual CPUs are unpaused, and all this can happen + * within a predefined downtime period. It's clear that this process may never + * converge if downtime is too short and/or the guest keeps changing a lot of + * memory pages. + * + * When migration is switched to post-copy mode, the virtual CPUs are paused + * immediately, only a minimum set of pages is transferred, and the CPUs are + * unpaused on destination. The source keeps sending all remaining memory pages + * to the destination while the guest is already running there. Whenever the + * guest tries to read a memory page which has not been migrated yet, the + * hypervisor has to tell the source to transfer that page in a priority + * channel. To minimize such page faults, it is a good idea to run at least one + * iteration of pre-copy migration before switching to post-copy. + * + * Post-copy migration is guaranteed to converge since each page is transferred + * at most once no matter how fast it changes. On the other hand once the + * guest is running on the destination host, the migration can no longer be + * rolled back because none of the hosts has complete state. If this happens, + * libvirt will leave the domain paused on both hosts with + * VIR_DOMAIN_PAUSED_POSTCOPY_FAILED reason. It's up to the upper layer to + * decide what to do in such case. + * + * The following domain life cycle events are emitted during post-copy + * migration: + * VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY (on the source) -- migration entered + * post-copy mode. + * VIR_DOMAIN_EVENT_RESUMED_POSTCOPY (on the destination) -- the guest is + * running on the destination host while some of its memory pages still + * remain on the source host; neither the source nor the destination host + * contain a complete guest state from this point until migration + * finishes. + * VIR_DOMAIN_EVENT_RESUMED_MIGRATED (on the destination), + * VIR_DOMAIN_EVENT_STOPPED_MIGRATED (on the source) -- migration finished + * successfully and the destination host holds a complete guest state. + * VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED (on the destination) -- emitted + * when migration fails in post-copy mode and it's unclear whether any + * of the hosts has a complete guest state. + * + * The progress of a post-copy migration can be monitored normally using + * virDomainGetJobStats on the source host. Fetching statistics of a completed + * post-copy migration can also be done on the source host (by calling + * virDomainGetJobStats or listening to VIR_DOMAIN_EVENT_ID_JOB_COMPLETED + * event, but (in contrast to pre-copy migration) the statistics are not + * available on the destination host. Thus, VIR_DOMAIN_EVENT_ID_JOB_COMPLETED + * event is the only way of getting statistics of a completed post-copy + * migration of a transient domain (because the domain is removed after + * migration and there's no domain to run virDomainGetJobStats on). + * + * Returns 0 in case of success, -1 otherwise. + */ +int +virDomainMigrateStartPostCopy(virDomainPtr domain, + unsigned int flags) +{ + virConnectPtr conn; + + VIR_DOMAIN_DEBUG(domain); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckReadOnlyGoto(conn->flags, error); + + if (conn->driver->domainMigrateStartPostCopy) { + if (conn->driver->domainMigrateStartPostCopy(domain, flags) < 0) + goto error; + return 0; + } + + virReportUnsupportedError(); + error: + virDispatchError(conn); + return -1; +} + + +/** * virConnectDomainEventRegisterAny: * @conn: pointer to the connection * @dom: pointer to the domain diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index dd94191..d98414d 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -725,4 +725,8 @@ LIBVIRT_1.2.19 { virDomainRename; } LIBVIRT_1.2.17; +LIBVIRT_1.3.2 { + virDomainMigrateStartPostCopy; +} LIBVIRT_1.2.19; + # .... define new API here using predicted next version number .... diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 2daa507..09de5f8 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8417,6 +8417,7 @@ static virHypervisorDriver hypervisor_driver = { .domainRename = remoteDomainRename, /* 1.2.19 */ .connectRegisterCloseCallback = remoteConnectRegisterCloseCallback, /* 1.3.2 */ .connectUnregisterCloseCallback = remoteConnectUnregisterCloseCallback, /* 1.3.2 */ + .domainMigrateStartPostCopy = remoteDomainMigrateStartPostCopy, /* 1.3.3 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 952686c..6adad7a 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -3238,6 +3238,11 @@ struct remote_domain_event_callback_job_completed_msg { remote_typed_param params<REMOTE_DOMAIN_JOB_STATS_MAX>; }; +struct remote_domain_migrate_start_post_copy_args { + remote_nonnull_domain dom; + unsigned int flags; +}; + /*----- Protocol. -----*/ /* Define the program number, protocol version and procedure numbers here. */ @@ -5740,5 +5745,11 @@ enum remote_procedure { * @generate: both * @acl: none */ - REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363 + REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_START_POST_COPY = 364 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index 070338c..9d59e0e 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -2708,6 +2708,10 @@ struct remote_domain_event_callback_job_completed_msg { remote_typed_param * params_val; } params; }; +struct remote_domain_migrate_start_post_copy_args { + remote_nonnull_domain dom; + u_int flags; +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -3072,4 +3076,5 @@ enum remote_procedure { REMOTE_PROC_CONNECT_CLOSE_CALLBACK_UNREGISTER = 361, REMOTE_PROC_CONNECT_EVENT_CONNECTION_CLOSED = 362, REMOTE_PROC_DOMAIN_EVENT_CALLBACK_JOB_COMPLETED = 363, + REMOTE_PROC_DOMAIN_MIGRATE_START_POST_COPY = 364, }; -- 2.7.2