5:06 p.m.
Add plumbing for QEMU's switchover-ack migration capability, which
helps lower the downtime during VFIO migrations. This capability is
enabled by default as long as both the source and destination support
it.
Note: switchover-ack depends on the return path capability, so this may
not be used when VIR_MIGRATE_TUNNELLED flag is set.
Extensive details about the qemu switchover-ack implementation are
available in the qemu series v6 cover letter [1] where the highlight is
the extreme reduction in guest visible downtime. In addition to the
original test results below, I saw a roughly ~20% reduction in downtime
for VFIO VGPU devices at minimum.
=== Test results ===
The below table shows the downtime of two identical migrations. In the
first migration swithcover ack is disabled and in the second it is
enabled. The migrated VM is assigned with a mlx5 VFIO device which has
300MB of device data to be migrated.
+----------------------+-----------------------+----------+
| Switchover ack | VFIO device data size | Downtime |
+----------------------+-----------------------+----------+
| Disabled | 300MB | 1900ms |
| Enabled | 300MB | 420ms |
+----------------------+-----------------------+----------+
Switchover ack gives a roughly 4.5 times improvement in downtime.
The 1480ms difference is time that is used for resource allocation for
the VFIO device in the destination. Without switchover ack, this time is
spent when the source VM is stopped and thus the downtime is much
higher. With switchover ack, the time is spent when the source VM is
still running.
[1]
https://patchwork.kernel.org/project/qemu-devel/cover/20230621111201.2972...
Signed-off-by: Jon Kohler <jon(a)nutanix.com>
Cc: Alex Williamson <alex.williamson(a)redhat.com>
Cc: Avihai Horon <avihaih(a)nvidia.com>
Cc: Markus Armbruster <armbru(a)redhat.com>
Cc: Peter Xu <peterx(a)redhat.com>
Cc: YangHang Liu <yanghliu(a)redhat.com>
---
include/libvirt/libvirt-domain.h | 11 +++++++++++
src/libvirt-domain.c | 20 ++++++++++++++++++++
src/qemu/qemu_migration.h | 1 +
src/qemu/qemu_migration_params.c | 8 +++++++-
src/qemu/qemu_migration_params.h | 1 +
5 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h
index 2f5b01bbfe..9543629f30 100644
--- a/include/libvirt/libvirt-domain.h
+++ b/include/libvirt/libvirt-domain.h
@@ -1100,6 +1100,17 @@ typedef enum {
* Since: 8.5.0
*/
VIR_MIGRATE_ZEROCOPY = (1 << 20),
+
+ /* Use switchover ack migration capability to reduce downtime on VFIO
+ * device migration. This prevents the source from stopping the VM and
+ * completing the migration until an ACK is received from the destination
+ * that it's OK to do so. Thus, a VFIO device can make sure that its
+ * initial bytes were sent and loaded in the destination before the
+ * source VM is stopped.
+ *
+ * Since: 10.5.0
+ */
+ VIR_MIGRATE_SWITCHOVER_ACK = (1 << 21),
} virDomainMigrateFlags;
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c
index 7c6b93963c..786fef317d 100644
--- a/src/libvirt-domain.c
+++ b/src/libvirt-domain.c
@@ -3822,6 +3822,10 @@ virDomainMigrate(virDomainPtr domain,
VIR_MIGRATE_PARALLEL,
error);
+ VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_TUNNELLED,
+ VIR_MIGRATE_SWITCHOVER_ACK,
+ error);
+
VIR_REQUIRE_FLAG_GOTO(VIR_MIGRATE_NON_SHARED_SYNCHRONOUS_WRITES,
VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC,
error);
@@ -4021,6 +4025,10 @@ virDomainMigrate2(virDomainPtr domain,
VIR_MIGRATE_PARALLEL,
error);
+ VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_TUNNELLED,
+ VIR_MIGRATE_SWITCHOVER_ACK,
+ error);
+
VIR_REQUIRE_FLAG_GOTO(VIR_MIGRATE_NON_SHARED_SYNCHRONOUS_WRITES,
VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC,
error);
@@ -4497,6 +4505,10 @@ virDomainMigrateToURI(virDomainPtr domain,
VIR_MIGRATE_PARALLEL,
error);
+ VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_TUNNELLED,
+ VIR_MIGRATE_SWITCHOVER_ACK,
+ error);
+
if (virDomainMigrateUnmanagedCheckCompat(domain, flags) < 0)
goto error;
@@ -4577,6 +4589,10 @@ virDomainMigrateToURI2(virDomainPtr domain,
VIR_MIGRATE_PARALLEL,
error);
+ VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_TUNNELLED,
+ VIR_MIGRATE_SWITCHOVER_ACK,
+ error);
+
if (virDomainMigrateUnmanagedCheckCompat(domain, flags) < 0)
goto error;
@@ -4656,6 +4672,10 @@ virDomainMigrateToURI3(virDomainPtr domain,
VIR_MIGRATE_PARALLEL,
error);
+ VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_TUNNELLED,
+ VIR_MIGRATE_SWITCHOVER_ACK,
+ error);
+
if (virDomainMigrateUnmanagedCheckCompat(domain, flags) < 0)
goto error;
diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h
index ed62fd4a91..cd89e100e1 100644
--- a/src/qemu/qemu_migration.h
+++ b/src/qemu/qemu_migration.h
@@ -62,6 +62,7 @@
VIR_MIGRATE_NON_SHARED_SYNCHRONOUS_WRITES | \
VIR_MIGRATE_POSTCOPY_RESUME | \
VIR_MIGRATE_ZEROCOPY | \
+ VIR_MIGRATE_SWITCHOVER_ACK | \
0)
/* All supported migration parameters and their types. */
diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c
index 48f8657f71..9593b6ba65 100644
--- a/src/qemu/qemu_migration_params.c
+++ b/src/qemu/qemu_migration_params.c
@@ -105,6 +105,7 @@ VIR_ENUM_IMPL(qemuMigrationCapability,
"return-path",
"zero-copy-send",
"postcopy-preempt",
+ "switchover-ack",
);
@@ -138,7 +139,7 @@ struct _qemuMigrationParamsAlwaysOnItem {
typedef struct _qemuMigrationParamsFlagMapItem qemuMigrationParamsFlagMapItem;
struct _qemuMigrationParamsFlagMapItem {
/* Describes what to do with the capability if @flag is found.
- * When se to QEMU_MIGRATION_FLAG_REQUIRED, the capability will be
+ * When set to QEMU_MIGRATION_FLAG_REQUIRED, the capability will be
* enabled iff the specified migration flag is enabled. On the other hand
* QEMU_MIGRATION_FLAG_FORBIDDEN will enable the capability as long as
* the specified migration flag is not enabled. */
@@ -215,6 +216,11 @@ static const qemuMigrationParamsFlagMapItem
qemuMigrationParamsFlagMap[] = {
.flag = VIR_MIGRATE_ZEROCOPY,
.cap = QEMU_MIGRATION_CAP_ZERO_COPY_SEND,
.party = QEMU_MIGRATION_SOURCE},
+
+ {.match = QEMU_MIGRATION_FLAG_FORBIDDEN,
+ .flag = VIR_MIGRATE_TUNNELLED,
+ .cap = QEMU_MIGRATION_CAP_SWITCHOVER_ACK,
+ .party = QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION},
};
/* Translation from VIR_MIGRATE_PARAM_* typed parameters to
diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h
index 91bc6792cd..df67f1fb92 100644
--- a/src/qemu/qemu_migration_params.h
+++ b/src/qemu/qemu_migration_params.h
@@ -41,6 +41,7 @@ typedef enum {
QEMU_MIGRATION_CAP_RETURN_PATH,
QEMU_MIGRATION_CAP_ZERO_COPY_SEND,
QEMU_MIGRATION_CAP_POSTCOPY_PREEMPT,
+ QEMU_MIGRATION_CAP_SWITCHOVER_ACK,
QEMU_MIGRATION_CAP_LAST
} qemuMigrationCapability;
--
2.43.0