10:16 p.m.
v2:
- mark the VIR_MIGRATE_DIRTY_LIMIT flag since 10.0.0
v1:
The dirty-limit functionality for live migration was
introduced since qemu>=8.1.
In the live migration scenario, it implements the force
convergence using the dirty-limit approach, which results
in better reliable read performance.
A straightforward dirty-limit capability for live migration
is added by this patchset. Users might not care about other
dirty-limit arguments like "x-vcpu-dirty-limit-period"
or "vcpu-dirty-limit," thus do not expose them to Libvirt
and Keep the default configurations and values in place.
For more details about dirty-limit, please see the following
reference:
https://lore.kernel.org/qemu-
devel/169024923116.19090.10825599068950039132-0(a)git.sr.ht/
Hyman Huang (4):
Add VIR_MIGRATE_DIRTY_LIMIT flag
qemu_migration: Implement VIR_MIGRATE_DIRTY_LIMIT flag
virsh: Add support for VIR_MIGRATE_DIRTY_LIMIT flag
NEWS: document support for dirty-limit live migration
NEWS.rst | 8 ++++++++
docs/manpages/virsh.rst | 10 +++++++++-
include/libvirt/libvirt-domain.h | 5 +++++
src/libvirt-domain.c | 8 ++++++++
src/qemu/qemu_migration.c | 8 ++++++++
src/qemu/qemu_migration.h | 1 +
src/qemu/qemu_migration_params.c | 6 ++++++
src/qemu/qemu_migration_params.h | 1 +
tools/virsh-domain.c | 10 ++++++++++
9 files changed, 56 insertions(+), 1 deletion(-)
--
2.39.1
10:16 p.m.
New subject: [PATCH v2 1/4] Add VIR_MIGRATE_DIRTY_LIMIT flag
The flag can be used to enable dirty-limit capability for
live migration.
Signed-off-by: Hyman Huang <yong.huang(a)smartx.com>
---
include/libvirt/libvirt-domain.h | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h
index a1902546bb..44af09823c 100644
--- a/include/libvirt/libvirt-domain.h
+++ b/include/libvirt/libvirt-domain.h
@@ -1100,6 +1100,11 @@ typedef enum {
* Since: 8.5.0
*/
VIR_MIGRATE_ZEROCOPY = (1 << 20),
+
+ /* Enable dirty-limit capability to make migration convergent.
+ * Since: 10.0.0
+ */
+ VIR_MIGRATE_DIRTY_LIMIT = (1 << 21),
} virDomainMigrateFlags;
--
2.39.1
10:16 p.m.
New subject: [PATCH v2 2/4] qemu_migration: Implement VIR_MIGRATE_DIRTY_LIMIT flag
Signed-off-by: Hyman Huang <yong.huang(a)smartx.com>
---
src/qemu/qemu_migration.c | 8 ++++++++
src/qemu/qemu_migration.h | 1 +
src/qemu/qemu_migration_params.c | 6 ++++++
src/qemu/qemu_migration_params.h | 1 +
4 files changed, 16 insertions(+)
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index a5488fd477..df3a4df0c1 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -2570,6 +2570,14 @@ qemuMigrationSrcBeginPhase(virQEMUDriver *driver,
if (!qemuMigrationSrcIsAllowed(vm, true, vm->job->asyncJob, flags))
return NULL;
+ if (flags & VIR_MIGRATE_DIRTY_LIMIT &&
+ (!(vm->def->features[VIR_DOMAIN_FEATURE_KVM] == VIR_TRISTATE_SWITCH_ON
&&
+ vm->def->kvm_features->features[VIR_DOMAIN_KVM_DIRTY_RING] ==
VIR_TRISTATE_SWITCH_ON))) {
+ virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
+ _("dirtylimit migration requires dirty-ring features
enabled"));
+ return NULL;
+ }
+
if (!(flags & (VIR_MIGRATE_UNSAFE | VIR_MIGRATE_OFFLINE)) &&
!qemuMigrationSrcIsSafe(vm->def, priv->qemuCaps,
nmigrate_disks, migrate_disks, flags))
diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h
index ed62fd4a91..a2a598fe28 100644
--- a/src/qemu/qemu_migration.h
+++ b/src/qemu/qemu_migration.h
@@ -62,6 +62,7 @@
VIR_MIGRATE_NON_SHARED_SYNCHRONOUS_WRITES | \
VIR_MIGRATE_POSTCOPY_RESUME | \
VIR_MIGRATE_ZEROCOPY | \
+ VIR_MIGRATE_DIRTY_LIMIT | \
0)
/* All supported migration parameters and their types. */
diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c
index 79fe6e97c8..797293d8e7 100644
--- a/src/qemu/qemu_migration_params.c
+++ b/src/qemu/qemu_migration_params.c
@@ -99,6 +99,7 @@ VIR_ENUM_IMPL(qemuMigrationCapability,
"dirty-bitmaps",
"return-path",
"zero-copy-send",
+ "dirty-limit",
);
@@ -194,6 +195,11 @@ static const qemuMigrationParamsFlagMapItem
qemuMigrationParamsFlagMap[] = {
VIR_MIGRATE_ZEROCOPY,
QEMU_MIGRATION_CAP_ZERO_COPY_SEND,
QEMU_MIGRATION_SOURCE},
+
+ {QEMU_MIGRATION_FLAG_REQUIRED,
+ VIR_MIGRATE_DIRTY_LIMIT,
+ QEMU_MIGRATION_CAP_DIRTY_LIMIT,
+ QEMU_MIGRATION_SOURCE},
};
/* Translation from VIR_MIGRATE_PARAM_* typed parameters to
diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h
index 5857673227..44a44aae07 100644
--- a/src/qemu/qemu_migration_params.h
+++ b/src/qemu/qemu_migration_params.h
@@ -40,6 +40,7 @@ typedef enum {
QEMU_MIGRATION_CAP_BLOCK_DIRTY_BITMAPS,
QEMU_MIGRATION_CAP_RETURN_PATH,
QEMU_MIGRATION_CAP_ZERO_COPY_SEND,
+ QEMU_MIGRATION_CAP_DIRTY_LIMIT,
QEMU_MIGRATION_CAP_LAST
} qemuMigrationCapability;
--
2.39.1
10:16 p.m.
New subject: [PATCH v2 3/4] virsh: Add support for VIR_MIGRATE_DIRTY_LIMIT flag
Introudce option to enable dirty-limit convergence algorithim
during live migration.
Signed-off-by: Hyman Huang <yong.huang(a)smartx.com>
---
docs/manpages/virsh.rst | 10 +++++++++-
src/libvirt-domain.c | 8 ++++++++
tools/virsh-domain.c | 10 ++++++++++
3 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst
index 3e7a4c6c22..d1d20cb5ba 100644
--- a/docs/manpages/virsh.rst
+++ b/docs/manpages/virsh.rst
@@ -3376,7 +3376,7 @@ migrate
[--compressed] [--comp-methods method-list]
[--comp-mt-level] [--comp-mt-threads] [--comp-mt-dthreads]
[--comp-xbzrle-cache] [--comp-zlib-level] [--comp-zstd-level]
- [--auto-converge] [auto-converge-initial]
+ [--dirty-limit] [--auto-converge] [auto-converge-initial]
[auto-converge-increment] [--persistent-xml file] [--tls]
[--postcopy-bandwidth bandwidth]
[--parallel [--parallel-connections connections]]
@@ -3432,6 +3432,14 @@ source or destination host and the ``migrate`` command will report
an error
leaving the domain active on both hosts. To recover from such situation repeat
the original ``migrate`` command with an additional *--postcopy-resume* flag.
+*--dirty-limit* forces convergence using dirty-limit algorithms during live
+migration. For QEMU/KVM, this means migration will throttle vCPUs as needed to
+keep their dirty page rate within the migration parameter ``vcpu-dirty-limit``
+(1 megabits/s by default). This can improve the responsiveness of large guests
+during live migration and result in more stable read performance. It requires
+dirty-ring size configuration and conflicts with the traditional
+``auto-converge`` algorithm.
+
*--auto-converge* forces convergence during live migration. The initial
guest CPU throttling rate can be set with *auto-converge-initial*. If the
initial throttling rate is not enough to ensure convergence, the rate is
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c
index 77a9682ecb..dde60ab05b 100644
--- a/src/libvirt-domain.c
+++ b/src/libvirt-domain.c
@@ -4227,6 +4227,10 @@ virDomainMigrate3(virDomainPtr domain,
/* Now checkout the destination */
virCheckReadOnlyGoto(dconn->flags, error);
+ VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_AUTO_CONVERGE,
+ VIR_MIGRATE_DIRTY_LIMIT,
+ error);
+
VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_NON_SHARED_DISK,
VIR_MIGRATE_NON_SHARED_INC,
error);
@@ -4652,6 +4656,10 @@ virDomainMigrateToURI3(virDomainPtr domain,
virCheckDomainReturn(domain, -1);
virCheckReadOnlyGoto(domain->conn->flags, error);
+ VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_AUTO_CONVERGE,
+ VIR_MIGRATE_DIRTY_LIMIT,
+ error);
+
VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_TUNNELLED,
VIR_MIGRATE_PARALLEL,
error);
diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c
index fa9d356e15..a40a7a3b7f 100644
--- a/tools/virsh-domain.c
+++ b/tools/virsh-domain.c
@@ -10981,6 +10981,10 @@ static const vshCmdOptDef opts_migrate[] = {
.type = VSH_OT_BOOL,
.help = N_("compress repeated pages during live migration")
},
+ {.name = "dirty-limit",
+ .type = VSH_OT_BOOL,
+ .help = N_("force convergence with dirty-limit algorithim during live
migration")
+ },
{.name = "auto-converge",
.type = VSH_OT_BOOL,
.help = N_("force convergence during live migration")
@@ -11167,6 +11171,7 @@ doMigrate(void *opaque)
{ "change-protection", VIR_MIGRATE_CHANGE_PROTECTION },
{ "unsafe", VIR_MIGRATE_UNSAFE },
{ "compressed", VIR_MIGRATE_COMPRESSED },
+ { "dirty-limit", VIR_MIGRATE_DIRTY_LIMIT },
{ "auto-converge", VIR_MIGRATE_AUTO_CONVERGE },
{ "rdma-pin-all", VIR_MIGRATE_RDMA_PIN_ALL },
{ "offline", VIR_MIGRATE_OFFLINE },
@@ -11193,6 +11198,11 @@ doMigrate(void *opaque)
flags |= flagmap[i].migflag;
}
+ if (flags & VIR_MIGRATE_DIRTY_LIMIT && flags &
VIR_MIGRATE_AUTO_CONVERGE) {
+ vshError(ctl, "'--dirty-limit' conflicts with
'--auto-converge'");
+ goto out;
+ }
+
if (flags & VIR_MIGRATE_NON_SHARED_SYNCHRONOUS_WRITES &&
!(flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC))) {
vshError(ctl, "'--copy-storage-synchronous-writes' requires one of
'--copy-storage-all', '--copy-storage-inc'");
--
2.39.1
10:16 p.m.
New subject: [PATCH v2 4/4] NEWS: document support for dirty-limit live migration
Signed-off-by: Hyman Huang <yong.huang(a)smartx.com>
---
NEWS.rst | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/NEWS.rst b/NEWS.rst
index dc40602c72..4d1ff489b0 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -17,6 +17,14 @@ v10.0.0 (unreleased)
* **New features**
+ * QEMU: Support dirty-limit live migration
+
+ Introduce ``VIR_MIGRATE_DIRTY_LIMIT`` flag for live migration API,
+ which can enable the dirty-limit convergence algorithm during live
+ migration and reap more stable read performance for guest vCPUs.
+
+ Also, add the corresponding ``--dirty-limit`` option to ``virsh migrate``.
+
* **Improvements**
* **Bug fixes**
--
2.39.1
350
days inactive
350
days old
4 comments
1 participants
participants (1)
-
Hyman Huang