[v2 0/4] Support for dirty-limit live migration
Hello, this is series version 2. Just certain that it won't be overlooked. Please review, thanks, Yong v2: - rebase on master v1: In qemu>=8.1, the dirty-limit functionality for live migration was included. In the live migration scenario, it implements the force convergence using the dirty-limit approach, which results in better reliable read performance. A straightforward dirty-limit capability for live migration is added by this patchset. Users might not care about other dirty-limit arguments like "x-vcpu-dirty-limit-period" or "vcpu-dirty-limit," thus do not expose them to Libvirt and leave the default settings in place. values as default. For more details about dirty-limit. please see the following reference: https://lore.kernel.org/qemu- devel/169024923116.19090.10825599068950039132-0@git.sr.ht/ Hyman Huang (4): Add VIR_MIGRATE_DIRTY_LIMIT flag qemu_migration: Implement VIR_MIGRATE_DIRTY_LIMIT flag virsh: Add support for VIR_MIGRATE_DIRTY_LIMIT flag NEWS: document support for dirty-limit live migration NEWS.rst | 8 ++++++++ docs/manpages/virsh.rst | 10 +++++++++- include/libvirt/libvirt-domain.h | 5 +++++ src/libvirt-domain.c | 8 ++++++++ src/qemu/qemu_migration.c | 8 ++++++++ src/qemu/qemu_migration.h | 1 + src/qemu/qemu_migration_params.c | 6 ++++++ src/qemu/qemu_migration_params.h | 1 + tools/virsh-domain.c | 10 ++++++++++ 9 files changed, 56 insertions(+), 1 deletion(-) -- 2.39.1
The flag can be used to enable dirty-limit capability for live migration. Signed-off-by: Hyman Huang <yong.huang@smartx.com> --- include/libvirt/libvirt-domain.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index a1902546bb..4a4209c01a 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -1100,6 +1100,11 @@ typedef enum { * Since: 8.5.0 */ VIR_MIGRATE_ZEROCOPY = (1 << 20), + + /* Enable dirty-limit capability to make migration convergent. + * Since: 9.9.0 + */ + VIR_MIGRATE_DIRTY_LIMIT = (1 << 21), } virDomainMigrateFlags; -- 2.39.1
Signed-off-by: Hyman Huang <yong.huang@smartx.com> --- src/qemu/qemu_migration.c | 8 ++++++++ src/qemu/qemu_migration.h | 1 + src/qemu/qemu_migration_params.c | 6 ++++++ src/qemu/qemu_migration_params.h | 1 + 4 files changed, 16 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index ac58aa1a8c..430a58a93d 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2558,6 +2558,14 @@ qemuMigrationSrcBeginPhase(virQEMUDriver *driver, if (!qemuMigrationSrcIsAllowed(vm, true, vm->job->asyncJob, flags)) return NULL; + if (flags & VIR_MIGRATE_DIRTY_LIMIT && + (!(vm->def->features[VIR_DOMAIN_FEATURE_KVM] == VIR_TRISTATE_SWITCH_ON && + vm->def->kvm_features->features[VIR_DOMAIN_KVM_DIRTY_RING] == VIR_TRISTATE_SWITCH_ON))) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("dirtylimit migration requires dirty-ring features enabled")); + return NULL; + } + if (!(flags & (VIR_MIGRATE_UNSAFE | VIR_MIGRATE_OFFLINE)) && !qemuMigrationSrcIsSafe(vm->def, priv->qemuCaps, nmigrate_disks, migrate_disks, flags)) diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index ed62fd4a91..a2a598fe28 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -62,6 +62,7 @@ VIR_MIGRATE_NON_SHARED_SYNCHRONOUS_WRITES | \ VIR_MIGRATE_POSTCOPY_RESUME | \ VIR_MIGRATE_ZEROCOPY | \ + VIR_MIGRATE_DIRTY_LIMIT | \ 0) /* All supported migration parameters and their types. */ diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 79fe6e97c8..797293d8e7 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -99,6 +99,7 @@ VIR_ENUM_IMPL(qemuMigrationCapability, "dirty-bitmaps", "return-path", "zero-copy-send", + "dirty-limit", ); @@ -194,6 +195,11 @@ static const qemuMigrationParamsFlagMapItem qemuMigrationParamsFlagMap[] = { VIR_MIGRATE_ZEROCOPY, QEMU_MIGRATION_CAP_ZERO_COPY_SEND, QEMU_MIGRATION_SOURCE}, + + {QEMU_MIGRATION_FLAG_REQUIRED, + VIR_MIGRATE_DIRTY_LIMIT, + QEMU_MIGRATION_CAP_DIRTY_LIMIT, + QEMU_MIGRATION_SOURCE}, }; /* Translation from VIR_MIGRATE_PARAM_* typed parameters to diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 5857673227..44a44aae07 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -40,6 +40,7 @@ typedef enum { QEMU_MIGRATION_CAP_BLOCK_DIRTY_BITMAPS, QEMU_MIGRATION_CAP_RETURN_PATH, QEMU_MIGRATION_CAP_ZERO_COPY_SEND, + QEMU_MIGRATION_CAP_DIRTY_LIMIT, QEMU_MIGRATION_CAP_LAST } qemuMigrationCapability; -- 2.39.1
Introudce option to enable dirty-limit convergence algorithim during live migration. Signed-off-by: Hyman Huang <yong.huang@smartx.com> --- docs/manpages/virsh.rst | 10 +++++++++- src/libvirt-domain.c | 8 ++++++++ tools/virsh-domain.c | 10 ++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index 3e7a4c6c22..d1d20cb5ba 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -3376,7 +3376,7 @@ migrate [--compressed] [--comp-methods method-list] [--comp-mt-level] [--comp-mt-threads] [--comp-mt-dthreads] [--comp-xbzrle-cache] [--comp-zlib-level] [--comp-zstd-level] - [--auto-converge] [auto-converge-initial] + [--dirty-limit] [--auto-converge] [auto-converge-initial] [auto-converge-increment] [--persistent-xml file] [--tls] [--postcopy-bandwidth bandwidth] [--parallel [--parallel-connections connections]] @@ -3432,6 +3432,14 @@ source or destination host and the ``migrate`` command will report an error leaving the domain active on both hosts. To recover from such situation repeat the original ``migrate`` command with an additional *--postcopy-resume* flag. +*--dirty-limit* forces convergence using dirty-limit algorithms during live +migration. For QEMU/KVM, this means migration will throttle vCPUs as needed to +keep their dirty page rate within the migration parameter ``vcpu-dirty-limit`` +(1 megabits/s by default). This can improve the responsiveness of large guests +during live migration and result in more stable read performance. It requires +dirty-ring size configuration and conflicts with the traditional +``auto-converge`` algorithm. + *--auto-converge* forces convergence during live migration. The initial guest CPU throttling rate can be set with *auto-converge-initial*. If the initial throttling rate is not enough to ensure convergence, the rate is diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 58e1e5ea8d..e5b9679644 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -4213,6 +4213,10 @@ virDomainMigrate3(virDomainPtr domain, /* Now checkout the destination */ virCheckReadOnlyGoto(dconn->flags, error); + VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_AUTO_CONVERGE, + VIR_MIGRATE_DIRTY_LIMIT, + error); + VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_NON_SHARED_DISK, VIR_MIGRATE_NON_SHARED_INC, error); @@ -4633,6 +4637,10 @@ virDomainMigrateToURI3(virDomainPtr domain, virCheckDomainReturn(domain, -1); virCheckReadOnlyGoto(domain->conn->flags, error); + VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_AUTO_CONVERGE, + VIR_MIGRATE_DIRTY_LIMIT, + error); + VIR_EXCLUSIVE_FLAGS_GOTO(VIR_MIGRATE_TUNNELLED, VIR_MIGRATE_PARALLEL, error); diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 66f933dead..3f34230e6b 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -10981,6 +10981,10 @@ static const vshCmdOptDef opts_migrate[] = { .type = VSH_OT_BOOL, .help = N_("compress repeated pages during live migration") }, + {.name = "dirty-limit", + .type = VSH_OT_BOOL, + .help = N_("force convergence with dirty-limit algorithim during live migration") + }, {.name = "auto-converge", .type = VSH_OT_BOOL, .help = N_("force convergence during live migration") @@ -11167,6 +11171,7 @@ doMigrate(void *opaque) { "change-protection", VIR_MIGRATE_CHANGE_PROTECTION }, { "unsafe", VIR_MIGRATE_UNSAFE }, { "compressed", VIR_MIGRATE_COMPRESSED }, + { "dirty-limit", VIR_MIGRATE_DIRTY_LIMIT }, { "auto-converge", VIR_MIGRATE_AUTO_CONVERGE }, { "rdma-pin-all", VIR_MIGRATE_RDMA_PIN_ALL }, { "offline", VIR_MIGRATE_OFFLINE }, @@ -11193,6 +11198,11 @@ doMigrate(void *opaque) flags |= flagmap[i].migflag; } + if (flags & VIR_MIGRATE_DIRTY_LIMIT && flags & VIR_MIGRATE_AUTO_CONVERGE) { + vshError(ctl, "'--dirty-limit' conflicts with '--auto-converge'"); + goto out; + } + if (flags & VIR_MIGRATE_NON_SHARED_SYNCHRONOUS_WRITES && !(flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_DISK))) { vshError(ctl, "'--copy-storage-synchronous-writes' requires one of '--copy-storage-all', '--copy-storage-inc'"); -- 2.39.1
Signed-off-by: Hyman Huang <yong.huang@smartx.com> --- NEWS.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 08e5a3d04a..655666e64b 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -35,6 +35,14 @@ v9.9.0 (2023-11-01) guest features to see if the current libvirt supports both deleting and reverting external snapshots. + * QEMU: Support dirty-limit live migration + + Introduce ``VIR_MIGRATE_DIRTY_LIMIT`` flag for live migration API, + which can enable the dirty-limit convergence algorithm during live + migration and reap more stable read performance for guest vCPUs. + + Also, add the corresponding ``--dirty-limit`` option to ``virsh migrate``. + * virsh: add ``console --resume`` support The ``virsh console`` subcommand now accepts a ``--resume`` option. This -- 2.39.1
participants (1)
-
Hyman Huang