[PATCH 00/12] migration/migrationpin:support migration pin
Support set the CPU affinity of the live migration thread to improve the migration performance in specific cases. By default, the migration thread shares CPU resources with the VM process. With those API, support pin migration thread to expected CPU list to avoid preempting CPU resources of VM process. New API 'pinMigrationThread' and virsh command 'virsh migrationpin' is used to pin migration thread to expected CPU list before or during migration. New migration param 'migration.pin' is used to support migration pin via interface 'virDomainMigrateToURI3'. Jiang Jiacheng (2): migration/migration-pin: support migration thread pin by virsh command migration/migration-pin/multifd-pin: add migration pin status handle zhengchuan (10): migration/migration-pin: get migration pid for migration pin migration/migration-pin: pin migration pid by given cpumap migration/migration-pin: add qemu monitor callback functions migration/migration-pin: add migrationpin for migration parameters migration/migration-pin: get cpumap from migration.pin migration/migration-pin: add domainMigrationPid for qemuMonitorCallbacks migration/multifd-pin: get multifd pid for migration pin migration/multifd-pin: pin multifd pid by given cpumap migration/multifd-pin: add qemu monitor callback functions migration/multifd-pin: support migration multifd thread pin include/libvirt/libvirt-domain.h | 19 ++ src/conf/domain_conf.c | 9 + src/conf/domain_conf.h | 11 ++ src/conf/virconftypes.h | 2 + src/driver-hypervisor.h | 16 ++ src/libvirt-domain.c | 144 +++++++++++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 7 + src/qemu/qemu_domain.c | 5 + src/qemu/qemu_domain.h | 5 + src/qemu/qemu_driver.c | 175 ++++++++++++++++++ src/qemu/qemu_migration.c | 5 + src/qemu/qemu_migration.h | 1 + src/qemu/qemu_migration_params.c | 21 +++ src/qemu/qemu_migration_params.h | 4 + src/qemu/qemu_monitor.c | 20 ++ src/qemu/qemu_monitor.h | 13 ++ src/qemu/qemu_monitor_json.c | 32 ++++ src/qemu/qemu_process.c | 303 +++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 15 ++ src/remote/remote_driver.c | 3 + src/remote/remote_protocol.x | 43 ++++- src/remote_protocol-structs | 28 +++ src/util/vircgroup.c | 3 + src/util/vircgroup.h | 1 + tools/virsh-domain.c | 69 +++++++ 26 files changed, 954 insertions(+), 1 deletion(-) -- 2.33.0
From: zhengchuan <zhengchuan@huawei.com> Firstly, we need to get migration pids, add virDomainMigrateGetMigrationPids() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 3 +++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 39 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 ++++ src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 31 +++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 8 +++++++ 10 files changed, 112 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 295fd30c93..e11f2795f1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6457,4 +6457,7 @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, int seconds, unsigned int flags); +char *virDomainMigrateGetMigrationPids(virDomainPtr domain, + unsigned int flags); + #endif /* LIBVIRT_DOMAIN_H */ diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 016d5cec7c..618f116012 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -1441,6 +1441,10 @@ typedef int int seconds, unsigned int flags); +typedef char * +(*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, + unsigned int flags); + typedef struct _virHypervisorDriver virHypervisorDriver; /** @@ -1712,4 +1716,5 @@ struct _virHypervisorDriver { virDrvDomainAuthorizedSSHKeysSet domainAuthorizedSSHKeysSet; virDrvDomainGetMessages domainGetMessages; virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; + virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 78c26b2219..fabfb2dd7f 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -13972,3 +13972,42 @@ virDomainStartDirtyRateCalc(virDomainPtr domain, virDispatchError(conn); return -1; } + +/** + * virDomainMigrateGetMigrationPids: + * @domain: pointer to domain object + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Get migration thread pid. + * + * Returns the migration pids which must be freed by the caller, or + * NULL if there was an error. + * + * Since: 9.1.0 + */ +char * +virDomainMigrateGetMigrationPids(virDomainPtr domain, + unsigned int flags) +{ + virConnectPtr conn; + VIR_DOMAIN_DEBUG(domain, "migration pids flags=0x%x", flags); + virResetLastError(); + virCheckDomainReturn(domain, NULL); + conn = domain->conn; + + virCheckReadOnlyGoto(domain->conn->flags, error); + + if (conn->driver->domainMigrateGetMigrationPids) { + char *ret; + ret = conn->driver->domainMigrateGetMigrationPids(domain, flags); + if (!ret) + goto error; + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return NULL; +} diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index 297a2c436a..f11fe1b26b 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -927,4 +927,9 @@ LIBVIRT_8.5.0 { virDomainAbortJobFlags; } LIBVIRT_8.4.0; +LIBVIRT_9.1.0 { + global: + virDomainMigrateGetMigrationPids; +} LIBVIRT_8.5.0; + # .... define new API here using predicted next version number .... diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 5c05032ce3..0bff24dc47 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1780,6 +1780,7 @@ qemuDomainObjPrivateDataClear(qemuDomainObjPrivate *priv) /* remove automatic pinning data */ g_clear_pointer(&priv->autoNodeset, virBitmapFree); g_clear_pointer(&priv->autoCpuset, virBitmapFree); + g_clear_pointer(&priv->pcpumap, virBitmapFree); g_clear_pointer(&priv->pciaddrs, virDomainPCIAddressSetFree); g_clear_pointer(&priv->usbaddrs, virDomainUSBAddressSetFree); g_clear_pointer(&priv->origCPU, virCPUDefFree); @@ -1827,6 +1828,7 @@ qemuDomainObjPrivateFree(void *data) virObjectUnref(priv->monConfig); g_free(priv->lockState); g_free(priv->origname); + g_free(priv->migrationPids); virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 2f027fad87..a804a1b46e 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -139,6 +139,7 @@ struct _qemuDomainObjPrivate { char *origname; int nbdPort; /* Port used for migration with NBD */ unsigned short migrationPort; + char *migrationPids; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case it was changed for the current @@ -163,6 +164,7 @@ struct _qemuDomainObjPrivate { /* Bitmaps below hold data from the auto NUMA feature */ virBitmap *autoNodeset; virBitmap *autoCpuset; + virBitmap *pcpumap; bool signalIOError; /* true if the domain condition should be signalled on I/O error */ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index d509582719..b9dc5f29f5 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20648,6 +20648,36 @@ qemuDomainStartDirtyRateCalc(virDomainPtr dom, return ret; } +static char * +qemuDomainMigrateGetMigrationPids(virDomainPtr dom, + unsigned int flags) +{ + char *ret = NULL; + virDomainObj *vm = NULL; + qemuDomainObjPrivate *priv = NULL; + + virCheckFlags(0, NULL); + + vm = qemuDomainObjFromDomain(dom); + if (!vm) + goto cleanup; + + if (virDomainMigrateGetMigrationPidsEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + priv = vm->privateData; + + if (priv->migrationPids) { + ret = g_strdup(priv->migrationPids); + if (!ret) + VIR_ERROR(_("failed to strdup migrationPids")); + } + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -20897,6 +20927,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainGetMessages = qemuDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index b0dba9057b..9d8d7142b7 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8651,6 +8651,7 @@ static virHypervisorDriver hypervisor_driver = { .domainGetMessages = remoteDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 7dfb4548f4..d1b799db13 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -2528,6 +2528,15 @@ struct remote_domain_migrate_set_compression_cache_args { unsigned int flags; }; +struct remote_domain_migrate_get_migration_pids_args { + remote_nonnull_domain dom; + unsigned int flags; +}; + +struct remote_domain_migrate_get_migration_pids_ret { + remote_nonnull_string migrationPids; +}; + struct remote_domain_migrate_set_max_speed_args { remote_nonnull_domain dom; unsigned hyper bandwidth; @@ -6961,5 +6970,11 @@ enum remote_procedure { * @generate: both * @acl: domain:write */ - REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442 + REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index ca5222439d..da4d799f42 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -3268,6 +3268,13 @@ struct remote_domain_event_memory_device_size_change_msg { remote_nonnull_string alias; uint64_t size; }; +struct remote_domain_migrate_get_migration_pids_args { + remote_nonnull_domain dom; + u_int flags; +}; +struct remote_domain_migrate_get_migration_pids_ret { + remote_nonnull_string migrationPids; +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -3711,4 +3718,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_SAVE_PARAMS = 440, REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441, REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, }; -- 2.33.0
On Tue, Jan 03, 2023 at 09:08:20PM +0800, Jiang Jiacheng wrote:
From: zhengchuan <zhengchuan@huawei.com>
Firstly, we need to get migration pids, add virDomainMigrateGetMigrationPids() for migration pin.
Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 3 +++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 39 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 ++++ src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 31 +++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 8 +++++++ 10 files changed, 112 insertions(+), 1 deletion(-)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 295fd30c93..e11f2795f1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6457,4 +6457,7 @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, int seconds, unsigned int flags);
+char *virDomainMigrateGetMigrationPids(virDomainPtr domain, + unsigned int flags);
Exposing QEMU thread PIDs in the public libvirt API is not something we should be doing. We've explicitly aimed to avoid exposing the notion of PIDs in our API in general. With regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On 2023/1/9 22:46, Daniel P. Berrangé wrote:
On Tue, Jan 03, 2023 at 09:08:20PM +0800, Jiang Jiacheng wrote:
From: zhengchuan <zhengchuan@huawei.com>
Firstly, we need to get migration pids, add virDomainMigrateGetMigrationPids() for migration pin.
Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 3 +++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 39 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 ++++ src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 31 +++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 8 +++++++ 10 files changed, 112 insertions(+), 1 deletion(-)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 295fd30c93..e11f2795f1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6457,4 +6457,7 @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, int seconds, unsigned int flags);
+char *virDomainMigrateGetMigrationPids(virDomainPtr domain, + unsigned int flags);
Exposing QEMU thread PIDs in the public libvirt API is not something we should be doing. We've explicitly aimed to avoid exposing the notion of PIDs in our API in general.
Thanks for your reply! As we should not exposing QEMU thread PIDs, is it a better way to proactively detect the PIDs of the qemu migration thread? For example, using QMP command during migration to detect the QEMU migration thread PIDs like iothreadpin does. Compared to QEMU event, detecting proactively may spend more time to complete migration pin, but i think its still makes sense for migration. Thanks, Jiang Jiacheng
With regards, Daniel
On Tue, Jan 10, 2023 at 07:30:08PM +0800, Jiang Jiacheng wrote:
On 2023/1/9 22:46, Daniel P. Berrangé wrote:
On Tue, Jan 03, 2023 at 09:08:20PM +0800, Jiang Jiacheng wrote:
From: zhengchuan <zhengchuan@huawei.com>
Firstly, we need to get migration pids, add virDomainMigrateGetMigrationPids() for migration pin.
Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 3 +++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 39 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 ++++ src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 31 +++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 8 +++++++ 10 files changed, 112 insertions(+), 1 deletion(-)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 295fd30c93..e11f2795f1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6457,4 +6457,7 @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, int seconds, unsigned int flags);
+char *virDomainMigrateGetMigrationPids(virDomainPtr domain, + unsigned int flags);
Exposing QEMU thread PIDs in the public libvirt API is not something we should be doing. We've explicitly aimed to avoid exposing the notion of PIDs in our API in general.
Thanks for your reply!
As we should not exposing QEMU thread PIDs, is it a better way to proactively detect the PIDs of the qemu migration thread? For example, using QMP command during migration to detect the QEMU migration thread PIDs like iothreadpin does. Compared to QEMU event, detecting proactively may spend more time to complete migration pin, but i think its still makes sense for migration.
iothreadpin doesn't take thread PIDs, it takes integer identifiers associated with the threads. Why does the mgmt app need to know the migration PIDs ? The API for setting pinning in the next patch doesn't accept a PID, and we don't need to set different pinning for each migration thread. Only libvirt QEMU driver code needs to know the PIDs, not the libvirt public API/mgmt app. With regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On 2023/1/10 19:44, Daniel P. Berrangé wrote:
On Tue, Jan 10, 2023 at 07:30:08PM +0800, Jiang Jiacheng wrote:
On 2023/1/9 22:46, Daniel P. Berrangé wrote:
On Tue, Jan 03, 2023 at 09:08:20PM +0800, Jiang Jiacheng wrote:
From: zhengchuan <zhengchuan@huawei.com>
Firstly, we need to get migration pids, add virDomainMigrateGetMigrationPids() for migration pin.
Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 3 +++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 39 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 ++++ src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 31 +++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 8 +++++++ 10 files changed, 112 insertions(+), 1 deletion(-)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 295fd30c93..e11f2795f1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6457,4 +6457,7 @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, int seconds, unsigned int flags);
+char *virDomainMigrateGetMigrationPids(virDomainPtr domain, + unsigned int flags);
Exposing QEMU thread PIDs in the public libvirt API is not something we should be doing. We've explicitly aimed to avoid exposing the notion of PIDs in our API in general.
Thanks for your reply!
As we should not exposing QEMU thread PIDs, is it a better way to proactively detect the PIDs of the qemu migration thread? For example, using QMP command during migration to detect the QEMU migration thread PIDs like iothreadpin does. Compared to QEMU event, detecting proactively may spend more time to complete migration pin, but i think its still makes sense for migration.
iothreadpin doesn't take thread PIDs, it takes integer identifiers associated with the threads.
Why does the mgmt app need to know the migration PIDs ? The API for setting pinning in the next patch doesn't accept a PID, and we don't need to set different pinning for each migration thread. Only libvirt QEMU driver code needs to know the PIDs, not the libvirt public API/mgmt app.
I have got that the pids shouldn't show up in libvirt public API/mgmt app, so I want to drop those changes related with PIDs in mgmt app and try to get the migration pid using qmp command like the way getting iothreads' pid in 'qemuProcessDetectIOThreadPIDs' in qemu_process.c. Thanks Jiang Jiacheng
With regards, Daniel
From: zhengchuan <zhengchuan@huawei.com> Secondly, we start to pin migration pid by given cpumap, add virDomainPinMigrationThread() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 4 ++ src/conf/domain_conf.c | 9 ++++ src/conf/domain_conf.h | 11 ++++ src/conf/virconftypes.h | 2 + src/driver-hypervisor.h | 6 +++ src/libvirt-domain.c | 65 +++++++++++++++++++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 1 + src/qemu/qemu_driver.c | 91 ++++++++++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 ++++- src/remote_protocol-structs | 5 ++ src/util/vircgroup.c | 3 ++ src/util/vircgroup.h | 1 + 14 files changed, 212 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index e11f2795f1..26af8292d3 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -2424,6 +2424,10 @@ int virDomainPinEmulator (virDomainPtr domain, int maplen, unsigned int flags); +int virDomainPinMigrationThread (virDomainPtr domain, + unsigned char *cpumap, + int maplen); + int virDomainGetEmulatorPinInfo (virDomainPtr domain, unsigned char *cpumaps, int maplen, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 6c088ff295..a94a4c1c88 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -30299,3 +30299,12 @@ virDomainDefHasSpiceGraphics(const virDomainDef *def) return false; } + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def) +{ + if (!def) + return; + virBitmapFree(def->cpumask); + VIR_FREE(def); +} diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 1404c55053..a8384a4f6e 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -27,6 +27,7 @@ #include <libxml/xpath.h> #include "internal.h" +#include "viralloc.h" #include "virconftypes.h" #include "capabilities.h" #include "cpu_conf.h" @@ -4353,3 +4354,13 @@ virDomainObjGetMessages(virDomainObj *vm, bool virDomainDefHasSpiceGraphics(const virDomainDef *def); + +struct _virDomainMigrationIDDef { + bool autofill; + int thread_id; + virBitmap *cpumask; + virDomainThreadSchedParam sched; +}; + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def); diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h index 154805091a..984c2181d8 100644 --- a/src/conf/virconftypes.h +++ b/src/conf/virconftypes.h @@ -255,3 +255,5 @@ typedef struct _virDomainXMLOption virDomainXMLOption; typedef struct _virDomainXMLPrivateDataCallbacks virDomainXMLPrivateDataCallbacks; typedef struct _virDomainXenbusControllerOpts virDomainXenbusControllerOpts; + +typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef; diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 618f116012..1d7f606c59 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -1441,6 +1441,11 @@ typedef int int seconds, unsigned int flags); +typedef int +(*virDrvDomainPinMigrationThread)(virDomainPtr domain, + unsigned char *cpumap, + int maplen); + typedef char * (*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, unsigned int flags); @@ -1716,5 +1721,6 @@ struct _virHypervisorDriver { virDrvDomainAuthorizedSSHKeysSet domainAuthorizedSSHKeysSet; virDrvDomainGetMessages domainGetMessages; virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; + virDrvDomainPinMigrationThread domainPinMigrationThread; virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index fabfb2dd7f..d35a8237ae 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14011,3 +14011,68 @@ virDomainMigrateGetMigrationPids(virDomainPtr domain, virDispatchError(domain->conn); return NULL; } + +/** + * virDomainPinMigrationThread: + * @domain: a domain object + * @cpumap: pointer to a bit map of real CPUs (in 8-bit bytes) (IN) + * Each bit set to 1 means that corresponding CPU is usable. + * Bytes are stored in little-endian order: CPU0-7, 8-15... + * In each byte, lowest CPU number is least significant bit. + * @maplen: number of bytes in cpumap, from 1 up to size of CPU map in + * underlying virtualization system (Xen...). + * If maplen < size, missing bytes are set to zero. + * If maplen > size, failure code is returned. + * + * Allocate the real CPUs to the migrationThread which will be launched + * at the beginning of migration. + * This allocation can be handled whether before the migration, or during + * the migration which performs as an instant change. + * + * This function may require privileged access to the hypervisor. + * + * Returns 0 in case of success, -1 in case of failure. + * + * Since: 9.1.0 + */ +int +virDomainPinMigrationThread(virDomainPtr domain, + unsigned char *cpumap, + int maplen) +{ + virConnectPtr conn; + g_autofree char *str = NULL; + + VIR_DOMAIN_DEBUG(domain, "migration: cpumap=%p, maplen=%d", + cpumap, maplen); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckNonNullArgGoto(cpumap, error); + virCheckPositiveArgGoto(maplen, error); + + str = virBitmapDataFormat(cpumap, maplen); + VIR_INFO("Begin to PinMigrationThread(domain=%s): new CPU Affinity:%s", + NULLSTR(domain->name), NULLSTR(str)); + + if (conn->driver->domainPinMigrationThread) { + int ret; + ret = conn->driver->domainPinMigrationThread(domain, cpumap, maplen); + if (ret < 0) { + VIR_ERROR(_("Failed to PinMigrationThread(domain=%s), ret=%d"), + NULLSTR(domain->name), ret); + goto error; + } + VIR_INFO("Success to PinMigrationThread(domain=%s)", NULLSTR(domain->name)); + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index ae746a2d51..ad19d37d95 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -516,6 +516,7 @@ virDomainMemoryModelTypeToString; virDomainMemoryRemove; virDomainMemorySourceTypeFromString; virDomainMemorySourceTypeToString; +virDomainMigrationIDDefFree; virDomainMouseModeTypeFromString; virDomainMouseModeTypeToString; virDomainNetAllocateActualDevice; diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index f11fe1b26b..61561f2367 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -929,6 +929,7 @@ LIBVIRT_8.5.0 { LIBVIRT_9.1.0 { global: + virDomainPinMigrationThread; virDomainMigrateGetMigrationPids; } LIBVIRT_8.5.0; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index b9dc5f29f5..7ae81f459f 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20678,6 +20678,96 @@ qemuDomainMigrateGetMigrationPids(virDomainPtr dom, return ret; } +static int +qemuDomainPinMigrationThread(virDomainPtr dom, + unsigned char *cpumap, + int maplen) +{ + int ret = -1; + virQEMUDriver *driver = dom->conn->privateData; + virQEMUDriverConfig *cfg = NULL; + virCaps *caps = NULL; + virDomainObj *vm = NULL; + virBitmap *pcpumap = NULL; + virCgroup *cgroup_migthread = NULL; + qemuDomainObjPrivate *priv = NULL; + int migration_id = 0; + virDomainMigrationIDDef *migration = NULL; + + cfg = virQEMUDriverGetConfig(driver); + + if (!(caps = virQEMUDriverGetCapabilities(driver, false))) + goto cleanup; + + if (!(vm = qemuDomainObjFromDomain(dom))) + goto cleanup; + + priv = vm->privateData; + + if (virDomainPinMigrationThreadEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0) + goto cleanup; + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto release; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty migration thread cpumap list for pinning")); + goto release; + } + + virBitmapFree(priv->pcpumap); + priv->pcpumap = pcpumap; + + migration = g_new0(virDomainMigrationIDDef, 1); + + if (priv->migrationPids != NULL) { + if (virStrToLong_i(priv->migrationPids, NULL, 10, &(migration->thread_id)) != 0) { + VIR_ERROR(_("migrationPid trans failure, migrationPids = %s"), priv->migrationPids); + goto endjob; + } + + if (pcpumap) { + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, + migration_id, false, &cgroup_migthread) < 0) + goto endjob; + if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for migration thread %d"), migration_id); + goto endjob; + } + } + + if (virProcessSetAffinity(migration->thread_id, pcpumap, false) < 0) + goto endjob; + } + } + + ret = 0; + goto endjob; + + release: + virBitmapFree(pcpumap); + + endjob: + virDomainObjEndJob(vm); + + cleanup: + if (cgroup_migthread) + virCgroupFree(cgroup_migthread); + virDomainMigrationIDDefFree(migration); + virDomainObjEndAPI(&vm); + virObjectUnref(caps); + virObjectUnref(cfg); + return ret; +} + static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -20927,6 +21017,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainGetMessages = qemuDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 9d8d7142b7..1c7cac6b29 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8651,6 +8651,7 @@ static virHypervisorDriver hypervisor_driver = { .domainGetMessages = remoteDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index d1b799db13..f86172b636 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -1291,6 +1291,11 @@ struct remote_domain_pin_iothread_args { unsigned int flags; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; + struct remote_domain_add_iothread_args { remote_nonnull_domain dom; unsigned int iothread_id; @@ -6976,5 +6981,11 @@ enum remote_procedure { * @generate: both * @acl: domain:migrate */ - REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443 + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, + + /** + * @generate: both + * @acl: domain:write + */ + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index da4d799f42..9ef27913f1 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -3268,6 +3268,10 @@ struct remote_domain_event_memory_device_size_change_msg { remote_nonnull_string alias; uint64_t size; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; struct remote_domain_migrate_get_migration_pids_args { remote_nonnull_domain dom; u_int flags; @@ -3719,4 +3723,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441, REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, }; diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c index 73675b4478..d4307c0d72 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -1123,6 +1123,9 @@ virCgroupNewThread(virCgroup *domain, case VIR_CGROUP_THREAD_IOTHREAD: name = g_strdup_printf("iothread%d", id); break; + case VIR_CGROUP_THREAD_MIGRATION_THREAD: + name = g_strdup_printf("migthread%d", id); + break; case VIR_CGROUP_THREAD_LAST: virReportError(VIR_ERR_INTERNAL_ERROR, _("unexpected name value %d"), nameval); diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h index 690f09465c..3c5c23965f 100644 --- a/src/util/vircgroup.h +++ b/src/util/vircgroup.h @@ -53,6 +53,7 @@ typedef enum { VIR_CGROUP_THREAD_VCPU = 0, VIR_CGROUP_THREAD_EMULATOR, VIR_CGROUP_THREAD_IOTHREAD, + VIR_CGROUP_THREAD_MIGRATION_THREAD, VIR_CGROUP_THREAD_LAST } virCgroupThreadName; -- 2.33.0
Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- tools/virsh-domain.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 2d162cf8c0..e17c526850 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -13760,6 +13760,69 @@ cmdDomDirtyRateCalc(vshControl *ctl, const vshCmd *cmd) } +/* + * "migrationpin" command + */ +static const vshCmdInfo info_migrationpin[] = { + {.name = "help", + .data = N_("control domain migrationThreads affinity") + }, + {.name = "desc", + .data = N_("Pin domain migrationThreads to host physical CPUs.") + }, + {.name = NULL} +}; + +static const vshCmdOptDef opts_migrationpin[] = { + VIRSH_COMMON_OPT_DOMAIN_FULL(0), + {.name = "cpulist", + .type = VSH_OT_DATA, + .flags = VSH_OFLAG_REQ, + .help = N_("host cpu number(s) to set") + }, + {.name = NULL} +}; + +static bool +cmdMigrationPin(vshControl *ctl, const vshCmd *cmd) +{ + g_autoptr(virshDomain) dom = NULL; + const char *name = NULL; + const char *cpulist = NULL; + int maxcpu; + g_autofree unsigned char *cpumap = NULL; + int cpumaplen; + virshControl *priv = ctl->privData; + + dom = virshCommandOptDomain(ctl, cmd, NULL); + if (!dom) + return false; + + if (vshCommandOptStringReq(ctl, cmd, "domain", &name) < 0) + return false; + + if (vshCommandOptStringReq(ctl, cmd, "cpulist", &cpulist) < 0) + return false; + + maxcpu = virshNodeGetCPUCount(priv->conn); + if (maxcpu < 0) + return false; + + cpumap = virshParseCPUList(ctl, &cpumaplen, cpulist, maxcpu); + if (!cpumap) + return false; + + if (virDomainPinMigrationThread(dom, cpumap, + cpumaplen) != 0) + return false; + + vshPrintExtra(ctl, _("Pin domain %s's migration thread " + "to cpulist %s success.\n"), name, cpulist); + + return true; +} + + const vshCmdDef domManagementCmds[] = { {.name = "attach-device", .handler = cmdAttachDevice, @@ -14417,5 +14480,11 @@ const vshCmdDef domManagementCmds[] = { .info = info_domdirtyrate_calc, .flags = 0 }, + {.name = "migrationpin", + .handler = cmdMigrationPin, + .opts = opts_migrationpin, + .info = info_migrationpin, + .flags = 0 + }, {.name = NULL} }; -- 2.33.0
From: zhengchuan <zhengchuan@huawei.com> add qemu monitor callback functions Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_monitor.c | 10 ++++++++++ src/qemu/qemu_monitor.h | 7 +++++++ src/qemu/qemu_monitor_json.c | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index 734364e070..36ec923861 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -1380,6 +1380,16 @@ qemuMonitorEmitPRManagerStatusChanged(qemuMonitor *mon, } +void +qemuMonitorEmitMigrationPid(qemuMonitor *mon, + int mpid) +{ + VIR_DEBUG("mon=%p, pass=%d", mon, mpid); + + QEMU_MONITOR_CALLBACK(mon, domainMigrationPid, mon->vm, mpid); +} + + void qemuMonitorEmitRdmaGidStatusChanged(qemuMonitor *mon, const char *netdev, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 906a919f52..4eddd79779 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -362,6 +362,10 @@ typedef void (*qemuMonitorDomainMemoryDeviceSizeChange)(qemuMonitor *mon, const char *alias, unsigned long long size); +typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitor *mon, + virDomainObj *vm, + int mcpid); + typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks; struct _qemuMonitorCallbacks { qemuMonitorEofNotifyCallback eofNotify; @@ -397,6 +401,7 @@ struct _qemuMonitorCallbacks { qemuMonitorDomainMemoryFailureCallback domainMemoryFailure; qemuMonitorDomainMemoryDeviceSizeChange domainMemoryDeviceSizeChange; qemuMonitorDomainDeviceUnplugErrCallback domainDeviceUnplugError; + qemuMonitorDomainMigrationPidCallback domainMigrationPid; }; qemuMonitor *qemuMonitorOpen(virDomainObj *vm, @@ -499,6 +504,8 @@ void qemuMonitorEmitMigrationStatus(qemuMonitor *mon, void qemuMonitorEmitMigrationPass(qemuMonitor *mon, int pass); +void qemuMonitorEmitMigrationPid(qemuMonitor *mon, int mpid); + void qemuMonitorEmitAcpiOstInfo(qemuMonitor *mon, const char *alias, const char *slotType, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index 9822097bd7..3129305107 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -84,6 +84,7 @@ static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitor *mon, virJSONV static void qemuMonitorJSONHandleMemoryFailure(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleMemoryDeviceSizeChange(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleDeviceUnplugErr(qemuMonitor *mon, virJSONValue *data); +static void qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, virJSONValue *data); typedef struct { const char *type; @@ -106,6 +107,7 @@ static qemuEventHandler eventHandlers[] = { { "MEMORY_FAILURE", qemuMonitorJSONHandleMemoryFailure, }, { "MIGRATION", qemuMonitorJSONHandleMigrationStatus, }, { "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, }, + { "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, }, { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, }, { "PR_MANAGER_STATUS_CHANGED", qemuMonitorJSONHandlePRManagerStatusChanged, }, { "RDMA_GID_STATUS_CHANGED", qemuMonitorJSONHandleRdmaGidStatusChanged, }, @@ -129,6 +131,20 @@ static qemuEventHandler eventHandlers[] = { /* We use bsearch, so keep this list sorted. */ }; +static void +qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, + virJSONValue *data) +{ + int mpid; + + if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) { + VIR_WARN("missing migration pid in migration-pid event"); + return; + } + + qemuMonitorEmitMigrationPid(mon, mpid); +} + static int qemuMonitorEventCompare(const void *key, const void *elt) { -- 2.33.0
From: zhengchuan <zhengchuan@huawei.com> Add a migrationpin to the migration parameters of live migration to bind cores to the migration thread during VM migration. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 10 ++++++++++ src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_migration.c | 3 +++ src/qemu/qemu_migration.h | 1 + src/qemu/qemu_migration_params.c | 21 +++++++++++++++++++++ src/qemu/qemu_migration_params.h | 4 ++++ 7 files changed, 41 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 26af8292d3..76744eb0f1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -1367,6 +1367,16 @@ typedef enum { */ # define VIR_MIGRATE_PARAM_TLS_DESTINATION "tls.destination" +/** + * VIR_MIGRATE_PARAM_MIGRATIONPIN: + * + * virDomainMigrate* params field: the pin of migration threads for + * migration as VIR_TYPED_PARAM_STRING. + * + * Since: 9.1.0 + */ +# define VIR_MIGRATE_PARAM_MIGRATIONPIN "migration.pin" + /* Domain migration. */ virDomainPtr virDomainMigrate (virDomainPtr domain, virConnectPtr dconn, unsigned long flags, const char *dname, diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 0bff24dc47..de90cf42f9 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1829,6 +1829,7 @@ qemuDomainObjPrivateFree(void *data) g_free(priv->lockState); g_free(priv->origname); g_free(priv->migrationPids); + g_free(priv->migrationThreadPinList); virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index a804a1b46e..5aa4eb5840 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -140,6 +140,7 @@ struct _qemuDomainObjPrivate { int nbdPort; /* Port used for migration with NBD */ unsigned short migrationPort; char *migrationPids; + char *migrationThreadPinList; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case it was changed for the current diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index f4441d61ae..7e2894eb76 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -3335,6 +3335,7 @@ qemuMigrationDstPrepareFresh(virQEMUDriver *driver, priv = vm->privateData; priv->origname = g_strdup(origname); + g_free(priv->migrationPids); if (taint_hook) { /* Domain XML has been altered by a hook script. */ @@ -4815,6 +4816,8 @@ qemuMigrationSrcRun(virQEMUDriver *driver, priv->migMaxBandwidth * 1024 * 1024) < 0) goto error; + qemuMigrationMigrationParamsToVM(migParams, vm); + if (qemuMigrationParamsApply(vm, VIR_ASYNC_JOB_MIGRATION_OUT, migParams, flags) < 0) goto error; diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 38a961f4e9..e40c251261 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -80,6 +80,7 @@ VIR_MIGRATE_PARAM_COMPRESSION_MT_LEVEL, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_THREADS, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_DTHREADS, VIR_TYPED_PARAM_INT, \ + VIR_MIGRATE_PARAM_MIGRATIONPIN, VIR_TYPED_PARAM_STRING, \ VIR_MIGRATE_PARAM_COMPRESSION_XBZRLE_CACHE, VIR_TYPED_PARAM_ULLONG, \ VIR_MIGRATE_PARAM_PERSIST_XML, VIR_TYPED_PARAM_STRING, \ VIR_MIGRATE_PARAM_AUTO_CONVERGE_INITIAL, VIR_TYPED_PARAM_INT, \ diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index bd09dcfb23..47d1e29712 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -104,6 +104,7 @@ VIR_ENUM_IMPL(qemuMigrationParam, "compress-level", "compress-threads", "decompress-threads", + "migrationpin", "cpu-throttle-initial", "cpu-throttle-increment", "tls-creds", @@ -212,6 +213,10 @@ static const qemuMigrationParamsTPMapItem qemuMigrationParamsTPMap[] = { .param = QEMU_MIGRATION_PARAM_DECOMPRESS_THREADS, .party = QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION}, + {.typedParam = VIR_MIGRATE_PARAM_MIGRATIONPIN, + .param = QEMU_MIGRATION_PARAM_MIGRATIONPIN, + .party = QEMU_MIGRATION_SOURCE}, + {.typedParam = VIR_MIGRATE_PARAM_COMPRESSION_XBZRLE_CACHE, .param = QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE, .party = QEMU_MIGRATION_SOURCE | QEMU_MIGRATION_DESTINATION}, @@ -240,6 +245,9 @@ static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = { [QEMU_MIGRATION_PARAM_DECOMPRESS_THREADS] = { .type = QEMU_MIGRATION_PARAM_TYPE_INT, }, + [QEMU_MIGRATION_PARAM_MIGRATIONPIN] = { + .type = QEMU_MIGRATION_PARAM_TYPE_STRING, + }, [QEMU_MIGRATION_PARAM_THROTTLE_INITIAL] = { .type = QEMU_MIGRATION_PARAM_TYPE_INT, }, @@ -578,6 +586,15 @@ qemuMigrationParamsSetCompression(virTypedParameterPtr params, return 0; } +void +qemuMigrationMigrationParamsToVM(const qemuMigrationParams *migParams, const virDomainObj *vm) +{ + if (migParams && migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].set) { + qemuDomainObjPrivate *priv = vm->privateData; + priv->migrationThreadPinList = g_strdup(migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].value.s); + } +} + void qemuMigrationParamsSetBlockDirtyBitmapMapping(qemuMigrationParams *migParams, @@ -798,6 +815,10 @@ qemuMigrationParamsToJSON(qemuMigrationParams *migParams, if (!pv->set) continue; + if (i == QEMU_MIGRATION_PARAM_MIGRATIONPIN) { + continue; + } + if (postcopyResume && !qemuMigrationParamInfo[i].applyOnPostcopyResume) continue; diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index e7c65f6a21..8ebebf87a5 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -49,6 +49,7 @@ typedef enum { QEMU_MIGRATION_PARAM_COMPRESS_LEVEL, QEMU_MIGRATION_PARAM_COMPRESS_THREADS, QEMU_MIGRATION_PARAM_DECOMPRESS_THREADS, + QEMU_MIGRATION_PARAM_MIGRATIONPIN, QEMU_MIGRATION_PARAM_THROTTLE_INITIAL, QEMU_MIGRATION_PARAM_THROTTLE_INCREMENT, QEMU_MIGRATION_PARAM_TLS_CREDS, @@ -74,6 +75,9 @@ typedef enum { virBitmap * qemuMigrationParamsGetAlwaysOnCaps(qemuMigrationParty party); +void +qemuMigrationMigrationParamsToVM(const qemuMigrationParams *migParams, const virDomainObj *vm); + qemuMigrationParams * qemuMigrationParamsFromFlags(virTypedParameterPtr params, int nparams, -- 2.33.0
From: zhengchuan <zhengchuan@huawei.com> Add qemuProcessGetPcpumap to get cpumap from migration parameters when 'virsh migrationpin' is not called. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_process.c | 79 +++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 4 +++ 2 files changed, 83 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index b6adcf2f2a..b2e9456b98 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3677,6 +3677,85 @@ qemuProcessRecoverMigration(virQEMUDriver *driver, } +unsigned char * +virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu) +{ + int lastcpu; + unsigned char *cpumap = NULL; + virBitmap *map = NULL; + + if (cpulist[0] == 'r') { + map = virBitmapNew(maxcpu); + if (!map) + return NULL; + virBitmapSetAll(map); + } else { + if (virBitmapParse(cpulist, &map, 1024) < 0 || + virBitmapIsAllClear(map)) { + goto cleanup; + } + + lastcpu = virBitmapLastSetBit(map); + if (lastcpu >= maxcpu) + goto cleanup; + } + + if (virBitmapToData(map, &cpumap, cpumaplen) < 0) + VIR_ERROR(_("Bitmap to data failure")); + + cleanup: + virBitmapFree(map); + return cpumap; +} + + +/* + * The value of "virsh migrationpin" is saved to priv->pcpumap + * If priv->pcpumap is NULL, it means migrationpin command is not called, + * otherwise we set the affinity of migration thread by migrationpin + */ +static virBitmap* +qemuProcessGetPcpumap(qemuDomainObjPrivate *priv) +{ + int cpumaplen = 0; + int maxcpu = 0; + virBitmap *pcpumap = NULL; + g_autofree unsigned char *cpumap = NULL; + + if (priv->pcpumap) + return priv->pcpumap; + + if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) { + VIR_ERROR(_("didn't set the migratin thread pin")); + return NULL; + } + + /* judge whether set_migration_pin is default value or not */ + if (STREQ(priv->migrationThreadPinList, "none")) + return NULL; + + maxcpu = virHostCPUGetCount(); + if (maxcpu < 0) { + VIR_ERROR(_("get the cpu count of host failure")); + return NULL; + } + + cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu); + if (!cpumap) { + VIR_ERROR(_("parse migration.pin param failure : migration.pin = %s"), + priv->migrationThreadPinList); + return NULL; + } + + if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) { + VIR_ERROR(_("Bitmap data failure")); + return NULL; + } + + return pcpumap; +} + + static int qemuProcessRecoverJob(virQEMUDriver *driver, virDomainObj *vm, diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index 9a24745f15..5b1e05b1f8 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -242,3 +242,7 @@ bool qemuProcessRebootAllowed(const virDomainDef *def); void qemuProcessCleanupMigrationJob(virQEMUDriver *driver, virDomainObj *vm); + +unsigned char *virParseCPUList(int *cpumaplen, + const char *cpulist, + int maxcpu); -- 2.33.0
From: zhengchuan <zhengchuan@huawei.com> add domainMigrationPid for qemuMonitorCallbacks Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_process.c | 107 ++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 7 +++ 2 files changed, 114 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index b2e9456b98..361daee081 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -1791,6 +1791,7 @@ static qemuMonitorCallbacks monitorCallbacks = { .domainMemoryFailure = qemuProcessHandleMemoryFailure, .domainMemoryDeviceSizeChange = qemuProcessHandleMemoryDeviceSizeChange, .domainDeviceUnplugError = qemuProcessHandleDeviceUnplugErr, + .domainMigrationPid = qemuProcessHandleMigrationPid, }; static void @@ -3677,6 +3678,20 @@ qemuProcessRecoverMigration(virQEMUDriver *driver, } +int +qemuProcessSetupMigration(virDomainObj *vm, + virDomainMigrationIDDef *migration) +{ + return qemuProcessSetupPid(vm, migration->thread_id, + VIR_CGROUP_THREAD_MIGRATION_THREAD, + 0, + vm->def->cputune.emulatorpin, + vm->def->cputune.emulator_period, + vm->def->cputune.emulator_quota, + &migration->sched); +} + + unsigned char * virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu) { @@ -3756,6 +3771,98 @@ qemuProcessGetPcpumap(qemuDomainObjPrivate *priv) } +/* + * In order to set migration thread affinity when vm is migrating, + * we should create the cgroup for migration thread. + */ +static void +qemuProcessSetMigthreadAffinity(qemuDomainObjPrivate *priv, + virBitmap *pcpumap, + int mpid) +{ + int migration_id = 0; + virCgroup *cgroup_migthread = NULL; + + if (!pcpumap) + return; + + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, + migration_id, false, &cgroup_migthread) < 0) + goto cleanup; + + if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for migration%d thread"), migration_id); + goto cleanup; + } + } + + if (virProcessSetAffinity(mpid, pcpumap, false) < 0) + VIR_WARN("failed to set affinity in migration"); + + cleanup: + if (cgroup_migthread) + virCgroupFree(cgroup_migthread); + return; +} + + +int +qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid) +{ + qemuDomainObjPrivate *priv = NULL; + char *mpidStr = NULL; + virDomainMigrationIDDef *migration = NULL; + virBitmap *pcpumap = NULL; + virObjectLock(vm); + + VIR_INFO("Migrating domain %p %s, migration pid %d", + vm, vm->def->name, mpid); + + priv = vm->privateData; + if (vm->job->asyncJob == VIR_ASYNC_JOB_NONE) { + VIR_DEBUG("got MIGRATION_PID event without a migration job"); + goto cleanup; + } + + migration = g_new0(virDomainMigrationIDDef, 1); + migration->thread_id = mpid; + + if (qemuProcessSetupMigration(vm, migration) < 0) { + VIR_ERROR(_("fail to setup migration cgroup")); + goto cleanup; + } + + mpidStr = g_strdup_printf("%d", mpid); + g_free(priv->migrationPids); + priv->migrationPids = mpidStr; + + pcpumap = qemuProcessGetPcpumap(priv); + + if (!pcpumap) + goto cleanup; + + qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid); + + cleanup: + /* + * If the value of pcpumap is setted by priv->migrationThreadPinList, + * we need to free pcpumap. + */ + if (pcpumap != priv->pcpumap) + virBitmapFree(pcpumap); + virDomainMigrationIDDefFree(migration); + virObjectUnlock(vm); + + return 0; +} + + static int qemuProcessRecoverJob(virQEMUDriver *driver, virDomainObj *vm, diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index 5b1e05b1f8..12e2cc1f48 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -246,3 +246,10 @@ void qemuProcessCleanupMigrationJob(virQEMUDriver *driver, unsigned char *virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu); + +int qemuProcessSetupMigration(virDomainObj *vm, + virDomainMigrationIDDef *migration); + +int qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid); -- 2.33.0
From: zhengchuan <zhengchuan@huawei.com> Firstly, we need to get multifd pids, add virDomainMigrateGetMigrationMultiFdPids() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 2 ++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 40 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 1 + src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 33 ++++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 15 ++++++++++++ 10 files changed, 117 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 76744eb0f1..cca35c1be1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6474,4 +6474,6 @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, char *virDomainMigrateGetMigrationPids(virDomainPtr domain, unsigned int flags); +char *virDomainMigrateGetMigrationMultiFdPids(virDomainPtr domain, + unsigned int flags); #endif /* LIBVIRT_DOMAIN_H */ diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 1d7f606c59..cb1d4241af 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -1450,6 +1450,10 @@ typedef char * (*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, unsigned int flags); +typedef char * +(*virDrvDomainMigrateGetMigrationMultiFdPids)(virDomainPtr domain, + unsigned int flags); + typedef struct _virHypervisorDriver virHypervisorDriver; /** @@ -1723,4 +1727,5 @@ struct _virHypervisorDriver { virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; virDrvDomainPinMigrationThread domainPinMigrationThread; virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; + virDrvDomainMigrateGetMigrationMultiFdPids domainMigrateGetMigrationMultiFdPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index d35a8237ae..c36e3ba9d6 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14012,6 +14012,46 @@ virDomainMigrateGetMigrationPids(virDomainPtr domain, return NULL; } + +/* + * virDomainMigrateGetMigrationMultiFdPids: + * @domain: pointer to domain object + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Get multifd thread pid. + * + * Returns the multifd1 and multifd2 pids which must be freed by the caller, or + * NULL if there was an error. + * + * Since: 9.1.0 + */ +char * +virDomainMigrateGetMigrationMultiFdPids(virDomainPtr domain, + unsigned int flags) +{ + virConnectPtr conn; + VIR_DOMAIN_DEBUG(domain, "MultiFd pids flags=0x%x", flags); + virResetLastError(); + virCheckDomainReturn(domain, NULL); + conn = domain->conn; + + virCheckReadOnlyGoto(domain->conn->flags, error); + + if (conn->driver->domainMigrateGetMigrationMultiFdPids) { + char *ret; + ret = conn->driver->domainMigrateGetMigrationMultiFdPids(domain, flags); + if (!ret) + goto error; + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return NULL; +} + /** * virDomainPinMigrationThread: * @domain: a domain object diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index 61561f2367..e31081354f 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -931,6 +931,7 @@ LIBVIRT_9.1.0 { global: virDomainPinMigrationThread; virDomainMigrateGetMigrationPids; + virDomainMigrateGetMigrationMultiFdPids; } LIBVIRT_8.5.0; # .... define new API here using predicted next version number .... diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index de90cf42f9..4ab7771a59 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1830,6 +1830,8 @@ qemuDomainObjPrivateFree(void *data) g_free(priv->origname); g_free(priv->migrationPids); g_free(priv->migrationThreadPinList); + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdCount = 0; virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 5aa4eb5840..36984e2bb1 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -141,6 +141,8 @@ struct _qemuDomainObjPrivate { unsigned short migrationPort; char *migrationPids; char *migrationThreadPinList; + char *migrationMultiFdPids; + unsigned int migrationMultiFdCount; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case it was changed for the current diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 7ae81f459f..92049c5bcd 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20678,6 +20678,38 @@ qemuDomainMigrateGetMigrationPids(virDomainPtr dom, return ret; } + +static char * +qemuDomainMigrateGetMigrationMultiFdPids(virDomainPtr dom, + unsigned int flags) +{ + char *ret = NULL; + virDomainObj *vm = NULL; + qemuDomainObjPrivate *priv = NULL; + + virCheckFlags(0, NULL); + + vm = qemuDomainObjFromDomain(dom); + if (!vm) + goto cleanup; + + if (virDomainMigrateGetMigrationMultiFdPidsEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + priv = vm->privateData; + + if (priv->migrationMultiFdPids) { + ret = g_strdup(priv->migrationMultiFdPids); + if (!ret) + VIR_ERROR(_("failed to strdup MultiFdPids")); + } + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + + static int qemuDomainPinMigrationThread(virDomainPtr dom, unsigned char *cpumap, @@ -21019,6 +21051,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ + .domainMigrateGetMigrationMultiFdPids = qemuDomainMigrateGetMigrationMultiFdPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 1c7cac6b29..f6ec3fb85e 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8653,6 +8653,7 @@ static virHypervisorDriver hypervisor_driver = { .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ + .domainMigrateGetMigrationMultiFdPids = remoteDomainMigrateGetMigrationMultiFdPids, /* 9.1.0 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index f86172b636..e83bc297cd 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -2542,6 +2542,15 @@ struct remote_domain_migrate_get_migration_pids_ret { remote_nonnull_string migrationPids; }; +struct remote_domain_migrate_get_migration_multi_fd_pids_args { + remote_nonnull_domain dom; + unsigned int flags; +}; + +struct remote_domain_migrate_get_migration_multi_fd_pids_ret { + remote_nonnull_string migrationMultiFdPids; +}; + struct remote_domain_migrate_set_max_speed_args { remote_nonnull_domain dom; unsigned hyper bandwidth; @@ -6987,5 +6996,11 @@ enum remote_procedure { * @generate: both * @acl: domain:write */ - REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_MULTI_FD_PIDS = 445 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index 9ef27913f1..a6cedb7682 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -3279,6 +3279,20 @@ struct remote_domain_migrate_get_migration_pids_args { struct remote_domain_migrate_get_migration_pids_ret { remote_nonnull_string migrationPids; }; +struct remote_domain_migrate_get_migration_pids_args { + remote_nonnull_domain dom; + u_int flags; +}; +struct remote_domain_migrate_get_migration_pids_ret { + remote_nonnull_string migrationPids; +}; +struct remote_domain_migrate_get_migration_multi_fd_pids_args { + remote_nonnull_domain dom; + u_int flags; +}; +struct remote_domain_migrate_get_migration_multi_fd_pids_ret { + remote_nonnull_string m igrationMultiFdPids; +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -3724,4 +3738,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_MULTI_FD_PIDS = 445, }; -- 2.33.0
On Tue, Jan 03, 2023 at 09:08:27PM +0800, Jiang Jiacheng wrote:
From: zhengchuan <zhengchuan@huawei.com>
Firstly, we need to get multifd pids, add virDomainMigrateGetMigrationMultiFdPids() for migration pin.
Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 2 ++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 40 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 1 + src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 33 ++++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 15 ++++++++++++ 10 files changed, 117 insertions(+), 1 deletion(-)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 76744eb0f1..cca35c1be1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6474,4 +6474,6 @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, char *virDomainMigrateGetMigrationPids(virDomainPtr domain, unsigned int flags);
+char *virDomainMigrateGetMigrationMultiFdPids(virDomainPtr domain, + unsigned int flags); #endif /* LIBVIRT_DOMAIN_H */
Again, we don't want to be exposing PIDs in our API, With regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
From: zhengchuan <zhengchuan@huawei.com> Secondly, we start to pin multifd pid by given cpumap. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_driver.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 92049c5bcd..d4597489d5 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20725,6 +20725,8 @@ qemuDomainPinMigrationThread(virDomainPtr dom, qemuDomainObjPrivate *priv = NULL; int migration_id = 0; virDomainMigrationIDDef *migration = NULL; + char **multiFdPids = NULL; + unsigned int multiFdIndex = 0; cfg = virQEMUDriverGetConfig(driver); @@ -20781,6 +20783,23 @@ qemuDomainPinMigrationThread(virDomainPtr dom, } } + /* + * set migration multiFd thread affinity + */ + if (priv->migrationMultiFdPids != NULL && pcpumap) { + multiFdPids = g_strsplit(priv->migrationMultiFdPids, "/", + priv->migrationMultiFdCount); + for (multiFdIndex = 0; multiFdIndex < (priv->migrationMultiFdCount); multiFdIndex++) { + if (virStrToLong_i(multiFdPids[multiFdIndex], NULL, 10, &(migration->thread_id)) != 0) { + VIR_ERROR(_("migrationMultiFdPids trans failure, migrationMultiFdPid = %s"), + multiFdPids[multiFdIndex]); + goto endjob; + } + if (virProcessSetAffinity(migration->thread_id, pcpumap, false) < 0) + goto endjob; + } + } + ret = 0; goto endjob; @@ -20797,6 +20816,7 @@ qemuDomainPinMigrationThread(virDomainPtr dom, virDomainObjEndAPI(&vm); virObjectUnref(caps); virObjectUnref(cfg); + g_strfreev(multiFdPids); return ret; } -- 2.33.0
From: zhengchuan <zhengchuan@huawei.com> add qemu monitor callback functions Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_monitor.c | 10 ++++++++++ src/qemu/qemu_monitor.h | 6 ++++++ src/qemu/qemu_monitor_json.c | 16 ++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index 36ec923861..17f00f13fd 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -1390,6 +1390,16 @@ qemuMonitorEmitMigrationPid(qemuMonitor *mon, } +void +qemuMonitorEmitMigrationMultiFdPids(qemuMonitor *mon, + int mpid) +{ + VIR_DEBUG("mon=%p, pass=%d", mon, mpid); + + QEMU_MONITOR_CALLBACK(mon, domainMigrationMultiFdPids, mon->vm, mpid); +} + + void qemuMonitorEmitRdmaGidStatusChanged(qemuMonitor *mon, const char *netdev, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 4eddd79779..f19f94eaf0 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -366,6 +366,10 @@ typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitor *mon, virDomainObj *vm, int mcpid); +typedef int (*qemuMonitorDomainMigrationMultiFdPidsCallback)(qemuMonitor *mon, + virDomainObj *vm, + int mcpid); + typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks; struct _qemuMonitorCallbacks { qemuMonitorEofNotifyCallback eofNotify; @@ -402,6 +406,7 @@ struct _qemuMonitorCallbacks { qemuMonitorDomainMemoryDeviceSizeChange domainMemoryDeviceSizeChange; qemuMonitorDomainDeviceUnplugErrCallback domainDeviceUnplugError; qemuMonitorDomainMigrationPidCallback domainMigrationPid; + qemuMonitorDomainMigrationMultiFdPidsCallback domainMigrationMultiFdPids; }; qemuMonitor *qemuMonitorOpen(virDomainObj *vm, @@ -505,6 +510,7 @@ void qemuMonitorEmitMigrationPass(qemuMonitor *mon, int pass); void qemuMonitorEmitMigrationPid(qemuMonitor *mon, int mpid); +void qemuMonitorEmitMigrationMultiFdPids(qemuMonitor *mon, int mpid); void qemuMonitorEmitAcpiOstInfo(qemuMonitor *mon, const char *alias, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index 3129305107..1fc8c23518 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -85,6 +85,7 @@ static void qemuMonitorJSONHandleMemoryFailure(qemuMonitor *mon, virJSONValue *d static void qemuMonitorJSONHandleMemoryDeviceSizeChange(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleDeviceUnplugErr(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, virJSONValue *data); +static void qemuMonitorJSONHandleMigrationMultiFdPids(qemuMonitor *mon, virJSONValue *data); typedef struct { const char *type; @@ -106,6 +107,7 @@ static qemuEventHandler eventHandlers[] = { { "MEMORY_DEVICE_SIZE_CHANGE", qemuMonitorJSONHandleMemoryDeviceSizeChange, }, { "MEMORY_FAILURE", qemuMonitorJSONHandleMemoryFailure, }, { "MIGRATION", qemuMonitorJSONHandleMigrationStatus, }, + { "MIGRATION_MULTIFD_PID", qemuMonitorJSONHandleMigrationMultiFdPids, }, { "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, }, { "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, }, { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, }, @@ -145,6 +147,20 @@ qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, qemuMonitorEmitMigrationPid(mon, mpid); } +static void +qemuMonitorJSONHandleMigrationMultiFdPids(qemuMonitor *mon, + virJSONValue *data) +{ + int mpid; + + if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) { + VIR_WARN("missing multifd pid in migration-multifd-pid event"); + return; + } + + qemuMonitorEmitMigrationMultiFdPids(mon, mpid); +} + static int qemuMonitorEventCompare(const void *key, const void *elt) { -- 2.33.0
From: zhengchuan <zhengchuan@huawei.com> support migration multifd thread pin by configuration. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_migration.c | 2 ++ src/qemu/qemu_process.c | 61 +++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 4 +++ 3 files changed, 67 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 7e2894eb76..fe7e2a0737 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -3336,6 +3336,8 @@ qemuMigrationDstPrepareFresh(virQEMUDriver *driver, priv = vm->privateData; priv->origname = g_strdup(origname); g_free(priv->migrationPids); + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdCount = 0; if (taint_hook) { /* Domain XML has been altered by a hook script. */ diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 361daee081..1914aaa992 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -1792,6 +1792,7 @@ static qemuMonitorCallbacks monitorCallbacks = { .domainMemoryDeviceSizeChange = qemuProcessHandleMemoryDeviceSizeChange, .domainDeviceUnplugError = qemuProcessHandleDeviceUnplugErr, .domainMigrationPid = qemuProcessHandleMigrationPid, + .domainMigrationMultiFdPids = qemuProcessHandleMigrationMultiFdPids, }; static void @@ -3862,6 +3863,66 @@ qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, return 0; } +int +qemuProcessHandleMigrationMultiFdPids(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid) +{ + qemuDomainObjPrivate *priv = NULL; + char *mpidOldStr = NULL; + char *mpidStr = NULL; + virDomainMigrationIDDef *migration = NULL; + virBitmap *pcpumap = NULL; + virObjectLock(vm); + + VIR_INFO("Migrating domain %p %s, migration-multifd pid %d", + vm, vm->def->name, mpid); + + priv = vm->privateData; + if (vm->job->asyncJob == VIR_ASYNC_JOB_NONE) { + VIR_DEBUG("got MIGRATION_MULTIFD_PID event without a migration job"); + goto cleanup; + } + + migration = g_new0(virDomainMigrationIDDef, 1); + migration->thread_id = mpid; + + if (qemuProcessSetupMigration(vm, migration) < 0) { + VIR_ERROR(_("fail to setup migration multiFd cgroup")); + goto cleanup; + } + + mpidOldStr = priv->migrationMultiFdPids; + if (!mpidOldStr) { + mpidStr = g_strdup_printf("%d", mpid); + } else { + mpidStr = g_strdup_printf("%s/%d", mpidOldStr, mpid); + } + + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdPids = mpidStr; + priv->migrationMultiFdCount++; + + pcpumap = qemuProcessGetPcpumap(priv); + + if (!pcpumap) + goto cleanup; + + qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid); + + cleanup: + /* + * If the value of pcpumap is setted by priv->migrationThreadPinList, + * we need to free pcpumap. + */ + if (pcpumap != priv->pcpumap) + virBitmapFree(pcpumap); + virDomainMigrationIDDefFree(migration); + virObjectUnlock(vm); + + return 0; +} + static int qemuProcessRecoverJob(virQEMUDriver *driver, diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index 12e2cc1f48..d41959bc97 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -253,3 +253,7 @@ int qemuProcessSetupMigration(virDomainObj *vm, int qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, virDomainObj *vm, int mpid); + +int qemuProcessHandleMigrationMultiFdPids(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid); -- 2.33.0
Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_process.c | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 1914aaa992..f5aff9db42 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -1411,6 +1411,61 @@ qemuProcessHandleSpiceMigrated(qemuMonitor *mon G_GNUC_UNUSED, } +static void +qemuProcessHandleMigrationPinStatus(virDomainObj *vm, int status) +{ + qemuDomainObjPrivate *priv = vm->privateData; + + if (vm->job->asyncJob != VIR_ASYNC_JOB_MIGRATION_OUT) + return; + + switch (status) { + case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_SETUP: + case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER: + case QEMU_MONITOR_MIGRATION_STATUS_DEVICE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY_PAUSED: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY_RECOVER: + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: + case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED: + case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG: + break; + case QEMU_MONITOR_MIGRATION_STATUS_ERROR: + /* + * migration thread is still running, + * so we can't delete migration Cgroup. + */ + g_free(priv->migrationPids); + g_free(priv->migrationMultiFdPids); + g_free(priv->migrationThreadPinList); + priv->migrationMultiFdCount = 0; + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + break; + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED: + g_free(priv->migrationPids); + g_free(priv->migrationMultiFdPids); + g_free(priv->migrationThreadPinList); + priv->migrationMultiFdCount = 0; + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + if (virCgroupDelThread(priv->cgroup, + VIR_CGROUP_THREAD_MIGRATION_THREAD, 0) < 0) + VIR_WARN("Failed to delete migration thread Cgroup!"); + VIR_INFO("success to free pcpumap and migrationPids"); + break; + default: + VIR_WARN("got unknown migration status'%s'", + qemuMonitorMigrationStatusTypeToString(status)); + break; + } + + return; +} + + static void qemuProcessHandleMigrationStatus(qemuMonitor *mon G_GNUC_UNUSED, virDomainObj *vm, @@ -1524,6 +1579,7 @@ qemuProcessHandleMigrationStatus(qemuMonitor *mon G_GNUC_UNUSED, default: break; } + qemuProcessHandleMigrationPinStatus(vm, status); cleanup: virObjectUnlock(vm); -- 2.33.0
participants (2)
-
Daniel P. Berrangé -
Jiang Jiacheng