On Tue, Jan 03, 2023 at 09:08:20PM +0800, Jiang Jiacheng wrote: Exposing QEMU thread PIDs in the public libvirt API is not something we should be doing. We've explicitly aimed to avoid exposing the notion of PIDs in our API in general. With regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

On Tue, Jan 10, 2023 at 07:30:08PM +0800, Jiang Jiacheng wrote: iothreadpin doesn't take thread PIDs, it takes integer identifiers associated with the threads. Why does the mgmt app need to know the migration PIDs ? The API for setting pinning in the next patch doesn't accept a PID, and we don't need to set different pinning for each migration thread. Only libvirt QEMU driver code needs to know the PIDs, not the libvirt public API/mgmt app. With regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

From: zhengchuan <zhengchuan(a)huawei.com&gt; Secondly, we start to pin migration pid by given cpumap, add virDomainPinMigrationThread() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 4 ++ src/conf/domain_conf.c | 9 ++++ src/conf/domain_conf.h | 11 ++++ src/conf/virconftypes.h | 2 + src/driver-hypervisor.h | 6 +++ src/libvirt-domain.c | 65 +++++++++++++++++++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 1 + src/qemu/qemu_driver.c | 91 ++++++++++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 ++++- src/remote_protocol-structs | 5 ++ src/util/vircgroup.c | 3 ++ src/util/vircgroup.h | 1 + 14 files changed, 212 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index e11f2795f1..26af8292d3 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -2424,6 +2424,10 @@ int virDomainPinEmulator (virDomainPtr domain, int maplen, unsigned int flags); +int virDomainPinMigrationThread (virDomainPtr domain, + unsigned char *cpumap, + int maplen); + int virDomainGetEmulatorPinInfo (virDomainPtr domain, unsigned char *cpumaps, int maplen, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 6c088ff295..a94a4c1c88 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -30299,3 +30299,12 @@ virDomainDefHasSpiceGraphics(const virDomainDef *def) return false; } + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def) +{ + if (!def) + return; + virBitmapFree(def->cpumask); + VIR_FREE(def); +} diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 1404c55053..a8384a4f6e 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -27,6 +27,7 @@ #include <libxml/xpath.h> #include "internal.h" +#include "viralloc.h" #include "virconftypes.h" #include "capabilities.h" #include "cpu_conf.h" @@ -4353,3 +4354,13 @@ virDomainObjGetMessages(virDomainObj *vm, bool virDomainDefHasSpiceGraphics(const virDomainDef *def); + +struct _virDomainMigrationIDDef { + bool autofill; + int thread_id; + virBitmap *cpumask; + virDomainThreadSchedParam sched; +}; + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def); diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h index 154805091a..984c2181d8 100644 --- a/src/conf/virconftypes.h +++ b/src/conf/virconftypes.h @@ -255,3 +255,5 @@ typedef struct _virDomainXMLOption virDomainXMLOption; typedef struct _virDomainXMLPrivateDataCallbacks virDomainXMLPrivateDataCallbacks; typedef struct _virDomainXenbusControllerOpts virDomainXenbusControllerOpts; + +typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef; diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 618f116012..1d7f606c59 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -1441,6 +1441,11 @@ typedef int int seconds, unsigned int flags); +typedef int +(*virDrvDomainPinMigrationThread)(virDomainPtr domain, + unsigned char *cpumap, + int maplen); + typedef char * (*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, unsigned int flags); @@ -1716,5 +1721,6 @@ struct _virHypervisorDriver { virDrvDomainAuthorizedSSHKeysSet domainAuthorizedSSHKeysSet; virDrvDomainGetMessages domainGetMessages; virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; + virDrvDomainPinMigrationThread domainPinMigrationThread; virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index fabfb2dd7f..d35a8237ae 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14011,3 +14011,68 @@ virDomainMigrateGetMigrationPids(virDomainPtr domain, virDispatchError(domain->conn); return NULL; } + +/** + * virDomainPinMigrationThread: + * @domain: a domain object + * @cpumap: pointer to a bit map of real CPUs (in 8-bit bytes) (IN) + * Each bit set to 1 means that corresponding CPU is usable. + * Bytes are stored in little-endian order: CPU0-7, 8-15... + * In each byte, lowest CPU number is least significant bit. + * @maplen: number of bytes in cpumap, from 1 up to size of CPU map in + * underlying virtualization system (Xen...). + * If maplen < size, missing bytes are set to zero. + * If maplen > size, failure code is returned. + * + * Allocate the real CPUs to the migrationThread which will be launched + * at the beginning of migration. + * This allocation can be handled whether before the migration, or during + * the migration which performs as an instant change. + * + * This function may require privileged access to the hypervisor. + * + * Returns 0 in case of success, -1 in case of failure. + * + * Since: 9.1.0 + */ +int +virDomainPinMigrationThread(virDomainPtr domain, + unsigned char *cpumap, + int maplen) +{ + virConnectPtr conn; + g_autofree char *str = NULL; + + VIR_DOMAIN_DEBUG(domain, "migration: cpumap=%p, maplen=%d", + cpumap, maplen); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckNonNullArgGoto(cpumap, error); + virCheckPositiveArgGoto(maplen, error); + + str = virBitmapDataFormat(cpumap, maplen); + VIR_INFO("Begin to PinMigrationThread(domain=%s): new CPU Affinity:%s", + NULLSTR(domain->name), NULLSTR(str)); + + if (conn->driver->domainPinMigrationThread) { + int ret; + ret = conn->driver->domainPinMigrationThread(domain, cpumap, maplen); + if (ret < 0) { + VIR_ERROR(_("Failed to PinMigrationThread(domain=%s), ret=%d"), + NULLSTR(domain->name), ret); + goto error; + } + VIR_INFO("Success to PinMigrationThread(domain=%s)", NULLSTR(domain->name)); + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index ae746a2d51..ad19d37d95 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -516,6 +516,7 @@ virDomainMemoryModelTypeToString; virDomainMemoryRemove; virDomainMemorySourceTypeFromString; virDomainMemorySourceTypeToString; +virDomainMigrationIDDefFree; virDomainMouseModeTypeFromString; virDomainMouseModeTypeToString; virDomainNetAllocateActualDevice; diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index f11fe1b26b..61561f2367 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -929,6 +929,7 @@ LIBVIRT_8.5.0 { LIBVIRT_9.1.0 { global: + virDomainPinMigrationThread; virDomainMigrateGetMigrationPids; } LIBVIRT_8.5.0; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index b9dc5f29f5..7ae81f459f 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20678,6 +20678,96 @@ qemuDomainMigrateGetMigrationPids(virDomainPtr dom, return ret; } +static int +qemuDomainPinMigrationThread(virDomainPtr dom, + unsigned char *cpumap, + int maplen) +{ + int ret = -1; + virQEMUDriver *driver = dom->conn->privateData; + virQEMUDriverConfig *cfg = NULL; + virCaps *caps = NULL; + virDomainObj *vm = NULL; + virBitmap *pcpumap = NULL; + virCgroup *cgroup_migthread = NULL; + qemuDomainObjPrivate *priv = NULL; + int migration_id = 0; + virDomainMigrationIDDef *migration = NULL; + + cfg = virQEMUDriverGetConfig(driver); + + if (!(caps = virQEMUDriverGetCapabilities(driver, false))) + goto cleanup; + + if (!(vm = qemuDomainObjFromDomain(dom))) + goto cleanup; + + priv = vm->privateData; + + if (virDomainPinMigrationThreadEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0) + goto cleanup; + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto release; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty migration thread cpumap list for pinning")); + goto release; + } + + virBitmapFree(priv->pcpumap); + priv->pcpumap = pcpumap; + + migration = g_new0(virDomainMigrationIDDef, 1); + + if (priv->migrationPids != NULL) { + if (virStrToLong_i(priv->migrationPids, NULL, 10, &(migration->thread_id)) != 0) { + VIR_ERROR(_("migrationPid trans failure, migrationPids = %s"), priv->migrationPids); + goto endjob; + } + + if (pcpumap) { + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, + migration_id, false, &cgroup_migthread) < 0) + goto endjob; + if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for migration thread %d"), migration_id); + goto endjob; + } + } + + if (virProcessSetAffinity(migration->thread_id, pcpumap, false) < 0) + goto endjob; + } + } + + ret = 0; + goto endjob; + + release: + virBitmapFree(pcpumap); + + endjob: + virDomainObjEndJob(vm); + + cleanup: + if (cgroup_migthread) + virCgroupFree(cgroup_migthread); + virDomainMigrationIDDefFree(migration); + virDomainObjEndAPI(&vm); + virObjectUnref(caps); + virObjectUnref(cfg); + return ret; +} + static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -20927,6 +21017,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainGetMessages = qemuDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 9d8d7142b7..1c7cac6b29 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8651,6 +8651,7 @@ static virHypervisorDriver hypervisor_driver = { .domainGetMessages = remoteDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index d1b799db13..f86172b636 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -1291,6 +1291,11 @@ struct remote_domain_pin_iothread_args { unsigned int flags; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; + struct remote_domain_add_iothread_args { remote_nonnull_domain dom; unsigned int iothread_id; @@ -6976,5 +6981,11 @@ enum remote_procedure { * @generate: both * @acl: domain:migrate */ - REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443 + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, + + /** + * @generate: both + * @acl: domain:write + */ + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index da4d799f42..9ef27913f1 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -3268,6 +3268,10 @@ struct remote_domain_event_memory_device_size_change_msg { remote_nonnull_string alias; uint64_t size; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; struct remote_domain_migrate_get_migration_pids_args { remote_nonnull_domain dom; u_int flags; @@ -3719,4 +3723,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441, REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, }; diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c index 73675b4478..d4307c0d72 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -1123,6 +1123,9 @@ virCgroupNewThread(virCgroup *domain, case VIR_CGROUP_THREAD_IOTHREAD: name = g_strdup_printf("iothread%d", id); break; + case VIR_CGROUP_THREAD_MIGRATION_THREAD: + name = g_strdup_printf("migthread%d", id); + break; case VIR_CGROUP_THREAD_LAST: virReportError(VIR_ERR_INTERNAL_ERROR, _("unexpected name value %d"), nameval); diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h index 690f09465c..3c5c23965f 100644 --- a/src/util/vircgroup.h +++ b/src/util/vircgroup.h @@ -53,6 +53,7 @@ typedef enum { VIR_CGROUP_THREAD_VCPU = 0, VIR_CGROUP_THREAD_EMULATOR, VIR_CGROUP_THREAD_IOTHREAD, + VIR_CGROUP_THREAD_MIGRATION_THREAD, VIR_CGROUP_THREAD_LAST } virCgroupThreadName; -- 2.33.0

From: zhengchuan <zhengchuan(a)huawei.com&gt; add qemu monitor callback functions Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_monitor.c | 10 ++++++++++ src/qemu/qemu_monitor.h | 7 +++++++ src/qemu/qemu_monitor_json.c | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index 734364e070..36ec923861 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -1380,6 +1380,16 @@ qemuMonitorEmitPRManagerStatusChanged(qemuMonitor *mon, } +void +qemuMonitorEmitMigrationPid(qemuMonitor *mon, + int mpid) +{ + VIR_DEBUG("mon=%p, pass=%d", mon, mpid); + + QEMU_MONITOR_CALLBACK(mon, domainMigrationPid, mon->vm, mpid); +} + + void qemuMonitorEmitRdmaGidStatusChanged(qemuMonitor *mon, const char *netdev, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 906a919f52..4eddd79779 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -362,6 +362,10 @@ typedef void (*qemuMonitorDomainMemoryDeviceSizeChange)(qemuMonitor *mon, const char *alias, unsigned long long size); +typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitor *mon, + virDomainObj *vm, + int mcpid); + typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks; struct _qemuMonitorCallbacks { qemuMonitorEofNotifyCallback eofNotify; @@ -397,6 +401,7 @@ struct _qemuMonitorCallbacks { qemuMonitorDomainMemoryFailureCallback domainMemoryFailure; qemuMonitorDomainMemoryDeviceSizeChange domainMemoryDeviceSizeChange; qemuMonitorDomainDeviceUnplugErrCallback domainDeviceUnplugError; + qemuMonitorDomainMigrationPidCallback domainMigrationPid; }; qemuMonitor *qemuMonitorOpen(virDomainObj *vm, @@ -499,6 +504,8 @@ void qemuMonitorEmitMigrationStatus(qemuMonitor *mon, void qemuMonitorEmitMigrationPass(qemuMonitor *mon, int pass); +void qemuMonitorEmitMigrationPid(qemuMonitor *mon, int mpid); + void qemuMonitorEmitAcpiOstInfo(qemuMonitor *mon, const char *alias, const char *slotType, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index 9822097bd7..3129305107 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -84,6 +84,7 @@ static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitor *mon, virJSONV static void qemuMonitorJSONHandleMemoryFailure(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleMemoryDeviceSizeChange(qemuMonitor *mon, virJSONValue *data); static void qemuMonitorJSONHandleDeviceUnplugErr(qemuMonitor *mon, virJSONValue *data); +static void qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, virJSONValue *data); typedef struct { const char *type; @@ -106,6 +107,7 @@ static qemuEventHandler eventHandlers[] = { { "MEMORY_FAILURE", qemuMonitorJSONHandleMemoryFailure, }, { "MIGRATION", qemuMonitorJSONHandleMigrationStatus, }, { "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, }, + { "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, }, { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, }, { "PR_MANAGER_STATUS_CHANGED", qemuMonitorJSONHandlePRManagerStatusChanged, }, { "RDMA_GID_STATUS_CHANGED", qemuMonitorJSONHandleRdmaGidStatusChanged, }, @@ -129,6 +131,20 @@ static qemuEventHandler eventHandlers[] = { /* We use bsearch, so keep this list sorted. */ }; +static void +qemuMonitorJSONHandleMigrationPid(qemuMonitor *mon, + virJSONValue *data) +{ + int mpid; + + if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) { + VIR_WARN("missing migration pid in migration-pid event"); + return; + } + + qemuMonitorEmitMigrationPid(mon, mpid); +} + static int qemuMonitorEventCompare(const void *key, const void *elt) { -- 2.33.0

From: zhengchuan <zhengchuan(a)huawei.com&gt; Add qemuProcessGetPcpumap to get cpumap from migration parameters when 'virsh migrationpin' is not called. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_process.c | 79 +++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 4 +++ 2 files changed, 83 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index b6adcf2f2a..b2e9456b98 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3677,6 +3677,85 @@ qemuProcessRecoverMigration(virQEMUDriver *driver, } +unsigned char * +virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu) +{ + int lastcpu; + unsigned char *cpumap = NULL; + virBitmap *map = NULL; + + if (cpulist[0] == 'r') { + map = virBitmapNew(maxcpu); + if (!map) + return NULL; + virBitmapSetAll(map); + } else { + if (virBitmapParse(cpulist, &map, 1024) < 0 || + virBitmapIsAllClear(map)) { + goto cleanup; + } + + lastcpu = virBitmapLastSetBit(map); + if (lastcpu >= maxcpu) + goto cleanup; + } + + if (virBitmapToData(map, &cpumap, cpumaplen) < 0) + VIR_ERROR(_("Bitmap to data failure")); + + cleanup: + virBitmapFree(map); + return cpumap; +} + + +/* + * The value of "virsh migrationpin" is saved to priv->pcpumap + * If priv->pcpumap is NULL, it means migrationpin command is not called, + * otherwise we set the affinity of migration thread by migrationpin + */ +static virBitmap* +qemuProcessGetPcpumap(qemuDomainObjPrivate *priv) +{ + int cpumaplen = 0; + int maxcpu = 0; + virBitmap *pcpumap = NULL; + g_autofree unsigned char *cpumap = NULL; + + if (priv->pcpumap) + return priv->pcpumap; + + if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) { + VIR_ERROR(_("didn't set the migratin thread pin")); + return NULL; + } + + /* judge whether set_migration_pin is default value or not */ + if (STREQ(priv->migrationThreadPinList, "none")) + return NULL; + + maxcpu = virHostCPUGetCount(); + if (maxcpu < 0) { + VIR_ERROR(_("get the cpu count of host failure")); + return NULL; + } + + cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu); + if (!cpumap) { + VIR_ERROR(_("parse migration.pin param failure : migration.pin = %s"), + priv->migrationThreadPinList); + return NULL; + } + + if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) { + VIR_ERROR(_("Bitmap data failure")); + return NULL; + } + + return pcpumap; +} + + static int qemuProcessRecoverJob(virQEMUDriver *driver, virDomainObj *vm, diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index 9a24745f15..5b1e05b1f8 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -242,3 +242,7 @@ bool qemuProcessRebootAllowed(const virDomainDef *def); void qemuProcessCleanupMigrationJob(virQEMUDriver *driver, virDomainObj *vm); + +unsigned char *virParseCPUList(int *cpumaplen, + const char *cpulist, + int maxcpu); -- 2.33.0

From: zhengchuan <zhengchuan(a)huawei.com&gt; Firstly, we need to get multifd pids, add virDomainMigrateGetMigrationMultiFdPids() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 2 ++ src/driver-hypervisor.h | 5 ++++ src/libvirt-domain.c | 40 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 1 + src/qemu/qemu_domain.c | 2 ++ src/qemu/qemu_domain.h | 2 ++ src/qemu/qemu_driver.c | 33 ++++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +++++++++++++- src/remote_protocol-structs | 15 ++++++++++++ 10 files changed, 117 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 76744eb0f1..cca35c1be1 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6474,4 +6474,6 @@ int virDomainStartDirtyRateCalc(virDomainPtr domain, char *virDomainMigrateGetMigrationPids(virDomainPtr domain, unsigned int flags); +char *virDomainMigrateGetMigrationMultiFdPids(virDomainPtr domain, + unsigned int flags); #endif /* LIBVIRT_DOMAIN_H */ diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 1d7f606c59..cb1d4241af 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -1450,6 +1450,10 @@ typedef char * (*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, unsigned int flags); +typedef char * +(*virDrvDomainMigrateGetMigrationMultiFdPids)(virDomainPtr domain, + unsigned int flags); + typedef struct _virHypervisorDriver virHypervisorDriver; /** @@ -1723,4 +1727,5 @@ struct _virHypervisorDriver { virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; virDrvDomainPinMigrationThread domainPinMigrationThread; virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; + virDrvDomainMigrateGetMigrationMultiFdPids domainMigrateGetMigrationMultiFdPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index d35a8237ae..c36e3ba9d6 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14012,6 +14012,46 @@ virDomainMigrateGetMigrationPids(virDomainPtr domain, return NULL; } + +/* + * virDomainMigrateGetMigrationMultiFdPids: + * @domain: pointer to domain object + * @flags: extra flags; not used yet, so callers should always pass 0 + * + * Get multifd thread pid. + * + * Returns the multifd1 and multifd2 pids which must be freed by the caller, or + * NULL if there was an error. + * + * Since: 9.1.0 + */ +char * +virDomainMigrateGetMigrationMultiFdPids(virDomainPtr domain, + unsigned int flags) +{ + virConnectPtr conn; + VIR_DOMAIN_DEBUG(domain, "MultiFd pids flags=0x%x", flags); + virResetLastError(); + virCheckDomainReturn(domain, NULL); + conn = domain->conn; + + virCheckReadOnlyGoto(domain->conn->flags, error); + + if (conn->driver->domainMigrateGetMigrationMultiFdPids) { + char *ret; + ret = conn->driver->domainMigrateGetMigrationMultiFdPids(domain, flags); + if (!ret) + goto error; + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return NULL; +} + /** * virDomainPinMigrationThread: * @domain: a domain object diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index 61561f2367..e31081354f 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -931,6 +931,7 @@ LIBVIRT_9.1.0 { global: virDomainPinMigrationThread; virDomainMigrateGetMigrationPids; + virDomainMigrateGetMigrationMultiFdPids; } LIBVIRT_8.5.0; # .... define new API here using predicted next version number .... diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index de90cf42f9..4ab7771a59 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1830,6 +1830,8 @@ qemuDomainObjPrivateFree(void *data) g_free(priv->origname); g_free(priv->migrationPids); g_free(priv->migrationThreadPinList); + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdCount = 0; virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 5aa4eb5840..36984e2bb1 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -141,6 +141,8 @@ struct _qemuDomainObjPrivate { unsigned short migrationPort; char *migrationPids; char *migrationThreadPinList; + char *migrationMultiFdPids; + unsigned int migrationMultiFdCount; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case it was changed for the current diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 7ae81f459f..92049c5bcd 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20678,6 +20678,38 @@ qemuDomainMigrateGetMigrationPids(virDomainPtr dom, return ret; } + +static char * +qemuDomainMigrateGetMigrationMultiFdPids(virDomainPtr dom, + unsigned int flags) +{ + char *ret = NULL; + virDomainObj *vm = NULL; + qemuDomainObjPrivate *priv = NULL; + + virCheckFlags(0, NULL); + + vm = qemuDomainObjFromDomain(dom); + if (!vm) + goto cleanup; + + if (virDomainMigrateGetMigrationMultiFdPidsEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + priv = vm->privateData; + + if (priv->migrationMultiFdPids) { + ret = g_strdup(priv->migrationMultiFdPids); + if (!ret) + VIR_ERROR(_("failed to strdup MultiFdPids")); + } + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + + static int qemuDomainPinMigrationThread(virDomainPtr dom, unsigned char *cpumap, @@ -21019,6 +21051,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ + .domainMigrateGetMigrationMultiFdPids = qemuDomainMigrateGetMigrationMultiFdPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 1c7cac6b29..f6ec3fb85e 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8653,6 +8653,7 @@ static virHypervisorDriver hypervisor_driver = { .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ + .domainMigrateGetMigrationMultiFdPids = remoteDomainMigrateGetMigrationMultiFdPids, /* 9.1.0 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index f86172b636..e83bc297cd 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -2542,6 +2542,15 @@ struct remote_domain_migrate_get_migration_pids_ret { remote_nonnull_string migrationPids; }; +struct remote_domain_migrate_get_migration_multi_fd_pids_args { + remote_nonnull_domain dom; + unsigned int flags; +}; + +struct remote_domain_migrate_get_migration_multi_fd_pids_ret { + remote_nonnull_string migrationMultiFdPids; +}; + struct remote_domain_migrate_set_max_speed_args { remote_nonnull_domain dom; unsigned hyper bandwidth; @@ -6987,5 +6996,11 @@ enum remote_procedure { * @generate: both * @acl: domain:write */ - REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, + + /** + * @generate: both + * @acl: domain:migrate + */ + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_MULTI_FD_PIDS = 445 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index 9ef27913f1..a6cedb7682 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -3279,6 +3279,20 @@ struct remote_domain_migrate_get_migration_pids_args { struct remote_domain_migrate_get_migration_pids_ret { remote_nonnull_string migrationPids; }; +struct remote_domain_migrate_get_migration_pids_args { + remote_nonnull_domain dom; + u_int flags; +}; +struct remote_domain_migrate_get_migration_pids_ret { + remote_nonnull_string migrationPids; +}; +struct remote_domain_migrate_get_migration_multi_fd_pids_args { + remote_nonnull_domain dom; + u_int flags; +}; +struct remote_domain_migrate_get_migration_multi_fd_pids_ret { + remote_nonnull_string m igrationMultiFdPids; +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -3724,4 +3738,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_MULTI_FD_PIDS = 445, }; -- 2.33.0

From: zhengchuan <zhengchuan(a)huawei.com&gt; Secondly, we start to pin multifd pid by given cpumap. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_driver.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 92049c5bcd..d4597489d5 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20725,6 +20725,8 @@ qemuDomainPinMigrationThread(virDomainPtr dom, qemuDomainObjPrivate *priv = NULL; int migration_id = 0; virDomainMigrationIDDef *migration = NULL; + char **multiFdPids = NULL; + unsigned int multiFdIndex = 0; cfg = virQEMUDriverGetConfig(driver); @@ -20781,6 +20783,23 @@ qemuDomainPinMigrationThread(virDomainPtr dom, } } + /* + * set migration multiFd thread affinity + */ + if (priv->migrationMultiFdPids != NULL && pcpumap) { + multiFdPids = g_strsplit(priv->migrationMultiFdPids, "/", + priv->migrationMultiFdCount); + for (multiFdIndex = 0; multiFdIndex < (priv->migrationMultiFdCount); multiFdIndex++) { + if (virStrToLong_i(multiFdPids[multiFdIndex], NULL, 10, &(migration->thread_id)) != 0) { + VIR_ERROR(_("migrationMultiFdPids trans failure, migrationMultiFdPid = %s"), + multiFdPids[multiFdIndex]); + goto endjob; + } + if (virProcessSetAffinity(migration->thread_id, pcpumap, false) < 0) + goto endjob; + } + } + ret = 0; goto endjob; @@ -20797,6 +20816,7 @@ qemuDomainPinMigrationThread(virDomainPtr dom, virDomainObjEndAPI(&vm); virObjectUnref(caps); virObjectUnref(cfg); + g_strfreev(multiFdPids); return ret; } -- 2.33.0

From: zhengchuan <zhengchuan(a)huawei.com&gt; support migration multifd thread pin by configuration. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- src/qemu/qemu_migration.c | 2 ++ src/qemu/qemu_process.c | 61 +++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 4 +++ 3 files changed, 67 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 7e2894eb76..fe7e2a0737 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -3336,6 +3336,8 @@ qemuMigrationDstPrepareFresh(virQEMUDriver *driver, priv = vm->privateData; priv->origname = g_strdup(origname); g_free(priv->migrationPids); + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdCount = 0; if (taint_hook) { /* Domain XML has been altered by a hook script. */ diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 361daee081..1914aaa992 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -1792,6 +1792,7 @@ static qemuMonitorCallbacks monitorCallbacks = { .domainMemoryDeviceSizeChange = qemuProcessHandleMemoryDeviceSizeChange, .domainDeviceUnplugError = qemuProcessHandleDeviceUnplugErr, .domainMigrationPid = qemuProcessHandleMigrationPid, + .domainMigrationMultiFdPids = qemuProcessHandleMigrationMultiFdPids, }; static void @@ -3862,6 +3863,66 @@ qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, return 0; } +int +qemuProcessHandleMigrationMultiFdPids(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid) +{ + qemuDomainObjPrivate *priv = NULL; + char *mpidOldStr = NULL; + char *mpidStr = NULL; + virDomainMigrationIDDef *migration = NULL; + virBitmap *pcpumap = NULL; + virObjectLock(vm); + + VIR_INFO("Migrating domain %p %s, migration-multifd pid %d", + vm, vm->def->name, mpid); + + priv = vm->privateData; + if (vm->job->asyncJob == VIR_ASYNC_JOB_NONE) { + VIR_DEBUG("got MIGRATION_MULTIFD_PID event without a migration job"); + goto cleanup; + } + + migration = g_new0(virDomainMigrationIDDef, 1); + migration->thread_id = mpid; + + if (qemuProcessSetupMigration(vm, migration) < 0) { + VIR_ERROR(_("fail to setup migration multiFd cgroup")); + goto cleanup; + } + + mpidOldStr = priv->migrationMultiFdPids; + if (!mpidOldStr) { + mpidStr = g_strdup_printf("%d", mpid); + } else { + mpidStr = g_strdup_printf("%s/%d", mpidOldStr, mpid); + } + + g_free(priv->migrationMultiFdPids); + priv->migrationMultiFdPids = mpidStr; + priv->migrationMultiFdCount++; + + pcpumap = qemuProcessGetPcpumap(priv); + + if (!pcpumap) + goto cleanup; + + qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid); + + cleanup: + /* + * If the value of pcpumap is setted by priv->migrationThreadPinList, + * we need to free pcpumap. + */ + if (pcpumap != priv->pcpumap) + virBitmapFree(pcpumap); + virDomainMigrationIDDefFree(migration); + virObjectUnlock(vm); + + return 0; +} + static int qemuProcessRecoverJob(virQEMUDriver *driver, diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index 12e2cc1f48..d41959bc97 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -253,3 +253,7 @@ int qemuProcessSetupMigration(virDomainObj *vm, int qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED, virDomainObj *vm, int mpid); + +int qemuProcessHandleMigrationMultiFdPids(qemuMonitor *mon ATTRIBUTE_UNUSED, + virDomainObj *vm, + int mpid); -- 2.33.0