[PATCH v2 0/8] migration/migrationpin:support migration pin
Support set the CPU affinity of the live migration threads to improve the migration performance in specific cases. By default, the migration thread shares CPU resources with the VM process. With those API, support pin migration thread to expected CPU list to avoid preempting CPU resources of VM process. Use the following methods to implement migration thread pin: - set the cpulist by virsh command 'virsh migrationpin' or migration param 'migration.pin'. These settings affect only the next migration - continuously query migration thread information[1] during the migration process until matches the thread number desired or migration finished unexpectedly - pin migration thread to expected cpulist, we can set/change the cpulist by 'virsh migrationpin' during migration at anytime Note that only support pin main thread of migration(live migration) and parallel send threads now. diff to v1: - add qmp command to query migration thread information - change the method of get qemu's migration thread info, from qemu event to qmp commands. - unified the interface of common migration and parallel migration. ref: [1]https://patchew.org/QEMU/20230203073519.2969147-1-jiangjiacheng@huawei.com/ Jiang Jiacheng (8): migration/migration-pin: Implemente the interface of obtaining migration thread information migration/migration-pin: get migration pid for migration pin migration/migration-pin: Introduce virDomainPinMigrationThread API migration/migration-pin: Implement qemuDomainPinMigrationThread migration/migration-pin: support migration pin by virsh command migration/migration-pin: add migrationpin for migration parameters migration/migration-pin: pin migration thread immediately after get thread info migration/migration-pin/multifd-pin: add migration pin status handle include/libvirt/libvirt-domain.h | 15 +++ src/conf/domain_conf.c | 9 ++ src/conf/domain_conf.h | 10 ++ src/conf/virconftypes.h | 2 + src/driver-hypervisor.h | 6 + src/libvirt-domain.c | 65 ++++++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 4 + src/qemu/qemu_domain.c | 26 ++++ src/qemu/qemu_domain.h | 13 ++ src/qemu/qemu_driver.c | 56 +++++++++ src/qemu/qemu_migration.c | 111 +++++++++++++++++ src/qemu/qemu_migration.h | 33 +++--- src/qemu/qemu_migration_params.c | 40 +++++++ src/qemu/qemu_migration_params.h | 9 ++ src/qemu/qemu_monitor.c | 25 ++++ src/qemu/qemu_monitor.h | 11 ++ src/qemu/qemu_monitor_json.c | 62 ++++++++++ src/qemu/qemu_monitor_json.h | 5 + src/qemu/qemu_process.c | 198 +++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 13 ++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 +- src/remote_protocol-structs | 5 + src/util/vircgroup.c | 3 + src/util/vircgroup.h | 1 + tools/virsh-domain.c | 69 +++++++++++ 27 files changed, 789 insertions(+), 17 deletions(-) -- 2.33.0
First, we need get migration thread info, implement the related interface to obtain migration threadinfo from qemu using qmp command. Implementation of the Interface for Obtaining Thread Information Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_domain.c | 24 ++++++++++++++ src/qemu/qemu_domain.h | 11 +++++++ src/qemu/qemu_monitor.c | 25 +++++++++++++++ src/qemu/qemu_monitor.h | 11 +++++++ src/qemu/qemu_monitor_json.c | 62 ++++++++++++++++++++++++++++++++++++ src/qemu/qemu_monitor_json.h | 5 +++ 6 files changed, 138 insertions(+) diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index aa567f652a..ba8d1055ae 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -797,6 +797,28 @@ qemuDomainSecretInfoDestroy(qemuDomainSecretInfo *secinfo) qemuDomainSecretInfoClear(secinfo, true); } +static void +qemuDomainMigThreadInfoFree(qemuDomainMigThreadInfo *migthread) +{ + if (!migthread) + return; + + g_free(migthread->thread_name); +} + +void +qemuDomainMigThreadsInfoFree(qemuDomainMigThreadInfo **migthreads, + int nmigthreads) +{ + size_t i; + if (!migthreads) + return; + + for (i = 0; i < nmigthreads; i++) { + g_clear_pointer(&migthreads[i], qemuDomainMigThreadInfoFree); + } + g_free(migthreads); +} static virClass *qemuDomainDiskPrivateClass; static void qemuDomainDiskPrivateDispose(void *obj); @@ -1842,6 +1864,8 @@ qemuDomainObjPrivateDataClear(qemuDomainObjPrivate *priv) priv->allowReboot = VIR_TRISTATE_BOOL_ABSENT; g_clear_pointer(&priv->migrationCaps, virBitmapFree); + qemuDomainMigThreadsInfoFree(priv->migThreadsInfo, priv->migThreadCount); + priv->migThreadCount = 0; virHashRemoveAll(priv->blockjobs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 550397ee50..42115a20ef 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -97,6 +97,12 @@ struct _qemuDomainSecretInfo { char *ciphertext; /* encoded/encrypted secret */ }; +typedef struct _qemuDomainMigThreadInfo qemuDomainMigThreadInfo; +struct _qemuDomainMigThreadInfo { + char *thread_name; + int thread_id; +}; + typedef struct _qemuDomainObjPrivate qemuDomainObjPrivate; struct _qemuDomainObjPrivate { virQEMUDriver *driver; @@ -138,6 +144,8 @@ struct _qemuDomainObjPrivate { unsigned long migMaxBandwidth; char *origname; int nbdPort; /* Port used for migration with NBD */ + int migThreadCount; + qemuDomainMigThreadInfo **migThreadsInfo; unsigned short migrationPort; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case @@ -925,6 +933,9 @@ int qemuDomainSecretChardevPrepare(virQEMUDriverConfig *cfg, ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(3) ATTRIBUTE_NONNULL(4); +void qemuDomainMigThreadsInfoFree(qemuDomainMigThreadInfo **migthreads, + int nmigthreads); + void qemuDomainCleanupStorageSourceFD(virStorageSource *src); void qemuDomainStartupCleanup(virDomainObj *vm); diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index 1fa35f03cc..ad35034db2 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -2222,6 +2222,31 @@ qemuMonitorGetMigrationStats(qemuMonitor *mon, return qemuMonitorJSONGetMigrationStats(mon, stats, error); } +void qemuMonitorMigThreadsInfoFree(qemuMonitorMigThreadInfo **migthreads, + int nmigthreads) +{ + size_t i; + + if (!migthreads) + return; + + for (i = 0; i < nmigthreads; i++) { + g_free(migthreads[i]->thread_name); + g_free(migthreads[i]); + } + + g_free(migthreads); +} + +int +qemuMonitorGetMigThreadsInfo(qemuMonitor *mon, + qemuMonitorMigThreadInfo ***migthreads, + int *nmigthreads) +{ + QEMU_CHECK_MONITOR(mon); + + return qemuMonitorJSONGetMigThreadsInfo(mon, migthreads, nmigthreads); +} int qemuMonitorMigrateToFd(qemuMonitor *mon, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 72db0c0838..076e32bdd3 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -823,6 +823,17 @@ int qemuMonitorGetMigrationCapabilities(qemuMonitor *mon, int qemuMonitorSetMigrationCapabilities(qemuMonitor *mon, virJSONValue **caps); +typedef struct _qemuMonitorMigThreadInfo qemuMonitorMigThreadInfo; +struct _qemuMonitorMigThreadInfo { + char *thread_name; + int thread_id; +}; +int qemuMonitorGetMigThreadsInfo(qemuMonitor *mon, + qemuMonitorMigThreadInfo ***migthreads, + int *nmigthreads); +void qemuMonitorMigThreadsInfoFree(qemuMonitorMigThreadInfo **migthreads, + int nmigthreads); + int qemuMonitorGetGICCapabilities(qemuMonitor *mon, virGICCapability **capabilities); diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index ee2ea68acb..4fd381859f 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -3124,6 +3124,68 @@ int qemuMonitorJSONGetMigrationStats(qemuMonitor *mon, return 0; } +int +qemuMonitorJSONGetMigThreadsInfo(qemuMonitor *mon, + qemuMonitorMigThreadInfo ***migthreads, + int *nmigthreads) +{ + int ret = -1; + g_autoptr(virJSONValue) cmd = NULL; + g_autoptr(virJSONValue) reply = NULL; + virJSONValue *data = NULL; + qemuMonitorMigThreadInfo **infolist = NULL; + size_t n = 0; + size_t i; + + *migthreads = NULL; + + if (!(cmd = qemuMonitorJSONMakeCommand("query-migrationthreads", NULL))) + return ret; + + if (qemuMonitorJSONCommand(mon, cmd, &reply) < 0) + goto cleanup; + + if (qemuMonitorJSONCheckReply(cmd, reply, VIR_JSON_TYPE_ARRAY) < 0) + goto cleanup; + + data = virJSONValueObjectGetArray(reply, "return"); + n = virJSONValueArraySize(data); + + infolist = g_new0(qemuMonitorMigThreadInfo *, n + 1); + + for (i = 0; i < n; i++) { + virJSONValue *child = virJSONValueArrayGet(data, i); + const char *tmp; + qemuMonitorMigThreadInfo *info; + + info = g_new0(qemuMonitorMigThreadInfo, 1); + + infolist[i] = info; + + if (!(tmp = virJSONValueObjectGetString(child, "name"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("query-migrationthreads reply data was missing 'name'")); + goto cleanup; + } + info->thread_name = g_strdup(tmp); + + if (virJSONValueObjectGetNumberInt(child, "thread-id", + &info->thread_id) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("query-migrationthreads reply has malformed " + "'thread-id' data")); + goto cleanup; + } + } + + *nmigthreads = n; + *migthreads = g_steal_pointer(&infolist); + ret = 0; + + cleanup: + qemuMonitorMigThreadsInfoFree(infolist, n); + return ret; +} int qemuMonitorJSONMigrate(qemuMonitor *mon, unsigned int flags, diff --git a/src/qemu/qemu_monitor_json.h b/src/qemu/qemu_monitor_json.h index 6f376cf9b7..1a0875bd2c 100644 --- a/src/qemu/qemu_monitor_json.h +++ b/src/qemu/qemu_monitor_json.h @@ -156,6 +156,11 @@ int qemuMonitorJSONSetMigrationCapabilities(qemuMonitor *mon, virJSONValue **caps); +int +qemuMonitorJSONGetMigThreadsInfo(qemuMonitor *mon, + qemuMonitorMigThreadInfo ***migthreads, + int *nmigthreads); + int qemuMonitorJSONGetGICCapabilities(qemuMonitor *mon, virGICCapability **capabilities); -- 2.33.0
Second, query migration thread information during migration process proactively. Migration should start after all thread be created, so querying the migration thread infomation before wating for completion and terminate the query in any abnormal situation. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_migration.c | 81 ++++++++++++++++++++++++++++++++ src/qemu/qemu_migration_params.c | 19 ++++++++ src/qemu/qemu_migration_params.h | 4 ++ 3 files changed, 104 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 2720f0b083..0cea055eec 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2003,6 +2003,81 @@ qemuMigrationAnyCompleted(virDomainObj *vm, return -1; } +/* Returns 0 on get migration thread info success, + * -1 on unexpected failure + */ +static int +qemuMigrationSrcWaitForThreadInfo(virDomainObj *vm, + qemuMigrationParams *migParams, + unsigned long migflags, + virDomainAsyncJob asyncJob, + virConnectPtr dconn, + unsigned int flags) +{ + int ret = -1; + qemuDomainObjPrivate *priv = vm->privateData; + qemuMonitorMigThreadInfo **infolist = NULL; + qemuDomainMigThreadInfo **migthreads = NULL; + int nmigthreads = 0; + int rc; + size_t i; + + /* migrationThreadCount = live_migration + multifd channels */ + priv->migThreadCount = qemuMigrationParamsGetMigThreadCount(migParams, + migflags); + while ((rc = qemuMigrationAnyCompleted(vm, asyncJob, + dconn, flags)) != 1) { + if (rc < 0) + return rc; + + if (qemuDomainObjEnterMonitorAsync(vm, asyncJob) < 0) + return ret; + rc = qemuMonitorGetMigThreadsInfo(priv->mon, &infolist, &nmigthreads); + qemuDomainObjExitMonitor(vm); + if (rc < 0) + goto cleanup; + + if (nmigthreads == priv->migThreadCount) { + break; + } else if (nmigthreads > priv->migThreadCount) { + VIR_WARN("migration threads is more than expected"); + goto cleanup; + } else { + struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000 * 1000ull }; + virObjectUnlock(vm); + nanosleep(&ts, NULL); + virObjectLock(vm); + } + } + + migthreads = g_new0(qemuDomainMigThreadInfo *, nmigthreads + 1); + + for (i = 0; i < nmigthreads; i++) { + qemuDomainMigThreadInfo *migthread; + migthread = g_new0(qemuDomainMigThreadInfo, nmigthreads + 1); + + migthread->thread_name = g_strdup(infolist[i]->thread_name); + migthread->thread_id = infolist[i]->thread_id; + + migthreads[i] = migthread; + } + + priv->migThreadsInfo = migthreads; + ret = 0; + + cleanup: + if (migthreads && (ret < 0)) { + qemuDomainMigThreadsInfoFree(migthreads, priv->migThreadCount); + priv->migThreadCount = 0; + } + + if (infolist) { + qemuMonitorMigThreadsInfoFree(infolist, nmigthreads); + } + + return ret; +} + /* Returns 0 on success, -2 when migration needs to be cancelled, or -1 when * QEMU reports failed migration. @@ -4917,6 +4992,12 @@ qemuMigrationSrcRun(virQEMUDriver *driver, if (flags & VIR_MIGRATE_POSTCOPY) waitFlags |= QEMU_MIGRATION_COMPLETED_POSTCOPY; + rc = qemuMigrationSrcWaitForThreadInfo(vm, migParams, flags, + VIR_ASYNC_JOB_MIGRATION_OUT, + dconn, waitFlags); + if (rc == -1) + goto error; + rc = qemuMigrationSrcWaitForCompletion(vm, VIR_ASYNC_JOB_MIGRATION_OUT, dconn, waitFlags); if (rc == -2) diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index bd09dcfb23..43bed2e618 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -1540,3 +1540,22 @@ qemuMigrationParamsGetTLSHostname(qemuMigrationParams *migParams) return migParams->params[QEMU_MIGRATION_PARAM_TLS_HOSTNAME].value.s; } + + +int +qemuMigrationParamsGetMigThreadCount(qemuMigrationParams *migParams, + unsigned long migflags) +{ + int nmigthreads = 1; /* live_migtion */ + + if (!(migflags & VIR_MIGRATE_PARALLEL)) { + return nmigthreads; + } else { + if (migParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set) + nmigthreads += migParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i; + else + nmigthreads += 2; /* default multifd channel is 2 */ + } + + return nmigthreads; +} diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index e7c65f6a21..975b6691d4 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -166,3 +166,7 @@ qemuMigrationCapsGet(virDomainObj *vm, const char * qemuMigrationParamsGetTLSHostname(qemuMigrationParams *migParams); + +int +qemuMigrationParamsGetMigThreadCount(qemuMigrationParams *migParams, + unsigned long migflags); -- 2.33.0
Then, pin migration thread with given cpumap. Introduce virDomainPinMigrationThread API, and it will be implement in next patch. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- include/libvirt/libvirt-domain.h | 5 +++ src/driver-hypervisor.h | 6 +++ src/libvirt-domain.c | 65 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 4 ++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 ++++++- src/remote_protocol-structs | 5 +++ 7 files changed, 98 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 5152ed4551..b6c01016d8 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -2429,6 +2429,11 @@ int virDomainGetEmulatorPinInfo (virDomainPtr domain, int maplen, unsigned int flags); +int virDomainPinMigrationThread (virDomainPtr domain, + unsigned char *cpumap, + int maplen); + + /** * virDomainIOThreadInfo: * diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 5219344b72..a8413b1a6c 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -1441,6 +1441,11 @@ typedef int int seconds, unsigned int flags); +typedef int +(*virDrvDomainPinMigrationThread)(virDomainPtr domain, + unsigned char *cpumap, + int maplen); + typedef int (*virDrvDomainFDAssociate)(virDomainPtr domain, const char *name, @@ -1720,4 +1725,5 @@ struct _virHypervisorDriver { virDrvDomainGetMessages domainGetMessages; virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; virDrvDomainFDAssociate domainFDAssociate; + virDrvDomainPinMigrationThread domainPinMigrationThread; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 379057d9a7..a18b7753c2 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14055,3 +14055,68 @@ virDomainFDAssociate(virDomainPtr domain, virDispatchError(conn); return -1; } + +/** + * virDomainPinMigrationThread: + * @domain: a domain object + * @cpumap: pointer to a bit map of real CPUs (in 8-bit bytes) (IN) + * Each bit set to 1 means that corresponding CPU is usable. + * Bytes are stored in little-endian order: CPU0-7, 8-15... + * In each byte, lowest CPU number is least significant bit. + * @maplen: number of bytes in cpumap, from 1 up to size of CPU map in + * underlying virtualization system (Xen...). + * If maplen < size, missing bytes are set to zero. + * If maplen > size, failure code is returned. + * + * Allocate the real CPUs to the migrationThread which will be launched + * at the beginning of migration. + * This allocation can be handled whether before the migration, or during + * the migration which performs as an instant change. + * + * This function may require privileged access to the hypervisor. + * + * Returns 0 in case of success, -1 in case of failure. + * + * Since: 9.1.0 + */ +int +virDomainPinMigrationThread(virDomainPtr domain, + unsigned char *cpumap, + int maplen) +{ + virConnectPtr conn; + g_autofree char *str = NULL; + + VIR_DOMAIN_DEBUG(domain, "migration: cpumap=%p, maplen=%d", + cpumap, maplen); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckNonNullArgGoto(cpumap, error); + virCheckPositiveArgGoto(maplen, error); + + str = virBitmapDataFormat(cpumap, maplen); + VIR_INFO("Begin to PinMigrationThread(domain=%s): new CPU Affinity:%s", + NULLSTR(domain->name), NULLSTR(str)); + + if (conn->driver->domainPinMigrationThread) { + int ret; + ret = conn->driver->domainPinMigrationThread(domain, cpumap, maplen); + if (ret < 0) { + VIR_ERROR(_("Failed to PinMigrationThread(domain=%s), ret=%d"), + NULLSTR(domain->name), ret); + goto error; + } + VIR_INFO("Success to PinMigrationThread(domain=%s)", NULLSTR(domain->name)); + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index 80742f268e..a5223786ed 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -932,4 +932,8 @@ LIBVIRT_9.0.0 { virDomainFDAssociate; } LIBVIRT_8.5.0; +LIBVIRT_9.1.0 { + global: + virDomainPinMigrationThread; +} LIBVIRT_9.0.0; # .... define new API here using predicted next version number .... diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index a4c60be3d7..54c5099731 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8126,6 +8126,7 @@ static virHypervisorDriver hypervisor_driver = { .domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ .domainFDAssociate = remoteDomainFDAssociate, /* 9.0.0 */ + .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 5d86a51116..358e44bd14 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -3935,6 +3935,11 @@ struct remote_domain_fd_associate_args { remote_nonnull_string name; unsigned int flags; }; + +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; /*----- Protocol. -----*/ /* Define the program number, protocol version and procedure numbers here. */ @@ -6974,5 +6979,11 @@ enum remote_procedure { * @generate: none * @acl: domain:write */ - REMOTE_PROC_DOMAIN_FD_ASSOCIATE = 443 + REMOTE_PROC_DOMAIN_FD_ASSOCIATE = 443, + + /** + * @generate: both + * @acl: domain:write + */ + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index 3c6c230a16..a355f3a76b 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -3273,6 +3273,10 @@ struct remote_domain_fd_associate_args { remote_nonnull_string name; u_int flags; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; enum remote_procedure { REMOTE_PROC_CONNECT_OPEN = 1, REMOTE_PROC_CONNECT_CLOSE = 2, @@ -3717,4 +3721,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441, REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_FD_ASSOCIATE = 443, + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, }; -- 2.33.0
Add cgroup for migation thread and implement qemuDomainPinMigrationThread to pin migraiton thread to given cpumap. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/conf/domain_conf.c | 9 ++++++ src/conf/domain_conf.h | 10 +++++++ src/conf/virconftypes.h | 2 ++ src/libvirt_private.syms | 1 + src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_driver.c | 56 ++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.c | 61 ++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 10 +++++++ src/util/vircgroup.c | 3 ++ src/util/vircgroup.h | 1 + 11 files changed, 155 insertions(+) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index f7f9ec3c0a..68200dbbab 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -30883,3 +30883,12 @@ virDomainWatchdogDefFind(const virDomainDef *def, return -1; } + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def) +{ + if (!def) + return; + virBitmapFree(def->cpumask); + VIR_FREE(def); +} diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 21ea4a48b3..fff846f26f 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -4469,3 +4469,13 @@ virDomainObjGetMessages(virDomainObj *vm, bool virDomainDefHasSpiceGraphics(const virDomainDef *def); + +struct _virDomainMigrationIDDef { + bool autofill; + int thread_id; + virBitmap *cpumask; + virDomainThreadSchedParam sched; +}; + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def); diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h index e07f967814..84a28a08b8 100644 --- a/src/conf/virconftypes.h +++ b/src/conf/virconftypes.h @@ -261,3 +261,5 @@ typedef struct _virDomainXMLOption virDomainXMLOption; typedef struct _virDomainXMLPrivateDataCallbacks virDomainXMLPrivateDataCallbacks; typedef struct _virDomainXenbusControllerOpts virDomainXenbusControllerOpts; + +typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef; diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index d5b1b9cb72..b5d9aa62ae 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -525,6 +525,7 @@ virDomainMemoryModelTypeToString; virDomainMemoryRemove; virDomainMemorySourceTypeFromString; virDomainMemorySourceTypeToString; +virDomainMigrationIDDefFree; virDomainMouseModeTypeFromString; virDomainMouseModeTypeToString; virDomainNetAllocateActualDevice; diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index ba8d1055ae..3d497dd5d0 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1853,6 +1853,7 @@ qemuDomainObjPrivateDataClear(qemuDomainObjPrivate *priv) /* remove automatic pinning data */ g_clear_pointer(&priv->autoNodeset, virBitmapFree); g_clear_pointer(&priv->autoCpuset, virBitmapFree); + g_clear_pointer(&priv->pcpumap, virBitmapFree); g_clear_pointer(&priv->pciaddrs, virDomainPCIAddressSetFree); g_clear_pointer(&priv->usbaddrs, virDomainUSBAddressSetFree); g_clear_pointer(&priv->origCPU, virCPUDefFree); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 42115a20ef..af986b5423 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -171,6 +171,7 @@ struct _qemuDomainObjPrivate { /* Bitmaps below hold data from the auto NUMA feature */ virBitmap *autoNodeset; virBitmap *autoCpuset; + virBitmap *pcpumap; bool signalIOError; /* true if the domain condition should be signalled on I/O error */ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index c546e35953..2b9a952c85 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20581,6 +20581,61 @@ qemuDomainFDAssociate(virDomainPtr domain, return ret; } +static int +qemuDomainPinMigrationThread(virDomainPtr dom, + unsigned char *cpumap, + int maplen) +{ + int ret = -1; + virQEMUDriver *driver = dom->conn->privateData; + g_autoptr(virQEMUDriverConfig) cfg = NULL; + virDomainObj *vm = NULL; + g_autoptr(virBitmap) pcpumap = NULL; + qemuDomainObjPrivate *priv = NULL; + qemuDomainMigThreadInfo **migthreads = NULL; + size_t i; + + cfg = virQEMUDriverGetConfig(driver); + + if (!(vm = qemuDomainObjFromDomain(dom))) + goto cleanup; + + priv = vm->privateData; + + if (virDomainPinMigrationThreadEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0) + goto cleanup; + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto endjob; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty migration thread cpumap list for pinning")); + goto endjob; + } + + virBitmapFree(priv->pcpumap); + priv->pcpumap = virBitmapNewCopy(pcpumap); + migthreads = priv->migThreadsInfo; + + if (migthreads && pcpumap) { + for (i = 0; i < priv->migThreadCount; i++) { + qemuProcessSetMigThreadAffinity(priv, vm, migthreads[i]->thread_id, pcpumap); + } + } + + ret = 0; + + endjob: + virDomainObjEndJob(vm); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -20831,6 +20886,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ .domainFDAssociate = qemuDomainFDAssociate, /* 9.0.0 */ + .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ }; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 9a612ca443..946aeb81b1 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -9547,3 +9547,64 @@ qemuProcessQMPStart(qemuProcessQMP *proc) return 0; } + +int +qemuProcessSetupMigration(virDomainObj *vm, + virDomainMigrationIDDef *migration) +{ + return qemuProcessSetupPid(vm, migration->thread_id, + VIR_CGROUP_THREAD_MIGRATION_THREAD, + 0, + vm->def->cputune.emulatorpin, + vm->def->cputune.emulator_period, + vm->def->cputune.emulator_quota, + &migration->sched); +} + +/* + * In order to set migration thread affinity when vm is migrating, + * we should create the cgroup for migration thread. + */ +void +qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv, + virDomainObj *vm, + int mpid, + virBitmap *pcpumap) +{ + virDomainMigrationIDDef *migration = NULL; + int migration_id = 0; + virCgroup *cgroup_migthread = NULL; + + if (!pcpumap) + return; + + migration = g_new0(virDomainMigrationIDDef, 1); + migration->thread_id = mpid; + if (qemuProcessSetupMigration(vm, migration) < 0) { + VIR_ERROR(_("fail to setup migration cgroup")); + goto cleanup; + } + + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, + migration_id, false, &cgroup_migthread) < 0) + goto cleanup; + + if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for migration%d thread"), migration_id); + goto cleanup; + } + } + + if (virProcessSetAffinity(mpid, pcpumap, false) < 0) + VIR_WARN("failed to set affinity in migration"); + + cleanup: + if (cgroup_migthread) + virCgroupFree(cgroup_migthread); + virDomainMigrationIDDefFree(migration); + return; +} diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index b171f0464c..2af1e2ee87 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -235,3 +235,13 @@ bool qemuProcessRebootAllowed(const virDomainDef *def); void qemuProcessCleanupMigrationJob(virQEMUDriver *driver, virDomainObj *vm); + +int +qemuProcessSetupMigration(virDomainObj *vm, + virDomainMigrationIDDef *migration); + +void +qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv, + virDomainObj *vm, + int mpid, + virBitmap *pcpumap); diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c index 73675b4478..d4307c0d72 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -1123,6 +1123,9 @@ virCgroupNewThread(virCgroup *domain, case VIR_CGROUP_THREAD_IOTHREAD: name = g_strdup_printf("iothread%d", id); break; + case VIR_CGROUP_THREAD_MIGRATION_THREAD: + name = g_strdup_printf("migthread%d", id); + break; case VIR_CGROUP_THREAD_LAST: virReportError(VIR_ERR_INTERNAL_ERROR, _("unexpected name value %d"), nameval); diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h index adf3850b22..c5f6ddd7a6 100644 --- a/src/util/vircgroup.h +++ b/src/util/vircgroup.h @@ -53,6 +53,7 @@ typedef enum { VIR_CGROUP_THREAD_VCPU = 0, VIR_CGROUP_THREAD_EMULATOR, VIR_CGROUP_THREAD_IOTHREAD, + VIR_CGROUP_THREAD_MIGRATION_THREAD, VIR_CGROUP_THREAD_LAST } virCgroupThreadName; -- 2.33.0
Add virsh command 'virsh migrationpin', its usage is like 'virsh migrationpin [vm] [cpulist]'. After using this command, if migration thread infomation have been got, pin it to given cpumap immediately, else wait for next migraion operation and pin migration thread after getting information. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- tools/virsh-domain.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index a4abcb1b03..43ecd651d8 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -13814,6 +13814,69 @@ cmdDomDirtyRateCalc(vshControl *ctl, const vshCmd *cmd) } +/* + * "migrationpin" command + */ +static const vshCmdInfo info_migrationpin[] = { + {.name = "help", + .data = N_("control domain migrationThreads affinity") + }, + {.name = "desc", + .data = N_("Pin domain migrationThreads to host physical CPUs.") + }, + {.name = NULL} +}; + +static const vshCmdOptDef opts_migrationpin[] = { + VIRSH_COMMON_OPT_DOMAIN_FULL(0), + {.name = "cpulist", + .type = VSH_OT_DATA, + .flags = VSH_OFLAG_REQ, + .help = N_("host cpu number(s) to set") + }, + {.name = NULL} +}; + +static bool +cmdMigrationPin(vshControl *ctl, const vshCmd *cmd) +{ + g_autoptr(virshDomain) dom = NULL; + const char *name = NULL; + const char *cpulist = NULL; + int maxcpu; + g_autofree unsigned char *cpumap = NULL; + int cpumaplen; + virshControl *priv = ctl->privData; + + dom = virshCommandOptDomain(ctl, cmd, NULL); + if (!dom) + return false; + + if (vshCommandOptStringReq(ctl, cmd, "domain", &name) < 0) + return false; + + if (vshCommandOptStringReq(ctl, cmd, "cpulist", &cpulist) < 0) + return false; + + maxcpu = virshNodeGetCPUCount(priv->conn); + if (maxcpu < 0) + return false; + + cpumap = virshParseCPUList(ctl, &cpumaplen, cpulist, maxcpu); + if (!cpumap) + return false; + + if (virDomainPinMigrationThread(dom, cpumap, + cpumaplen) != 0) + return false; + + vshPrintExtra(ctl, _("Pin domain %s's migration thread " + "to cpulist %s success.\n"), name, cpulist); + + return true; +} + + const vshCmdDef domManagementCmds[] = { {.name = "attach-device", .handler = cmdAttachDevice, @@ -14477,5 +14540,11 @@ const vshCmdDef domManagementCmds[] = { .info = info_dom_fd_associate, .flags = 0 }, + {.name = "migrationpin", + .handler = cmdMigrationPin, + .opts = opts_migrationpin, + .info = info_migrationpin, + .flags = 0 + }, {.name = NULL} }; -- 2.33.0
Add 'migrationpin' to migration parameters to provide another method to set the cpulist required for migration thread pin. 'migrationpin' can only be set before migration and won't effect if 'virsh migrationpin' is set. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- include/libvirt/libvirt-domain.h | 10 ++++++++++ src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_migration.c | 2 ++ src/qemu/qemu_migration.h | 33 ++++++++++++++++---------------- src/qemu/qemu_migration_params.c | 21 ++++++++++++++++++++ src/qemu/qemu_migration_params.h | 5 +++++ 7 files changed, 57 insertions(+), 16 deletions(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index b6c01016d8..5a203d97dc 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -1367,6 +1367,16 @@ typedef enum { */ # define VIR_MIGRATE_PARAM_TLS_DESTINATION "tls.destination" +/** + * VIR_MIGRATE_PARAM_MIGRATIONPIN: + * + * virDomainMigrate* params field: the pin of migration threads for + * migration as VIR_TYPED_PARAM_STRING. + * + * Since: 9.1.0 + */ +# define VIR_MIGRATE_PARAM_MIGRATIONPIN "migration.pin" + /* Domain migration. */ virDomainPtr virDomainMigrate (virDomainPtr domain, virConnectPtr dconn, unsigned long flags, const char *dname, diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 3d497dd5d0..04e89bae95 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1903,6 +1903,7 @@ qemuDomainObjPrivateFree(void *data) virObjectUnref(priv->monConfig); g_free(priv->lockState); g_free(priv->origname); + g_free(priv->migrationThreadPinList); virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index af986b5423..f52f2c9957 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -146,6 +146,7 @@ struct _qemuDomainObjPrivate { int nbdPort; /* Port used for migration with NBD */ int migThreadCount; qemuDomainMigThreadInfo **migThreadsInfo; + char *migrationThreadPinList; unsigned short migrationPort; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 0cea055eec..1fb091255f 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -4891,6 +4891,8 @@ qemuMigrationSrcRun(virQEMUDriver *driver, priv->migMaxBandwidth * 1024 * 1024) < 0) goto error; + qemuMigrationMigrationParamsToVM(migParams, vm); + if (qemuMigrationParamsApply(vm, VIR_ASYNC_JOB_MIGRATION_OUT, migParams, flags) < 0) goto error; diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index d21b6f67e8..f416462c9d 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -66,28 +66,29 @@ /* All supported migration parameters and their types. */ #define QEMU_MIGRATION_PARAMETERS \ - VIR_MIGRATE_PARAM_URI, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_DEST_NAME, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_DEST_XML, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_BANDWIDTH, VIR_TYPED_PARAM_ULLONG, \ - VIR_MIGRATE_PARAM_GRAPHICS_URI, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_LISTEN_ADDRESS, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_MIGRATE_DISKS, VIR_TYPED_PARAM_STRING | \ - VIR_TYPED_PARAM_MULTIPLE, \ - VIR_MIGRATE_PARAM_DISKS_PORT, VIR_TYPED_PARAM_INT, \ - VIR_MIGRATE_PARAM_COMPRESSION, VIR_TYPED_PARAM_STRING | \ - VIR_TYPED_PARAM_MULTIPLE, \ + VIR_MIGRATE_PARAM_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_DEST_NAME, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_DEST_XML, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_BANDWIDTH, VIR_TYPED_PARAM_ULLONG, \ + VIR_MIGRATE_PARAM_GRAPHICS_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_LISTEN_ADDRESS, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_MIGRATE_DISKS, VIR_TYPED_PARAM_STRING | \ + VIR_TYPED_PARAM_MULTIPLE, \ + VIR_MIGRATE_PARAM_DISKS_PORT, VIR_TYPED_PARAM_INT, \ + VIR_MIGRATE_PARAM_COMPRESSION, VIR_TYPED_PARAM_STRING | \ + VIR_TYPED_PARAM_MULTIPLE, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_LEVEL, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_THREADS, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_MT_DTHREADS, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_COMPRESSION_XBZRLE_CACHE, VIR_TYPED_PARAM_ULLONG, \ - VIR_MIGRATE_PARAM_PERSIST_XML, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_PERSIST_XML, VIR_TYPED_PARAM_STRING, \ VIR_MIGRATE_PARAM_AUTO_CONVERGE_INITIAL, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_AUTO_CONVERGE_INCREMENT, VIR_TYPED_PARAM_INT, \ - VIR_MIGRATE_PARAM_BANDWIDTH_POSTCOPY, VIR_TYPED_PARAM_ULLONG, \ - VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, VIR_TYPED_PARAM_INT, \ - VIR_MIGRATE_PARAM_TLS_DESTINATION, VIR_TYPED_PARAM_STRING, \ - VIR_MIGRATE_PARAM_DISKS_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_BANDWIDTH_POSTCOPY, VIR_TYPED_PARAM_ULLONG, \ + VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, VIR_TYPED_PARAM_INT, \ + VIR_MIGRATE_PARAM_TLS_DESTINATION, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_DISKS_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_MIGRATIONPIN, VIR_TYPED_PARAM_STRING, \ NULL diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 43bed2e618..7852f3e295 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -114,6 +114,7 @@ VIR_ENUM_IMPL(qemuMigrationParam, "xbzrle-cache-size", "max-postcopy-bandwidth", "multifd-channels", + "migrationpin", ); typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem; @@ -228,6 +229,10 @@ static const qemuMigrationParamsTPMapItem qemuMigrationParamsTPMap[] = { {.typedParam = VIR_MIGRATE_PARAM_TLS_DESTINATION, .param = QEMU_MIGRATION_PARAM_TLS_HOSTNAME, .party = QEMU_MIGRATION_SOURCE}, + + {.typedParam = VIR_MIGRATE_PARAM_MIGRATIONPIN, + .param = QEMU_MIGRATION_PARAM_MIGRATIONPIN, + .party = QEMU_MIGRATION_SOURCE}, }; static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = { @@ -271,6 +276,9 @@ static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = { [QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS] = { .type = QEMU_MIGRATION_PARAM_TYPE_INT, }, + [QEMU_MIGRATION_PARAM_MIGRATIONPIN] = { + .type = QEMU_MIGRATION_PARAM_TYPE_STRING, + }, }; G_STATIC_ASSERT(G_N_ELEMENTS(qemuMigrationParamInfo) == QEMU_MIGRATION_PARAM_LAST); @@ -798,6 +806,10 @@ qemuMigrationParamsToJSON(qemuMigrationParams *migParams, if (!pv->set) continue; + if (i == QEMU_MIGRATION_PARAM_MIGRATIONPIN) { + continue; + } + if (postcopyResume && !qemuMigrationParamInfo[i].applyOnPostcopyResume) continue; @@ -1559,3 +1571,12 @@ qemuMigrationParamsGetMigThreadCount(qemuMigrationParams *migParams, return nmigthreads; } + +void +qemuMigrationMigrationParamsToVM(const qemuMigrationParams *migParams, const virDomainObj *vm) +{ + if (migParams && migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].set) { + qemuDomainObjPrivate *priv = vm->privateData; + priv->migrationThreadPinList = g_strdup(migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].value.s); + } +} diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 975b6691d4..d8321cace8 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -59,6 +59,7 @@ typedef enum { QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE, QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH, QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS, + QEMU_MIGRATION_PARAM_MIGRATIONPIN, QEMU_MIGRATION_PARAM_LAST } qemuMigrationParam; @@ -170,3 +171,7 @@ qemuMigrationParamsGetTLSHostname(qemuMigrationParams *migParams); int qemuMigrationParamsGetMigThreadCount(qemuMigrationParams *migParams, unsigned long migflags); + +void +qemuMigrationMigrationParamsToVM(const qemuMigrationParams *migParams, + const virDomainObj *vm); -- 2.33.0
If pcpumap is set before migration, pin migration thread to expected cpulist immediately after get thread info during migration. Add 'qemuProcessGetPcpumap' to get pcpumap, try get the result of 'virsh migratepin' first, and if it's not set, try to get the pcpumap from migration parameter. Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_migration.c | 28 ++++++++++++++ src/qemu/qemu_process.c | 81 +++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_process.h | 3 ++ 3 files changed, 112 insertions(+) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 1fb091255f..30f41e8d25 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2003,6 +2003,23 @@ qemuMigrationAnyCompleted(virDomainObj *vm, return -1; } + +static void +qemuMigrationSrcThreadPin(virDomainObj *vm, + virBitmap *pcpumap, + int nmigthreads) +{ + qemuDomainObjPrivate *priv = vm->privateData; + qemuDomainMigThreadInfo **migthreads = priv->migThreadsInfo; + size_t i; + + + for (i = 0; i < nmigthreads; i++) { + qemuProcessSetMigThreadAffinity(priv, vm, migthreads[i]->thread_id, pcpumap); + } +} + + /* Returns 0 on get migration thread info success, * -1 on unexpected failure */ @@ -2018,6 +2035,7 @@ qemuMigrationSrcWaitForThreadInfo(virDomainObj *vm, qemuDomainObjPrivate *priv = vm->privateData; qemuMonitorMigThreadInfo **infolist = NULL; qemuDomainMigThreadInfo **migthreads = NULL; + virBitmap *pcpumap = NULL; int nmigthreads = 0; int rc; size_t i; @@ -2065,7 +2083,17 @@ qemuMigrationSrcWaitForThreadInfo(virDomainObj *vm, priv->migThreadsInfo = migthreads; ret = 0; + /* Try get pcpumap for migration thread pin, + * but thread pin failed shouldn't effect migration */ + if (!(pcpumap = qemuProcessGetPcpumap(priv))) + goto cleanup; + + qemuMigrationSrcThreadPin(vm, pcpumap, nmigthreads); + cleanup: + if (pcpumap != priv->pcpumap) + virBitmapFree(pcpumap); + if (migthreads && (ret < 0)) { qemuDomainMigThreadsInfoFree(migthreads, priv->migThreadCount); priv->migThreadCount = 0; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 946aeb81b1..e787db8b24 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -9608,3 +9608,84 @@ qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv, virDomainMigrationIDDefFree(migration); return; } + + +static unsigned char * +virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu) +{ + int lastcpu; + unsigned char *cpumap = NULL; + virBitmap *map = NULL; + + if (cpulist[0] == 'r') { + map = virBitmapNew(maxcpu); + if (!map) + return NULL; + virBitmapSetAll(map); + } else { + if (virBitmapParse(cpulist, &map, 1024) < 0 || + virBitmapIsAllClear(map)) { + goto cleanup; + } + + lastcpu = virBitmapLastSetBit(map); + if (lastcpu >= maxcpu) + goto cleanup; + } + + if (virBitmapToData(map, &cpumap, cpumaplen) < 0) + VIR_ERROR(_("Bitmap to data failure")); + + cleanup: + virBitmapFree(map); + return cpumap; +} + + +/* + * The value of "virsh migrationpin" is saved to priv->pcpumap + * If priv->pcpumap is NULL, it means migrationpin command is not called, + * otherwise we set the affinity of migration thread by migrationpin + */ +virBitmap* +qemuProcessGetPcpumap(qemuDomainObjPrivate *priv) +{ + int cpumaplen = 0; + int maxcpu = 0; + virBitmap *pcpumap = NULL; + g_autofree unsigned char *cpumap = NULL; + + /* priv->pcpumap = NULL means migrationpin is not set, try to get pcpumap + * from migration parameter. */ + if (priv->pcpumap) + return priv->pcpumap; + + if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) { + VIR_ERROR(_("didn't set the migratin thread pin")); + return NULL; + } + + /* judge whether set_migration_pin is default value or not */ + if (STREQ(priv->migrationThreadPinList, "none")) + return NULL; + + maxcpu = virHostCPUGetCount(); + if (maxcpu < 0) { + VIR_ERROR(_("get the cpu count of host failure")); + return NULL; + } + + cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu); + if (!cpumap) { + VIR_ERROR(_("parse migration.pin param failure : migration.pin = %s"), + priv->migrationThreadPinList); + return NULL; + } + + if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) { + VIR_ERROR(_("Bitmap data failure")); + return NULL; + } + + return pcpumap; +} diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index 2af1e2ee87..74e96ccb07 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -245,3 +245,6 @@ qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv, virDomainObj *vm, int mpid, virBitmap *pcpumap); + +virBitmap* +qemuProcessGetPcpumap(qemuDomainObjPrivate *priv); -- 2.33.0
Signed-off-by: zhengchuan<zhengchuan@huawei.com> Signed-off-by: Jiang Jiacheng <jiangjiacheng@huawei.com> --- src/qemu/qemu_process.c | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index e787db8b24..85a22c7e51 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -1438,6 +1438,61 @@ qemuProcessHandleSpiceMigrated(qemuMonitor *mon G_GNUC_UNUSED, } +static void +qemuProcessHandleMigrationPinStatus(virDomainObj *vm, int status) +{ + qemuDomainObjPrivate *priv = vm->privateData; + + if (vm->job->asyncJob != VIR_ASYNC_JOB_MIGRATION_OUT) + return; + + switch (status) { + case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_SETUP: + case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: + case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER: + case QEMU_MONITOR_MIGRATION_STATUS_DEVICE: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY_PAUSED: + case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY_RECOVER: + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: + case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED: + case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG: + break; + case QEMU_MONITOR_MIGRATION_STATUS_ERROR: + /* + * migration thread is still running, + * so we can't delete migration Cgroup. + */ + qemuDomainMigThreadsInfoFree(priv->migThreadsInfo, priv->migThreadCount); + priv->migThreadsInfo = NULL; + priv->migThreadCount = 0; + g_free(priv->migrationThreadPinList); + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + break; + case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED: + qemuDomainMigThreadsInfoFree(priv->migThreadsInfo, priv->migThreadCount); + priv->migThreadsInfo = NULL; + priv->migThreadCount = 0; + g_free(priv->migrationThreadPinList); + virBitmapFree(priv->pcpumap); + priv->pcpumap = NULL; + if (virCgroupDelThread(priv->cgroup, + VIR_CGROUP_THREAD_MIGRATION_THREAD, 0) < 0) + VIR_WARN("Failed to delete migration thread Cgroup!"); + VIR_INFO("success to free pcpumap and migrationPids"); + break; + default: + VIR_WARN("got unknown migration status'%s'", + qemuMonitorMigrationStatusTypeToString(status)); + break; + } + + return; +} + + static void qemuProcessHandleMigrationStatus(qemuMonitor *mon G_GNUC_UNUSED, virDomainObj *vm, @@ -1552,6 +1607,7 @@ qemuProcessHandleMigrationStatus(qemuMonitor *mon G_GNUC_UNUSED, default: break; } + qemuProcessHandleMigrationPinStatus(vm, status); cleanup: virObjectUnlock(vm); -- 2.33.0
participants (1)
-
Jiang Jiacheng