[PATCH 02/12] migration/migration-pin: pin migration pid by given cpumap

From: zhengchuan <zhengchuan(a)huawei.com&gt; Secondly, we start to pin migration pid by given cpumap, add virDomainPinMigrationThread() for migration pin. Signed-off-by:zhengchuan<zhengchuan@huawei.com> --- include/libvirt/libvirt-domain.h | 4 ++ src/conf/domain_conf.c | 9 ++++ src/conf/domain_conf.h | 11 ++++ src/conf/virconftypes.h | 2 + src/driver-hypervisor.h | 6 +++ src/libvirt-domain.c | 65 +++++++++++++++++++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 1 + src/qemu/qemu_driver.c | 91 ++++++++++++++++++++++++++++++++ src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 13 ++++- src/remote_protocol-structs | 5 ++ src/util/vircgroup.c | 3 ++ src/util/vircgroup.h | 1 + 14 files changed, 212 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index e11f2795f1..26af8292d3 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -2424,6 +2424,10 @@ int virDomainPinEmulator (virDomainPtr domain, int maplen, unsigned int flags); +int virDomainPinMigrationThread (virDomainPtr domain, + unsigned char *cpumap, + int maplen); + int virDomainGetEmulatorPinInfo (virDomainPtr domain, unsigned char *cpumaps, int maplen, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 6c088ff295..a94a4c1c88 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -30299,3 +30299,12 @@ virDomainDefHasSpiceGraphics(const virDomainDef *def) return false; } + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def) +{ + if (!def) + return; + virBitmapFree(def->cpumask); + VIR_FREE(def); +} diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 1404c55053..a8384a4f6e 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -27,6 +27,7 @@ #include <libxml/xpath.h> #include "internal.h" +#include "viralloc.h" #include "virconftypes.h" #include "capabilities.h" #include "cpu_conf.h" @@ -4353,3 +4354,13 @@ virDomainObjGetMessages(virDomainObj *vm, bool virDomainDefHasSpiceGraphics(const virDomainDef *def); + +struct _virDomainMigrationIDDef { + bool autofill; + int thread_id; + virBitmap *cpumask; + virDomainThreadSchedParam sched; +}; + +void +virDomainMigrationIDDefFree(virDomainMigrationIDDef *def); diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h index 154805091a..984c2181d8 100644 --- a/src/conf/virconftypes.h +++ b/src/conf/virconftypes.h @@ -255,3 +255,5 @@ typedef struct _virDomainXMLOption virDomainXMLOption; typedef struct _virDomainXMLPrivateDataCallbacks virDomainXMLPrivateDataCallbacks; typedef struct _virDomainXenbusControllerOpts virDomainXenbusControllerOpts; + +typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef; diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 618f116012..1d7f606c59 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -1441,6 +1441,11 @@ typedef int int seconds, unsigned int flags); +typedef int +(*virDrvDomainPinMigrationThread)(virDomainPtr domain, + unsigned char *cpumap, + int maplen); + typedef char * (*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain, unsigned int flags); @@ -1716,5 +1721,6 @@ struct _virHypervisorDriver { virDrvDomainAuthorizedSSHKeysSet domainAuthorizedSSHKeysSet; virDrvDomainGetMessages domainGetMessages; virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc; + virDrvDomainPinMigrationThread domainPinMigrationThread; virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index fabfb2dd7f..d35a8237ae 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14011,3 +14011,68 @@ virDomainMigrateGetMigrationPids(virDomainPtr domain, virDispatchError(domain->conn); return NULL; } + +/** + * virDomainPinMigrationThread: + * @domain: a domain object + * @cpumap: pointer to a bit map of real CPUs (in 8-bit bytes) (IN) + * Each bit set to 1 means that corresponding CPU is usable. + * Bytes are stored in little-endian order: CPU0-7, 8-15... + * In each byte, lowest CPU number is least significant bit. + * @maplen: number of bytes in cpumap, from 1 up to size of CPU map in + * underlying virtualization system (Xen...). + * If maplen < size, missing bytes are set to zero. + * If maplen > size, failure code is returned. + * + * Allocate the real CPUs to the migrationThread which will be launched + * at the beginning of migration. + * This allocation can be handled whether before the migration, or during + * the migration which performs as an instant change. + * + * This function may require privileged access to the hypervisor. + * + * Returns 0 in case of success, -1 in case of failure. + * + * Since: 9.1.0 + */ +int +virDomainPinMigrationThread(virDomainPtr domain, + unsigned char *cpumap, + int maplen) +{ + virConnectPtr conn; + g_autofree char *str = NULL; + + VIR_DOMAIN_DEBUG(domain, "migration: cpumap=%p, maplen=%d", + cpumap, maplen); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckNonNullArgGoto(cpumap, error); + virCheckPositiveArgGoto(maplen, error); + + str = virBitmapDataFormat(cpumap, maplen); + VIR_INFO("Begin to PinMigrationThread(domain=%s): new CPU Affinity:%s", + NULLSTR(domain->name), NULLSTR(str)); + + if (conn->driver->domainPinMigrationThread) { + int ret; + ret = conn->driver->domainPinMigrationThread(domain, cpumap, maplen); + if (ret < 0) { + VIR_ERROR(_("Failed to PinMigrationThread(domain=%s), ret=%d"), + NULLSTR(domain->name), ret); + goto error; + } + VIR_INFO("Success to PinMigrationThread(domain=%s)", NULLSTR(domain->name)); + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index ae746a2d51..ad19d37d95 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -516,6 +516,7 @@ virDomainMemoryModelTypeToString; virDomainMemoryRemove; virDomainMemorySourceTypeFromString; virDomainMemorySourceTypeToString; +virDomainMigrationIDDefFree; virDomainMouseModeTypeFromString; virDomainMouseModeTypeToString; virDomainNetAllocateActualDevice; diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index f11fe1b26b..61561f2367 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -929,6 +929,7 @@ LIBVIRT_8.5.0 { LIBVIRT_9.1.0 { global: + virDomainPinMigrationThread; virDomainMigrateGetMigrationPids; } LIBVIRT_8.5.0; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index b9dc5f29f5..7ae81f459f 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -20678,6 +20678,96 @@ qemuDomainMigrateGetMigrationPids(virDomainPtr dom, return ret; } +static int +qemuDomainPinMigrationThread(virDomainPtr dom, + unsigned char *cpumap, + int maplen) +{ + int ret = -1; + virQEMUDriver *driver = dom->conn->privateData; + virQEMUDriverConfig *cfg = NULL; + virCaps *caps = NULL; + virDomainObj *vm = NULL; + virBitmap *pcpumap = NULL; + virCgroup *cgroup_migthread = NULL; + qemuDomainObjPrivate *priv = NULL; + int migration_id = 0; + virDomainMigrationIDDef *migration = NULL; + + cfg = virQEMUDriverGetConfig(driver); + + if (!(caps = virQEMUDriverGetCapabilities(driver, false))) + goto cleanup; + + if (!(vm = qemuDomainObjFromDomain(dom))) + goto cleanup; + + priv = vm->privateData; + + if (virDomainPinMigrationThreadEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0) + goto cleanup; + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto release; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty migration thread cpumap list for pinning")); + goto release; + } + + virBitmapFree(priv->pcpumap); + priv->pcpumap = pcpumap; + + migration = g_new0(virDomainMigrationIDDef, 1); + + if (priv->migrationPids != NULL) { + if (virStrToLong_i(priv->migrationPids, NULL, 10, &(migration->thread_id)) != 0) { + VIR_ERROR(_("migrationPid trans failure, migrationPids = %s"), priv->migrationPids); + goto endjob; + } + + if (pcpumap) { + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, + migration_id, false, &cgroup_migthread) < 0) + goto endjob; + if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for migration thread %d"), migration_id); + goto endjob; + } + } + + if (virProcessSetAffinity(migration->thread_id, pcpumap, false) < 0) + goto endjob; + } + } + + ret = 0; + goto endjob; + + release: + virBitmapFree(pcpumap); + + endjob: + virDomainObjEndJob(vm); + + cleanup: + if (cgroup_migthread) + virCgroupFree(cgroup_migthread); + virDomainMigrationIDDefFree(migration); + virDomainObjEndAPI(&vm); + virObjectUnref(caps); + virObjectUnref(cfg); + return ret; +} + static virHypervisorDriver qemuHypervisorDriver = { .name = QEMU_DRIVER_NAME, @@ -20927,6 +21017,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainGetMessages = qemuDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 9d8d7142b7..1c7cac6b29 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8651,6 +8651,7 @@ static virHypervisorDriver hypervisor_driver = { .domainGetMessages = remoteDomainGetMessages, /* 7.1.0 */ .domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */ .domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */ + .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */ .domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */ }; diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index d1b799db13..f86172b636 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -1291,6 +1291,11 @@ struct remote_domain_pin_iothread_args { unsigned int flags; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; + struct remote_domain_add_iothread_args { remote_nonnull_domain dom; unsigned int iothread_id; @@ -6976,5 +6981,11 @@ enum remote_procedure { * @generate: both * @acl: domain:migrate */ - REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443 + REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, + + /** + * @generate: both + * @acl: domain:write + */ + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index da4d799f42..9ef27913f1 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -3268,6 +3268,10 @@ struct remote_domain_event_memory_device_size_change_msg { remote_nonnull_string alias; uint64_t size; }; +struct remote_domain_pin_migration_thread_args { + remote_nonnull_domain dom; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ +}; struct remote_domain_migrate_get_migration_pids_args { remote_nonnull_domain dom; u_int flags; @@ -3719,4 +3723,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441, REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442, REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443, + REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444, }; diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c index 73675b4478..d4307c0d72 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -1123,6 +1123,9 @@ virCgroupNewThread(virCgroup *domain, case VIR_CGROUP_THREAD_IOTHREAD: name = g_strdup_printf("iothread%d", id); break; + case VIR_CGROUP_THREAD_MIGRATION_THREAD: + name = g_strdup_printf("migthread%d", id); + break; case VIR_CGROUP_THREAD_LAST: virReportError(VIR_ERR_INTERNAL_ERROR, _("unexpected name value %d"), nameval); diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h index 690f09465c..3c5c23965f 100644 --- a/src/util/vircgroup.h +++ b/src/util/vircgroup.h @@ -53,6 +53,7 @@ typedef enum { VIR_CGROUP_THREAD_VCPU = 0, VIR_CGROUP_THREAD_EMULATOR, VIR_CGROUP_THREAD_IOTHREAD, + VIR_CGROUP_THREAD_MIGRATION_THREAD, VIR_CGROUP_THREAD_LAST } virCgroupThreadName; -- 2.33.0