On 03/06/2015 07:39 AM, Ján Tomko wrote: Hmm... I'll have to go back and look again as this was pretty much following the VcpuPin or EmulatorPin examples with a touch of GetCPUMap since this code returns the cpumap rather than expecting one on input. I'll revisit, but again I was keeping in line with other examples. I'll address these in an update John

Add the 'iothreadsinfo' command to display IOThread Info data. Allow for [--live] or [--config] options in order to display live or config data for an active domain. $ virsh iothreadsinfo --help NAME iothreadsinfo - view domain IOThreads SYNOPSIS iothreadsinfo <domain> [--config] [--live] [--current] DESCRIPTION Returns basic information about the domain IOThreads. OPTIONS [--domain] <string> domain name, id or uuid --config affect next boot --live affect running domain --current affect current domain An active domain may return: $ virsh iothreads $dom IOThread ID CPU Affinity --------------------------------------------------- 1 2 2 3 3 0 $ echo $? 0 For domains which don't have IOThreads the following is returned: $ virsh iothreads $dom No IOThreads found for the domain $ echo $? 0 For domains which are not running the following is returned: $ virsh iothreads $dom --live error: Unable to get domain IOThreads information error: Requested operation is not valid: domain is not running $ echo $? 1 Editing a domains configuration and modifying the iothreadpin data for thread 3 from nothing provided to setting a cpuset of '0-1' and then displaying using --config would display: $ virsh iothreads f18iothr --config IOThread ID CPU Affinity ---------------------------- 1 2 2 3 3 0-1 $ Signed-off-by: John Ferlan <jferlan(a)redhat.com&gt; --- tools/virsh-domain.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tools/virsh.pod | 12 +++++++ 2 files changed, 106 insertions(+) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 55c269c..0cce57b 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -6790,6 +6790,94 @@ cmdSetvcpus(vshControl *ctl, const vshCmd *cmd) } /* + * "iothreadsinfo" command + */ +static const vshCmdInfo info_iothreads[] = { + {.name = "help", + .data = N_("view domain IOThreads") + }, + {.name = "desc", + .data = N_("Returns basic information about the domain IOThreads.") + }, + {.name = NULL} +}; +static const vshCmdOptDef opts_iothreads[] = { + {.name = "domain", + .type = VSH_OT_DATA, + .flags = VSH_OFLAG_REQ, + .help = N_("domain name, id or uuid") + }, + {.name = "config", + .type = VSH_OT_BOOL, + .help = N_("affect next boot") + }, + {.name = "live", + .type = VSH_OT_BOOL, + .help = N_("affect running domain") + }, + {.name = "current", + .type = VSH_OT_BOOL, + .help = N_("affect current domain") + }, + {.name = NULL} +}; + +static bool +cmdIOThreadsInfo(vshControl *ctl, const vshCmd *cmd) +{ + virDomainPtr dom; + bool config = vshCommandOptBool(cmd, "config"); + bool live = vshCommandOptBool(cmd, "live"); + bool current = vshCommandOptBool(cmd, "current"); + int niothreads = 0; + virDomainIOThreadInfoPtr *info; + size_t i; + int maxcpu; + unsigned int flags = VIR_DOMAIN_AFFECT_CURRENT; + + VSH_EXCLUSIVE_OPTIONS_VAR(current, live); + VSH_EXCLUSIVE_OPTIONS_VAR(current, config); + + if (config) + flags |= VIR_DOMAIN_AFFECT_CONFIG; + if (live) + flags |= VIR_DOMAIN_AFFECT_LIVE; + + if (!(dom = vshCommandOptDomain(ctl, cmd, NULL))) + return false; + + if ((maxcpu = vshNodeGetCPUCount(ctl->conn)) < 0) + goto cleanup; + + if ((niothreads = virDomainGetIOThreadsInfo(dom, &info, flags)) < 0) { + vshError(ctl, _("Unable to get domain IOThreads information")); + goto cleanup; + } + + if (niothreads == 0) { + vshPrintExtra(ctl, _("No IOThreads found for the domain")); + goto cleanup; + } + + vshPrintExtra(ctl, " %-15s %-15s\n", + _("IOThread ID"), _("CPU Affinity")); + vshPrintExtra(ctl, "---------------------------------------------------\n"); + for (i = 0; i < niothreads; i++) { + + vshPrint(ctl, " %-15u ", info[i]->iothread_id); + ignore_value(vshPrintPinInfo(info[i]->cpumap, info[i]->cpumaplen, + maxcpu, 0)); + vshPrint(ctl, "\n"); + virDomainIOThreadsInfoFree(info[i]); + } + VIR_FREE(info); + + cleanup: + virDomainFree(dom); + return niothreads >= 0; +} + +/* * "cpu-compare" command */ static const vshCmdInfo info_cpu_compare[] = { @@ -12702,6 +12790,12 @@ const vshCmdDef domManagementCmds[] = { .info = info_inject_nmi, .flags = 0 }, + {.name = "iothreadsinfo", + .handler = cmdIOThreadsInfo, + .opts = opts_iothreads, + .info = info_iothreads, + .flags = 0 + }, {.name = "send-key", .handler = cmdSendKey, .opts = opts_send_key, diff --git a/tools/virsh.pod b/tools/virsh.pod index 1f71c91..f19601e 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -1361,6 +1361,18 @@ If I<--timeout> is specified, the command gives up waiting for events after I<seconds> have elapsed. With I<--loop>, the command prints all events until a timeout or interrupt key. +=item B<iothreadsinfo> I<domain> [[I<--live>] [I<--config>] | [I<--current>]] + +Display basic domain IOThreads information including the IOThread ID and +the CPU Affinity for each IOThread. + +If I<--live> is specified, get the IOThreads data from the running guest. If +the guest is not running, an error is returned. +If I<--config> is specified, get the IOThreads data from the next boot of +a persistent guest. +If I<--current> is specified or I<--live> and I<--config> are not specified, +then get the IOThread data based on the current guest state. + =item B<managedsave> I<domain> [I<--bypass-cache>] [{I<--running> | I<--paused>}] [I<--verbose>] -- 2.1.0

Add virDomainGetIOThreadPin to fetch the pinned CPU affinity map for one IOThread. Add virDomainPinIOThread to allow setting the CPU affinity for a specific IOThread. Signed-off-by: John Ferlan <jferlan(a)redhat.com&gt; --- include/libvirt/libvirt-domain.h | 10 +++ src/driver-hypervisor.h | 16 +++++ src/libvirt-domain.c | 152 +++++++++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 2 + 4 files changed, 180 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 9487b80..fc35cd2 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -1609,6 +1609,16 @@ void virDomainIOThreadsInfoFree(virDomainIOThreadInfoPtr info); int virDomainGetIOThreadsInfo(virDomainPtr domain, virDomainIOThreadInfoPtr **info, unsigned int flags); +int virDomainGetIOThreadPin (virDomainPtr domain, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags); +int virDomainPinIOThread(virDomainPtr domain, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags); /** * VIR_USE_CPU: diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index c2b4cd8..d62ce4b 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -386,6 +386,20 @@ typedef int unsigned int flags); typedef int +(*virDrvDomainGetIOThreadPin)(virDomainPtr domain, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags); + +typedef int +(*virDrvDomainPinIOThread)(virDomainPtr domain, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags); + +typedef int (*virDrvDomainGetSecurityLabel)(virDomainPtr domain, virSecurityLabelPtr seclabel); @@ -1260,6 +1274,8 @@ struct _virHypervisorDriver { virDrvDomainGetVcpus domainGetVcpus; virDrvDomainGetMaxVcpus domainGetMaxVcpus; virDrvDomainGetIOThreadsInfo domainGetIOThreadsInfo; + virDrvDomainGetIOThreadPin domainGetIOThreadPin; + virDrvDomainPinIOThread domainPinIOThread; virDrvDomainGetSecurityLabel domainGetSecurityLabel; virDrvDomainGetSecurityLabelList domainGetSecurityLabelList; virDrvNodeGetSecurityModel nodeGetSecurityModel; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 27f6abe..ee3659a 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -7964,6 +7964,158 @@ virDomainIOThreadsInfoFree(virDomainIOThreadInfoPtr info) /** + * virDomainGetIOThreadPin: + * @domain: pointer to domain object, or NULL for Domain0 + * @iothread_id: An IOThread ID of which to get the pin information + * @cpumap: pointer to a bit map of real CPUs for the IOThread of this + * domain (in 8-bit bytes) (OUT) + * The memory allocated to cpumap must be maplen bytes + * Must not be NULL. + * @maplen: the number of bytes in the cpumap, from 1 up to size of CPU map. + * Must be positive. + * @flags: bitwise-OR of virDomainModificationImpact + * Must not be VIR_DOMAIN_AFFECT_LIVE and + * VIR_DOMAIN_AFFECT_CONFIG concurrently. + * + * Query the CPU affinity setting of an IOThread ID of the domain, store + * it in cpumap. + * + * Returns 1 in case of success, + * 0 in case of no IOThread threads are pinned to pcpus, + * -1 in case of failure. + */ +int +virDomainGetIOThreadPin(virDomainPtr domain, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags) +{ + virConnectPtr conn; + + VIR_DOMAIN_DEBUG(domain, "iothread_id=%u cpumap=%p, maplen=%d, flags=%x", + iothread_id, cpumap, maplen, flags); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + if ((unsigned short) iothread_id != iothread_id) { + virReportError(VIR_ERR_OVERFLOW, _("input too large: %u"), + iothread_id); + goto error; + } + virCheckPositiveArgGoto(iothread_id, error); + virCheckNonNullArgGoto(cpumap, error); + virCheckPositiveArgGoto(maplen, error); + + /* At most one of these two flags should be set. */ + if ((flags & VIR_DOMAIN_AFFECT_LIVE) && + (flags & VIR_DOMAIN_AFFECT_CONFIG)) { + virReportInvalidArg(flags, + _("flags 'affect live' and 'affect config' in %s " + "are mutually exclusive"), + __FUNCTION__); + goto error; + } + conn = domain->conn; + + if (conn->driver->domainGetIOThreadPin) { + int ret; + ret = conn->driver->domainGetIOThreadPin(domain, iothread_id, + cpumap, maplen, flags); + if (ret < 0) + goto error; + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} + + +/** + * virDomainPinIOThread: + * @domain: a domain object + * @iothread_id: either the thread_id to modify or a count of IOThreads + * to be added or removed from the domain depending on the @flags setting + * @cpumap: pointer to a bit map of real CPUs (in 8-bit bytes) (IN) + * Each bit set to 1 means that corresponding CPU is usable. + * Bytes are stored in little-endian order: CPU0-7, 8-15... + * In each byte, lowest CPU number is least significant bit. + * @maplen: number of bytes in cpumap, from 1 up to size of CPU map in + * underlying virtualization system (Xen...). + * If maplen < size, missing bytes are set to zero. + * If maplen > size, failure code is returned. + * @flags: bitwise-OR of virDomainModificationImpact + * + * Dynamically change the real CPUs which can be allocated to an IOThread. + * This function may require privileged access to the hypervisor. + * + * @flags may include VIR_DOMAIN_AFFECT_LIVE or VIR_DOMAIN_AFFECT_CONFIG. + * Both flags may be set. + * If VIR_DOMAIN_AFFECT_LIVE is set, the change affects a running domain + * and may fail if domain is not alive. + * If VIR_DOMAIN_AFFECT_CONFIG is set, the change affects persistent state, + * and will fail for transient domains. If neither flag is specified (that is, + * @flags is VIR_DOMAIN_AFFECT_CURRENT), then an inactive domain modifies + * persistent setup, while an active domain is hypervisor-dependent on whether + * just live or both live and persistent state is changed. + * Not all hypervisors can support all flag combinations. + * + * See also virDomainGetIOThreadsInfo for querying this information. + * + * Returns 0 in case of success, -1 in case of failure. + */ +int +virDomainPinIOThread(virDomainPtr domain, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags) +{ + virConnectPtr conn; + + VIR_DOMAIN_DEBUG(domain, "iothread_id=%u, cpumap=%p, maplen=%d", + iothread_id, cpumap, maplen); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckReadOnlyGoto(conn->flags, error); + if ((unsigned short) iothread_id != iothread_id) { + virReportError(VIR_ERR_OVERFLOW, _("input too large: %u"), + iothread_id); + goto error; + } + virCheckPositiveArgGoto(iothread_id, error); + virCheckNonNullArgGoto(cpumap, error); + virCheckPositiveArgGoto(maplen, error); + + if (conn->driver->domainPinIOThread) { + int ret; + ret = conn->driver->domainPinIOThread(domain, iothread_id, + cpumap, maplen, flags); + if (ret < 0) + goto error; + return ret; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} + + +/** * virDomainGetSecurityLabel: * @domain: a domain object * @seclabel: pointer to a virSecurityLabel structure diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index 34852c1..e966bdd 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -699,6 +699,8 @@ LIBVIRT_1.2.14 { global: virDomainIOThreadsInfoFree; virDomainGetIOThreadsInfo; + virDomainGetIOThreadPin; + virDomainPinIOThread; } LIBVIRT_1.2.12; # .... define new API here using predicted next version number .... -- 2.1.0

On 03/06/2015 04:46 AM, Daniel P. Berrange wrote: Yes - although based on reviews from yesterday it wasn't totally clear if having a GetIOThreadPin in addition to GetIOThreadsInfo was requested, so I added it to be "safe", but can remove it. There is a GetVcpus and a GetVcpuPinInfo and the IOThreads code mirrors that. Tks - John

Implement the remote plumbing to get an IOThread's pinned CPU data or to allow pinning an IOThread to a specific CPU set Signed-off-by: John Ferlan <jferlan(a)redhat.com&gt; --- daemon/remote.c | 49 ++++++++++++++++++++++++++++++++++ src/remote/remote_driver.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ src/remote/remote_protocol.x | 35 ++++++++++++++++++++++++- src/remote_protocol-structs | 24 +++++++++++++++++ src/rpc/gendispatch.pl | 1 + 5 files changed, 170 insertions(+), 1 deletion(-) diff --git a/daemon/remote.c b/daemon/remote.c index ff64eeb..8907b7e 100644 --- a/daemon/remote.c +++ b/daemon/remote.c @@ -2341,6 +2341,55 @@ remoteDispatchDomainGetIOThreadsInfo(virNetServerPtr server ATTRIBUTE_UNUSED, } static int +remoteDispatchDomainGetIOThreadPin(virNetServerPtr server ATTRIBUTE_UNUSED, + virNetServerClientPtr client ATTRIBUTE_UNUSED, + virNetMessagePtr msg ATTRIBUTE_UNUSED, + virNetMessageErrorPtr rerr, + remote_domain_get_iothread_pin_args *args, + remote_domain_get_iothread_pin_ret *ret) +{ + virDomainPtr dom = NULL; + unsigned char *cpumap = NULL; + int r; + int rv = -1; + struct daemonClientPrivate *priv = + virNetServerClientGetPrivateData(client); + + if (!priv->conn) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("connection not open")); + goto cleanup; + } + + if (!(dom = get_nonnull_domain(priv->conn, args->dom))) + goto cleanup; + + /* Allocate buffers to take the results */ + if (args->maplen > 0 && + VIR_ALLOC_N(cpumap, args->maplen) < 0) + goto cleanup; + + if ((r = virDomainGetIOThreadPin(dom, + args->iothread_id, + cpumap, + args->maplen, + args->flags)) < 0) + goto cleanup; + + ret->ret = r; + ret->cpumap.cpumap_len = args->maplen; + ret->cpumap.cpumap_val = (char *) cpumap; + cpumap = NULL; + + rv = 0; + + cleanup: + if (rv < 0) + virNetMessageSaveError(rerr); + VIR_FREE(cpumap); + virObjectUnref(dom); + return rv; +} +static int remoteDispatchDomainMigratePrepare(virNetServerPtr server ATTRIBUTE_UNUSED, virNetServerClientPtr client ATTRIBUTE_UNUSED, virNetMessagePtr msg ATTRIBUTE_UNUSED, diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 42dab9d..fa99a31 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -2394,6 +2394,66 @@ remoteDomainGetIOThreadsInfo(virDomainPtr dom, } static int +remoteDomainGetIOThreadPin(virDomainPtr domain, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags) +{ + int rv = -1; + size_t i; + remote_domain_get_iothread_pin_args args; + remote_domain_get_iothread_pin_ret ret; + struct private_data *priv = domain->conn->privateData; + + remoteDriverLock(priv); + + /* There is only one cpumap for each IOThread thread */ + if (maplen > REMOTE_CPUMAPS_MAX) { + virReportError(VIR_ERR_RPC, + _("IOThread map buffer length exceeds maximum: %d > %d"), + maplen, REMOTE_CPUMAPS_MAX); + goto done; + } + + make_nonnull_domain(&args.dom, domain); + args.iothread_id = iothread_id; + args.maplen = maplen; + args.flags = flags; + + memset(&ret, 0, sizeof(ret)); + + if (call(domain->conn, priv, 0, REMOTE_PROC_DOMAIN_GET_IOTHREAD_PIN, + (xdrproc_t) xdr_remote_domain_get_iothread_pin_args, + (char *) &args, + (xdrproc_t) xdr_remote_domain_get_iothread_pin_ret, + (char *) &ret) == -1) + goto done; + + if (ret.cpumap.cpumap_len > maplen) { + virReportError(VIR_ERR_RPC, + _("host reports map buffer length exceeds maximum: %d > %d"), + ret.cpumap.cpumap_len, maplen); + goto cleanup; + } + + memset(cpumap, 0, maplen); + + for (i = 0; i < ret.cpumap.cpumap_len; ++i) + cpumap[i] = ret.cpumap.cpumap_val[i]; + + rv = ret.ret; + + cleanup: + xdr_free((xdrproc_t) xdr_remote_domain_get_iothread_pin_ret, + (char *) &ret); + + done: + remoteDriverUnlock(priv); + return rv; +} + +static int remoteDomainGetSecurityLabel(virDomainPtr domain, virSecurityLabelPtr seclabel) { remote_domain_get_security_label_args args; @@ -8106,6 +8166,8 @@ static virHypervisorDriver hypervisor_driver = { .domainGetVcpus = remoteDomainGetVcpus, /* 0.3.0 */ .domainGetMaxVcpus = remoteDomainGetMaxVcpus, /* 0.3.0 */ .domainGetIOThreadsInfo = remoteDomainGetIOThreadsInfo, /* 1.2.14 */ + .domainGetIOThreadPin = remoteDomainGetIOThreadPin, /* 1.2.14 */ + .domainPinIOThread = remoteDomainPinIOThread, /* 1.2.14 */ .domainGetSecurityLabel = remoteDomainGetSecurityLabel, /* 0.6.1 */ .domainGetSecurityLabelList = remoteDomainGetSecurityLabelList, /* 0.10.0 */ .nodeGetSecurityModel = remoteNodeGetSecurityModel, /* 0.6.1 */ diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 4ea535e..dc916f3 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -1199,6 +1199,25 @@ struct remote_domain_get_iothreads_info_ret { unsigned int ret; }; +struct remote_domain_get_iothread_pin_args { + remote_nonnull_domain dom; + unsigned int iothread_id; + int maplen; + unsigned int flags; +}; + +struct remote_domain_get_iothread_pin_ret { + opaque cpumap<REMOTE_CPUMAPS_MAX>; + int ret; +}; + +struct remote_domain_pin_iothread_args { + remote_nonnull_domain dom; + unsigned int iothreads_id; + opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */ + unsigned int flags; +}; + struct remote_domain_get_security_label_args { remote_nonnull_domain dom; }; @@ -5593,5 +5612,19 @@ enum remote_procedure { * @generate: none * @acl: domain:read */ - REMOTE_PROC_DOMAIN_GET_IOTHREADS_INFO = 351 + REMOTE_PROC_DOMAIN_GET_IOTHREADS_INFO = 351, + + /** + * @generate: none + * @acl: domain:read + */ + REMOTE_PROC_DOMAIN_GET_IOTHREAD_PIN = 352, + + /** + * @generate: both + * @acl: domain:write + * @acl: domain:save:!VIR_DOMAIN_AFFECT_CONFIG|VIR_DOMAIN_AFFECT_LIVE + * @acl: domain:save:VIR_DOMAIN_AFFECT_CONFIG + */ + REMOTE_PROC_DOMAIN_PIN_IOTHREAD = 353 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index 907bcd4..4fa710c 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -825,6 +825,28 @@ struct remote_domain_get_iothreads_info_ret { } info; u_int ret; }; +struct remote_domain_get_iothread_pin_args { + remote_nonnull_domain dom; + u_int iothread_id; + int maplen; + u_int flags; +}; +struct remote_domain_get_iothread_pin_ret { + struct { + u_int cpumap_len; + char * cpumap_val; + } cpumap; + int ret; +}; +struct remote_domain_pin_iothread_args { + remote_nonnull_domain dom; + u_int iothreads_id; + struct { + u_int cpumap_len; + char * cpumap_val; + } cpumap; + u_int flags; +}; struct remote_domain_get_security_label_args { remote_nonnull_domain dom; }; @@ -2982,4 +3004,6 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_GET_FSINFO = 349, REMOTE_PROC_DOMAIN_DEFINE_XML_FLAGS = 350, REMOTE_PROC_DOMAIN_GET_IOTHREADS_INFO = 351, + REMOTE_PROC_DOMAIN_GET_IOTHREAD_PIN = 352, + REMOTE_PROC_DOMAIN_PIN_IOTHREAD = 353, }; diff --git a/src/rpc/gendispatch.pl b/src/rpc/gendispatch.pl index 78cb415..8b488eb 100755 --- a/src/rpc/gendispatch.pl +++ b/src/rpc/gendispatch.pl @@ -68,6 +68,7 @@ sub fixup_name { $name =~ s/Fsthaw$/FSThaw/; $name =~ s/Fsinfo$/FSInfo/; $name =~ s/Iothreads$/IOThreads/; + $name =~ s/Iothread$/IOThread/; $name =~ s/Scsi/SCSI/; $name =~ s/Wwn$/WWN/; $name =~ s/Dhcp$/DHCP/; -- 2.1.0

Add qemuDomainGetIOThreadPin to handle getting the CPU Affinity for one specific IOThread Add qemuDomainPinIOThread to handle setting the cpuset for a specific IOThread Signed-off-by: John Ferlan <jferlan(a)redhat.com&gt; --- include/libvirt/libvirt-domain.h | 9 ++ src/qemu/qemu_driver.c | 302 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 311 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index fc35cd2..e81389a 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -3223,6 +3223,15 @@ typedef void (*virConnectDomainEventDeviceRemovedCallback)(virConnectPtr conn, # define VIR_DOMAIN_TUNABLE_CPU_EMULATORPIN "cputune.emulatorpin" /** + * VIR_DOMAIN_TUNABLE_CPU_IOTHREADSPIN: + * + * Macro represents formatted pinning for one IOThread specified by id which is + * appended to the parameter name, for example "cputune.iothreadpin1", + * as VIR_TYPED_PARAM_STRING. + */ +# define VIR_DOMAIN_TUNABLE_CPU_IOTHREADSPIN "cputune.iothreadpin%u" + +/** * VIR_DOMAIN_TUNABLE_CPU_CPU_SHARES: * * Macro represents proportional weight of the scheduler used on the diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index d8a761d..5c8040e 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -5780,6 +5780,306 @@ qemuDomainGetIOThreadsInfo(virDomainPtr dom, return ret; } +static int +qemuDomainGetIOThreadPin(virDomainPtr dom, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags) +{ + virQEMUDriverPtr driver = dom->conn->privateData; + virDomainObjPtr vm = NULL; + virDomainDefPtr targetDef = NULL; + int ret = -1; + virCapsPtr caps = NULL; + int maxcpu, hostcpus, pcpu; + virBitmapPtr cpumask = NULL; + bool pinned; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG, -1); + + if (!(vm = qemuDomObjFromDomain(dom))) + goto cleanup; + + if (virDomainGetIOThreadPinEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (!(caps = virQEMUDriverGetCapabilities(driver, false))) + goto cleanup; + + if (virDomainLiveConfigHelperMethod(caps, driver->xmlopt, + vm, &flags, &targetDef) < 0) + goto cleanup; + + if (flags & VIR_DOMAIN_AFFECT_LIVE) + targetDef = vm->def; + + /* Coverity didn't realize that targetDef must be set if we got here. */ + sa_assert(targetDef); + + if ((hostcpus = nodeGetCPUCount()) < 0) + goto cleanup; + + maxcpu = maplen * 8; + if (maxcpu > hostcpus) + maxcpu = hostcpus; + + /* initialize cpumap */ + memset(cpumap, 0xff, maplen); + if (maxcpu % 8) + cpumap[maplen - 1] &= (1 << maxcpu % 8) - 1; + + if (targetDef->cputune.iothreadspin) { + if (iothread_id > targetDef->iothreads) { + virReportError(VIR_ERR_INVALID_ARG, + _("iothread value out of range %d > %d"), + iothread_id, targetDef->iothreads); + goto cleanup; + } + + cpumask = targetDef->cputune.iothreadspin[iothread_id - 1]->cpumask; + } else if (targetDef->cpumask) { + cpumask = targetDef->cpumask; + } else { + ret = 0; + goto cleanup; + } + + for (pcpu = 0; pcpu < maxcpu; pcpu++) { + if (virBitmapGetBit(cpumask, pcpu, &pinned) < 0) + goto cleanup; + if (!pinned) + VIR_UNUSE_CPU(cpumap, pcpu); + } + ret = 1; + + cleanup: + qemuDomObjEndAPI(&vm); + virObjectUnref(caps); + return ret; +} + +static int +qemuDomainPinIOThread(virDomainPtr dom, + unsigned int iothread_id, + unsigned char *cpumap, + int maplen, + unsigned int flags) +{ + int ret = -1; + virQEMUDriverPtr driver = dom->conn->privateData; + virQEMUDriverConfigPtr cfg = NULL; + virDomainObjPtr vm; + virCapsPtr caps = NULL; + virDomainDefPtr persistentDef = NULL; + virBitmapPtr pcpumap = NULL; + bool doReset = false; + qemuDomainObjPrivatePtr priv; + virDomainVcpuPinDefPtr *newIOThreadsPin = NULL; + size_t newIOThreadsPinNum = 0; + virCgroupPtr cgroup_iothread = NULL; + virObjectEventPtr event = NULL; + char paramField[VIR_TYPED_PARAM_FIELD_LENGTH] = ""; + char *str = NULL; + virTypedParameterPtr eventParams = NULL; + int eventNparams = 0; + int eventMaxparams = 0; + + virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | + VIR_DOMAIN_AFFECT_CONFIG, -1); + + cfg = virQEMUDriverGetConfig(driver); + + if (!(vm = qemuDomObjFromDomain(dom))) + goto cleanup; + priv = vm->privateData; + + if (virDomainPinIOThreadEnsureACL(dom->conn, vm->def, flags) < 0) + goto cleanup; + + if (!(caps = virQEMUDriverGetCapabilities(driver, false))) + goto cleanup; + + if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("Changing affinity for IOThread dynamically is " + "not allowed when CPU placement is 'auto'")); + goto cleanup; + } + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + goto cleanup; + + if (virDomainLiveConfigHelperMethod(caps, driver->xmlopt, vm, &flags, + &persistentDef) < 0) + goto endjob; + + if (flags & VIR_DOMAIN_AFFECT_LIVE) + persistentDef = vm->def; + + /* Coverity didn't realize that targetDef must be set if we got here. */ + sa_assert(persistentDef); + + if (!(pcpumap = virBitmapNewData(cpumap, maplen))) + goto endjob; + + if (virBitmapIsAllClear(pcpumap)) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Empty iothread cpumap list for pinning")); + goto endjob; + } + + /* pinning to all physical cpus means resetting, + * so check if we can reset setting. + */ + if (virBitmapIsAllSet(pcpumap)) + doReset = true; + + if (flags & VIR_DOMAIN_AFFECT_LIVE) { + + if (priv->iothreadpids == NULL) { + virReportError(VIR_ERR_OPERATION_INVALID, + "%s", _("IOThread affinity is not supported")); + goto endjob; + } + + if (iothread_id > priv->niothreadpids) { + virReportError(VIR_ERR_INVALID_ARG, + _("iothread value out of range %d > %d"), + iothread_id, priv->niothreadpids); + goto endjob; + } + + if (vm->def->cputune.iothreadspin) { + /* The VcpuPinDefCopy works for IOThreads too */ + newIOThreadsPin = + virDomainVcpuPinDefCopy(vm->def->cputune.iothreadspin, + vm->def->cputune.niothreadspin); + if (!newIOThreadsPin) + goto endjob; + + newIOThreadsPinNum = vm->def->cputune.niothreadspin; + } else { + if (VIR_ALLOC(newIOThreadsPin) < 0) + goto endjob; + newIOThreadsPinNum = 0; + } + + if (virDomainIOThreadsPinAdd(&newIOThreadsPin, &newIOThreadsPinNum, + cpumap, maplen, iothread_id) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to update iothreadspin")); + virDomainVcpuPinDefArrayFree(newIOThreadsPin, newIOThreadsPinNum); + goto endjob; + } + + /* Configure the corresponding cpuset cgroup before set affinity. */ + if (virCgroupHasController(priv->cgroup, + VIR_CGROUP_CONTROLLER_CPUSET)) { + if (virCgroupNewIOThread(priv->cgroup, iothread_id, + false, &cgroup_iothread) < 0) + goto endjob; + if (qemuSetupCgroupIOThreadsPin(cgroup_iothread, + newIOThreadsPin, + newIOThreadsPinNum, + iothread_id) < 0) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("failed to set cpuset.cpus in cgroup" + " for iothread %d"), iothread_id); + goto endjob; + } + } else { + if (virProcessSetAffinity(priv->iothreadpids[iothread_id - 1], + pcpumap) < 0) { + virReportError(VIR_ERR_SYSTEM_ERROR, + _("failed to set cpu affinity for IOThread %d"), + iothread_id); + goto endjob; + } + } + + if (doReset) { + virDomainIOThreadsPinDel(vm->def, iothread_id); + } else { + if (vm->def->cputune.iothreadspin) + virDomainVcpuPinDefArrayFree(vm->def->cputune.iothreadspin, + vm->def->cputune.niothreadspin); + + vm->def->cputune.iothreadspin = newIOThreadsPin; + vm->def->cputune.niothreadspin = newIOThreadsPinNum; + newIOThreadsPin = NULL; + } + + if (newIOThreadsPin) + virDomainVcpuPinDefArrayFree(newIOThreadsPin, newIOThreadsPinNum); + + if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm) < 0) + goto endjob; + + if (snprintf(paramField, VIR_TYPED_PARAM_FIELD_LENGTH, + VIR_DOMAIN_TUNABLE_CPU_IOTHREADSPIN, iothread_id) < 0) { + goto endjob; + } + + str = virBitmapFormat(pcpumap); + if (virTypedParamsAddString(&eventParams, &eventNparams, + &eventMaxparams, paramField, str) < 0) + goto endjob; + + event = virDomainEventTunableNewFromDom(dom, eventParams, eventNparams); + } + + if (flags & VIR_DOMAIN_AFFECT_CONFIG) { + if (iothread_id > persistentDef->iothreads) { + virReportError(VIR_ERR_INVALID_ARG, + _("iothread value out of range %d > %d"), + iothread_id, persistentDef->iothreads); + goto endjob; + } + + if (doReset) { + virDomainIOThreadsPinDel(persistentDef, iothread_id); + } else { + if (!persistentDef->cputune.iothreadspin) { + if (VIR_ALLOC(persistentDef->cputune.iothreadspin) < 0) + goto endjob; + persistentDef->cputune.niothreadspin = 0; + } + if (virDomainIOThreadsPinAdd(&persistentDef->cputune.iothreadspin, + &persistentDef->cputune.niothreadspin, + cpumap, + maplen, + iothread_id) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to update or add iothreadspin xml " + "of a persistent domain")); + goto endjob; + } + } + + ret = virDomainSaveConfig(cfg->configDir, persistentDef); + goto endjob; + } + + ret = 0; + + endjob: + qemuDomainObjEndJob(driver, vm); + + cleanup: + if (cgroup_iothread) + virCgroupFree(&cgroup_iothread); + if (event) + qemuDomainEventQueue(driver, event); + VIR_FREE(str); + virBitmapFree(pcpumap); + qemuDomObjEndAPI(&vm); + virObjectUnref(caps); + virObjectUnref(cfg); + return ret; +} + static int qemuDomainGetSecurityLabel(virDomainPtr dom, virSecurityLabelPtr seclabel) { virQEMUDriverPtr driver = dom->conn->privateData; @@ -19332,6 +19632,8 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainGetVcpus = qemuDomainGetVcpus, /* 0.4.4 */ .domainGetMaxVcpus = qemuDomainGetMaxVcpus, /* 0.4.4 */ .domainGetIOThreadsInfo = qemuDomainGetIOThreadsInfo, /* 1.2.14 */ + .domainGetIOThreadPin = qemuDomainGetIOThreadPin, /* 1.2.14 */ + .domainPinIOThread = qemuDomainPinIOThread, /* 1.2.14 */ .domainGetSecurityLabel = qemuDomainGetSecurityLabel, /* 0.6.1 */ .domainGetSecurityLabelList = qemuDomainGetSecurityLabelList, /* 0.10.0 */ .nodeGetSecurityModel = qemuNodeGetSecurityModel, /* 0.6.1 */ -- 2.1.0