This patch adds some helpers to support cpu hotplug in libvirt. virHotplugGetCpuidFromMsg(): Get cpuid from cpu hotplug netlink message. The message is of the following format: {online|offline}@/devices/system/cpu/cpuxx (xx is cpuid) virHotplugUpdateCgroupCpuset(): Update the cpuset.cpus in the specified cgroup when a cpu is hotpluged. virHotplugSetCpusetToLastest(): Copy the cpuset from root to the specified cgroup. This function could be used when there are cpu houplug events occurred when libvirtd is not running. Signed-off-by: Tang Chen <tangchen(a)cn.fujitsu.com&gt; --- include/libvirt/virterror.h | 2 + src/Makefile.am | 1 + src/libvirt_private.syms | 6 ++ src/util/cgroup.c | 4 +- src/util/cgroup.h | 3 + src/util/hotplug.c | 240 +++++++++++++++++++++++++++++++++++++++++++ src/util/hotplug.h | 44 ++++++++ src/util/virterror.c | 3 +- 8 files changed, 300 insertions(+), 3 deletions(-) create mode 100644 src/util/hotplug.c create mode 100644 src/util/hotplug.h diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h index 5140c38..aabda04 100644 --- a/include/libvirt/virterror.h +++ b/include/libvirt/virterror.h @@ -114,6 +114,8 @@ typedef enum { VIR_FROM_SSH = 50, /* Error from libssh2 connection transport */ + VIR_FROM_HOTPLUG = 51, /* Error from Hotplug driver */ + # ifdef VIR_ENUM_SENTINELS VIR_ERR_DOMAIN_LAST # endif diff --git a/src/Makefile.am b/src/Makefile.am index 95e1bea..c65ee37 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -60,6 +60,7 @@ UTIL_SOURCES = \ util/event.c util/event.h \ util/event_poll.c util/event_poll.h \ util/hooks.c util/hooks.h \ + util/hotplug.c util/hotplug.h \ util/iptables.c util/iptables.h \ util/ebtables.c util/ebtables.h \ util/dnsmasq.c util/dnsmasq.h \ diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 6f14763..2ff7e0e 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -94,6 +94,7 @@ virCgroupKillPainfully; virCgroupKillRecursive; virCgroupMounted; virCgroupMoveTask; +virCgroupNew; virCgroupPathOfController; virCgroupRemove; virCgroupSetBlkioDeviceWeight; @@ -643,6 +644,11 @@ virHookInitialize; virHookPresent; +# hotplug.h +virHotplugUpdateCgroupCpuset; +virHotplugSetCpusetToLastest; + + # interface_conf.h virInterfaceAssignDef; virInterfaceDefFormat; diff --git a/src/util/cgroup.c b/src/util/cgroup.c index 5dc0764..393d439 100644 --- a/src/util/cgroup.c +++ b/src/util/cgroup.c @@ -608,8 +608,8 @@ static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group, } -static int virCgroupNew(const char *path, - virCgroupPtr *group) +int virCgroupNew(const char *path, + virCgroupPtr *group) { int rc = 0; char *typpath = NULL; diff --git a/src/util/cgroup.h b/src/util/cgroup.h index 68ac232..e294495 100644 --- a/src/util/cgroup.h +++ b/src/util/cgroup.h @@ -68,6 +68,9 @@ int virCgroupPathOfController(virCgroupPtr group, const char *key, char **path); +int virCgroupNew(const char *path, + virCgroupPtr *group); + int virCgroupAddTask(virCgroupPtr group, pid_t pid); int virCgroupAddTaskController(virCgroupPtr group, diff --git a/src/util/hotplug.c b/src/util/hotplug.c new file mode 100644 index 0000000..7c44299 --- /dev/null +++ b/src/util/hotplug.c @@ -0,0 +1,240 @@ +/* + * Copyright (C) 2012 FUJITSU, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Tang Chen <tangchen(a)cn.fujitsu.com&gt; + */ + +#include <sys/types.h> +#include <sys/socket.h> +#include <string.h> +#include <stdlib.h> + +#include "hotplug.h" +#include "cgroup.h" +#include "buf.h" +#include "virterror_internal.h" +#include "virnetlink.h" +#include "logging.h" +#include "memory.h" +#include "util.h" + +#define VIR_FROM_THIS VIR_FROM_HOTPLUG + +#ifdef __linux__ + +/** + * CPU hotplug message is of the following format: + * {online|offline}@/devices/system/cpu/cpuxx (xx is cpuid) + */ +# define CPU_ONLINE_MSG "online@/devices/system/cpu/cpu" +# define CPU_OFFLINE_MSG "offline@/devices/system/cpu/cpu" + +/** + * virHotplugGetCpuidFromMsg: + * + * @msg: The buffer containing the received netlink message + * @length: The length of the received netlink message + * @cpuid: Contains the cpuid in the message + * + * This function get the cpuid from the following netlink message, + * {online|offline}@/devices/system/cpu/cpuxx (xx is cpuid) + * + * Returns 0 on success, or -1 on error. + */ +static int virHotplugGetCpuidFromMsg(unsigned char *msg, + int length, + char **cpuid) +{ + char *p = NULL; + size_t len_online = MIN(length, strlen(CPU_ONLINE_MSG)); + size_t len_offline = MIN(length, strlen(CPU_OFFLINE_MSG)); + + if (VIR_ALLOC_N(*cpuid, 64) < 0) + goto memory_error; + + /** + * For now, we aren't sure if the message is a '/0' ended string. + * So we only test the first len_online or len_offline characters. + */ + if (strncmp((const char *)msg, CPU_ONLINE_MSG, len_online) == 0 || + strncmp((const char *)msg, CPU_OFFLINE_MSG, len_offline) == 0) { + p = strrchr((const char *)msg, '/'); + p = p + 4; + strcpy(*cpuid, p); + return 0; + } else { + VIR_DEBUG("Event is not a cpu hotplug event."); + VIR_FREE(*cpuid); + return 0; + } + +memory_error: + virReportOOMError(); + return -1; +} + +/** + * virHotplugUpdateCgroupCpuset: + * + * @msg: The buffer containing the received netlink message + * @length: The length of the received netlink message + * @cgroup: Contains a cgroup identifier to be modified + * + * Cpu hotplug netlink event handler. It is called when libvirtd + * receives netlink message from kernel, and modifies cpuset.cpus + * of specified cgroup. + * + * Return 0 on success. + */ +int +virHotplugUpdateCgroupCpuset(unsigned char *msg, + int length, + virCgroupPtr cgroup) +{ + int rc = 0; + char *cpuid = NULL; + char *cpus = NULL; + + if (!cgroup) { + virReportSystemError(EINVAL, + "%s", + _("invalid argument")); + return -EINVAL; + } + + if (VIR_ALLOC_N(cpus, 1024) < 0) { + virReportOOMError(); + return -ENOMEM; + } + + rc = virHotplugGetCpuidFromMsg(msg, length, &cpuid); + if (rc < 0) + goto error; + + /** + * If virHotplugGetCpuidFromMsg() succeeds, we are sure the + * netlink message is a '/0' ended string. + */ + VIR_DEBUG("netlink (cpu hotplug): %s", msg); + + if (msg == strstr((const char *)msg, "online")) { + VIR_DEBUG("CPU %s online message received.", cpuid); + + if (virCgroupGetCpusetCpus(cgroup, &cpus) < 0) { + virReportSystemError(errno, + "%s", + _("Unable to get cpuset.cpus")); + rc = -1; + goto error; + } + + if (virAsprintf(&cpus, "%s,%s", cpus, cpuid) < 0) { + virReportOOMError(); + rc = -ENOMEM; + goto error; + } + + if (virCgroupSetCpusetCpus(cgroup, cpus) < 0) { + virReportSystemError(errno, + "%s", + _("Unable to set cpuset.cpus")); + rc = -1; + goto error; + } + + } else if (msg == strstr((const char *)msg, "offline")) { + /* For now, nothing to do. */ + VIR_DEBUG("CPU %s offline message received.", cpuid); + } + +error: + VIR_FREE(cpuid); + VIR_FREE(cpus); + + return rc; +} + +/** + * virHotplugSetCpusetToLastest: + * + * @cgroup: The cgroup to be reset + * + * This function copies the cpuset from root to the specified cgroup. + * + * Return 0 on success or errno on error. + */ +int +virHotplugSetCpusetToLastest(virCgroupPtr cgroup) +{ + int rc = 0; + char *cpus = NULL; + virCgroupPtr mpgrp = NULL; + + /* mpgrp here is just the cpuset controller's mount point, + * which always contains all the online cpus. + */ + rc = virCgroupNew("/", &mpgrp); + if (rc < 0) + goto out; + + rc = virCgroupGetCpusetCpus(mpgrp, &cpus); + if (rc < 0) + goto error; + + rc = virCgroupSetCpusetCpus(cgroup, cpus); + if (rc < 0) + goto error; + +error: + virCgroupFree(&mpgrp); + VIR_FREE(cpus); + +out: + return rc; +} + +#else + +static const char *unsupported = N_("Not a linux system."); + +/** + * virHotplugUpdateCgroupCpuset: This function is called when libvirtd + * receives netlink message from kernel, and modifies cpuset.cpus of + * specified cgroup. + */ +int +virHotplugUpdateCgroupCpuset(unsigned char *msg ATTRIBUTE_UNUSED, + int length ATTRIBUTE_UNUSED, + virCgroupPtr cgroup ATTRIBUTE_UNUSED) +{ + VIR_DEBUG("%s", _(unsupported)); + return 0; +} + +/** + * virHotplugSetCpusetToLastest: This function copies the cpuset from + * root to the specified cgroup. + */ +int +virHotplugSetCpusetToLastest(virCgroupPtr cgroup ATTRIBUTE_UNUSED) +{ + VIR_DEBUG("%s", _(unsupported)); + return 0; +} + +#endif /* __linux__ */ diff --git a/src/util/hotplug.h b/src/util/hotplug.h new file mode 100644 index 0000000..ef3e279 --- /dev/null +++ b/src/util/hotplug.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2012 FUJITSU, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Tang Chen <tangchen(a)cn.fujitsu.com&gt; + */ + +#ifndef __HOTPLUG_H_ +# define __HOTPLUG_H_ + +# include "internal.h" +# include "cgroup.h" + +/** + * virHotplugUpdateCgroupCpuset: This function is called when libvirtd + * receives netlink message from kernel, and modifies cpuset.cpus of + * specified cgroup. + */ +int +virHotplugUpdateCgroupCpuset(unsigned char *msg, int length, + virCgroupPtr opaque); + +/** + * virHotplugSetCpusetToLastest: This function copies the cpuset from + * root to the specified cgroup. + */ +int +virHotplugSetCpusetToLastest(virCgroupPtr cgroup); + +#endif diff --git a/src/util/virterror.c b/src/util/virterror.c index 7caa69e..eff52ad 100644 --- a/src/util/virterror.c +++ b/src/util/virterror.c @@ -115,7 +115,8 @@ VIR_ENUM_IMPL(virErrorDomain, VIR_ERR_DOMAIN_LAST, "Parallels Cloud Server", "Device Config", - "SSH transport layer" /* 50 */ + "SSH transport layer", /* 50 */ + "Hotplug driver" /* 51 */ ) -- 1.7.10.1

This patch sets cpuset.cpus to lastest for qemu driver when the driver is being initialized, and then registers 2 cpu hotplug events handlers for qemu driver: - qemuNetlinkCpuHotplugHandleCallback() - qemuNetlinkCpuHotplugRemoveCallback() Signed-off-by: Tang Chen <tangchen(a)cn.fujitsu.com&gt; --- src/qemu/qemu_driver.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 53d6e5b..08f7be1 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -92,6 +92,9 @@ #include "virnodesuspend.h" #include "virtime.h" #include "virtypedparam.h" +#include "hotplug.h" +#include "cgroup.h" +#include "virnetlink.h" #define VIR_FROM_THIS VIR_FROM_QEMU @@ -525,6 +528,43 @@ static void qemuDomainNetsRestart(void *payload, virDomainObjUnlock(vm); } +static void +qemuNetlinkCpuHotplugHandleCallback(unsigned char *msg, + int length, + struct sockaddr_nl *peer ATTRIBUTE_UNUSED, + bool *handled ATTRIBUTE_UNUSED, + void *opaque) +{ + virCgroupPtr cgroup = NULL; + + /* opaque is a cgroup identifier pointing to libvirt root cgroup. */ + if (!opaque) { + virReportSystemError(EINVAL, + "%s", + _("invalid argument")); + return; + } + + cgroup = (virCgroupPtr)opaque; + + if (virHotplugUpdateCgroupCpuset(msg, length, cgroup) < 0) { + virReportSystemError(errno, + "%s", + _("Unable to update cpuset in cgroup")); + } + + return; +} + +static void +qemuNetlinkCpuHotplugRemoveCallback(int watch ATTRIBUTE_UNUSED, + const virMacAddrPtr macaddr ATTRIBUTE_UNUSED, + void *opaque ATTRIBUTE_UNUSED) +{ + /* For now, nothing to do. */ + VIR_INFO("CPU hotplug netlink handler has been removed."); +} + /** * qemudStartup: * @@ -682,6 +722,30 @@ qemudStartup(int privileged) { virStrerror(-rc, ebuf, sizeof(ebuf))); } +#if defined(__linux__) && defined(NETLINK_KOBJECT_UEVENT) + /** + * If we had ever missed any cpu hotplug event, we need to update + * cpuset.cpus to the lastest value + */ + rc = virHotplugSetCpusetToLastest(qemu_driver->cgroup); + if (rc < 0) { + VIR_INFO("Unable to reset cgroup for driver: %s", + virStrerror(-rc, ebuf, sizeof(ebuf))); + } + + /* Register cpu hotplug event handler for qemu driver */ + if (virNetlinkEventServiceIsRunning(NETLINK_KOBJECT_UEVENT)) { + rc = virNetlinkEventAddClient(qemuNetlinkCpuHotplugHandleCallback, + qemuNetlinkCpuHotplugRemoveCallback, + qemu_driver->cgroup, NULL, + NETLINK_KOBJECT_UEVENT); + if (rc < 0) { + VIR_INFO("Unable to register cpu hotplug event handler for driver: %s", + virStrerror(-rc, ebuf, sizeof(ebuf))); + } + } +#endif + if (qemudLoadDriverConfig(qemu_driver, driverConf) < 0) { goto error; } -- 1.7.10.1