We need to perform a sanity check on the udev monitor before every use so that we know nothing has changed in the meantime. The reason for moving the code to a separate helper is to enhance readability and shift the focus on the important stuff within the udevEventHandleCallback handler. Signed-off-by: Erik Skultety <eskultet(a)redhat.com&gt; --- src/node_device/node_device_udev.c | 43 ++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/node_device/node_device_udev.c b/src/node_device/node_device_udev.c index f4177455c..a9a4c9b6b 100644 --- a/src/node_device/node_device_udev.c +++ b/src/node_device/node_device_udev.c @@ -1615,24 +1615,20 @@ udevHandleOneDevice(struct udev_device *device) } -static void -udevEventHandleCallback(int watch ATTRIBUTE_UNUSED, - int fd, - int events ATTRIBUTE_UNUSED, - void *data ATTRIBUTE_UNUSED) +static bool +udevEventMonitorSanityCheck(struct udev_monitor *udev_monitor, + int fd) { - struct udev_device *device = NULL; - struct udev_monitor *udev_monitor = DRV_STATE_UDEV_MONITOR(driver); - int udev_fd = -1; + int rc = -1; - udev_fd = udev_monitor_get_fd(udev_monitor); - if (fd != udev_fd) { + rc = udev_monitor_get_fd(udev_monitor); + if (fd != rc) { udevPrivate *priv = driver->privateData; virReportError(VIR_ERR_INTERNAL_ERROR, _("File descriptor returned by udev %d does not " "match node device file descriptor %d"), - fd, udev_fd); + fd, rc); /* this is a non-recoverable error, let's remove the handle, so that we * don't get in here again because of some spurious behaviour and report @@ -1641,21 +1637,36 @@ udevEventHandleCallback(int watch ATTRIBUTE_UNUSED, virEventRemoveHandle(priv->watch); priv->watch = -1; - goto cleanup; + return false; } + return true; +} + + +static void +udevEventHandleCallback(int watch ATTRIBUTE_UNUSED, + int fd, + int events ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + struct udev_device *device = NULL; + struct udev_monitor *udev_monitor = NULL; + + udev_monitor = DRV_STATE_UDEV_MONITOR(driver); + + if (!udevEventMonitorSanityCheck(udev_monitor, fd)) + return; + device = udev_monitor_receive_device(udev_monitor); if (device == NULL) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("udev_monitor_receive_device returned NULL")); - goto cleanup; + return; } udevHandleOneDevice(device); - - cleanup: udev_device_unref(device); - return; } -- 2.13.6

Since there's going to be a worker thread which needs to have some data protected by a lock, the whole code would just simply get unnecessary complex, since two sets of locks would be necessary, driver lock (for udev monitor and event handle) and a mutex protecting thread-local data. Given the future thread will need to access the udev monitor socket as well, why not protect everything with a single lock, even better, by converting the driver's private data to a lockable object, we get the automatic object disposal feature for free. Signed-off-by: Erik Skultety <eskultet(a)redhat.com&gt; --- src/node_device/node_device_udev.c | 140 ++++++++++++++++++++++++------------- src/node_device/node_device_udev.h | 3 - 2 files changed, 93 insertions(+), 50 deletions(-) diff --git a/src/node_device/node_device_udev.c b/src/node_device/node_device_udev.c index a9a4c9b6b..bb9787fdb 100644 --- a/src/node_device/node_device_udev.c +++ b/src/node_device/node_device_udev.c @@ -53,12 +53,78 @@ VIR_LOG_INIT("node_device.node_device_udev"); # define TYPE_RAID 12 #endif -struct _udevPrivate { +typedef struct _udevEventData udevEventData; +typedef udevEventData *udevEventDataPtr; + +struct _udevEventData { + virObjectLockable parent; + struct udev_monitor *udev_monitor; int watch; bool privileged; }; +static virClassPtr udevEventDataClass; + +static void +udevEventDataDispose(void *obj) +{ + struct udev *udev = NULL; + udevEventDataPtr data = obj; + + if (data->watch != -1) + virEventRemoveHandle(data->watch); + + if (data->udev_monitor) + udev = udev_monitor_get_udev(data->udev_monitor); + + udev_unref(udev); + udev_monitor_unref(data->udev_monitor); +} + + +static int +udevEventDataOnceInit(void) +{ + if (!(udevEventDataClass = virClassNew(virClassForObjectLockable(), + "udevEventData", + sizeof(udevEventData), + udevEventDataDispose))) + return -1; + + return 0; +} + +VIR_ONCE_GLOBAL_INIT(udevEventData) + +static udevEventDataPtr +udevEventDataNew(void) +{ + udevEventDataPtr ret = NULL; + + if (udevEventDataInitialize() < 0) + return NULL; + + if (!(ret = virObjectLockableNew(udevEventDataClass))) + return NULL; + + ret->watch = -1; + return ret; +} + + +static bool +udevEventDataIsPrivileged(udevEventDataPtr data) +{ + bool privileged; + + virObjectLock(data); + privileged = data->privileged; + virObjectUnlock(data); + + return privileged; +} + static bool udevHasDeviceProperty(struct udev_device *dev, @@ -447,7 +513,7 @@ udevProcessPCI(struct udev_device *device, virNodeDevCapPCIDevPtr pci_dev = &def->caps->data.pci_dev; virPCIEDeviceInfoPtr pci_express = NULL; virPCIDevicePtr pciDev = NULL; - udevPrivate *priv = driver->privateData; + udevEventDataPtr priv = driver->privateData; int ret = -1; char *p; @@ -498,7 +564,7 @@ udevProcessPCI(struct udev_device *device, goto cleanup; /* We need to be root to read PCI device configs */ - if (priv->privileged) { + if (udevEventDataIsPrivileged(priv)) { if (virPCIGetHeaderType(pciDev, &pci_dev->hdrType) < 0) goto cleanup; @@ -1559,39 +1625,18 @@ udevPCITranslateDeinit(void) static int nodeStateCleanup(void) { - udevPrivate *priv = NULL; - struct udev_monitor *udev_monitor = NULL; - struct udev *udev = NULL; - if (!driver) return -1; nodeDeviceLock(); + virObjectUnref(driver->privateData); virObjectUnref(driver->nodeDeviceEventState); - priv = driver->privateData; - - if (priv) { - if (priv->watch != -1) - virEventRemoveHandle(priv->watch); - - udev_monitor = DRV_STATE_UDEV_MONITOR(driver); - - if (udev_monitor != NULL) { - udev = udev_monitor_get_udev(udev_monitor); - udev_monitor_unref(udev_monitor); - } - } - - if (udev != NULL) - udev_unref(udev); - virNodeDeviceObjListFree(driver->devs); nodeDeviceUnlock(); virMutexDestroy(&driver->lock); VIR_FREE(driver); - VIR_FREE(priv); udevPCITranslateDeinit(); return 0; @@ -1615,16 +1660,17 @@ udevHandleOneDevice(struct udev_device *device) } +/* the caller must be holding the udevEventData object lock prior to calling + * this function + */ static bool -udevEventMonitorSanityCheck(struct udev_monitor *udev_monitor, +udevEventMonitorSanityCheck(udevEventDataPtr priv, int fd) { int rc = -1; - rc = udev_monitor_get_fd(udev_monitor); + rc = udev_monitor_get_fd(priv->udev_monitor); if (fd != rc) { - udevPrivate *priv = driver->privateData; - virReportError(VIR_ERR_INTERNAL_ERROR, _("File descriptor returned by udev %d does not " "match node device file descriptor %d"), @@ -1650,15 +1696,19 @@ udevEventHandleCallback(int watch ATTRIBUTE_UNUSED, int events ATTRIBUTE_UNUSED, void *data ATTRIBUTE_UNUSED) { + udevEventDataPtr priv = driver->privateData; struct udev_device *device = NULL; - struct udev_monitor *udev_monitor = NULL; - udev_monitor = DRV_STATE_UDEV_MONITOR(driver); + virObjectLock(priv); - if (!udevEventMonitorSanityCheck(udev_monitor, fd)) + if (!udevEventMonitorSanityCheck(priv, fd)) { + virObjectUnlock(priv); return; + } + + device = udev_monitor_receive_device(priv->udev_monitor); + virObjectUnlock(priv); - device = udev_monitor_receive_device(udev_monitor); if (device == NULL) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("udev_monitor_receive_device returned NULL")); @@ -1675,12 +1725,13 @@ udevEventHandleCallback(int watch ATTRIBUTE_UNUSED, static void udevGetDMIData(virNodeDevCapSystemPtr syscap) { + udevEventDataPtr priv = driver->privateData; struct udev *udev = NULL; struct udev_device *device = NULL; virNodeDevCapSystemHardwarePtr hardware = &syscap->hardware; virNodeDevCapSystemFirmwarePtr firmware = &syscap->firmware; - udev = udev_monitor_get_udev(DRV_STATE_UDEV_MONITOR(driver)); + udev = udev_monitor_get_udev(priv->udev_monitor); device = udev_device_new_from_syspath(udev, DMI_DEVPATH); if (device == NULL) { @@ -1791,15 +1842,9 @@ nodeStateInitialize(bool privileged, virStateInhibitCallback callback ATTRIBUTE_UNUSED, void *opaque ATTRIBUTE_UNUSED) { - udevPrivate *priv = NULL; + udevEventDataPtr priv = NULL; struct udev *udev = NULL; - if (VIR_ALLOC(priv) < 0) - return -1; - - priv->watch = -1; - priv->privileged = privileged; - if (VIR_ALLOC(driver) < 0) { VIR_FREE(priv); return -1; @@ -1813,12 +1858,13 @@ nodeStateInitialize(bool privileged, return -1; } + nodeDeviceLock(); + + if (!(driver->devs = virNodeDeviceObjListNew()) || + !(priv = udevEventDataNew())) + goto unlock; + driver->privateData = priv; - nodeDeviceLock(); - - if (!(driver->devs = virNodeDeviceObjListNew())) - goto unlock; - driver->nodeDeviceEventState = virObjectEventStateNew(); if (udevPCITranslateInit(privileged) < 0) @@ -1836,7 +1882,7 @@ nodeStateInitialize(bool privileged, #endif priv->udev_monitor = udev_monitor_new_from_netlink(udev, "udev"); - if (priv->udev_monitor == NULL) { + if (!priv->udev_monitor) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("udev_monitor_new_from_netlink returned NULL")); goto unlock; diff --git a/src/node_device/node_device_udev.h b/src/node_device/node_device_udev.h index 9a07ab77e..f15e5204c 100644 --- a/src/node_device/node_device_udev.h +++ b/src/node_device/node_device_udev.h @@ -23,9 +23,6 @@ #include <libudev.h> #include <stdint.h> -typedef struct _udevPrivate udevPrivate; - #define SYSFS_DATA_SIZE 4096 -#define DRV_STATE_UDEV_MONITOR(ds) (((udevPrivate *)((ds)->privateData))->udev_monitor) #define DMI_DEVPATH "/sys/devices/virtual/dmi/id" #define DMI_DEVPATH_FALLBACK "/sys/class/dmi/id" -- 2.13.6

[...] I like the notion of generic private data per backend, even though one of them doesn't use it all. But in principle, I agree with you, I'd just wait 'till we decide that we can safely deprecate hal backend in upstream completely. Good point, noted. libudev handles NULLs gracefully... I'll go with this version then, I did it this way, because udev keeps a pointer to the context withing the monitor, but doesn't touch it when freeing, leaving the responsibility to the caller. But I agree that releasing the monitor before releasing the context makes more sense. Good observation, a patch will be added. No problem. [..] This would only be necessary if that was a user's input, but it's us who's completely responsible for that and @data are only touched by stateCleanup, which runs after the eventloop ends, so this check would be essentially useless. this is called from within Initialize which holds the lock almost the whole time, so this would cause a deadlock (I only realized this when I tried it). So because what you say makes more sense, I'll add a separate patch that will move the object lock in Initialize before trying to set up the system node (there's no need to hold it the whole time) and then we can go with this suggestion. Great eyes ;). It's always mistakes like these that are so obvious yet they can be only spotted during a review :), thanks. Also, I appreciate the RBs but since I'll be adding 3 more patches (even though trivial ones) I'd still appreciate one more (final) review. Erik

On 10/11/2017 10:52 AM, Erik Skultety wrote: To go along with my thought from patch 2, should there be a "if (!priv)" check here too? I believe it doesn't matter yet because there are driver level locks that would seemingly prevent the thread code from running if Cleanup occurs, but soon enough that won't be the case. Reviewed-by: John Ferlan <jferlan(a)redhat.com&gt; John

Adjust udevEventHandleThread to be a proper thread routine running in an infinite loop handling devices. The handler thread pulls all available data from the udev monitor and only then waits until a wakeup signal for new incoming data has been emitted by udevEventHandleCallback. Signed-off-by: Erik Skultety <eskultet(a)redhat.com&gt; --- src/node_device/node_device_udev.c | 125 +++++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 32 deletions(-) diff --git a/src/node_device/node_device_udev.c b/src/node_device/node_device_udev.c index f7646cd8a..a6d7e6d70 100644 --- a/src/node_device/node_device_udev.c +++ b/src/node_device/node_device_udev.c @@ -62,6 +62,12 @@ struct _udevEventData { struct udev_monitor *udev_monitor; int watch; bool privileged; + + /* Thread data */ + virThread th; + virCond threadCond; + bool threadQuit; + bool dataReady; }; static virClassPtr udevEventDataClass; @@ -80,6 +86,8 @@ udevEventDataDispose(void *obj) udev_unref(udev); udev_monitor_unref(data->udev_monitor); + + virCondDestroy(&data->threadCond); } @@ -108,9 +116,15 @@ udevEventDataNew(void) if (!(ret = virObjectLockableNew(udevEventDataClass))) return NULL; + if (virCondInit(&ret->threadCond) < 0) { + virObjectUnref(ret); + return NULL; + } + ret->watch = -1; return ret; -} +}; + static bool @@ -1625,15 +1639,25 @@ udevPCITranslateDeinit(void) static int nodeStateCleanup(void) { + udevEventDataPtr priv = NULL; + if (!driver) return -1; + priv = driver->privateData; + nodeDeviceLock(); - virObjectUnref(driver->privateData); + virObjectLock(priv); + priv->threadQuit = true; + virCondSignal(&priv->threadCond); + virObjectUnlock(priv); + virThreadJoin(&priv->th); + + virObjectUnref(priv); virObjectUnref(driver->nodeDeviceEventState); - virNodeDeviceObjListFree(driver->devs); + nodeDeviceUnlock(); virMutexDestroy(&driver->lock); VIR_FREE(driver); @@ -1691,30 +1715,60 @@ udevEventMonitorSanityCheck(udevEventDataPtr priv, static void -udevEventHandleThread(void *opaque) +udevEventHandleThread(void *opaque ATTRIBUTE_UNUSED) { udevEventDataPtr priv = driver->privateData; - int fd = (intptr_t) opaque; struct udev_device *device = NULL; - virObjectLock(priv); + /* continue rather than break from the loop on non-fatal errors */ + while (1) { + virObjectLock(priv); + while (!priv->dataReady && !priv->threadQuit) { + if (virCondWait(&priv->threadCond, &priv->parent.lock)) { + virReportSystemError(errno, "%s", + _("handler failed to wait on condition")); + virObjectUnlock(priv); + return; + } + } - if (!udevEventMonitorSanityCheck(priv, fd)) { + if (priv->threadQuit) { + virObjectUnlock(priv); + return; + } + + errno = 0; + device = udev_monitor_receive_device(priv->udev_monitor); virObjectUnlock(priv); - return; - } - device = udev_monitor_receive_device(priv->udev_monitor); - virObjectUnlock(priv); + if (!device) { + if (errno == 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("udev_monitor_receive_device failed")); + return; + } - if (device == NULL) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("udev_monitor_receive_device returned NULL")); - return; - } + /* POSIX allows both EAGAIN and EWOULDBLOCK to be used + * interchangeably when the read would block or timeout was fired + */ + VIR_WARNINGS_NO_WLOGICALOP_EQUAL_EXPR + if (errno != EAGAIN && errno != EWOULDBLOCK) { + VIR_WARNINGS_RESET + virReportSystemError(errno, "%s", + _("udev_monitor_receive_device failed")); + return; + } + + virObjectLock(priv); + priv->dataReady = false; + virObjectUnlock(priv); - udevHandleOneDevice(device); - udev_device_unref(device); + continue; + } + + udevHandleOneDevice(device); + udev_device_unref(device); + } } @@ -1722,18 +1776,19 @@ static void udevEventHandleCallback(int watch ATTRIBUTE_UNUSED, int fd, int events ATTRIBUTE_UNUSED, - void *data ATTRIBUTE_UNUSED) + void *opaque ATTRIBUTE_UNUSED) { udevEventDataPtr priv = driver->privateData; virObjectLock(priv); - if (!udevEventMonitorSanityCheck(priv, fd)) { - virObjectUnlock(priv); - return; - } + + if (!udevEventMonitorSanityCheck(priv, fd)) + priv->threadQuit = true; + else + priv->dataReady = true; + + virCondSignal(&priv->threadCond); virObjectUnlock(priv); - - udevEventHandleThread((void *)(intptr_t) fd); } @@ -1875,29 +1930,29 @@ nodeStateInitialize(bool privileged, return -1; } - nodeDeviceLock(); - if (!(driver->devs = virNodeDeviceObjListNew()) || !(priv = udevEventDataNew())) - goto unlock; + goto cleanup; driver->privateData = priv; driver->nodeDeviceEventState = virObjectEventStateNew(); if (udevPCITranslateInit(privileged) < 0) - goto unlock; + goto cleanup; udev = udev_new(); if (!udev) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("failed to create udev context")); - goto unlock; + goto cleanup; } #if HAVE_UDEV_LOGGING /* cast to get rid of missing-format-attribute warning */ udev_set_log_fn(udev, (udevLogFunctionPtr) udevLogFunction); #endif + virObjectLock(priv); + priv->udev_monitor = udev_monitor_new_from_netlink(udev, "udev"); if (!priv->udev_monitor) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", @@ -1916,6 +1971,12 @@ nodeStateInitialize(bool privileged, 128 * 1024 * 1024); #endif + if (virThreadCreate(&priv->th, true, udevEventHandleThread, NULL) < 0) { + virReportSystemError(errno, "%s", + _("failed to create udev handler thread")); + goto unlock; + } + /* We register the monitor with the event callback so we are * notified by udev of device changes before we enumerate existing * devices because libvirt will simply recreate the device if we @@ -1934,7 +1995,7 @@ nodeStateInitialize(bool privileged, if (udevSetupSystemDev() != 0) goto unlock; - nodeDeviceUnlock(); + virObjectUnlock(priv); /* Populate with known devices */ if (udevEnumerateDevices(udev) != 0) @@ -1947,7 +2008,7 @@ nodeStateInitialize(bool privileged, return -1; unlock: - nodeDeviceUnlock(); + virObjectUnlock(priv); goto cleanup; } -- 2.13.6

On Sun, Oct 15, 2017 at 10:23:56AM -0400, John Ferlan wrote: Like I said, patch will be added. [...] Here it's actually quite possible, when allocation of the private data in Initialize fails, cleanup is performed, so this would get a SIGSEGV, thanks. Erik Yeah, I'll handle the locking prior to adding this thread-related stuff.