We are going to call qemuDomainDetachDeviceLive when usb device is unplugged from host. But later when DEVICE_DELETED event is delivered we need to keep device in libvirt config. For this purpuse let's save delete intention in device config. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/conf/domain_conf.h | 14 ++++++++++++++ src/qemu/qemu_driver.c | 4 ++-- src/qemu/qemu_hotplug.c | 23 ++++++++++++++++++++--- src/qemu/qemu_hotplug.h | 3 ++- tests/qemuhotplugtest.c | 2 +- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 33cef5b75c..19a5b21462 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -326,6 +326,19 @@ struct _virDomainHostdevCaps { } u; }; +typedef enum { + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_NONE = 0, + /* delete associated device from libvirt config + * as intended by client API call */ + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE, + /* keep associated device in libvirt config as + * qemu device is deleted as a result of unplugging + * device from host */ + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG, + + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_LAST +} virDomainHostdevDeleteActionType; + /* basic device for direct passthrough */ struct _virDomainHostdevDef { @@ -343,6 +356,7 @@ struct _virDomainHostdevDef { bool missing; bool readonly; bool shareable; + virDomainHostdevDeleteActionType deleteAction; union { virDomainHostdevSubsys subsys; virDomainHostdevCaps caps; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 78f5471b79..2378a2e7d0 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -9123,7 +9123,7 @@ qemuDomainDetachDeviceLiveAndConfig(virQEMUDriverPtr driver, if (flags & VIR_DOMAIN_AFFECT_LIVE) { int rc; - if ((rc = qemuDomainDetachDeviceLive(vm, dev_copy, driver, false)) < 0) + if ((rc = qemuDomainDetachDeviceLive(vm, dev_copy, driver, false, false)) < 0) goto cleanup; if (rc == 0 && qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0) @@ -9212,7 +9212,7 @@ qemuDomainDetachDeviceAliasLiveAndConfig(virQEMUDriverPtr driver, if (virDomainDefFindDevice(def, alias, &dev, true) < 0) goto cleanup; - if ((rc = qemuDomainDetachDeviceLive(vm, &dev, driver, true)) < 0) + if ((rc = qemuDomainDetachDeviceLive(vm, &dev, driver, true, false)) < 0) goto cleanup; if (rc == 0 && qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0) diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 63acb9c451..559254b234 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -5348,7 +5348,8 @@ qemuDomainDetachPrepController(virDomainObjPtr vm, static int qemuDomainDetachPrepHostdev(virDomainObjPtr vm, virDomainHostdevDefPtr match, - virDomainHostdevDefPtr *detach) + virDomainHostdevDefPtr *detach, + bool unplug) { virDomainHostdevSubsysPtr subsys = &match->source.subsys; virDomainHostdevSubsysUSBPtr usbsrc = &subsys->u.usb; @@ -5420,6 +5421,20 @@ qemuDomainDetachPrepHostdev(virDomainObjPtr vm, return -1; } + /* + * Why having additional check in second branch? Suppose client + * asks for device detaching and we delete device from qemu + * but don't get DEVICE_DELETED event yet. Next USB is unplugged + * from host and we have this function called again. If we reset + * delete action to 'unplug' then device will be left in + * libvirt config after handling DEVICE_DELETED event while + * it should not as client asked to detach the device before. + */ + if (!unplug) + hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE; + else if (hostdev->deleteAction != VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE) + hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG; + return 0; } @@ -5721,7 +5736,8 @@ int qemuDomainDetachDeviceLive(virDomainObjPtr vm, virDomainDeviceDefPtr match, virQEMUDriverPtr driver, - bool async) + bool async, + bool unplug) { virDomainDeviceDef detach = { .type = match->type }; virDomainDeviceInfoPtr info = NULL; @@ -5766,7 +5782,8 @@ qemuDomainDetachDeviceLive(virDomainObjPtr vm, break; case VIR_DOMAIN_DEVICE_HOSTDEV: if (qemuDomainDetachPrepHostdev(vm, match->data.hostdev, - &detach.data.hostdev) < 0) { + &detach.data.hostdev, + unplug) < 0) { return -1; } break; diff --git a/src/qemu/qemu_hotplug.h b/src/qemu/qemu_hotplug.h index 896e6c7b98..f7ebf3f505 100644 --- a/src/qemu/qemu_hotplug.h +++ b/src/qemu/qemu_hotplug.h @@ -116,7 +116,8 @@ int qemuDomainAttachRNGDevice(virQEMUDriverPtr driver, int qemuDomainDetachDeviceLive(virDomainObjPtr vm, virDomainDeviceDefPtr match, virQEMUDriverPtr driver, - bool async); + bool async, + bool unplug); void qemuDomainRemoveVcpuAlias(virQEMUDriverPtr driver, virDomainObjPtr vm, diff --git a/tests/qemuhotplugtest.c b/tests/qemuhotplugtest.c index b6aad330a9..ef91d4f131 100644 --- a/tests/qemuhotplugtest.c +++ b/tests/qemuhotplugtest.c @@ -151,7 +151,7 @@ testQemuHotplugDetach(virDomainObjPtr vm, case VIR_DOMAIN_DEVICE_CHR: case VIR_DOMAIN_DEVICE_SHMEM: case VIR_DOMAIN_DEVICE_WATCHDOG: - ret = qemuDomainDetachDeviceLive(vm, dev, &driver, async); + ret = qemuDomainDetachDeviceLive(vm, dev, &driver, async, false); break; default: VIR_TEST_VERBOSE("device type '%s' cannot be detached", -- 2.23.0

Handle host usb device unplug in DEVICE_DELETED handle execution path. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/qemu/qemu_hotplug.c | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 559254b234..b045735022 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -4366,7 +4366,8 @@ qemuDomainRemoveUSBHostDevice(virQEMUDriverPtr driver, virDomainHostdevDefPtr hostdev) { qemuHostdevReAttachUSBDevices(driver, vm->def->name, &hostdev, 1); - qemuDomainReleaseDeviceAddress(vm, hostdev->info); + if (hostdev->deleteAction != VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG) + qemuDomainReleaseDeviceAddress(vm, hostdev->info); } static void @@ -4408,6 +4409,7 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, char *drivealias = NULL; char *objAlias = NULL; bool is_vfio = false; + bool unplug = hostdev->deleteAction == VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG; VIR_DEBUG("Removing host device %s from domain %p %s", hostdev->info->alias, vm, vm->def->name); @@ -4454,16 +4456,24 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, } } - for (i = 0; i < vm->def->nhostdevs; i++) { - if (vm->def->hostdevs[i] == hostdev) { - virDomainHostdevRemove(vm->def, i); - break; + if (!unplug) { + for (i = 0; i < vm->def->nhostdevs; i++) { + if (vm->def->hostdevs[i] == hostdev) { + virDomainHostdevRemove(vm->def, i); + break; + } } } virDomainAuditHostdev(vm, hostdev, "detach", true); - if (!is_vfio && + /* + * In case of unplug the attempt to restore label will fail. But we don't + * need to restore the label! In case of separate mount namespace for the + * domain we remove device file later in this function. In case of global + * mount namespace the device file is deleted or being deleted by systemd. + */ + if (!is_vfio && !unplug && qemuSecurityRestoreHostdevLabel(driver, vm, hostdev) < 0) VIR_WARN("Failed to restore host device labelling"); @@ -4497,7 +4507,13 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, break; } - virDomainHostdevDefFree(hostdev); + if (unplug) { + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + usbsrc->bus = 0; + usbsrc->device = 0; + } else { + virDomainHostdevDefFree(hostdev); + } if (net) { if (net->type == VIR_DOMAIN_NET_TYPE_NETWORK) { @@ -4512,6 +4528,8 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, virDomainNetDefFree(net); } + hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_NONE; + ret = 0; cleanup: @@ -4981,6 +4999,7 @@ qemuDomainRemoveDevice(virQEMUDriverPtr driver, virDomainDeviceInfoPtr info; virObjectEventPtr event; VIR_AUTOFREE(char *) alias = NULL; + bool unplug; /* * save the alias to use when sending a DEVICE_REMOVED event after @@ -5021,8 +5040,13 @@ qemuDomainRemoveDevice(virQEMUDriverPtr driver, return -1; break; case VIR_DOMAIN_DEVICE_HOSTDEV: + unplug = dev->data.hostdev->deleteAction == VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG; + if (qemuDomainRemoveHostDevice(driver, vm, dev->data.hostdev) < 0) return -1; + + if (unplug) + return 0; break; case VIR_DOMAIN_DEVICE_RNG: if (qemuDomainRemoveRNGDevice(driver, vm, dev->data.rng) < 0) -- 2.23.0

We are going to use qemuDomainAttachHostUSBDevice when host usb device is plugged back to node. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/qemu/qemu_hotplug.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index b045735022..ea82cb54ef 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -2437,8 +2437,18 @@ qemuDomainAttachHostUSBDevice(virQEMUDriverPtr driver, bool teardownlabel = false; bool teardowndevice = false; int ret = -1; + bool replug; + size_t i; + + for (i = 0; i < vm->def->nhostdevs; i++) { + if (vm->def->hostdevs[i] == hostdev) { + replug = true; + break; + } + } - if (virDomainUSBAddressEnsure(priv->usbaddrs, hostdev->info) < 0) + if (!replug && + virDomainUSBAddressEnsure(priv->usbaddrs, hostdev->info) < 0) return -1; if (qemuHostdevPrepareUSBDevices(driver, vm->def->name, &hostdev, 1, 0) < 0) @@ -2463,7 +2473,7 @@ qemuDomainAttachHostUSBDevice(virQEMUDriverPtr driver, if (!(devstr = qemuBuildUSBHostdevDevStr(vm->def, hostdev, priv->qemuCaps))) goto cleanup; - if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs+1) < 0) + if (!replug && VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs+1) < 0) goto cleanup; qemuDomainObjEnterMonitor(driver, vm); @@ -2476,7 +2486,8 @@ qemuDomainAttachHostUSBDevice(virQEMUDriverPtr driver, if (ret < 0) goto cleanup; - vm->def->hostdevs[vm->def->nhostdevs++] = hostdev; + if (!replug) + vm->def->hostdevs[vm->def->nhostdevs++] = hostdev; ret = 0; cleanup: @@ -2489,9 +2500,17 @@ qemuDomainAttachHostUSBDevice(virQEMUDriverPtr driver, if (teardowndevice && qemuDomainNamespaceTeardownHostdev(vm, hostdev) < 0) VIR_WARN("Unable to remove host device from /dev"); - if (added) + if (added) { qemuHostdevReAttachUSBDevices(driver, vm->def->name, &hostdev, 1); - virDomainUSBAddressRelease(priv->usbaddrs, hostdev->info); + + if (replug) { + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + usbsrc->bus = 0; + usbsrc->device = 0; + } + } + if (!replug) + virDomainUSBAddressRelease(priv->usbaddrs, hostdev->info); } VIR_FREE(devstr); return ret; -- 2.23.0

Now when code handling attaching/detaching usb hostdev is appropriately changed use it to handle host usb device udev add/del events. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/qemu/Makefile.inc.am | 2 + src/qemu/qemu_conf.h | 3 + src/qemu/qemu_domain.c | 2 + src/qemu/qemu_domain.h | 2 + src/qemu/qemu_driver.c | 351 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 359 insertions(+), 1 deletion(-) diff --git a/src/qemu/Makefile.inc.am b/src/qemu/Makefile.inc.am index d16b315ebc..8be0dee396 100644 --- a/src/qemu/Makefile.inc.am +++ b/src/qemu/Makefile.inc.am @@ -85,6 +85,7 @@ libvirt_driver_qemu_impl_la_CFLAGS = \ -I$(srcdir)/conf \ -I$(srcdir)/secret \ $(AM_CFLAGS) \ + $(UDEV_CFLAGS) \ $(NULL) libvirt_driver_qemu_impl_la_LDFLAGS = $(AM_LDFLAGS) libvirt_driver_qemu_impl_la_LIBADD = \ @@ -93,6 +94,7 @@ libvirt_driver_qemu_impl_la_LIBADD = \ $(LIBNL_LIBS) \ $(SELINUX_LIBS) \ $(LIBXML_LIBS) \ + $(UDEV_LIBS) \ $(NULL) libvirt_driver_qemu_impl_la_SOURCES = $(QEMU_DRIVER_SOURCES) diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index 0cbddd7a9c..2e50bb0950 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -294,6 +294,9 @@ struct _virQEMUDriver { /* Immutable pointer, self-locking APIs */ virHashAtomicPtr migrationErrors; + + struct udev_monitor *udev_monitor; + int udev_watch; }; virQEMUDriverConfigPtr virQEMUDriverConfigNew(bool privileged); diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 657f3ecfe4..4784804d1e 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -15034,6 +15034,8 @@ qemuProcessEventFree(struct qemuProcessEvent *event) case QEMU_PROCESS_EVENT_SERIAL_CHANGED: case QEMU_PROCESS_EVENT_BLOCK_JOB: case QEMU_PROCESS_EVENT_MONITOR_EOF: + case QEMU_PROCESS_EVENT_USB_REMOVED: + case QEMU_PROCESS_EVENT_USB_ADDED: VIR_FREE(event->data); break; case QEMU_PROCESS_EVENT_JOB_STATUS_CHANGE: diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index d097f23342..94aea62693 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -521,6 +521,8 @@ typedef enum { QEMU_PROCESS_EVENT_MONITOR_EOF, QEMU_PROCESS_EVENT_PR_DISCONNECT, QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED, + QEMU_PROCESS_EVENT_USB_REMOVED, + QEMU_PROCESS_EVENT_USB_ADDED, QEMU_PROCESS_EVENT_LAST } qemuProcessEventType; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 2378a2e7d0..ce41b0a8d9 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -34,6 +34,7 @@ #include <sys/ioctl.h> #include <sys/un.h> #include <byteswap.h> +#include <libudev.h> #include "qemu_driver.h" @@ -719,6 +720,254 @@ qemuDomainFindMaxID(virDomainObjPtr vm, } +struct qemuUdevUSBRemoveData { + unsigned int bus; + unsigned int device; +}; + +struct qemuUdevUSBAddData { + unsigned int vendor; + unsigned int product; +}; + +struct qemuUdevUSBEventData { + union { + struct qemuUdevUSBRemoveData remove; + struct qemuUdevUSBAddData add; + } data; + bool found; + bool remove; +}; + +static int +qemuUdevUSBHandleEvent(virDomainObjPtr vm, void *opaque) +{ + struct qemuUdevUSBEventData *data = opaque; + struct qemuProcessEvent *event = NULL; + size_t i; + + if (data->found) + return 0; + + virObjectLock(vm); + + if (!virDomainObjIsActive(vm)) + goto cleanup; + + for (i = 0; i < vm->def->nhostdevs; i++) { + virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i]; + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + if (data->remove) { + if (usbsrc->bus != data->data.remove.bus || + usbsrc->device != data->data.remove.device) + continue; + } else { + if (usbsrc->vendor != data->data.add.vendor || + usbsrc->product != data->data.add.product) + continue; + } + + data->found = true; + + if (VIR_ALLOC(event) < 0) + goto cleanup; + + if (data->remove) { + struct qemuUdevUSBRemoveData *rm_data; + + + if (VIR_ALLOC(rm_data) < 0) + goto cleanup; + + *rm_data = data->data.remove; + event->data = rm_data; + event->eventType = QEMU_PROCESS_EVENT_USB_REMOVED; + } else { + struct qemuUdevUSBAddData *add_data; + + if (VIR_ALLOC(add_data) < 0) + goto cleanup; + + *add_data = data->data.add; + event->data = add_data; + event->eventType = QEMU_PROCESS_EVENT_USB_ADDED; + } + + event->vm = virObjectRef(vm); + + if (virThreadPoolSendJob(qemu_driver->workerPool, 0, event) < 0) { + virObjectUnref(vm); + goto cleanup; + } + + event = NULL; + + break; + } + + cleanup: + virObjectUnlock(vm); + + qemuProcessEventFree(event); + + return 0; +} + + +static void +qemuUdevEventHandleCallback(int watch ATTRIBUTE_UNUSED, + int fd ATTRIBUTE_UNUSED, + int events ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + struct qemuUdevUSBEventData event_data; + struct udev_device *dev = NULL; + const char *action; + const char *devtype; + const char *tmp; + + /* libvirtd daemon do not run event loop before full state drivers + * initialization. Also state drivers uninitialized only after + * full stop of event loop. In short driver initialization/uninitialization + * and handling events occurs in same main loop thread. Thus we + * don't need any locking here. */ + + if (!(dev = udev_monitor_receive_device(qemu_driver->udev_monitor))) { + VIR_WARNINGS_NO_WLOGICALOP_EQUAL_EXPR + if (errno == EAGAIN || errno == EWOULDBLOCK) { + VIR_WARNINGS_RESET + return; + } + + virReportSystemError(errno, "%s", + _("failed to receive device from udev monitor")); + return; + } + + devtype = udev_device_get_devtype(dev); + + if (STRNEQ_NULLABLE(devtype, "usb_device")) + goto cleanup; + + if (!(action = udev_device_get_action(dev))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive action from udev monitor")); + goto cleanup; + } + + if (STREQ(action, "remove")) { + struct qemuUdevUSBRemoveData *rm_data = &event_data.data.remove; + + if (!(tmp = udev_device_get_property_value(dev, "BUSNUM"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive busnum from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 10, &rm_data->bus) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert busnum to int")); + goto cleanup; + } + + if (!(tmp = udev_device_get_property_value(dev, "DEVNUM"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive devnum from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 10, &rm_data->device) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert devnum to int")); + goto cleanup; + } + event_data.remove = true; + } else if (STREQ(action, "add")) { + struct qemuUdevUSBAddData *add_data = &event_data.data.add; + + if (!(tmp = udev_device_get_property_value(dev, "ID_VENDOR_ID"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive vendor from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 16, &add_data->vendor) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert vendor to int")); + goto cleanup; + } + + if (!(tmp = udev_device_get_property_value(dev, "ID_MODEL_ID"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive product from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 16, &add_data->product) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert product to int")); + goto cleanup; + } + event_data.remove = false; + } + + event_data.found = false; + virDomainObjListForEach(qemu_driver->domains, qemuUdevUSBHandleEvent, &event_data); + + cleanup: + udev_device_unref(dev); +} + + +static int +qemuUdevInitialize(void) +{ + struct udev *udev; + + if (!(udev = udev_new())) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to create udev context")); + return -1; + } + + if (!(qemu_driver->udev_monitor = udev_monitor_new_from_netlink(udev, "udev"))) { + udev_unref(udev); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("udev_monitor_new_from_netlink returned NULL")); + return -1; + } + + udev_monitor_enable_receiving(qemu_driver->udev_monitor); + + qemu_driver->udev_watch = virEventAddHandle(udev_monitor_get_fd(qemu_driver->udev_monitor), + VIR_EVENT_HANDLE_READABLE, + qemuUdevEventHandleCallback, NULL, NULL); + + if (qemu_driver->udev_watch < 0) + return -1; + + return 0; +} + + +static void +qemuUdevCleanup(void) +{ + if (qemu_driver->udev_monitor) { + struct udev *udev = udev_monitor_get_udev(qemu_driver->udev_monitor); + + udev_monitor_unref(qemu_driver->udev_monitor); + udev_unref(udev); + qemu_driver->udev_monitor = NULL; + } + + if (qemu_driver->udev_watch > 0) { + virEventRemoveHandle(qemu_driver->udev_watch); + qemu_driver->udev_watch = 0; + } +} + + /** * qemuStateInitialize: * @@ -1030,6 +1279,9 @@ qemuStateInitialize(bool privileged, if (!(qemu_driver->closeCallbacks = virCloseCallbacksNew())) goto error; + if (qemuUdevInitialize() < 0) + goto error; + /* Get all the running persistent or transient configs first */ if (virDomainObjListLoadAllConfigs(qemu_driver->domains, cfg->stateDir, @@ -1239,6 +1491,8 @@ qemuStateCleanup(void) virLockManagerPluginUnref(qemu_driver->lockManager); + qemuUdevCleanup(); + virMutexDestroy(&qemu_driver->lock); VIR_FREE(qemu_driver); @@ -5011,7 +5265,96 @@ processRdmaGidStatusChangedEvent(virDomainObjPtr vm, } -static void qemuProcessEventHandler(void *data, void *opaque) +static void +processUSBAddedEvent(virQEMUDriverPtr driver, + virDomainObjPtr vm, + struct qemuUdevUSBAddData *data) +{ + virDomainHostdevDefPtr hostdev; + virDomainHostdevSubsysUSBPtr usbsrc; + size_t i; + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + return; + + if (!virDomainObjIsActive(vm)) { + VIR_DEBUG("Domain is not running"); + goto cleanup; + } + + for (i = 0; i < vm->def->nhostdevs; i++) { + hostdev = vm->def->hostdevs[i]; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + usbsrc = &hostdev->source.subsys.u.usb; + + if (usbsrc->vendor == data->vendor && usbsrc->product == data->product) + break; + } + + if (i == vm->def->nhostdevs) + goto cleanup; + + if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) + goto cleanup; + + cleanup: + qemuDomainObjEndJob(driver, vm); +} + + +static void +processUSBRemovedEvent(virQEMUDriverPtr driver, + virDomainObjPtr vm, + struct qemuUdevUSBRemoveData *data) +{ + size_t i; + virDomainHostdevDefPtr hostdev; + virDomainDeviceDef dev = { .type = VIR_DOMAIN_DEVICE_HOSTDEV }; + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + return; + + if (!virDomainObjIsActive(vm)) { + VIR_DEBUG("Domain is not running"); + goto cleanup; + } + + for (i = 0; i < vm->def->nhostdevs; i++) { + virDomainHostdevSubsysUSBPtr usbsrc; + + hostdev = vm->def->hostdevs[i]; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + usbsrc = &hostdev->source.subsys.u.usb; + + /* don't mess with devices that don't use stable host addressing + * with respect to unplug/plug to host + */ + if (!usbsrc->vendor || !usbsrc->product) + continue; + + if (usbsrc->bus == data->bus && usbsrc->device == data->device) + break; + } + + if (i == vm->def->nhostdevs) + goto cleanup; + + dev.data.hostdev = hostdev; + if (qemuDomainDetachDeviceLive(vm, &dev, driver, true, true) < 0) + goto cleanup; + + cleanup: + qemuDomainObjEndJob(driver, vm); +} + + +static void +qemuProcessEventHandler(void *data, void *opaque) { struct qemuProcessEvent *processEvent = data; virDomainObjPtr vm = processEvent->vm; @@ -5057,6 +5400,12 @@ static void qemuProcessEventHandler(void *data, void *opaque) case QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED: processRdmaGidStatusChangedEvent(vm, processEvent->data); break; + case QEMU_PROCESS_EVENT_USB_REMOVED: + processUSBRemovedEvent(driver, vm, processEvent->data); + break; + case QEMU_PROCESS_EVENT_USB_ADDED: + processUSBAddedEvent(driver, vm, processEvent->data); + break; case QEMU_PROCESS_EVENT_LAST: break; } -- 2.23.0

On Mon, Sep 09, 2019 at 14:33:07 +0300, Nikolay Shirokovskiy wrote: I don't see any XML change related to this in this series. IMO it's unacceptable to re-plug ANY device by default and we must introduce a flag where the admin gives explicit consent for a device to be re-plugged on the host hotplug event. You must note that these two instances may be long time apart and thus the user might not notice that the device is re-grabbed by the VM. Also due to the nature of USB devices it may be a completely different device (e.g. a USB Flash drive of the same make/model but with different data). Allowing this by default would lead to confusion. Is this executed from the event loop thread? If yes we can't do this as qemuUdevUSBHandleEvent is taking domain locks and thus potentially waiting indefinitely for any stuck VM. AFAIK this will remove the definition form the XML so how is the re-plug going to be facilitated then?

It is possible for libvirtd to go down before DEVICE_DELETED event is delivered upon usb hostdev unplug and to receive the event after the libvirtd is up. In order to handle this case we need to save usb hostdev deleteAction is status file. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/conf/domain_conf.c | 26 ++++++++++++++++++++++++++ src/conf/domain_conf.h | 1 + src/qemu/qemu_driver.c | 20 ++++++++++++++++++-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index b7a342bb91..c200af050c 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -1236,6 +1236,13 @@ VIR_ENUM_IMPL(virDomainShmemModel, "ivshmem-doorbell", ); +VIR_ENUM_IMPL(virDomainHostdevDeleteAction, + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_LAST, + "none", + "delete", + "unplug" +); + VIR_ENUM_IMPL(virDomainLaunchSecurity, VIR_DOMAIN_LAUNCH_SECURITY_LAST, "", @@ -7533,6 +7540,7 @@ virDomainHostdevSubsysUSBDefParseXML(xmlNodePtr node, virDomainHostdevSubsysUSBPtr usbsrc = &def->source.subsys.u.usb; VIR_AUTOFREE(char *) startupPolicy = NULL; VIR_AUTOFREE(char *) autoAddress = NULL; + VIR_AUTOFREE(char *) deleteAction = NULL; if ((startupPolicy = virXMLPropString(node, "startupPolicy"))) { def->startupPolicy = @@ -7550,6 +7558,18 @@ virDomainHostdevSubsysUSBDefParseXML(xmlNodePtr node, usbsrc->autoAddress = true; } + if ((deleteAction = virXMLPropString(node, "deleteAction"))) { + def->deleteAction = + virDomainHostdevDeleteActionTypeFromString(deleteAction); + + if (def->deleteAction <= 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Unknown deleteAction '%s'"), + deleteAction); + goto out; + } + } + /* Product can validly be 0, so we need some extra help to determine * if it is uninitialized*/ got_product = false; @@ -24905,6 +24925,12 @@ virDomainHostdevDefFormatSubsys(virBufferPtr buf, if (def->missing && !(flags & VIR_DOMAIN_DEF_FORMAT_INACTIVE)) virBufferAddLit(buf, " missing='yes'"); + + if (def->deleteAction && (flags & VIR_DOMAIN_DEF_FORMAT_STATUS)) { + const char *deleteAction; + deleteAction = virDomainHostdevDeleteActionTypeToString(def->deleteAction); + virBufferAsprintf(buf, " deleteAction='%s'", deleteAction); + } } if (def->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI && diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 19a5b21462..df88790ac0 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -338,6 +338,7 @@ typedef enum { VIR_DOMAIN_HOSTDEV_DELETE_ACTION_LAST } virDomainHostdevDeleteActionType; +VIR_ENUM_DECL(virDomainHostdevDeleteAction); /* basic device for direct passthrough */ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index ce41b0a8d9..f1802b5d44 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -5272,10 +5272,13 @@ processUSBAddedEvent(virQEMUDriverPtr driver, { virDomainHostdevDefPtr hostdev; virDomainHostdevSubsysUSBPtr usbsrc; + virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver); size_t i; - if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) { + virObjectUnref(cfg); return; + } if (!virDomainObjIsActive(vm)) { VIR_DEBUG("Domain is not running"); @@ -5300,8 +5303,13 @@ processUSBAddedEvent(virQEMUDriverPtr driver, if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) goto cleanup; + if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm, driver->caps) < 0) + VIR_WARN("unable to save domain status after plugging device %s", + hostdev->info->alias); + cleanup: qemuDomainObjEndJob(driver, vm); + virObjectUnref(cfg); } @@ -5313,9 +5321,12 @@ processUSBRemovedEvent(virQEMUDriverPtr driver, size_t i; virDomainHostdevDefPtr hostdev; virDomainDeviceDef dev = { .type = VIR_DOMAIN_DEVICE_HOSTDEV }; + virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver); - if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) { + virObjectUnref(cfg); return; + } if (!virDomainObjIsActive(vm)) { VIR_DEBUG("Domain is not running"); @@ -5348,8 +5359,13 @@ processUSBRemovedEvent(virQEMUDriverPtr driver, if (qemuDomainDetachDeviceLive(vm, &dev, driver, true, true) < 0) goto cleanup; + if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm, driver->caps) < 0) + VIR_WARN("unable to save domain status after unplugging device %s", + hostdev->info->alias); + cleanup: qemuDomainObjEndJob(driver, vm); + virObjectUnref(cfg); } -- 2.23.0

Imagine host usb device is unplugged from host and as a result we send command to qemu to delete appropriate device. Then before qemu device is deleted host usb device is plugged back. Currenly code supposes there is no remnant device in qemu and will try to add new device and the attempt will fail. Instead let's check the device is not yet deleted and postpone adding qemu device to device_deleted event handler. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/qemu/qemu_driver.c | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index f1802b5d44..21640e49c7 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -4671,6 +4671,44 @@ processGuestPanicEvent(virQEMUDriverPtr driver, } +static int +qemuCheckHostdevPlugged(virQEMUDriverPtr driver, + virDomainObjPtr vm, + const char *devAlias) +{ + virDomainHostdevDefPtr hostdev; + virDomainHostdevSubsysUSBPtr usbsrc; + virDomainDeviceDef dev; + int num; + + if (virDomainDefFindDevice(vm->def, devAlias, &dev, false) < 0) + return 0; + + if (dev.type != VIR_DOMAIN_DEVICE_HOSTDEV) + return 0; + + hostdev = dev.data.hostdev; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + return 0; + + usbsrc = &hostdev->source.subsys.u.usb; + if (!usbsrc->vendor || !usbsrc->product) + return 0; + + if ((num = virUSBDeviceFindByVendor(usbsrc->vendor, usbsrc->product, + NULL, false, NULL)) < 0) + return -1; + + if (num == 0) + return 0; + + if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) + return -1; + + return 0; +} + + static void processDeviceDeletedEvent(virQEMUDriverPtr driver, virDomainObjPtr vm, @@ -4698,6 +4736,11 @@ processDeviceDeletedEvent(virQEMUDriverPtr driver, if (qemuDomainRemoveDevice(driver, vm, &dev) < 0) goto endjob; + + /* Fall thru and save status file even on error condition because + * device is removed successfully and changed configuration need + * to be saved in status file. */ + qemuCheckHostdevPlugged(driver, vm, devAlias); } if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm, driver->caps) < 0) @@ -5300,6 +5343,12 @@ processUSBAddedEvent(virQEMUDriverPtr driver, if (i == vm->def->nhostdevs) goto cleanup; + /* if device is not yet even deleted from qemu then handle plugging later. + * Or we failed handling host usb device unplugging, then another attempt of + * unplug/plug could help. */ + if (usbsrc->bus || usbsrc->device) + goto cleanup; + if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) goto cleanup; -- 2.23.0

On 12.09.2019 23:37, Daniel Henrique Barboza wrote: Yeah this is because remove() in case of unplugging does not actually remove device from libvirt config, so both find()s find same device. Nikolay

Somebody can easily unplug usb device from host while libvirtd is being stopped. Also usb device can be plugged or unplugged/plugged back and so forth. Let's handle such cases. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/qemu/qemu_process.c | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index c9921646e9..8bec36fe2c 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3749,6 +3749,58 @@ qemuProcessUpdateDevices(virQEMUDriverPtr driver, return ret; } + +static int +qemuProcessReattachUSBDevices(virQEMUDriverPtr driver, + virDomainObjPtr vm) +{ + size_t i; + + for (i = 0; i < vm->def->nhostdevs; i++) { + virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i]; + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + /* don't mess with devices that don't use stable host addressing + * with respect to unplug/plug to host + */ + if (!usbsrc->vendor || !usbsrc->product) + continue; + + if (!usbsrc->bus && !usbsrc->device) { + int num; + + if ((num = virUSBDeviceFindByVendor(usbsrc->vendor, usbsrc->product, + NULL, false, NULL)) < 0) + return -1; + + if (num > 0 && + qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) + return -1; + } else { + virUSBDevicePtr usb; + + if (virUSBDeviceFindByBus(usbsrc->bus, usbsrc->device, + NULL, false, &usb) < 0) + return -1; + + if (!usb) { + virDomainDeviceDef dev = { .type = VIR_DOMAIN_DEVICE_HOSTDEV }; + + dev.data.hostdev = hostdev; + if (qemuDomainDetachDeviceLive(vm, &dev, driver, true, true) < 0) + return -1; + } + virUSBDeviceFree(usb); + } + } + + return 0; +} + + static int qemuDomainPerfRestart(virDomainObjPtr vm) { @@ -8206,6 +8258,9 @@ qemuProcessReconnect(void *opaque) if (qemuProcessUpdateDevices(driver, obj) < 0) goto error; + if (qemuProcessReattachUSBDevices(driver, obj) < 0) + goto error; + if (qemuRefreshPRManagerState(driver, obj) < 0) goto error; -- 2.23.0

First I don't want to add code to handle dummy device that is used when host usb device is not present at the moment of starting/migrating etc. Second supporting non mandatory policies would require to handle races when host usb device is plugged to host and libvirtd starts adding device but if in the meanwhile host usb device it unplugged back then current code will use dummy device which is not desired in this case. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/qemu/qemu_driver.c | 8 ++++++++ src/qemu/qemu_process.c | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index b646642c99..fe5fd94ac5 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -5337,6 +5337,10 @@ processUSBAddedEvent(virQEMUDriverPtr driver, if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) continue; + if (hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_OPTIONAL || + hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_REQUISITE) + continue; + usbsrc = &hostdev->source.subsys.u.usb; if (usbsrc->vendor == data->vendor && usbsrc->product == data->product) @@ -5392,6 +5396,10 @@ processUSBRemovedEvent(virQEMUDriverPtr driver, if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) continue; + if (hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_OPTIONAL || + hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_REQUISITE) + continue; + usbsrc = &hostdev->source.subsys.u.usb; /* don't mess with devices that don't use stable host addressing diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 8bec36fe2c..d87fb637ac 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3769,6 +3769,10 @@ qemuProcessReattachUSBDevices(virQEMUDriverPtr driver, if (!usbsrc->vendor || !usbsrc->product) continue; + if (hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_OPTIONAL || + hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_REQUISITE) + continue; + if (!usbsrc->bus && !usbsrc->device) { int num; -- 2.23.0

If hostdev is unplugged we don't need to delete it's correspondent device from qemu etc. Just remove it from config immediately. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/qemu/qemu_hotplug.c | 16 +++++++++++++++- src/util/virhostdev.c | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index ea82cb54ef..cd1c1216fa 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -4429,6 +4429,7 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, char *objAlias = NULL; bool is_vfio = false; bool unplug = hostdev->deleteAction == VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG; + bool unplugged = false; VIR_DEBUG("Removing host device %s from domain %p %s", hostdev->info->alias, vm, vm->def->name); @@ -4486,13 +4487,17 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, virDomainAuditHostdev(vm, hostdev, "detach", true); + if (hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) { + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + unplugged = !usbsrc->bus && !usbsrc->device; + } /* * In case of unplug the attempt to restore label will fail. But we don't * need to restore the label! In case of separate mount namespace for the * domain we remove device file later in this function. In case of global * mount namespace the device file is deleted or being deleted by systemd. */ - if (!is_vfio && !unplug && + if (!is_vfio && !unplug && !unplugged && qemuSecurityRestoreHostdevLabel(driver, vm, hostdev) < 0) VIR_WARN("Failed to restore host device labelling"); @@ -5829,6 +5834,15 @@ qemuDomainDetachDeviceLive(virDomainObjPtr vm, unplug) < 0) { return -1; } + if (!unplug) { + virDomainHostdevDefPtr hostdev = detach.data.hostdev; + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + + if (hostdev->startupPolicy != VIR_DOMAIN_STARTUP_POLICY_OPTIONAL && + hostdev->startupPolicy != VIR_DOMAIN_STARTUP_POLICY_REQUISITE && + usbsrc->device == 0 && usbsrc->bus == 0) + return qemuDomainRemoveDevice(driver, vm, &detach); + } break; case VIR_DOMAIN_DEVICE_RNG: if (qemuDomainDetachPrepRNG(vm, match->data.rng, diff --git a/src/util/virhostdev.c b/src/util/virhostdev.c index d710193b94..90e4d2fb45 100644 --- a/src/util/virhostdev.c +++ b/src/util/virhostdev.c @@ -1794,6 +1794,8 @@ virHostdevReAttachUSBDevices(virHostdevManagerPtr mgr, continue; if (hostdev->missing) continue; + if (!usbsrc->bus && !usbsrc->device) + continue; if (!(usb = virUSBDeviceNew(usbsrc->bus, usbsrc->device, NULL))) { VIR_WARN("Unable to reattach USB device %03d.%03d on domain %s", -- 2.23.0

On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote: Reviewed-by: Daniel Henrique Barboza <danielhb413(a)gmail.com&gt;