I had to amend the patch to apply it to the current master (commit e9d51a221c). git am was putting the 'if (!unplug)' block in the wrong function and the code didn't compile. Here's what I did: ------- $ git diff diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 96b9ff998d..08ad8ef8cc 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -5458,20 +5458,6 @@ qemuDomainDetachPrepController(virDomainObjPtr vm,          return -1;      } -    /* -     * Why having additional check in second branch? Suppose client -     * asks for device detaching and we delete device from qemu -     * but don't get DEVICE_DELETED event yet. Next USB is unplugged -     * from host and we have this function called again. If we reset -     * delete action to 'unplug' then device will be left in -     * libvirt config after handling DEVICE_DELETED event while -     * it should not as client asked to detach the device before. -     */ -     */ -    if (!unplug) -        hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE; -    else if (hostdev->deleteAction != VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE) -        hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG; -      return 0;  } @@ -5553,6 +5539,20 @@ qemuDomainDetachPrepHostdev(virDomainObjPtr vm,          return -1;      } +    /* +     * Why having additional check in second branch? Suppose client +     * asks for device detaching and we delete device from qemu +     * but don't get DEVICE_DELETED event yet. Next USB is unplugged +     * from host and we have this function called again. If we reset +     * delete action to 'unplug' then device will be left in +     * libvirt config after handling DEVICE_DELETED event while +     * it should not as client asked to detach the device before. +     */ +    if (!unplug) +        hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE; +    else if (hostdev->deleteAction != VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE) +        hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG; +      return 0;  } diff --git a/tests/qemuhotplugtest.c b/tests/qemuhotplugtest.c index 3c177c6622..4e7851aa62 100644 --- a/tests/qemuhotplugtest.c +++ b/tests/qemuhotplugtest.c @@ -158,7 +158,7 @@ testQemuHotplugDetach(virDomainObjPtr vm,      case VIR_DOMAIN_DEVICE_SHMEM:      case VIR_DOMAIN_DEVICE_WATCHDOG:      case VIR_DOMAIN_DEVICE_HOSTDEV: -        ret = qemuDomainDetachDeviceLive(vm, dev, &driver, async); +        ret = qemuDomainDetachDeviceLive(vm, dev, &driver, async, false);          break;      default:          VIR_TEST_VERBOSE("device type '%s' cannot be detached", ------- A few more nits below: On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote:

We are going to call qemuDomainDetachDeviceLive when usb device is unplugged from host. But later when DEVICE_DELETED event is delivered we need to keep device in libvirt config. For this purpuse let's save

s/purpuse/purpose

delete intention in device config.

There's an explanation of why are you making this change in the comment block you put in qemuDomainDetachPrepHostdev (and across the other commit messages of the series). It would be good to put more a bit more info about the motivations in this first commit message as well (like you did in patch 06), since this patch is the backbone of the design change you're making.

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/conf/domain_conf.h | 14 ++++++++++++++ src/qemu/qemu_driver.c | 4 ++-- src/qemu/qemu_hotplug.c | 23 ++++++++++++++++++++--- src/qemu/qemu_hotplug.h | 3 ++- tests/qemuhotplugtest.c | 2 +- 5 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 33cef5b75c..19a5b21462 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -326,6 +326,19 @@ struct _virDomainHostdevCaps { } u; };

+typedef enum { + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_NONE = 0, + /* delete associated device from libvirt config + * as intended by client API call */ + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE, + /* keep associated device in libvirt config as + * qemu device is deleted as a result of unplugging + * device from host */ + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG, + + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_LAST +} virDomainHostdevDeleteActionType; +

/* basic device for direct passthrough */ struct _virDomainHostdevDef { @@ -343,6 +356,7 @@ struct _virDomainHostdevDef { bool missing; bool readonly; bool shareable; + virDomainHostdevDeleteActionType deleteAction; union { virDomainHostdevSubsys subsys; virDomainHostdevCaps caps; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 78f5471b79..2378a2e7d0 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -9123,7 +9123,7 @@ qemuDomainDetachDeviceLiveAndConfig(virQEMUDriverPtr driver, if (flags & VIR_DOMAIN_AFFECT_LIVE) { int rc;

- if ((rc = qemuDomainDetachDeviceLive(vm, dev_copy, driver, false)) < 0) + if ((rc = qemuDomainDetachDeviceLive(vm, dev_copy, driver, false, false)) < 0) goto cleanup;

if (rc == 0 && qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0) @@ -9212,7 +9212,7 @@ qemuDomainDetachDeviceAliasLiveAndConfig(virQEMUDriverPtr driver, if (virDomainDefFindDevice(def, alias, &dev, true) < 0) goto cleanup;

- if ((rc = qemuDomainDetachDeviceLive(vm, &dev, driver, true)) < 0) + if ((rc = qemuDomainDetachDeviceLive(vm, &dev, driver, true, false)) < 0) goto cleanup;

if (rc == 0 && qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0) diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 63acb9c451..559254b234 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -5348,7 +5348,8 @@ qemuDomainDetachPrepController(virDomainObjPtr vm, static int qemuDomainDetachPrepHostdev(virDomainObjPtr vm, virDomainHostdevDefPtr match, - virDomainHostdevDefPtr *detach) + virDomainHostdevDefPtr *detach, + bool unplug) { virDomainHostdevSubsysPtr subsys = &match->source.subsys; virDomainHostdevSubsysUSBPtr usbsrc = &subsys->u.usb; @@ -5420,6 +5421,20 @@ qemuDomainDetachPrepHostdev(virDomainObjPtr vm, return -1; }

+ /* + * Why having additional check in second branch? Suppose client + * asks for device detaching and we delete device from qemu + * but don't get DEVICE_DELETED event yet. Next USB is unplugged + * from host and we have this function called again. If we reset + * delete action to 'unplug' then device will be left in + * libvirt config after handling DEVICE_DELETED event while + * it should not as client asked to detach the device before. + */ + if (!unplug) + hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE; + else if (hostdev->deleteAction != VIR_DOMAIN_HOSTDEV_DELETE_ACTION_DELETE) + hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG; + return 0; }

@@ -5721,7 +5736,8 @@ int qemuDomainDetachDeviceLive(virDomainObjPtr vm, virDomainDeviceDefPtr match, virQEMUDriverPtr driver, - bool async) + bool async, + bool unplug) { virDomainDeviceDef detach = { .type = match->type }; virDomainDeviceInfoPtr info = NULL; @@ -5766,7 +5782,8 @@ qemuDomainDetachDeviceLive(virDomainObjPtr vm, break; case VIR_DOMAIN_DEVICE_HOSTDEV: if (qemuDomainDetachPrepHostdev(vm, match->data.hostdev, - &detach.data.hostdev) < 0) { + &detach.data.hostdev, + unplug) < 0) { return -1; } break; diff --git a/src/qemu/qemu_hotplug.h b/src/qemu/qemu_hotplug.h index 896e6c7b98..f7ebf3f505 100644 --- a/src/qemu/qemu_hotplug.h +++ b/src/qemu/qemu_hotplug.h @@ -116,7 +116,8 @@ int qemuDomainAttachRNGDevice(virQEMUDriverPtr driver, int qemuDomainDetachDeviceLive(virDomainObjPtr vm, virDomainDeviceDefPtr match, virQEMUDriverPtr driver, - bool async); + bool async, + bool unplug);

void qemuDomainRemoveVcpuAlias(virQEMUDriverPtr driver, virDomainObjPtr vm, diff --git a/tests/qemuhotplugtest.c b/tests/qemuhotplugtest.c index b6aad330a9..ef91d4f131 100644 --- a/tests/qemuhotplugtest.c +++ b/tests/qemuhotplugtest.c @@ -151,7 +151,7 @@ testQemuHotplugDetach(virDomainObjPtr vm, case VIR_DOMAIN_DEVICE_CHR: case VIR_DOMAIN_DEVICE_SHMEM: case VIR_DOMAIN_DEVICE_WATCHDOG: - ret = qemuDomainDetachDeviceLive(vm, dev, &driver, async); + ret = qemuDomainDetachDeviceLive(vm, dev, &driver, async, false); break; default: VIR_TEST_VERBOSE("device type '%s' cannot be detached",

On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote:

Handle host usb device unplug in DEVICE_DELETED handle execution path.

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> ---

Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

src/qemu/qemu_hotplug.c | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 559254b234..b045735022 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -4366,7 +4366,8 @@ qemuDomainRemoveUSBHostDevice(virQEMUDriverPtr driver, virDomainHostdevDefPtr hostdev) { qemuHostdevReAttachUSBDevices(driver, vm->def->name, &hostdev, 1); - qemuDomainReleaseDeviceAddress(vm, hostdev->info); + if (hostdev->deleteAction != VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG) + qemuDomainReleaseDeviceAddress(vm, hostdev->info); }

static void @@ -4408,6 +4409,7 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, char *drivealias = NULL; char *objAlias = NULL; bool is_vfio = false; + bool unplug = hostdev->deleteAction == VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG;

VIR_DEBUG("Removing host device %s from domain %p %s", hostdev->info->alias, vm, vm->def->name); @@ -4454,16 +4456,24 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, } }

- for (i = 0; i < vm->def->nhostdevs; i++) { - if (vm->def->hostdevs[i] == hostdev) { - virDomainHostdevRemove(vm->def, i); - break; + if (!unplug) { + for (i = 0; i < vm->def->nhostdevs; i++) { + if (vm->def->hostdevs[i] == hostdev) { + virDomainHostdevRemove(vm->def, i); + break; + } } }

virDomainAuditHostdev(vm, hostdev, "detach", true);

- if (!is_vfio && + /* + * In case of unplug the attempt to restore label will fail. But we don't + * need to restore the label! In case of separate mount namespace for the + * domain we remove device file later in this function. In case of global + * mount namespace the device file is deleted or being deleted by systemd. + */ + if (!is_vfio && !unplug && qemuSecurityRestoreHostdevLabel(driver, vm, hostdev) < 0) VIR_WARN("Failed to restore host device labelling");

@@ -4497,7 +4507,13 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, break; }

- virDomainHostdevDefFree(hostdev); + if (unplug) { + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + usbsrc->bus = 0; + usbsrc->device = 0; + } else { + virDomainHostdevDefFree(hostdev); + }

if (net) { if (net->type == VIR_DOMAIN_NET_TYPE_NETWORK) { @@ -4512,6 +4528,8 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver, virDomainNetDefFree(net); }

+ hostdev->deleteAction = VIR_DOMAIN_HOSTDEV_DELETE_ACTION_NONE; + ret = 0;

cleanup: @@ -4981,6 +4999,7 @@ qemuDomainRemoveDevice(virQEMUDriverPtr driver, virDomainDeviceInfoPtr info; virObjectEventPtr event; VIR_AUTOFREE(char *) alias = NULL; + bool unplug;

/* * save the alias to use when sending a DEVICE_REMOVED event after @@ -5021,8 +5040,13 @@ qemuDomainRemoveDevice(virQEMUDriverPtr driver, return -1; break; case VIR_DOMAIN_DEVICE_HOSTDEV: + unplug = dev->data.hostdev->deleteAction == VIR_DOMAIN_HOSTDEV_DELETE_ACTION_UNPLUG; + if (qemuDomainRemoveHostDevice(driver, vm, dev->data.hostdev) < 0) return -1; + + if (unplug) + return 0; break; case VIR_DOMAIN_DEVICE_RNG: if (qemuDomainRemoveRNGDevice(driver, vm, dev->data.rng) < 0)

On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote:

We are going to use qemuDomainAttachHostUSBDevice when host usb device is plugged back to node.

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/qemu/qemu_hotplug.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index b045735022..ea82cb54ef 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -2437,8 +2437,18 @@ qemuDomainAttachHostUSBDevice(virQEMUDriverPtr driver, bool teardownlabel = false; bool teardowndevice = false; int ret = -1; + bool replug; + size_t i; + + for (i = 0; i < vm->def->nhostdevs; i++) { + if (vm->def->hostdevs[i] == hostdev) { + replug = true; + break; + } + }

- if (virDomainUSBAddressEnsure(priv->usbaddrs, hostdev->info) < 0) + if (!replug && + virDomainUSBAddressEnsure(priv->usbaddrs, hostdev->info) < 0)

Compilation didn't complain about it, but for clarity, you can initialize 'replug' up there to 'false' to avoid using an uninitialized value here (since replug will not necessarily be set to 'true' in the for loop). Looks good otherwise. Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

return -1;

if (qemuHostdevPrepareUSBDevices(driver, vm->def->name, &hostdev, 1, 0) < 0) @@ -2463,7 +2473,7 @@ qemuDomainAttachHostUSBDevice(virQEMUDriverPtr driver, if (!(devstr = qemuBuildUSBHostdevDevStr(vm->def, hostdev, priv->qemuCaps))) goto cleanup;

- if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs+1) < 0) + if (!replug && VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs+1) < 0) goto cleanup;

qemuDomainObjEnterMonitor(driver, vm); @@ -2476,7 +2486,8 @@ qemuDomainAttachHostUSBDevice(virQEMUDriverPtr driver, if (ret < 0) goto cleanup;

- vm->def->hostdevs[vm->def->nhostdevs++] = hostdev; + if (!replug) + vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;

ret = 0; cleanup: @@ -2489,9 +2500,17 @@ qemuDomainAttachHostUSBDevice(virQEMUDriverPtr driver, if (teardowndevice && qemuDomainNamespaceTeardownHostdev(vm, hostdev) < 0) VIR_WARN("Unable to remove host device from /dev"); - if (added) + if (added) { qemuHostdevReAttachUSBDevices(driver, vm->def->name, &hostdev, 1); - virDomainUSBAddressRelease(priv->usbaddrs, hostdev->info); + + if (replug) { + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + usbsrc->bus = 0; + usbsrc->device = 0; + } + } + if (!replug) + virDomainUSBAddressRelease(priv->usbaddrs, hostdev->info); } VIR_FREE(devstr); return ret;

On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote:

Now when code handling attaching/detaching usb hostdev is appropriately changed use it to handle host usb device udev add/del events.

Assuming we're ok with adding an extra dependency (libudev), LGTM. Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com> Small detail below:

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/qemu/Makefile.inc.am | 2 + src/qemu/qemu_conf.h | 3 + src/qemu/qemu_domain.c | 2 + src/qemu/qemu_domain.h | 2 + src/qemu/qemu_driver.c | 351 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 359 insertions(+), 1 deletion(-)

diff --git a/src/qemu/Makefile.inc.am b/src/qemu/Makefile.inc.am index d16b315ebc..8be0dee396 100644 --- a/src/qemu/Makefile.inc.am +++ b/src/qemu/Makefile.inc.am @@ -85,6 +85,7 @@ libvirt_driver_qemu_impl_la_CFLAGS = \ -I$(srcdir)/conf \ -I$(srcdir)/secret \ $(AM_CFLAGS) \ + $(UDEV_CFLAGS) \ $(NULL) libvirt_driver_qemu_impl_la_LDFLAGS = $(AM_LDFLAGS) libvirt_driver_qemu_impl_la_LIBADD = \ @@ -93,6 +94,7 @@ libvirt_driver_qemu_impl_la_LIBADD = \ $(LIBNL_LIBS) \ $(SELINUX_LIBS) \ $(LIBXML_LIBS) \ + $(UDEV_LIBS) \ $(NULL) libvirt_driver_qemu_impl_la_SOURCES = $(QEMU_DRIVER_SOURCES)

diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index 0cbddd7a9c..2e50bb0950 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -294,6 +294,9 @@ struct _virQEMUDriver {

/* Immutable pointer, self-locking APIs */ virHashAtomicPtr migrationErrors; + + struct udev_monitor *udev_monitor; + int udev_watch; };

virQEMUDriverConfigPtr virQEMUDriverConfigNew(bool privileged); diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 657f3ecfe4..4784804d1e 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -15034,6 +15034,8 @@ qemuProcessEventFree(struct qemuProcessEvent *event) case QEMU_PROCESS_EVENT_SERIAL_CHANGED: case QEMU_PROCESS_EVENT_BLOCK_JOB: case QEMU_PROCESS_EVENT_MONITOR_EOF: + case QEMU_PROCESS_EVENT_USB_REMOVED: + case QEMU_PROCESS_EVENT_USB_ADDED: VIR_FREE(event->data); break; case QEMU_PROCESS_EVENT_JOB_STATUS_CHANGE: diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index d097f23342..94aea62693 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -521,6 +521,8 @@ typedef enum { QEMU_PROCESS_EVENT_MONITOR_EOF, QEMU_PROCESS_EVENT_PR_DISCONNECT, QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED, + QEMU_PROCESS_EVENT_USB_REMOVED, + QEMU_PROCESS_EVENT_USB_ADDED,

QEMU_PROCESS_EVENT_LAST } qemuProcessEventType; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 2378a2e7d0..ce41b0a8d9 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -34,6 +34,7 @@ #include <sys/ioctl.h> #include <sys/un.h> #include <byteswap.h> +#include <libudev.h>

#include "qemu_driver.h" @@ -719,6 +720,254 @@ qemuDomainFindMaxID(virDomainObjPtr vm, }

+struct qemuUdevUSBRemoveData { + unsigned int bus; + unsigned int device; +}; + +struct qemuUdevUSBAddData { + unsigned int vendor; + unsigned int product; +}; + +struct qemuUdevUSBEventData { + union { + struct qemuUdevUSBRemoveData remove; + struct qemuUdevUSBAddData add; + } data; + bool found; + bool remove; +}; + +static int +qemuUdevUSBHandleEvent(virDomainObjPtr vm, void *opaque) +{ + struct qemuUdevUSBEventData *data = opaque; + struct qemuProcessEvent *event = NULL; + size_t i; + + if (data->found) + return 0; + + virObjectLock(vm); + + if (!virDomainObjIsActive(vm)) + goto cleanup; + + for (i = 0; i < vm->def->nhostdevs; i++) { + virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i]; + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + if (data->remove) { + if (usbsrc->bus != data->data.remove.bus || + usbsrc->device != data->data.remove.device) + continue; + } else { + if (usbsrc->vendor != data->data.add.vendor || + usbsrc->product != data->data.add.product) + continue; + } + + data->found = true; + + if (VIR_ALLOC(event) < 0) + goto cleanup; + + if (data->remove) { + struct qemuUdevUSBRemoveData *rm_data; + + + if (VIR_ALLOC(rm_data) < 0) + goto cleanup; + + *rm_data = data->data.remove; + event->data = rm_data; + event->eventType = QEMU_PROCESS_EVENT_USB_REMOVED; + } else { + struct qemuUdevUSBAddData *add_data; + + if (VIR_ALLOC(add_data) < 0) + goto cleanup; + + *add_data = data->data.add; + event->data = add_data; + event->eventType = QEMU_PROCESS_EVENT_USB_ADDED; + } + + event->vm = virObjectRef(vm); + + if (virThreadPoolSendJob(qemu_driver->workerPool, 0, event) < 0) { + virObjectUnref(vm); + goto cleanup; + } + + event = NULL; + + break; + } + + cleanup: + virObjectUnlock(vm); + + qemuProcessEventFree(event); + + return 0; +} + + +static void +qemuUdevEventHandleCallback(int watch ATTRIBUTE_UNUSED, + int fd ATTRIBUTE_UNUSED, + int events ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + struct qemuUdevUSBEventData event_data; + struct udev_device *dev = NULL; + const char *action; + const char *devtype; + const char *tmp; + + /* libvirtd daemon do not run event loop before full state drivers + * initialization. Also state drivers uninitialized only after + * full stop of event loop. In short driver initialization/uninitialization + * and handling events occurs in same main loop thread. Thus we + * don't need any locking here. */ + + if (!(dev = udev_monitor_receive_device(qemu_driver->udev_monitor))) { + VIR_WARNINGS_NO_WLOGICALOP_EQUAL_EXPR + if (errno == EAGAIN || errno == EWOULDBLOCK) { + VIR_WARNINGS_RESET + return; + } + + virReportSystemError(errno, "%s", + _("failed to receive device from udev monitor")); + return; + } + + devtype = udev_device_get_devtype(dev); + + if (STRNEQ_NULLABLE(devtype, "usb_device")) + goto cleanup; + + if (!(action = udev_device_get_action(dev))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive action from udev monitor")); + goto cleanup; + } + + if (STREQ(action, "remove")) { + struct qemuUdevUSBRemoveData *rm_data = &event_data.data.remove; + + if (!(tmp = udev_device_get_property_value(dev, "BUSNUM"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive busnum from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 10, &rm_data->bus) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert busnum to int")); + goto cleanup; + } + + if (!(tmp = udev_device_get_property_value(dev, "DEVNUM"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive devnum from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 10, &rm_data->device) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert devnum to int")); + goto cleanup; + } + event_data.remove = true; + } else if (STREQ(action, "add")) { + struct qemuUdevUSBAddData *add_data = &event_data.data.add; + + if (!(tmp = udev_device_get_property_value(dev, "ID_VENDOR_ID"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive vendor from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 16, &add_data->vendor) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert vendor to int")); + goto cleanup; + } + + if (!(tmp = udev_device_get_property_value(dev, "ID_MODEL_ID"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive product from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 16, &add_data->product) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert product to int")); + goto cleanup; + } + event_data.remove = false; + } + + event_data.found = false; + virDomainObjListForEach(qemu_driver->domains, qemuUdevUSBHandleEvent, &event_data);

This API recently changed. You'll need to add a 'false' boolean value in the second parameter.

+ + cleanup: + udev_device_unref(dev); +} + + +static int +qemuUdevInitialize(void) +{ + struct udev *udev; + + if (!(udev = udev_new())) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to create udev context")); + return -1; + } + + if (!(qemu_driver->udev_monitor = udev_monitor_new_from_netlink(udev, "udev"))) { + udev_unref(udev); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("udev_monitor_new_from_netlink returned NULL")); + return -1; + } + + udev_monitor_enable_receiving(qemu_driver->udev_monitor); + + qemu_driver->udev_watch = virEventAddHandle(udev_monitor_get_fd(qemu_driver->udev_monitor), + VIR_EVENT_HANDLE_READABLE, + qemuUdevEventHandleCallback, NULL, NULL); + + if (qemu_driver->udev_watch < 0) + return -1; + + return 0; +} + + +static void +qemuUdevCleanup(void) +{ + if (qemu_driver->udev_monitor) { + struct udev *udev = udev_monitor_get_udev(qemu_driver->udev_monitor); + + udev_monitor_unref(qemu_driver->udev_monitor); + udev_unref(udev); + qemu_driver->udev_monitor = NULL; + } + + if (qemu_driver->udev_watch > 0) { + virEventRemoveHandle(qemu_driver->udev_watch); + qemu_driver->udev_watch = 0; + } +} + + /** * qemuStateInitialize: * @@ -1030,6 +1279,9 @@ qemuStateInitialize(bool privileged, if (!(qemu_driver->closeCallbacks = virCloseCallbacksNew())) goto error;

+ if (qemuUdevInitialize() < 0) + goto error; + /* Get all the running persistent or transient configs first */ if (virDomainObjListLoadAllConfigs(qemu_driver->domains, cfg->stateDir, @@ -1239,6 +1491,8 @@ qemuStateCleanup(void)

virLockManagerPluginUnref(qemu_driver->lockManager);

+ qemuUdevCleanup(); + virMutexDestroy(&qemu_driver->lock); VIR_FREE(qemu_driver);

@@ -5011,7 +5265,96 @@ processRdmaGidStatusChangedEvent(virDomainObjPtr vm, }

-static void qemuProcessEventHandler(void *data, void *opaque) +static void +processUSBAddedEvent(virQEMUDriverPtr driver, + virDomainObjPtr vm, + struct qemuUdevUSBAddData *data) +{ + virDomainHostdevDefPtr hostdev; + virDomainHostdevSubsysUSBPtr usbsrc; + size_t i; + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + return; + + if (!virDomainObjIsActive(vm)) { + VIR_DEBUG("Domain is not running"); + goto cleanup; + } + + for (i = 0; i < vm->def->nhostdevs; i++) { + hostdev = vm->def->hostdevs[i]; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + usbsrc = &hostdev->source.subsys.u.usb; + + if (usbsrc->vendor == data->vendor && usbsrc->product == data->product) + break; + } + + if (i == vm->def->nhostdevs) + goto cleanup; + + if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) + goto cleanup; + + cleanup: + qemuDomainObjEndJob(driver, vm); +} + + +static void +processUSBRemovedEvent(virQEMUDriverPtr driver, + virDomainObjPtr vm, + struct qemuUdevUSBRemoveData *data) +{ + size_t i; + virDomainHostdevDefPtr hostdev; + virDomainDeviceDef dev = { .type = VIR_DOMAIN_DEVICE_HOSTDEV }; + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + return; + + if (!virDomainObjIsActive(vm)) { + VIR_DEBUG("Domain is not running"); + goto cleanup; + } + + for (i = 0; i < vm->def->nhostdevs; i++) { + virDomainHostdevSubsysUSBPtr usbsrc; + + hostdev = vm->def->hostdevs[i]; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + usbsrc = &hostdev->source.subsys.u.usb; + + /* don't mess with devices that don't use stable host addressing + * with respect to unplug/plug to host + */ + if (!usbsrc->vendor || !usbsrc->product) + continue; + + if (usbsrc->bus == data->bus && usbsrc->device == data->device) + break; + } + + if (i == vm->def->nhostdevs) + goto cleanup; + + dev.data.hostdev = hostdev; + if (qemuDomainDetachDeviceLive(vm, &dev, driver, true, true) < 0) + goto cleanup; + + cleanup: + qemuDomainObjEndJob(driver, vm); +} + + +static void +qemuProcessEventHandler(void *data, void *opaque) { struct qemuProcessEvent *processEvent = data; virDomainObjPtr vm = processEvent->vm; @@ -5057,6 +5400,12 @@ static void qemuProcessEventHandler(void *data, void *opaque) case QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED: processRdmaGidStatusChangedEvent(vm, processEvent->data); break; + case QEMU_PROCESS_EVENT_USB_REMOVED: + processUSBRemovedEvent(driver, vm, processEvent->data); + break; + case QEMU_PROCESS_EVENT_USB_ADDED: + processUSBAddedEvent(driver, vm, processEvent->data); + break; case QEMU_PROCESS_EVENT_LAST: break; }

On 16.09.2019 11:26, Peter Krempa wrote:

On Mon, Sep 09, 2019 at 14:33:07 +0300, Nikolay Shirokovskiy wrote:

...

Now when code handling attaching/detaching usb hostdev is appropriately changed use it to handle host usb device udev add/del events.

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/qemu/Makefile.inc.am | 2 + src/qemu/qemu_conf.h | 3 + src/qemu/qemu_domain.c | 2 + src/qemu/qemu_domain.h | 2 + src/qemu/qemu_driver.c | 351 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 359 insertions(+), 1 deletion(-)

diff --git a/src/qemu/Makefile.inc.am b/src/qemu/Makefile.inc.am index d16b315ebc..8be0dee396 100644 --- a/src/qemu/Makefile.inc.am +++ b/src/qemu/Makefile.inc.am @@ -85,6 +85,7 @@ libvirt_driver_qemu_impl_la_CFLAGS = \ -I$(srcdir)/conf \ -I$(srcdir)/secret \ $(AM_CFLAGS) \ + $(UDEV_CFLAGS) \ $(NULL) libvirt_driver_qemu_impl_la_LDFLAGS = $(AM_LDFLAGS) libvirt_driver_qemu_impl_la_LIBADD = \ @@ -93,6 +94,7 @@ libvirt_driver_qemu_impl_la_LIBADD = \ $(LIBNL_LIBS) \ $(SELINUX_LIBS) \ $(LIBXML_LIBS) \ + $(UDEV_LIBS) \ $(NULL) libvirt_driver_qemu_impl_la_SOURCES = $(QEMU_DRIVER_SOURCES)

diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index 0cbddd7a9c..2e50bb0950 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -294,6 +294,9 @@ struct _virQEMUDriver {

/* Immutable pointer, self-locking APIs */ virHashAtomicPtr migrationErrors; + + struct udev_monitor *udev_monitor; + int udev_watch; };

virQEMUDriverConfigPtr virQEMUDriverConfigNew(bool privileged); diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 657f3ecfe4..4784804d1e 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -15034,6 +15034,8 @@ qemuProcessEventFree(struct qemuProcessEvent *event) case QEMU_PROCESS_EVENT_SERIAL_CHANGED: case QEMU_PROCESS_EVENT_BLOCK_JOB: case QEMU_PROCESS_EVENT_MONITOR_EOF: + case QEMU_PROCESS_EVENT_USB_REMOVED: + case QEMU_PROCESS_EVENT_USB_ADDED: VIR_FREE(event->data); break; case QEMU_PROCESS_EVENT_JOB_STATUS_CHANGE: diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index d097f23342..94aea62693 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -521,6 +521,8 @@ typedef enum { QEMU_PROCESS_EVENT_MONITOR_EOF, QEMU_PROCESS_EVENT_PR_DISCONNECT, QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED, + QEMU_PROCESS_EVENT_USB_REMOVED, + QEMU_PROCESS_EVENT_USB_ADDED,

QEMU_PROCESS_EVENT_LAST } qemuProcessEventType; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 2378a2e7d0..ce41b0a8d9 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -34,6 +34,7 @@ #include <sys/ioctl.h> #include <sys/un.h> #include <byteswap.h> +#include <libudev.h>

#include "qemu_driver.h" @@ -719,6 +720,254 @@ qemuDomainFindMaxID(virDomainObjPtr vm, }

+struct qemuUdevUSBRemoveData { + unsigned int bus; + unsigned int device; +}; + +struct qemuUdevUSBAddData { + unsigned int vendor; + unsigned int product; +}; + +struct qemuUdevUSBEventData { + union { + struct qemuUdevUSBRemoveData remove; + struct qemuUdevUSBAddData add; + } data; + bool found; + bool remove; +}; + +static int +qemuUdevUSBHandleEvent(virDomainObjPtr vm, void *opaque) +{ + struct qemuUdevUSBEventData *data = opaque; + struct qemuProcessEvent *event = NULL; + size_t i; + + if (data->found) + return 0; + + virObjectLock(vm); + + if (!virDomainObjIsActive(vm)) + goto cleanup; + + for (i = 0; i < vm->def->nhostdevs; i++) { + virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i]; + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + if (data->remove) { + if (usbsrc->bus != data->data.remove.bus || + usbsrc->device != data->data.remove.device) + continue; + } else { + if (usbsrc->vendor != data->data.add.vendor || + usbsrc->product != data->data.add.product) + continue; + }

I don't see any XML change related to this in this series.

IMO it's unacceptable to re-plug ANY device by default and we must introduce a flag where the admin gives explicit consent for a device to be re-plugged on the host hotplug event.

You must note that these two instances may be long time apart and thus the user might not notice that the device is re-grabbed by the VM.

Also due to the nature of USB devices it may be a completely different device (e.g. a USB Flash drive of the same make/model but with different data).

Allowing this by default would lead to confusion.

Fair enough.

...

+ + data->found = true; + + if (VIR_ALLOC(event) < 0) + goto cleanup; + + if (data->remove) { + struct qemuUdevUSBRemoveData *rm_data; + + + if (VIR_ALLOC(rm_data) < 0) + goto cleanup; + + *rm_data = data->data.remove; + event->data = rm_data; + event->eventType = QEMU_PROCESS_EVENT_USB_REMOVED; + } else { + struct qemuUdevUSBAddData *add_data; + + if (VIR_ALLOC(add_data) < 0) + goto cleanup; + + *add_data = data->data.add; + event->data = add_data; + event->eventType = QEMU_PROCESS_EVENT_USB_ADDED; + } + + event->vm = virObjectRef(vm); + + if (virThreadPoolSendJob(qemu_driver->workerPool, 0, event) < 0) { + virObjectUnref(vm); + goto cleanup; + } + + event = NULL; + + break; + } + + cleanup: + virObjectUnlock(vm); + + qemuProcessEventFree(event); + + return 0; +} + + +static void +qemuUdevEventHandleCallback(int watch ATTRIBUTE_UNUSED, + int fd ATTRIBUTE_UNUSED, + int events ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + struct qemuUdevUSBEventData event_data; + struct udev_device *dev = NULL; + const char *action; + const char *devtype; + const char *tmp; + + /* libvirtd daemon do not run event loop before full state drivers + * initialization. Also state drivers uninitialized only after + * full stop of event loop. In short driver initialization/uninitialization + * and handling events occurs in same main loop thread. Thus we + * don't need any locking here. */ + + if (!(dev = udev_monitor_receive_device(qemu_driver->udev_monitor))) { + VIR_WARNINGS_NO_WLOGICALOP_EQUAL_EXPR + if (errno == EAGAIN || errno == EWOULDBLOCK) { + VIR_WARNINGS_RESET + return; + } + + virReportSystemError(errno, "%s", + _("failed to receive device from udev monitor")); + return; + } + + devtype = udev_device_get_devtype(dev); + + if (STRNEQ_NULLABLE(devtype, "usb_device")) + goto cleanup; + + if (!(action = udev_device_get_action(dev))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive action from udev monitor")); + goto cleanup; + } + + if (STREQ(action, "remove")) { + struct qemuUdevUSBRemoveData *rm_data = &event_data.data.remove; + + if (!(tmp = udev_device_get_property_value(dev, "BUSNUM"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive busnum from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 10, &rm_data->bus) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert busnum to int")); + goto cleanup; + } + + if (!(tmp = udev_device_get_property_value(dev, "DEVNUM"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive devnum from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 10, &rm_data->device) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert devnum to int")); + goto cleanup; + } + event_data.remove = true; + } else if (STREQ(action, "add")) { + struct qemuUdevUSBAddData *add_data = &event_data.data.add; + + if (!(tmp = udev_device_get_property_value(dev, "ID_VENDOR_ID"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive vendor from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 16, &add_data->vendor) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert vendor to int")); + goto cleanup; + } + + if (!(tmp = udev_device_get_property_value(dev, "ID_MODEL_ID"))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to receive product from udev monitor")); + goto cleanup; + } + if (virStrToLong_ui(tmp, NULL, 16, &add_data->product) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to convert product to int")); + goto cleanup; + } + event_data.remove = false; + } + + event_data.found = false; + virDomainObjListForEach(qemu_driver->domains, qemuUdevUSBHandleEvent, &event_data);

Is this executed from the event loop thread? If yes we can't do this as qemuUdevUSBHandleEvent is taking domain locks and thus potentially waiting indefinitely for any stuck VM.

Yes, this is executed in the event loop thread. But I guess we can take domain lock here as this lock is intended to be grabbed only for short periods of time by design (as stated in THREADS.txt). There are a lot of qemu event handlers that grab domain lock (qemuProcessHandleMonitorEOF for example). AFAIK we use offloading to thread pool if we need to start a job which can take long time because of already running job. And this is offloading is done in qemuUdevUSBHandleEvent.

...

+ + cleanup: + udev_device_unref(dev); +} + + +static int +qemuUdevInitialize(void) +{ + struct udev *udev; + + if (!(udev = udev_new())) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to create udev context")); + return -1; + } + + if (!(qemu_driver->udev_monitor = udev_monitor_new_from_netlink(udev, "udev"))) { + udev_unref(udev); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("udev_monitor_new_from_netlink returned NULL")); + return -1; + } + + udev_monitor_enable_receiving(qemu_driver->udev_monitor); + + qemu_driver->udev_watch = virEventAddHandle(udev_monitor_get_fd(qemu_driver->udev_monitor), + VIR_EVENT_HANDLE_READABLE, + qemuUdevEventHandleCallback, NULL, NULL); + + if (qemu_driver->udev_watch < 0) + return -1; + + return 0; +} + + +static void +qemuUdevCleanup(void) +{ + if (qemu_driver->udev_monitor) { + struct udev *udev = udev_monitor_get_udev(qemu_driver->udev_monitor); + + udev_monitor_unref(qemu_driver->udev_monitor); + udev_unref(udev); + qemu_driver->udev_monitor = NULL; + } + + if (qemu_driver->udev_watch > 0) { + virEventRemoveHandle(qemu_driver->udev_watch); + qemu_driver->udev_watch = 0; + } +} + + /** * qemuStateInitialize: * @@ -1030,6 +1279,9 @@ qemuStateInitialize(bool privileged, if (!(qemu_driver->closeCallbacks = virCloseCallbacksNew())) goto error;

+ if (qemuUdevInitialize() < 0) + goto error; + /* Get all the running persistent or transient configs first */ if (virDomainObjListLoadAllConfigs(qemu_driver->domains, cfg->stateDir, @@ -1239,6 +1491,8 @@ qemuStateCleanup(void)

virLockManagerPluginUnref(qemu_driver->lockManager);

+ qemuUdevCleanup(); + virMutexDestroy(&qemu_driver->lock); VIR_FREE(qemu_driver);

@@ -5011,7 +5265,96 @@ processRdmaGidStatusChangedEvent(virDomainObjPtr vm, }

-static void qemuProcessEventHandler(void *data, void *opaque) +static void +processUSBAddedEvent(virQEMUDriverPtr driver, + virDomainObjPtr vm, + struct qemuUdevUSBAddData *data) +{ + virDomainHostdevDefPtr hostdev; + virDomainHostdevSubsysUSBPtr usbsrc; + size_t i; + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + return; + + if (!virDomainObjIsActive(vm)) { + VIR_DEBUG("Domain is not running"); + goto cleanup; + } + + for (i = 0; i < vm->def->nhostdevs; i++) { + hostdev = vm->def->hostdevs[i]; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + usbsrc = &hostdev->source.subsys.u.usb; + + if (usbsrc->vendor == data->vendor && usbsrc->product == data->product) + break; + } + + if (i == vm->def->nhostdevs) + goto cleanup; + + if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) + goto cleanup; + + cleanup: + qemuDomainObjEndJob(driver, vm); +} + + +static void +processUSBRemovedEvent(virQEMUDriverPtr driver, + virDomainObjPtr vm, + struct qemuUdevUSBRemoveData *data) +{ + size_t i; + virDomainHostdevDefPtr hostdev; + virDomainDeviceDef dev = { .type = VIR_DOMAIN_DEVICE_HOSTDEV }; + + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + return; + + if (!virDomainObjIsActive(vm)) { + VIR_DEBUG("Domain is not running"); + goto cleanup; + } + + for (i = 0; i < vm->def->nhostdevs; i++) { + virDomainHostdevSubsysUSBPtr usbsrc; + + hostdev = vm->def->hostdevs[i]; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + usbsrc = &hostdev->source.subsys.u.usb; + + /* don't mess with devices that don't use stable host addressing + * with respect to unplug/plug to host + */ + if (!usbsrc->vendor || !usbsrc->product) + continue; + + if (usbsrc->bus == data->bus && usbsrc->device == data->device) + break; + } + + if (i == vm->def->nhostdevs) + goto cleanup; + + dev.data.hostdev = hostdev; + if (qemuDomainDetachDeviceLive(vm, &dev, driver, true, true) < 0)

AFAIK this will remove the definition form the XML so how is the re-plug going to be facilitated then?

"[PATCH v2 02/11] qemu: support host usb device unplug" changes removing logic to keep unplugged device in the libvirt config. Nikolay

...

+ goto cleanup; + + cleanup: + qemuDomainObjEndJob(driver, vm); +} + + +static void +qemuProcessEventHandler(void *data, void *opaque) { struct qemuProcessEvent *processEvent = data; virDomainObjPtr vm = processEvent->vm;

On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote:

It is possible for libvirtd to go down before DEVICE_DELETED event is delivered upon usb hostdev unplug and to receive the event after the libvirtd is up. In order to handle this case we need to save usb hostdev deleteAction is status file.

nit: s/is/in ? Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> --- src/conf/domain_conf.c | 26 ++++++++++++++++++++++++++ src/conf/domain_conf.h | 1 + src/qemu/qemu_driver.c | 20 ++++++++++++++++++-- 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index b7a342bb91..c200af050c 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -1236,6 +1236,13 @@ VIR_ENUM_IMPL(virDomainShmemModel, "ivshmem-doorbell", );

+VIR_ENUM_IMPL(virDomainHostdevDeleteAction, + VIR_DOMAIN_HOSTDEV_DELETE_ACTION_LAST, + "none", + "delete", + "unplug" +); + VIR_ENUM_IMPL(virDomainLaunchSecurity, VIR_DOMAIN_LAUNCH_SECURITY_LAST, "", @@ -7533,6 +7540,7 @@ virDomainHostdevSubsysUSBDefParseXML(xmlNodePtr node, virDomainHostdevSubsysUSBPtr usbsrc = &def->source.subsys.u.usb; VIR_AUTOFREE(char *) startupPolicy = NULL; VIR_AUTOFREE(char *) autoAddress = NULL; + VIR_AUTOFREE(char *) deleteAction = NULL;

if ((startupPolicy = virXMLPropString(node, "startupPolicy"))) { def->startupPolicy = @@ -7550,6 +7558,18 @@ virDomainHostdevSubsysUSBDefParseXML(xmlNodePtr node, usbsrc->autoAddress = true; }

+ if ((deleteAction = virXMLPropString(node, "deleteAction"))) { + def->deleteAction = + virDomainHostdevDeleteActionTypeFromString(deleteAction); + + if (def->deleteAction <= 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("Unknown deleteAction '%s'"), + deleteAction); + goto out; + } + } + /* Product can validly be 0, so we need some extra help to determine * if it is uninitialized*/ got_product = false; @@ -24905,6 +24925,12 @@ virDomainHostdevDefFormatSubsys(virBufferPtr buf,

if (def->missing && !(flags & VIR_DOMAIN_DEF_FORMAT_INACTIVE)) virBufferAddLit(buf, " missing='yes'"); + + if (def->deleteAction && (flags & VIR_DOMAIN_DEF_FORMAT_STATUS)) { + const char *deleteAction; + deleteAction = virDomainHostdevDeleteActionTypeToString(def->deleteAction); + virBufferAsprintf(buf, " deleteAction='%s'", deleteAction); + } }

if (def->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI && diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 19a5b21462..df88790ac0 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -338,6 +338,7 @@ typedef enum {

VIR_DOMAIN_HOSTDEV_DELETE_ACTION_LAST } virDomainHostdevDeleteActionType; +VIR_ENUM_DECL(virDomainHostdevDeleteAction);

/* basic device for direct passthrough */ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index ce41b0a8d9..f1802b5d44 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -5272,10 +5272,13 @@ processUSBAddedEvent(virQEMUDriverPtr driver, { virDomainHostdevDefPtr hostdev; virDomainHostdevSubsysUSBPtr usbsrc; + virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver); size_t i;

- if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) { + virObjectUnref(cfg); return; + }

if (!virDomainObjIsActive(vm)) { VIR_DEBUG("Domain is not running"); @@ -5300,8 +5303,13 @@ processUSBAddedEvent(virQEMUDriverPtr driver, if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) goto cleanup;

+ if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm, driver->caps) < 0) + VIR_WARN("unable to save domain status after plugging device %s", + hostdev->info->alias); + cleanup: qemuDomainObjEndJob(driver, vm); + virObjectUnref(cfg); }

@@ -5313,9 +5321,12 @@ processUSBRemovedEvent(virQEMUDriverPtr driver, size_t i; virDomainHostdevDefPtr hostdev; virDomainDeviceDef dev = { .type = VIR_DOMAIN_DEVICE_HOSTDEV }; + virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);

- if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) + if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0) { + virObjectUnref(cfg); return; + }

if (!virDomainObjIsActive(vm)) { VIR_DEBUG("Domain is not running"); @@ -5348,8 +5359,13 @@ processUSBRemovedEvent(virQEMUDriverPtr driver, if (qemuDomainDetachDeviceLive(vm, &dev, driver, true, true) < 0) goto cleanup;

+ if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm, driver->caps) < 0) + VIR_WARN("unable to save domain status after unplugging device %s", + hostdev->info->alias); + cleanup: qemuDomainObjEndJob(driver, vm); + virObjectUnref(cfg); }

On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote:

Imagine host usb device is unplugged from host and as a result we send command to qemu to delete appropriate device. Then before qemu device is deleted host usb device is plugged back. Currenly code supposes there is

s/Currenly/Currently

no remnant device in qemu and will try to add new device and the attempt will fail.

Instead let's check the device is not yet deleted and postpone adding qemu device to device_deleted event handler.

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> ---

Honestly, I tried to give a NACK to this patch, not because of coding issues, but because I find it quite convoluted what you're doing here. qemuCheckHostdevPlugged() is calling qemuDomainAttachHostDevice() if the USB device is found. Problem is that you're calling this check function it right after qemuDomainRemoveDevice(), inside a function that is supposed to handle delete events. The result is a flow like this: - inside processDeviceDeletedEvent:     virDomainDefFindDevice(vm->def, devAlias, &dev, true)     qemuDomainRemoveDevice(driver, vm, &dev)     qemuCheckHostdevPlugged(driver, vm, devAlias); - inside qemuCheckHostdevPlugged:   virDomainDefFindDevice(vm->def, devAlias, &dev, false) <---- same find you just did ( .... other code that verifies if the USB device exists in the host ...)   qemuDomainAttachHostDevice(driver, vm, hostdev) So in short, you are executing a find(), then a remove(), then the same find() again, some code to assert that the USB is plugged in the host, then attach(). Problem is that I didn't come up with a cleaner solution for the problem you're solving here, at least considering the changes from the previous patches and the current code base. That said: Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

src/qemu/qemu_driver.c | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+)

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index f1802b5d44..21640e49c7 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -4671,6 +4671,44 @@ processGuestPanicEvent(virQEMUDriverPtr driver, }

+static int +qemuCheckHostdevPlugged(virQEMUDriverPtr driver, + virDomainObjPtr vm, + const char *devAlias) +{ + virDomainHostdevDefPtr hostdev; + virDomainHostdevSubsysUSBPtr usbsrc; + virDomainDeviceDef dev; + int num; + + if (virDomainDefFindDevice(vm->def, devAlias, &dev, false) < 0) + return 0; + + if (dev.type != VIR_DOMAIN_DEVICE_HOSTDEV) + return 0; + + hostdev = dev.data.hostdev; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + return 0; + + usbsrc = &hostdev->source.subsys.u.usb; + if (!usbsrc->vendor || !usbsrc->product) + return 0; + + if ((num = virUSBDeviceFindByVendor(usbsrc->vendor, usbsrc->product, + NULL, false, NULL)) < 0) + return -1; + + if (num == 0) + return 0; + + if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) + return -1; + + return 0; +} + + static void processDeviceDeletedEvent(virQEMUDriverPtr driver, virDomainObjPtr vm, @@ -4698,6 +4736,11 @@ processDeviceDeletedEvent(virQEMUDriverPtr driver,

if (qemuDomainRemoveDevice(driver, vm, &dev) < 0) goto endjob; + + /* Fall thru and save status file even on error condition because + * device is removed successfully and changed configuration need + * to be saved in status file. */ + qemuCheckHostdevPlugged(driver, vm, devAlias); }

if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm, driver->caps) < 0) @@ -5300,6 +5343,12 @@ processUSBAddedEvent(virQEMUDriverPtr driver, if (i == vm->def->nhostdevs) goto cleanup;

+ /* if device is not yet even deleted from qemu then handle plugging later. + * Or we failed handling host usb device unplugging, then another attempt of + * unplug/plug could help. */ + if (usbsrc->bus || usbsrc->device) + goto cleanup; + if (qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) goto cleanup;

On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote:

Somebody can easily unplug usb device from host while libvirtd is being stopped. Also usb device can be plugged or unplugged/plugged back and so forth. Let's handle such cases.

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com> ---

Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

src/qemu/qemu_process.c | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+)

diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index c9921646e9..8bec36fe2c 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3749,6 +3749,58 @@ qemuProcessUpdateDevices(virQEMUDriverPtr driver, return ret; }

+ +static int +qemuProcessReattachUSBDevices(virQEMUDriverPtr driver, + virDomainObjPtr vm) +{ + size_t i; + + for (i = 0; i < vm->def->nhostdevs; i++) { + virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i]; + virDomainHostdevSubsysUSBPtr usbsrc = &hostdev->source.subsys.u.usb; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) + continue; + + /* don't mess with devices that don't use stable host addressing + * with respect to unplug/plug to host + */ + if (!usbsrc->vendor || !usbsrc->product) + continue; + + if (!usbsrc->bus && !usbsrc->device) { + int num; + + if ((num = virUSBDeviceFindByVendor(usbsrc->vendor, usbsrc->product, + NULL, false, NULL)) < 0) + return -1; + + if (num > 0 && + qemuDomainAttachHostDevice(driver, vm, hostdev) < 0) + return -1; + } else { + virUSBDevicePtr usb; + + if (virUSBDeviceFindByBus(usbsrc->bus, usbsrc->device, + NULL, false, &usb) < 0) + return -1; + + if (!usb) { + virDomainDeviceDef dev = { .type = VIR_DOMAIN_DEVICE_HOSTDEV }; + + dev.data.hostdev = hostdev; + if (qemuDomainDetachDeviceLive(vm, &dev, driver, true, true) < 0) + return -1; + } + virUSBDeviceFree(usb); + } + } + + return 0; +} + + static int qemuDomainPerfRestart(virDomainObjPtr vm) { @@ -8206,6 +8258,9 @@ qemuProcessReconnect(void *opaque) if (qemuProcessUpdateDevices(driver, obj) < 0) goto error;

+ if (qemuProcessReattachUSBDevices(driver, obj) < 0) + goto error; + if (qemuRefreshPRManagerState(driver, obj) < 0) goto error;

On 9/9/19 8:33 AM, Nikolay Shirokovskiy wrote:

First I don't want to add code to handle dummy device that is used when host usb device is not present at the moment of starting/migrating etc. Second supporting non mandatory policies would require to handle races when host usb device is plugged to host and libvirtd starts adding device but if in the meanwhile host usb device it unplugged back then current code will use dummy device which is not desired in this case.

Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy@virtuozzo.com>

Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

--- src/qemu/qemu_driver.c | 8 ++++++++ src/qemu/qemu_process.c | 4 ++++ 2 files changed, 12 insertions(+)

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index b646642c99..fe5fd94ac5 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -5337,6 +5337,10 @@ processUSBAddedEvent(virQEMUDriverPtr driver, if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) continue;

+ if (hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_OPTIONAL || + hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_REQUISITE) + continue; + usbsrc = &hostdev->source.subsys.u.usb;

if (usbsrc->vendor == data->vendor && usbsrc->product == data->product) @@ -5392,6 +5396,10 @@ processUSBRemovedEvent(virQEMUDriverPtr driver, if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB) continue;

+ if (hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_OPTIONAL || + hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_REQUISITE) + continue; + usbsrc = &hostdev->source.subsys.u.usb;

/* don't mess with devices that don't use stable host addressing diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 8bec36fe2c..d87fb637ac 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3769,6 +3769,10 @@ qemuProcessReattachUSBDevices(virQEMUDriverPtr driver, if (!usbsrc->vendor || !usbsrc->product) continue;

+ if (hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_OPTIONAL || + hostdev->startupPolicy == VIR_DOMAIN_STARTUP_POLICY_REQUISITE) + continue; + if (!usbsrc->bus && !usbsrc->device) { int num;