[PATCH v2 0/5] qemu: support async live vCPU unplug
This series adds support for asynchronous live vCPU unplug in the QEMU driver. For live vCPU unplug libvirt currently waits only for a short synchronous completion window. If the unplug completes later there is no dedicated event reporting successful removal of the vCPU. This series adds a new domain event for successful unplug completion and extends the public APIs to allow requesting asynchronous unplug. As recommended, this changeset is split into five parts- Patch 1 adds the new event and the required public/internal plumbing. Patch 2 wires the event into the existing QEMU unplug completion path. Patch 3 adds the internal QEMU plumbing needed to skip waiting for unplug completion, but leaves it unused for now. Patches 4 and 5 then expose the new behaviour through virDomainSetVcpusFlags() and virDomainSetVcpu() together with the corresponding virsh changes. The new public flags are limited to live unplug. Rejected unplug requests continue to be reported via device-removal-failed. Changes from v1: - split the original patch into five pieces separating event plumbing, event emission, internal helper changes, virDomainSetVcpusFlags(), and virDomainSetVcpu() - rename VIR_DOMAIN_VCPU_ASYNC to VIR_DOMAIN_VCPU_ASYNC_UNPLUG - add a dedicated virDomainSetVcpuBehaviour enum with VIR_DOMAIN_SETVCPU_ASYNC_UNPLUG for virDomainSetVcpu() - add virsh manpage documentation for the new --async option - this flag is a no-op if the action does not involve a live vcpu hot-unplug, and has been documented as such for the relevant functions Akash Kulhalli (5): conf,remote: add vcpu-removed domain event qemu: emit vcpu-removed event on unplug completion qemu: thread async vcpu unplug through internal helpers API/qemu: add async unplug flag to virDomainSetVcpusFlags API/qemu: add async unplug flag to virDomainSetVcpu Related discussion: RFC: https://lists.libvirt.org/archives/list/devel@lists.libvirt.org/thread/2J53U... v1: https://lists.libvirt.org/archives/list/devel@lists.libvirt.org/thread/2J3VV...
Add a new domain event for completed vCPU removal. Wire the event through the internal event framework and extend the remote protocol so remote clients can receive it. Update virsh and the event-test example accordingly. The event is not emitted anywhere yet. Signed-off-by: Akash Kulhalli <akash.kulhalli@oracle.com> --- examples/c/misc/event-test.c | 12 ++++++ include/libvirt/libvirt-domain.h | 22 ++++++++++ src/conf/domain_event.c | 66 +++++++++++++++++++++++++++++ src/conf/domain_event.h | 6 +++ src/libvirt_private.syms | 2 + src/remote/remote_daemon_dispatch.c | 26 ++++++++++++ src/remote/remote_driver.c | 29 +++++++++++++ src/remote/remote_protocol.x | 14 +++++- src/remote_protocol-structs | 6 +++ tools/virsh-domain-event.c | 16 +++++++ 10 files changed, 198 insertions(+), 1 deletion(-) diff --git a/examples/c/misc/event-test.c b/examples/c/misc/event-test.c index 2ce82ca9e088..f9e65c55f064 100644 --- a/examples/c/misc/event-test.c +++ b/examples/c/misc/event-test.c @@ -1039,6 +1039,17 @@ myDomainEventDeviceRemovalFailedCallback(virConnectPtr conn G_GNUC_UNUSED, return 0; } +static int +myDomainEventVcpuRemovedCallback(virConnectPtr conn G_GNUC_UNUSED, + virDomainPtr dom, + unsigned int vcpuid, + void *opaque G_GNUC_UNUSED) +{ + printf("%s EVENT: Domain %s(%d) vcpu removed: %u\n", + __func__, virDomainGetName(dom), virDomainGetID(dom), vcpuid); + return 0; +} + static const char * metadataTypeToStr(int status) @@ -1183,6 +1194,7 @@ struct domainEventData domainEvents[] = { DOMAIN_EVENT(VIR_DOMAIN_EVENT_ID_MEMORY_FAILURE, myDomainEventMemoryFailureCallback), DOMAIN_EVENT(VIR_DOMAIN_EVENT_ID_MEMORY_DEVICE_SIZE_CHANGE, myDomainEventMemoryDeviceSizeChangeCallback), DOMAIN_EVENT(VIR_DOMAIN_EVENT_ID_NIC_MAC_CHANGE, myDomainEventNICMACChangeCallback), + DOMAIN_EVENT(VIR_DOMAIN_EVENT_ID_VCPU_REMOVED, myDomainEventVcpuRemovedCallback), }; struct storagePoolEventData { diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 4a8e3114b35d..92d5ecb32d65 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -6965,6 +6965,27 @@ typedef void (*virConnectDomainEventDeviceRemovalFailedCallback)(virConnectPtr c const char *devAlias, void *opaque); +/** + * virConnectDomainEventVcpuRemovedCallback: + * @conn: connection object + * @dom: domain on which the event occurred + * @vcpuid: libvirt XML vCPU id + * @opaque: application specified data + * + * This callback occurs when a vCPU is removed from the domain. + * + * The @vcpuid value matches the ``<vcpu id='...'>`` value from the domain XML. + * + * The callback signature to use when registering for an event of type + * VIR_DOMAIN_EVENT_ID_VCPU_REMOVED with virConnectDomainEventRegisterAny(). + * + * Since: 12.3.0 + */ +typedef void (*virConnectDomainEventVcpuRemovedCallback)(virConnectPtr conn, + virDomainPtr dom, + unsigned int vcpuid, + void *opaque); + /** * virConnectDomainEventMetadataChangeCallback: * @conn: connection object @@ -7617,6 +7638,7 @@ typedef enum { VIR_DOMAIN_EVENT_ID_MEMORY_FAILURE = 25, /* virConnectDomainEventMemoryFailureCallback (Since: 6.9.0) */ VIR_DOMAIN_EVENT_ID_MEMORY_DEVICE_SIZE_CHANGE = 26, /* virConnectDomainEventMemoryDeviceSizeChangeCallback (Since: 7.9.0) */ VIR_DOMAIN_EVENT_ID_NIC_MAC_CHANGE = 27, /* virConnectDomainEventNICMACChangeCallback (Since: 11.2.0) */ + VIR_DOMAIN_EVENT_ID_VCPU_REMOVED = 28, /* virConnectDomainEventVcpuRemovedCallback (Since: 12.3.0) */ # ifdef VIR_ENUM_SENTINELS VIR_DOMAIN_EVENT_ID_LAST diff --git a/src/conf/domain_event.c b/src/conf/domain_event.c index 88087bad4f21..17ad4a0d2c91 100644 --- a/src/conf/domain_event.c +++ b/src/conf/domain_event.c @@ -53,6 +53,7 @@ static virClass *virDomainEventDeviceAddedClass; static virClass *virDomainEventMigrationIterationClass; static virClass *virDomainEventJobCompletedClass; static virClass *virDomainEventDeviceRemovalFailedClass; +static virClass *virDomainEventVcpuRemovedClass; static virClass *virDomainEventMetadataChangeClass; static virClass *virDomainEventBlockThresholdClass; static virClass *virDomainEventMemoryFailureClass; @@ -78,6 +79,7 @@ static void virDomainEventDeviceAddedDispose(void *obj); static void virDomainEventMigrationIterationDispose(void *obj); static void virDomainEventJobCompletedDispose(void *obj); static void virDomainEventDeviceRemovalFailedDispose(void *obj); +static void virDomainEventVcpuRemovedDispose(void *obj); static void virDomainEventMetadataChangeDispose(void *obj); static void virDomainEventBlockThresholdDispose(void *obj); static void virDomainEventMemoryFailureDispose(void *obj); @@ -251,6 +253,13 @@ struct _virDomainEventDeviceRemovalFailed { }; typedef struct _virDomainEventDeviceRemovalFailed virDomainEventDeviceRemovalFailed; +struct _virDomainEventVcpuRemoved { + virDomainEvent parent; + + unsigned int vcpuid; +}; +typedef struct _virDomainEventVcpuRemoved virDomainEventVcpuRemoved; + struct _virDomainEventMetadataChange { virDomainEvent parent; @@ -337,6 +346,8 @@ virDomainEventsOnceInit(void) return -1; if (!VIR_CLASS_NEW(virDomainEventDeviceRemovalFailed, virDomainEventClass)) return -1; + if (!VIR_CLASS_NEW(virDomainEventVcpuRemoved, virDomainEventClass)) + return -1; if (!VIR_CLASS_NEW(virDomainEventMetadataChange, virDomainEventClass)) return -1; if (!VIR_CLASS_NEW(virDomainEventBlockThreshold, virDomainEventClass)) @@ -484,6 +495,13 @@ virDomainEventDeviceRemovalFailedDispose(void *obj) g_free(event->devAlias); } +static void +virDomainEventVcpuRemovedDispose(void *obj) +{ + virDomainEventVcpuRemoved *event = obj; + VIR_DEBUG("obj=%p", event); +} + static void virDomainEventPMDispose(void *obj) @@ -1382,6 +1400,43 @@ virDomainEventDeviceRemovalFailedNewFromDom(virDomainPtr dom, devAlias); } +static virObjectEvent * +virDomainEventVcpuRemovedNew(int id, + const char *name, + unsigned char *uuid, + unsigned int vcpuid) +{ + virDomainEventVcpuRemoved *ev; + + if (virDomainEventsInitialize() < 0) + return NULL; + + if (!(ev = virDomainEventNew(virDomainEventVcpuRemovedClass, + VIR_DOMAIN_EVENT_ID_VCPU_REMOVED, + id, name, uuid))) + return NULL; + + ev->vcpuid = vcpuid; + + return (virObjectEvent *)ev; +} + +virObjectEvent * +virDomainEventVcpuRemovedNewFromObj(virDomainObj *obj, + unsigned int vcpuid) +{ + return virDomainEventVcpuRemovedNew(obj->def->id, obj->def->name, + obj->def->uuid, vcpuid); +} + +virObjectEvent * +virDomainEventVcpuRemovedNewFromDom(virDomainPtr dom, + unsigned int vcpuid) +{ + return virDomainEventVcpuRemovedNew(dom->id, dom->name, dom->uuid, + vcpuid); +} + static virObjectEvent * virDomainEventAgentLifecycleNew(int id, @@ -2134,6 +2189,17 @@ virDomainEventDispatchDefaultFunc(virConnectPtr conn, goto cleanup; } + case VIR_DOMAIN_EVENT_ID_VCPU_REMOVED: + { + virDomainEventVcpuRemoved *vcpuRemovedEvent; + + vcpuRemovedEvent = (virDomainEventVcpuRemoved *)event; + ((virConnectDomainEventVcpuRemovedCallback)cb)(conn, dom, + vcpuRemovedEvent->vcpuid, + cbopaque); + goto cleanup; + } + case VIR_DOMAIN_EVENT_ID_LAST: break; } diff --git a/src/conf/domain_event.h b/src/conf/domain_event.h index f31cfb9e42ad..1b1b16095a77 100644 --- a/src/conf/domain_event.h +++ b/src/conf/domain_event.h @@ -192,6 +192,12 @@ virDomainEventDeviceRemovalFailedNewFromObj(virDomainObj *obj, virObjectEvent * virDomainEventDeviceRemovalFailedNewFromDom(virDomainPtr dom, const char *devAlias); +virObjectEvent * +virDomainEventVcpuRemovedNewFromObj(virDomainObj *obj, + unsigned int vcpuid); +virObjectEvent * +virDomainEventVcpuRemovedNewFromDom(virDomainPtr dom, + unsigned int vcpuid); virObjectEvent * virDomainEventTunableNewFromObj(virDomainObj *obj, diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index cf0e71cc6af1..f22b5895db12 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -811,6 +811,8 @@ virDomainEventTrayChangeNewFromDom; virDomainEventTrayChangeNewFromObj; virDomainEventTunableNewFromDom; virDomainEventTunableNewFromObj; +virDomainEventVcpuRemovedNewFromDom; +virDomainEventVcpuRemovedNewFromObj; virDomainEventWatchdogNewFromDom; virDomainEventWatchdogNewFromObj; virDomainQemuMonitorEventNew; diff --git a/src/remote/remote_daemon_dispatch.c b/src/remote/remote_daemon_dispatch.c index 7e74ff063f5b..81b0ed00da1a 100644 --- a/src/remote/remote_daemon_dispatch.c +++ b/src/remote/remote_daemon_dispatch.c @@ -1322,6 +1322,31 @@ remoteRelayDomainEventMemoryDeviceSizeChange(virConnectPtr conn, return 0; } +static int +remoteRelayDomainEventVcpuRemoved(virConnectPtr conn, + virDomainPtr dom, + unsigned int vcpuid, + void *opaque) +{ + daemonClientEventCallback *callback = opaque; + remote_domain_event_vcpu_removed_msg data; + + if (callback->callbackID < 0 || + !remoteRelayDomainEventCheckACL(callback->client, conn, dom)) + return -1; + + memset(&data, 0, sizeof(data)); + data.callbackID = callback->callbackID; + data.vcpuid = vcpuid; + make_nonnull_domain(&data.dom, dom); + + remoteDispatchObjectEventSend(callback->client, remoteProgram, + REMOTE_PROC_DOMAIN_EVENT_VCPU_REMOVED, + (xdrproc_t)xdr_remote_domain_event_vcpu_removed_msg, + &data); + return 0; +} + static int remoteRelayDomainEventNICMACChange(virConnectPtr conn, @@ -1383,6 +1408,7 @@ static virConnectDomainEventGenericCallback domainEventCallbacks[] = { VIR_DOMAIN_EVENT_CALLBACK(remoteRelayDomainEventMemoryFailure), VIR_DOMAIN_EVENT_CALLBACK(remoteRelayDomainEventMemoryDeviceSizeChange), VIR_DOMAIN_EVENT_CALLBACK(remoteRelayDomainEventNICMACChange), + VIR_DOMAIN_EVENT_CALLBACK(remoteRelayDomainEventVcpuRemoved), }; G_STATIC_ASSERT(G_N_ELEMENTS(domainEventCallbacks) == VIR_DOMAIN_EVENT_ID_LAST); diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index ec71eaed8762..c8a4e3f6da98 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -428,6 +428,10 @@ remoteDomainBuildEventMemoryDeviceSizeChange(virNetClientProgram *prog, virNetClient *client, void *evdata, void *opaque); static void +remoteDomainBuildEventVcpuRemoved(virNetClientProgram *prog, + virNetClient *client, + void *evdata, void *opaque); +static void remoteConnectNotifyEventConnectionClosed(virNetClientProgram *prog G_GNUC_UNUSED, virNetClient *client G_GNUC_UNUSED, void *evdata, void *opaque); @@ -659,6 +663,10 @@ static virNetClientProgramEvent remoteEvents[] = { remoteDomainBuildEventNICMACChange, sizeof(remote_domain_event_nic_mac_change_msg), (xdrproc_t)xdr_remote_domain_event_nic_mac_change_msg }, + { REMOTE_PROC_DOMAIN_EVENT_VCPU_REMOVED, + remoteDomainBuildEventVcpuRemoved, + sizeof(remote_domain_event_vcpu_removed_msg), + (xdrproc_t)xdr_remote_domain_event_vcpu_removed_msg }, }; static void @@ -5138,6 +5146,27 @@ remoteDomainBuildEventMemoryDeviceSizeChange(virNetClientProgram *prog G_GNUC_UN virObjectEventStateQueueRemote(priv->eventState, event, msg->callbackID); } +static void +remoteDomainBuildEventVcpuRemoved(virNetClientProgram *prog G_GNUC_UNUSED, + virNetClient *client G_GNUC_UNUSED, + void *evdata, void *opaque) +{ + virConnectPtr conn = opaque; + remote_domain_event_vcpu_removed_msg *msg = evdata; + struct private_data *priv = conn->privateData; + virDomainPtr dom; + virObjectEvent *event = NULL; + + if (!(dom = get_nonnull_domain(conn, msg->dom))) + return; + + event = virDomainEventVcpuRemovedNewFromDom(dom, msg->vcpuid); + + virObjectUnref(dom); + + virObjectEventStateQueueRemote(priv->eventState, event, msg->callbackID); +} + static void remoteDomainBuildEventNICMACChange(virNetClientProgram *prog G_GNUC_UNUSED, diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 38a83c64eadb..23699e99a60a 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -3981,6 +3981,12 @@ struct remote_domain_event_memory_device_size_change_msg { unsigned hyper size; }; +struct remote_domain_event_vcpu_removed_msg { + int callbackID; + remote_nonnull_domain dom; + unsigned int vcpuid; +}; + struct remote_domain_fd_associate_args { remote_nonnull_domain dom; @@ -7120,5 +7126,11 @@ enum remote_procedure { * @generate: both * @acl: none */ - REMOTE_PROC_DOMAIN_EVENT_NIC_MAC_CHANGE = 453 + REMOTE_PROC_DOMAIN_EVENT_NIC_MAC_CHANGE = 453, + + /** + * @generate: both + * @acl: none + */ + REMOTE_PROC_DOMAIN_EVENT_VCPU_REMOVED = 454 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index 0f87d13a5ae1..75c3d0cb2e06 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -3315,6 +3315,11 @@ struct remote_domain_event_memory_device_size_change_msg { remote_nonnull_string alias; uint64_t size; }; +struct remote_domain_event_vcpu_removed_msg { + int callbackID; + remote_nonnull_domain dom; + u_int vcpuid; +}; struct remote_domain_fd_associate_args { remote_nonnull_domain dom; remote_nonnull_string name; @@ -3791,4 +3796,5 @@ enum remote_procedure { REMOTE_PROC_DOMAIN_SET_THROTTLE_GROUP = 451, REMOTE_PROC_DOMAIN_DEL_THROTTLE_GROUP = 452, REMOTE_PROC_DOMAIN_EVENT_NIC_MAC_CHANGE = 453, + REMOTE_PROC_DOMAIN_EVENT_VCPU_REMOVED = 454, }; diff --git a/tools/virsh-domain-event.c b/tools/virsh-domain-event.c index b9d1cdf019ca..4a9a831b4525 100644 --- a/tools/virsh-domain-event.c +++ b/tools/virsh-domain-event.c @@ -705,6 +705,20 @@ virshEventDeviceRemovalFailedPrint(virConnectPtr conn G_GNUC_UNUSED, virshEventPrint(opaque, &buf); } +static void +virshEventVcpuRemovedPrint(virConnectPtr conn G_GNUC_UNUSED, + virDomainPtr dom, + unsigned int vcpuid, + void *opaque) +{ + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; + + virBufferAsprintf(&buf, + _("event 'vcpu-removed' for domain '%1$s': vcpu %2$u\n"), + virDomainGetName(dom), vcpuid); + virshEventPrint(opaque, &buf); +} + VIR_ENUM_DECL(virshEventMetadataChangeType); VIR_ENUM_IMPL(virshEventMetadataChangeType, VIR_DOMAIN_METADATA_LAST, @@ -873,6 +887,8 @@ virshDomainEventCallback virshDomainEventCallbacks[] = { VIR_DOMAIN_EVENT_CALLBACK(virshEventMemoryDeviceSizeChangePrint), }, { "nic-mac-change", VIR_DOMAIN_EVENT_CALLBACK(virshEventNICMACChangePrint), }, + { "vcpu-removed", + VIR_DOMAIN_EVENT_CALLBACK(virshEventVcpuRemovedPrint), }, }; G_STATIC_ASSERT(VIR_DOMAIN_EVENT_ID_LAST == G_N_ELEMENTS(virshDomainEventCallbacks)); -- 2.47.3
Emit the vcpu-removed event when QEMU vCPU unplug completes. Hook this into qemuDomainRemoveVcpu(), which covers both the synchronous completion path and the DEVICE_DELETED-driven alias removal path. Signed-off-by: Akash Kulhalli <akash.kulhalli@oracle.com> --- src/qemu/qemu_hotplug.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index b7a282b96e52..7828b7d73821 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -6728,6 +6728,7 @@ qemuDomainRemoveVcpu(virDomainObj *vm, unsigned int nvcpus = vcpupriv->vcpus; size_t i; ssize_t offlineVcpuWithTid = -1; + virObjectEvent *event = NULL; if (qemuDomainRefreshVcpuInfo(vm, VIR_ASYNC_JOB_NONE, false) < 0) return -1; @@ -6757,6 +6758,10 @@ qemuDomainRemoveVcpu(virDomainObj *vm, virDomainAuditVcpu(vm, oldvcpus, oldvcpus - nvcpus, "update", true); + /* fire the libvirt `vcpu-removed` event */ + event = virDomainEventVcpuRemovedNewFromObj(vm, vcpu); + virObjectEventStateQueue(priv->driver->domainEventState, event); + return 0; } -- 2.47.3
Thread an async_unplug flag through the internal QEMU vCPU unplug helpers. When set, the unplug path returns after QEMU accepts the device deletion request and leaves final completion to the existing DEVICE_DELETED handling routines. All callers still pass false, so this does not change behaviour yet. Signed-off-by: Akash Kulhalli <akash.kulhalli@oracle.com> --- src/qemu/qemu_driver.c | 5 ++-- src/qemu/qemu_hotplug.c | 52 +++++++++++++++++++++++++++++++++-------- src/qemu/qemu_hotplug.h | 6 +++-- tests/qemuhotplugtest.c | 5 ++-- 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index d227ac58cdb4..4d9be3d3a9e5 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -4303,7 +4303,7 @@ qemuDomainSetVcpusFlags(virDomainPtr dom, ret = qemuDomainSetVcpusMax(driver, vm, def, persistentDef, nvcpus); else ret = qemuDomainSetVcpusInternal(driver, vm, def, persistentDef, - nvcpus, hotpluggable); + nvcpus, hotpluggable, false); endjob: if (useAgent) @@ -19220,7 +19220,8 @@ qemuDomainSetVcpu(virDomainPtr dom, } } - ret = qemuDomainSetVcpuInternal(driver, vm, def, persistentDef, map, !!state); + ret = qemuDomainSetVcpuInternal(driver, vm, def, persistentDef, map, + !!state, false); endjob: virDomainObjEndJob(vm); diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 7828b7d73821..919224ed69f7 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -6793,7 +6793,8 @@ static int qemuDomainHotplugDelVcpu(virQEMUDriver *driver, virQEMUDriverConfig *cfg, virDomainObj *vm, - unsigned int vcpu) + unsigned int vcpu, + bool async_unplug) { virDomainVcpuDef *vcpuinfo = virDomainDefGetVcpu(vm->def, vcpu); qemuDomainVcpuPrivate *vcpupriv = QEMU_DOMAIN_VCPU_PRIVATE(vcpuinfo); @@ -6808,7 +6809,8 @@ qemuDomainHotplugDelVcpu(virQEMUDriver *driver, return -1; } - qemuDomainMarkDeviceAliasForRemoval(vm, vcpupriv->alias); + if (!async_unplug) + qemuDomainMarkDeviceAliasForRemoval(vm, vcpupriv->alias); rc = qemuDomainDeleteDevice(vm, vcpupriv->alias); if (rc < 0) { @@ -6817,6 +6819,12 @@ qemuDomainHotplugDelVcpu(virQEMUDriver *driver, virDomainAuditVcpu(vm, oldvcpus, oldvcpus - nvcpus, "update", false); goto cleanup; } + } else if (async_unplug) { + /* + * Let DEVICE_DELETED finish the unplug asynchronously when qemu + * accepted the delete request. + */ + return 0; } else { if ((rc = qemuDomainWaitForDeviceRemoval(vm)) <= 0) { if (rc == 0) @@ -6837,7 +6845,8 @@ qemuDomainHotplugDelVcpu(virQEMUDriver *driver, ret = 0; cleanup: - qemuDomainResetDeviceRemoval(vm); + if (!async_unplug) + qemuDomainResetDeviceRemoval(vm); return ret; } @@ -7008,7 +7017,8 @@ qemuDomainSetVcpusLive(virQEMUDriver *driver, virQEMUDriverConfig *cfg, virDomainObj *vm, virBitmap *vcpumap, - bool enable) + bool enable, + bool async) { qemuDomainObjPrivate *priv = vm->privateData; virCgroupEmulatorAllNodesData *emulatorCgroup = NULL; @@ -7028,7 +7038,7 @@ qemuDomainSetVcpusLive(virQEMUDriver *driver, if (!virBitmapIsBitSet(vcpumap, nextvcpu)) continue; - if (qemuDomainHotplugDelVcpu(driver, cfg, vm, nextvcpu) < 0) + if (qemuDomainHotplugDelVcpu(driver, cfg, vm, nextvcpu, async) < 0) goto cleanup; } } @@ -7119,7 +7129,8 @@ qemuDomainSetVcpusInternal(virQEMUDriver *driver, virDomainDef *def, virDomainDef *persistentDef, unsigned int nvcpus, - bool hotpluggable) + bool hotpluggable, + bool async_unplug) { g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); g_autoptr(virBitmap) vcpumap = NULL; @@ -7144,7 +7155,8 @@ qemuDomainSetVcpusInternal(virQEMUDriver *driver, &enable))) return -1; - if (qemuDomainSetVcpusLive(driver, cfg, vm, vcpumap, enable) < 0) + if (qemuDomainSetVcpusLive(driver, cfg, vm, vcpumap, enable, + async_unplug) < 0) return -1; } @@ -7287,14 +7299,33 @@ qemuDomainVcpuValidateConfig(virDomainDef *def, return 0; } - +/** + * qemuDomainSetVcpuInternal: + * + * When @async_unplug is set to true, libvirt will not wait for + * the guest to comply with the unplug request but instead return + * immediately after receiving the acknowledgement from QEMU. Otherwise, + * libvirt will wait for a brief moment (defined by qemuDomainGetUnplugTimeout) + * before giving up and returning control to the caller. + * + * If the request results in adding a vcpu, this parameter is ignored. + * + * @param driver the QEMU driver object + * @param vm the domain object + * @param def the live domain definition + * @param persistentDef the persistent (config) domain definition + * @param map a bitmap of cpus to be set to state @state + * @param state enable/disable the vcpus marked in @map + * @param async_unplug only used in case of unplug (i.e. @state=false) + */ int qemuDomainSetVcpuInternal(virQEMUDriver *driver, virDomainObj *vm, virDomainDef *def, virDomainDef *persistentDef, virBitmap *map, - bool state) + bool state, + bool async_unplug) { g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); g_autoptr(virBitmap) livevcpus = NULL; @@ -7326,7 +7357,8 @@ qemuDomainSetVcpuInternal(virQEMUDriver *driver, } if (livevcpus && - qemuDomainSetVcpusLive(driver, cfg, vm, livevcpus, state) < 0) + qemuDomainSetVcpusLive(driver, cfg, vm, livevcpus, state, + async_unplug) < 0) return -1; if (persistentDef) { diff --git a/src/qemu/qemu_hotplug.h b/src/qemu/qemu_hotplug.h index e6c90253e416..60ed0e174c21 100644 --- a/src/qemu/qemu_hotplug.h +++ b/src/qemu/qemu_hotplug.h @@ -106,7 +106,8 @@ qemuDomainSetVcpusInternal(virQEMUDriver *driver, virDomainDef *def, virDomainDef *persistentDef, unsigned int nvcpus, - bool hotpluggable); + bool hotpluggable, + bool async_unplug); int qemuDomainSetVcpuInternal(virQEMUDriver *driver, @@ -114,7 +115,8 @@ qemuDomainSetVcpuInternal(virQEMUDriver *driver, virDomainDef *def, virDomainDef *persistentDef, virBitmap *vcpus, - bool state); + bool state, + bool async_unplug); unsigned long long qemuDomainGetUnplugTimeout(virDomainObj *vm) ATTRIBUTE_MOCKABLE; diff --git a/tests/qemuhotplugtest.c b/tests/qemuhotplugtest.c index ea9d3243f8b1..7e49b9ad3661 100644 --- a/tests/qemuhotplugtest.c +++ b/tests/qemuhotplugtest.c @@ -420,7 +420,7 @@ testQemuHotplugCpuGroup(const void *opaque) rc = qemuDomainSetVcpusInternal(&driver, data->vm, data->vm->def, data->vm->newDef, params->newcpus, - true); + true, false); if (params->fail) { if (rc == 0) @@ -458,7 +458,8 @@ testQemuHotplugCpuIndividual(const void *opaque) goto cleanup; rc = qemuDomainSetVcpuInternal(&driver, data->vm, data->vm->def, - data->vm->newDef, map, params->state); + data->vm->newDef, map, params->state, + false); if (params->fail) { if (rc == 0) -- 2.47.3
Add VIR_DOMAIN_VCPU_ASYNC_UNPLUG for virDomainSetVcpusFlags(). With this flag, success indicates that QEMU accepted the unplug request, while final completion is reported by the vcpu-removed event. Rejected requests continue to be reported by the device-removal-failed event. Wire the flag through the QEMU driver, document its semantics, and add virsh support for setvcpus --async. Signed-off-by: Akash Kulhalli <akash.kulhalli@oracle.com> --- docs/manpages/virsh.rst | 13 +++++++++++-- include/libvirt/libvirt-domain.h | 1 + src/libvirt-domain.c | 10 ++++++++++ src/qemu/qemu_driver.c | 12 ++++++++++-- tools/virsh-domain.c | 8 ++++++++ 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index 80b0ea14a8b3..f4c8e573cd47 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -179,7 +179,8 @@ Most ``virsh`` commands act synchronously, except maybe shutdown, setvcpus and setmem. In those cases the fact that the ``virsh`` program returned, may not mean the action is complete and you must poll periodically to detect that the guest completed the -operation. +operation. This asynchronous behaviour can be enforced with the +``--async`` flag to ``setvcpus`` only. ``virsh`` strives for backward compatibility. Although the ``help`` command only lists the preferred usage of a command, if an older @@ -4810,7 +4811,7 @@ setvcpus :: - setvcpus domain count [--maximum] [[--config] [--live] | [--current]] [--guest] [--hotpluggable] + setvcpus domain count [--maximum] [[--config] [--live] | [--current]] [--guest] [--hotpluggable] [--async] Change the number of virtual CPUs active in a guest domain. By default, this command works on active guest domains. To change the settings for an @@ -4839,6 +4840,14 @@ is up to the hypervisor whether the *--config* flag is also assumed, and therefore whether the XML configuration is adjusted to make the change persistent. +If *--async* is specified, live vCPU unplug requests are fired without waiting +for the guest to comply. This affects only the live unplug part of the +operation; it may optionally be combined with *--config*. Final completion is +reported by the ``vcpu-removed`` domain event, while rejected unplug requests +continue to be reported by ``device-removal-failed``. If the requested count +increases the live vCPU count, or only affects the persistent configuration, +*--async* has no effect. This flag cannot be combined with *--guest*. + If *--guest* is specified, then the count of cpus is modified in the guest instead of the hypervisor. This flag is usable only for live domains and may require guest agent to be configured in the guest. diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 92d5ecb32d65..016b6ea700e9 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -2605,6 +2605,7 @@ typedef enum { VIR_DOMAIN_VCPU_MAXIMUM = (1 << 2), /* Max rather than current count (Since: 0.8.5) */ VIR_DOMAIN_VCPU_GUEST = (1 << 3), /* Modify state of the cpu in the guest (Since: 1.1.0) */ VIR_DOMAIN_VCPU_HOTPLUGGABLE = (1 << 4), /* Make vcpus added hot(un)pluggable (Since: 2.4.0) */ + VIR_DOMAIN_VCPU_ASYNC_UNPLUG = (1 << 5), /* Don't wait for the guest to comply with unplug request(s) (Since: 12.3.0) */ } virDomainVcpuFlags; int virDomainSetVcpus (virDomainPtr domain, diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index db9eea57745c..97af1099f939 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -7773,6 +7773,16 @@ virDomainSetVcpus(virDomainPtr domain, unsigned int nvcpus) * be used with live guests and is incompatible with VIR_DOMAIN_VCPU_MAXIMUM. * The usage of this flag may require a guest agent configured. * + * If @flags includes VIR_DOMAIN_VCPU_ASYNC_UNPLUG, live vCPU hot-unplug + * request(s) are fired without waiting for the guest to comply. Success in + * this mode only means that the unplug request(s) were accepted. Final + * completion is reported by VIR_DOMAIN_EVENT_ID_VCPU_REMOVED, carrying the + * XML ``<vcpu id='...'>`` value for each removed vCPU. Rejected unplug + * requests continue to be reported through the event + * VIR_DOMAIN_EVENT_ID_DEVICE_REMOVAL_FAILED. The success event may be + * delivered before this API call returns. This flag has no effect when this + * operation results in an increase in the live vCPU count. + * * Not all hypervisors can support all flag combinations. * * Returns 0 in case of success, -1 in case of failure. diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 4d9be3d3a9e5..c9c72b29617f 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -4269,6 +4269,7 @@ qemuDomainSetVcpusFlags(virDomainPtr dom, virDomainObj *vm = NULL; virDomainDef *def; virDomainDef *persistentDef; + bool async_unplug = !!(flags & VIR_DOMAIN_VCPU_ASYNC_UNPLUG); bool hotpluggable = !!(flags & VIR_DOMAIN_VCPU_HOTPLUGGABLE); bool useAgent = !!(flags & VIR_DOMAIN_VCPU_GUEST); int ret = -1; @@ -4277,7 +4278,8 @@ qemuDomainSetVcpusFlags(virDomainPtr dom, VIR_DOMAIN_AFFECT_CONFIG | VIR_DOMAIN_VCPU_MAXIMUM | VIR_DOMAIN_VCPU_GUEST | - VIR_DOMAIN_VCPU_HOTPLUGGABLE, -1); + VIR_DOMAIN_VCPU_HOTPLUGGABLE | + VIR_DOMAIN_VCPU_ASYNC_UNPLUG, -1); if (!(vm = qemuDomainObjFromDomain(dom))) goto cleanup; @@ -4297,13 +4299,19 @@ qemuDomainSetVcpusFlags(virDomainPtr dom, if (virDomainObjGetDefs(vm, flags, &def, &persistentDef) < 0) goto endjob; + if (async_unplug && (useAgent || !def)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("asynchronous mode is supported only for live vcpu unplug")); + goto endjob; + } + if (useAgent) ret = qemuDomainSetVcpusAgent(vm, nvcpus); else if (flags & VIR_DOMAIN_VCPU_MAXIMUM) ret = qemuDomainSetVcpusMax(driver, vm, def, persistentDef, nvcpus); else ret = qemuDomainSetVcpusInternal(driver, vm, def, persistentDef, - nvcpus, hotpluggable, false); + nvcpus, hotpluggable, async_unplug); endjob: if (useAgent) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 08a1ce395378..1fccee4bc9ed 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -7667,6 +7667,10 @@ static const vshCmdOptDef opts_setvcpus[] = { .type = VSH_OT_BOOL, .help = N_("make added vcpus hot(un)pluggable") }, + {.name = "async", + .type = VSH_OT_BOOL, + .help = N_("return after firing vcpu unplug request(s)") + }, {.name = NULL} }; @@ -7681,11 +7685,13 @@ cmdSetvcpus(vshControl *ctl, const vshCmd *cmd) bool current = vshCommandOptBool(cmd, "current"); bool guest = vshCommandOptBool(cmd, "guest"); bool hotpluggable = vshCommandOptBool(cmd, "hotpluggable"); + bool async = vshCommandOptBool(cmd, "async"); unsigned int flags = VIR_DOMAIN_AFFECT_CURRENT; VSH_EXCLUSIVE_OPTIONS_VAR(current, live); VSH_EXCLUSIVE_OPTIONS_VAR(current, config); VSH_EXCLUSIVE_OPTIONS_VAR(guest, config); + VSH_EXCLUSIVE_OPTIONS_VAR(async, guest); VSH_REQUIRE_OPTION_VAR(maximum, config); @@ -7699,6 +7705,8 @@ cmdSetvcpus(vshControl *ctl, const vshCmd *cmd) flags |= VIR_DOMAIN_VCPU_MAXIMUM; if (hotpluggable) flags |= VIR_DOMAIN_VCPU_HOTPLUGGABLE; + if (async) + flags |= VIR_DOMAIN_VCPU_ASYNC_UNPLUG; if (!(dom = virshCommandOptDomain(ctl, cmd, NULL))) return false; -- 2.47.3
Add VIR_DOMAIN_SETVCPU_ASYNC_UNPLUG for virDomainSetVcpu(). Define a dedicated virDomainSetVcpuBehaviour flag type and wire the new flag through the QEMU driver. As with setvcpus async unplug, success indicates request acceptance while final completion is reported by the vcpu-removed event. Update the API documentation and add virsh support for setvcpu --async. Signed-off-by: Akash Kulhalli <akash.kulhalli@oracle.com> --- docs/manpages/virsh.rst | 10 ++++++++-- include/libvirt/libvirt-domain.h | 14 ++++++++++++++ src/libvirt-domain.c | 11 ++++++++++- src/qemu/qemu_driver.c | 12 ++++++++++-- tools/virsh-domain.c | 8 ++++++++ 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index f4c8e573cd47..7b3c853aea27 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -180,7 +180,7 @@ setvcpus and setmem. In those cases the fact that the ``virsh`` program returned, may not mean the action is complete and you must poll periodically to detect that the guest completed the operation. This asynchronous behaviour can be enforced with the -``--async`` flag to ``setvcpus`` only. +``--async`` flag to ``setvcpus`` and ``setvcpu`` only. ``virsh`` strives for backward compatibility. Although the ``help`` command only lists the preferred usage of a command, if an older @@ -4871,7 +4871,7 @@ setvcpu :: - setvcpu domain vcpulist [--enable] | [--disable] + setvcpu domain vcpulist [--enable] | [--disable] [--async] [[--live] [--config] | [--current]] Change state of individual vCPUs using hot(un)plug mechanism. @@ -4891,6 +4891,12 @@ If *--current* is specified, it is equivalent to either *--live* or default. Both *--live* and *--config* flags may be given, but *--current* is exclusive. +If *--async* is specified with *--disable*, live vCPU unplug requests are +fired without waiting for the guest to comply. This may optionally be +combined with *--config*. Final completion of this operation is reported +by the ``vcpu-removed`` domain event, while rejected unplug requests continue +to be reported by ``device-removal-failed``. + shutdown -------- diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 016b6ea700e9..2b6c498ed5ab 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -329,6 +329,20 @@ typedef enum { /* 1 << 2 is reserved for virTypedParameterFlags */ } virDomainModificationImpact; +/** + * virDomainSetVcpuBehaviour: + * + * These flags must be used when calling the `virDomainSetVcpu` API. + * + * Since: 12.3.0 + */ +typedef enum { + VIR_DOMAIN_SETVCPU_AFFECT_CURRENT = VIR_DOMAIN_AFFECT_CURRENT, /* Affect current domain state. (Since: 0.9.2) */ + VIR_DOMAIN_SETVCPU_AFFECT_LIVE = VIR_DOMAIN_AFFECT_LIVE, /* Affect running domain state. (Since: 0.9.2) */ + VIR_DOMAIN_SETVCPU_AFFECT_CONFIG = VIR_DOMAIN_AFFECT_CONFIG, /* Affect persistent domain state. (Since: 0.9.2) */ + VIR_DOMAIN_SETVCPU_ASYNC_UNPLUG = 1 << 2 /* Do not wait for the guest to comply with the request (Since: 12.3.0) */ +} virDomainSetVcpuBehaviour; + /** * virDomainInfo: * diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 97af1099f939..e3d1b76bcd01 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -13135,7 +13135,7 @@ virDomainSetGuestVcpus(virDomainPtr domain, * @domain: pointer to domain object * @vcpumap: text representation of a bitmap of vcpus to set * @state: 0 to disable/1 to enable cpus described by @vcpumap - * @flags: bitwise-OR of virDomainModificationImpact + * @flags: bitwise-OR of virDomainSetVcpuBehaviour * * Enables/disables individual vcpus described by @vcpumap in the hypervisor. * @@ -13144,6 +13144,15 @@ virDomainSetGuestVcpus(virDomainPtr domain, * * Note that OSes and hypervisors may require vCPU 0 to stay online. * + * If @flags includes VIR_DOMAIN_SETVCPU_ASYNC_UNPLUG, live vCPU disable (hot-unplug) + * request(s) are fired without waiting for the guest to comply. Success in + * this mode means only that the unplug request(s) were accepted. Final + * completion is reported by VIR_DOMAIN_EVENT_ID_VCPU_REMOVED, carrying the + * XML ``<vcpu id='...'>`` value for each removed vCPU. Rejected unplug + * requests continue to be reported by VIR_DOMAIN_EVENT_ID_DEVICE_REMOVAL_FAILED. + * The success event may be delivered before this API call returns. This flag + * has no effect when the selected vCPUs are enabled. + * * Returns 0 on success, -1 on error. * * Since: 3.1.0 diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index c9c72b29617f..9a20486725cb 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -19177,12 +19177,14 @@ qemuDomainSetVcpu(virDomainPtr dom, virDomainObj *vm = NULL; virDomainDef *def = NULL; virDomainDef *persistentDef = NULL; + bool async_unplug = !!(flags & VIR_DOMAIN_SETVCPU_ASYNC_UNPLUG); g_autoptr(virBitmap) map = NULL; ssize_t lastvcpu; int ret = -1; virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | - VIR_DOMAIN_AFFECT_CONFIG, -1); + VIR_DOMAIN_AFFECT_CONFIG | + VIR_DOMAIN_SETVCPU_ASYNC_UNPLUG, -1); if (state != 0 && state != 1) { virReportInvalidArg(state, "%s", _("unsupported state value")); @@ -19228,8 +19230,14 @@ qemuDomainSetVcpu(virDomainPtr dom, } } + if (async_unplug && (!def)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("asynchronous mode is supported only for live vcpu unplug")); + goto endjob; + } + ret = qemuDomainSetVcpuInternal(driver, vm, def, persistentDef, map, - !!state, false); + !!state, async_unplug); endjob: virDomainObjEndJob(vm); diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 1fccee4bc9ed..76369e8694ce 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -7837,6 +7837,10 @@ static const vshCmdOptDef opts_setvcpu[] = { .type = VSH_OT_BOOL, .help = N_("disable cpus specified by cpumap") }, + {.name = "async", + .type = VSH_OT_BOOL, + .help = N_("return after firing vcpu unplug request") + }, VIRSH_COMMON_OPT_DOMAIN_CONFIG, VIRSH_COMMON_OPT_DOMAIN_LIVE, VIRSH_COMMON_OPT_DOMAIN_CURRENT, @@ -7851,6 +7855,7 @@ cmdSetvcpu(vshControl *ctl, const vshCmd *cmd) bool disable = vshCommandOptBool(cmd, "disable"); bool config = vshCommandOptBool(cmd, "config"); bool live = vshCommandOptBool(cmd, "live"); + bool async = vshCommandOptBool(cmd, "async"); const char *vcpulist = NULL; int state = 0; unsigned int flags = VIR_DOMAIN_AFFECT_CURRENT; @@ -7859,11 +7864,14 @@ cmdSetvcpu(vshControl *ctl, const vshCmd *cmd) VSH_EXCLUSIVE_OPTIONS("current", "live"); VSH_EXCLUSIVE_OPTIONS("current", "config"); + VSH_EXCLUSIVE_OPTIONS("async", "enable"); if (config) flags |= VIR_DOMAIN_AFFECT_CONFIG; if (live) flags |= VIR_DOMAIN_AFFECT_LIVE; + if (async) + flags |= VIR_DOMAIN_SETVCPU_ASYNC_UNPLUG; if (!(enable || disable)) { vshError(ctl, "%s", _("one of --enable, --disable is required")); -- 2.47.3
participants (1)
-
Akash Kulhalli