[PATCH v1 0/4] Adding support for client-based FD migrations.
Libvirt supports FD-based live migrations in the form of tunnelled migrations. However, this isn't as performant as of now, since it involves extra copies. This patchset proposes adding support for externally provided file descriptors for live migration. By allowing the client to supply the migration FD directly, the extra copy can be avoided, improving overall migration performance. The intent is to gather early feedback on the approach and its feasibility. Comments, concerns, and suggestions are very welcome :) regards, Tejus Tejus GK (4): qemu: fd: introduce qemuFDTuple for FD storage lib: introduce virDomainFDStore API qemu: support incoming "fd" uri qemu: add support for fd based live migrations include/libvirt/libvirt-domain.h | 9 +++ src/driver-hypervisor.h | 7 ++ src/libvirt-domain.c | 62 +++++++++++++++ src/libvirt_public.syms | 5 ++ src/qemu/qemu_conf.h | 4 + src/qemu/qemu_driver.c | 41 ++++++++++ src/qemu/qemu_fd.c | 23 ++++++ src/qemu/qemu_fd.h | 14 ++++ src/qemu/qemu_migration.c | 117 +++++++++++++++++++++++++++- src/remote/remote_daemon_dispatch.c | 36 +++++++++ src/remote/remote_driver.c | 22 ++++++ src/remote/remote_protocol.x | 12 ++- 12 files changed, 348 insertions(+), 4 deletions(-) -- 2.43.7
Introduce a simple struct to store an array of file descriptors. This is loosely based on how virStorageSourceFDTuple is implemented. This will be used later on to store FDs for FD based live migrations of qemu domains. Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- src/qemu/qemu_fd.c | 23 +++++++++++++++++++++++ src/qemu/qemu_fd.h | 14 ++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/qemu/qemu_fd.c b/src/qemu/qemu_fd.c index ef0a8d8127..2bfaac500c 100644 --- a/src/qemu/qemu_fd.c +++ b/src/qemu/qemu_fd.c @@ -509,3 +509,26 @@ qemuFDPassDirectGetPath(qemuFDPassDirect *fdpass) return fdpass->name; } + + +void +qemuFDTupleFree(qemuFDTuple *tuple) +{ + size_t i; + + if (!tuple) + return; + + for (i = 0; i < tuple->nfds; i++) + VIR_FORCE_CLOSE(tuple->fds[i]); + + g_free(tuple->fds); + g_free(tuple); +} + + +qemuFDTuple * +qemuFDTupleNew(void) +{ + return g_new0(qemuFDTuple, 1); +} diff --git a/src/qemu/qemu_fd.h b/src/qemu/qemu_fd.h index d873e110a8..396ff946cb 100644 --- a/src/qemu/qemu_fd.h +++ b/src/qemu/qemu_fd.h @@ -83,3 +83,17 @@ qemuFDPassDirectTransferMonitorRollback(qemuFDPassDirect *fdpass, qemuMonitor *mon); const char * qemuFDPassDirectGetPath(qemuFDPassDirect *fdpass); + + +typedef struct _qemuFDTuple qemuFDTuple; +struct _qemuFDTuple { + int *fds; + size_t nfds; +}; + +void +qemuFDTupleFree(qemuFDTuple *tuple); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(qemuFDTuple, qemuFDTupleFree); + +qemuFDTuple * +qemuFDTupleNew(void); -- 2.43.7
This API allows callers to pass and store file descriptors inside libvirt using a hash table, with a provided "name" serving as the key. The stored file descriptors can be queried and reused later, enabling use cases such as FD-based live migration. Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- include/libvirt/libvirt-domain.h | 9 +++++ src/driver-hypervisor.h | 7 ++++ src/libvirt-domain.c | 62 +++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 +++ src/qemu/qemu_conf.h | 4 ++ src/qemu/qemu_driver.c | 41 +++++++++++++++++++ src/remote/remote_daemon_dispatch.c | 36 +++++++++++++++++ src/remote/remote_driver.c | 22 ++++++++++ src/remote/remote_protocol.x | 12 +++++- 9 files changed, 197 insertions(+), 1 deletion(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index b0acd0083b..be40a127d6 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -8751,5 +8751,14 @@ virDomainDelThrottleGroup(virDomainPtr dom, const char *group, unsigned int flags); +/** +* virDomainFDStore: +* +* since: 12.0.0 +*/ +int virDomainFDStore(virConnectPtr conn, + const char *name, + unsigned int nfds, + int *fds); #endif /* LIBVIRT_DOMAIN_H */ diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 6a43688b0c..0c2a9041fe 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -1473,6 +1473,12 @@ typedef int const char *groupname, unsigned int flags); +typedef int +(*virDrvDomainFDStore)(virConnectPtr conn, + const char *name, + unsigned int nfds, + int *fds); + typedef struct _virHypervisorDriver virHypervisorDriver; /** @@ -1750,4 +1756,5 @@ struct _virHypervisorDriver { virDrvDomainGraphicsReload domainGraphicsReload; virDrvDomainSetThrottleGroup domainSetThrottleGroup; virDrvDomainDelThrottleGroup domainDelThrottleGroup; + virDrvDomainFDStore domainFDStore; }; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 74c70a0a43..7e194f7193 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14269,3 +14269,65 @@ virDomainDelThrottleGroup(virDomainPtr dom, virDispatchError(dom->conn); return -1; } + + +/** + * virDomainFDStore: + * @conn: a connection object + * @name: name for the file descriptor group + * @nfds: number of fds in @fds + * @fds: file descriptors to store + * + * Store the FDs in @fds with @conn under @name. The FDs persist unitl + * explicitly consumed (e.g. during a live migration) or until libvirt + * shuts down/ restarts. + * + * Unlike virDomainFDAssociate, this does not require a domain to exist + * at the time of storing the FDs. + * + * Returns 0 on success, -1 on error. + * + * Since: 12.0.0 + */ + int + virDomainFDStore(virConnectPtr conn, + const char *name, + unsigned int nfds, + int *fds) + { + int rc; + + VIR_DEBUG("conn=%p, name='%s', nfds=%u, fds=%p", + conn, name, nfds, fds); + + virResetLastError(); + + virCheckConnectReturn(conn, -1); + virCheckNonNullArgGoto(name, error); + virCheckNonZeroArgGoto(nfds, error); + virCheckNonNullArgGoto(fds, error); + + if ((rc = VIR_DRV_SUPPORTS_FEATURE(conn->driver, conn, + VIR_DRV_FEATURE_FD_PASSING)) < 0) + goto error; + + if (rc == 0) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("fd passing is not supported by this connection")); + goto error; + } + + if (!conn->driver->domainFDStore) { + virReportUnsupportedError(); + goto error; + } + + if ((rc = conn->driver->domainFDStore(conn, name, nfds, fds)) < 0) + goto error; + + return rc; + + error: + virDispatchError(conn); + return -1; + } diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index c506acd2ed..3b3a4bb4e1 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -956,4 +956,9 @@ LIBVIRT_11.2.0 { virDomainDelThrottleGroup; } LIBVIRT_10.2.0; +LIBVIRT_12.0.0 { + global: + virDomainFDStore; +} LIBVIRT_11.2.0; + # .... define new API here using predicted next version number .... diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index c284e108a1..cc655811f7 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -349,6 +349,10 @@ struct _virQEMUDriver { /* Immutable pointer, self-locking APIs */ virFileCache *nbdkitCapsCache; + + /* Hash table mapping FD names (string) to qemuFDTuple objects */ + GHashTable *domainFDs; + }; virQEMUDriverConfig *virQEMUDriverConfigNew(bool privileged, diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 3f154969b8..94a7b17090 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -35,6 +35,7 @@ #include "qemu_alias.h" #include "qemu_block.h" #include "qemu_conf.h" +#include "qemu_fd.h" #include "qemu_capabilities.h" #include "qemu_command.h" #include "qemu_hostdev.h" @@ -558,6 +559,11 @@ qemuStateInitialize(bool privileged, if (!(qemu_driver->domains = virDomainObjListNew())) goto error; + qemu_driver->domainFDs = g_hash_table_new_full(g_str_hash, + g_str_equal, + g_free, + (GDestroyNotify) qemuFDTupleFree); + /* Init domain events */ qemu_driver->domainEventState = virObjectEventStateNew(); if (!qemu_driver->domainEventState) @@ -1033,6 +1039,7 @@ qemuStateCleanup(void) virObjectUnref(qemu_driver->domains); virObjectUnref(qemu_driver->nbdkitCapsCache); virInhibitorFree(qemu_driver->inhibitor); + g_clear_pointer(&qemu_driver->domainFDs, g_hash_table_unref); if (qemu_driver->lockFD != -1) virPidFileRelease(qemu_driver->config->stateDir, "driver", qemu_driver->lockFD); @@ -20259,6 +20266,39 @@ qemuDomainFDAssociate(virDomainPtr domain, return ret; } +static int +qemuDomainFDStore(virConnectPtr conn, + const char *name, + unsigned int nfds, + int *fds) +{ + virQEMUDriver *driver = conn->privateData; + g_autoptr(qemuFDTuple) new = NULL; + size_t i; + + if (nfds == 0) + return 0; + + new = qemuFDTupleNew(); + new->nfds = nfds; + new->fds = g_new0(int, new->nfds); + + for (i = 0; i < new->nfds; i++) { + if ((new->fds[i] = dup(fds[i])) < 0) { + virReportSystemError(errno, + _("failed to duplicate passed fd with index '%1$zu'"), + i); + return -1; + } + } + + VIR_WITH_MUTEX_LOCK_GUARD(&driver->lock) { + g_hash_table_insert(driver->domainFDs, g_strdup(name), g_steal_pointer(&new)); + } + + return 0; +} + static int qemuDomainGraphicsReload(virDomainPtr domain, unsigned int type, @@ -20804,6 +20844,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainSetAutostartOnce = qemuDomainSetAutostartOnce, /* 11.2.0 */ .domainSetThrottleGroup = qemuDomainSetThrottleGroup, /* 11.2.0 */ .domainDelThrottleGroup = qemuDomainDelThrottleGroup, /* 11.2.0 */ + .domainFDStore = qemuDomainFDStore, /* 12.0.0 */ }; diff --git a/src/remote/remote_daemon_dispatch.c b/src/remote/remote_daemon_dispatch.c index 7e74ff063f..bbf39be748 100644 --- a/src/remote/remote_daemon_dispatch.c +++ b/src/remote/remote_daemon_dispatch.c @@ -7163,6 +7163,42 @@ remoteDispatchNetworkPortGetParameters(virNetServer *server G_GNUC_UNUSED, return rv; } +static int +remoteDispatchDomainFdStore(virNetServer *server G_GNUC_UNUSED, + virNetServerClient *client, + virNetMessage *msg G_GNUC_UNUSED, + struct virNetMessageError *rerr, + remote_domain_fd_store_args *args) +{ + int *fds = NULL; + unsigned int nfds = 0; + int rv = -1; + virConnectPtr conn = remoteGetHypervisorConn(client); + size_t i; + + if (!conn) + goto cleanup; + + fds = g_new0(int, msg->nfds); + for (i = 0; i < msg->nfds; i++) { + if ((fds[i] = virNetMessageDupFD(msg, i)) < 0) + goto cleanup; + nfds++; + } + + if (virDomainFDStore(conn, args->name, nfds, fds) < 0) + goto cleanup; + + rv = 0; + + cleanup: + for (i = 0; i < nfds; i++) + VIR_FORCE_CLOSE(fds[i]); + g_free(fds); + if (rv < 0) + virNetMessageSaveError(rerr); + return rv; +} /*----- Helpers. -----*/ diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index ec71eaed87..db255ac125 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -7488,6 +7488,27 @@ remoteDomainFDAssociate(virDomainPtr domain, return 0; } +static int +remoteDomainFDStore(virConnectPtr conn, + const char *name, + unsigned int nfds, + int *fds) +{ + remote_domain_fd_store_args args = {0}; + struct private_data *priv = conn->privateData; + VIR_LOCK_GUARD lock = remoteDriverLock(priv); + + args.name = (char *)name; + + if (callFull(conn, priv, 0, fds, nfds, NULL, NULL, + REMOTE_PROC_DOMAIN_FD_STORE, + (xdrproc_t) xdr_remote_domain_fd_store_args, (char *) &args, + (xdrproc_t) xdr_void, (char *) NULL) == -1) + return -1; + + return 0; +} + /* get_nonnull_domain and get_nonnull_network turn an on-wire * (name, uuid) pair into virDomainPtr or virNetworkPtr object. @@ -7935,6 +7956,7 @@ static virHypervisorDriver hypervisor_driver = { .domainGraphicsReload = remoteDomainGraphicsReload, /* 10.2.0 */ .domainSetThrottleGroup = remoteDomainSetThrottleGroup, /* 11.2.0 */ .domainDelThrottleGroup = remoteDomainDelThrottleGroup, /* 11.2.0 */ + .domainFDStore = remoteDomainFDStore, /* 12.0.0 */ }; static virNetworkDriver network_driver = { diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 3c93203210..3ddfdd8bec 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -4008,6 +4008,10 @@ struct remote_domain_event_nic_mac_change_msg { remote_nonnull_string newMAC; }; +struct remote_domain_fd_store_args { + remote_nonnull_string name; +}; + /*----- Protocol. -----*/ /* Define the program number, protocol version and procedure numbers here. */ @@ -7119,5 +7123,11 @@ enum remote_procedure { * @generate: both * @acl: none */ - REMOTE_PROC_DOMAIN_EVENT_NIC_MAC_CHANGE = 453 + REMOTE_PROC_DOMAIN_EVENT_NIC_MAC_CHANGE = 453, + + /** + * @generate: none + * @acl: none + */ + REMOTE_PROC_DOMAIN_FD_STORE = 454 }; -- 2.43.7
On Wed, Jan 14, 2026 at 10:42:02 +0000, Tejus GK wrote:
This API allows callers to pass and store file descriptors inside libvirt using a hash table, with a provided "name" serving as the key. The stored file descriptors can be queried and reused later, enabling use cases such as FD-based live migration.
Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- include/libvirt/libvirt-domain.h | 9 +++++ src/driver-hypervisor.h | 7 ++++ src/libvirt-domain.c | 62 +++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 +++ src/qemu/qemu_conf.h | 4 ++ src/qemu/qemu_driver.c | 41 +++++++++++++++++++ src/remote/remote_daemon_dispatch.c | 36 +++++++++++++++++ src/remote/remote_driver.c | 22 ++++++++++ src/remote/remote_protocol.x | 12 +++++- 9 files changed, 197 insertions(+), 1 deletion(-)
[...]
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 74c70a0a43..7e194f7193 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14269,3 +14269,65 @@ virDomainDelThrottleGroup(virDomainPtr dom, virDispatchError(dom->conn); return -1; } + + +/** + * virDomainFDStore: + * @conn: a connection object + * @name: name for the file descriptor group + * @nfds: number of fds in @fds + * @fds: file descriptors to store + * + * Store the FDs in @fds with @conn under @name. The FDs persist unitl + * explicitly consumed (e.g. during a live migration) or until libvirt + * shuts down/ restarts. + * + * Unlike virDomainFDAssociate, this does not require a domain to exist + * at the time of storing the FDs. + * + * Returns 0 on success, -1 on error. + * + * Since: 12.0.0 + */
This looks identical to as long as you are about to use the FD with a VM object: /** * virDomainFDAssociate: * @domain: a domain object * @name: name for the file descriptor group * @nfds: number of fds in @fds * @fds: file descriptors to associate with domain * @flags: optional flags; bitwise-OR of supported virDomainFDAssociateFlags * * Associate the FDs in @fd with @domain under @name. The FDs are associated as * long as the connection used to associated exists and are disposed of * afterwards. FD may still be kept open by the hypervisor for as long as it's * needed. * * Security labelling (e.g. via the selinux) may be applied on the passed FDs * when required for usage by the VM. By default libvirt does not restore the * seclabels on the FDs afterwards to avoid keeping it open unnecessarily. * * Restoring of the security label can be requested by passing either * VIR_DOMAIN_FD_ASSOCIATE_SECLABEL_RESTORE for a best-effort attempt to restore * the security label after use. * Requesting the restore of security label will require that the file * descriptors are kept open for the whole time they are used by the hypervisor, * or other additional overhead. * * In certain cases usage of the fd group would imply read-only access. Passing * VIR_DOMAIN_FD_ASSOCIATE_SECLABEL_WRITABLE in @flags ensures that a writable * security label is picked in case when the file represented by the fds may * be used in write mode. * * Returns 0 on success, -1 on error. * * Since: 9.0.0 */ int virDomainFDAssociate(virDomainPtr domain, const char *name, unsigned int nfds, int *fds, unsigned int flags)
+ int + virDomainFDStore(virConnectPtr conn, + const char *name, + unsigned int nfds, + int *fds) + { + int rc; + + VIR_DEBUG("conn=%p, name='%s', nfds=%u, fds=%p", + conn, name, nfds, fds);
On Wed, Jan 14, 2026 at 13:18:51 +0100, Peter Krempa via Devel wrote:
On Wed, Jan 14, 2026 at 10:42:02 +0000, Tejus GK wrote:
This API allows callers to pass and store file descriptors inside libvirt using a hash table, with a provided "name" serving as the key. The stored file descriptors can be queried and reused later, enabling use cases such as FD-based live migration.
Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- include/libvirt/libvirt-domain.h | 9 +++++ src/driver-hypervisor.h | 7 ++++ src/libvirt-domain.c | 62 +++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 +++ src/qemu/qemu_conf.h | 4 ++ src/qemu/qemu_driver.c | 41 +++++++++++++++++++ src/remote/remote_daemon_dispatch.c | 36 +++++++++++++++++ src/remote/remote_driver.c | 22 ++++++++++ src/remote/remote_protocol.x | 12 +++++- 9 files changed, 197 insertions(+), 1 deletion(-)
[...]
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 74c70a0a43..7e194f7193 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14269,3 +14269,65 @@ virDomainDelThrottleGroup(virDomainPtr dom, virDispatchError(dom->conn); return -1; } + + +/** + * virDomainFDStore: + * @conn: a connection object + * @name: name for the file descriptor group + * @nfds: number of fds in @fds + * @fds: file descriptors to store + * + * Store the FDs in @fds with @conn under @name. The FDs persist unitl + * explicitly consumed (e.g. during a live migration) or until libvirt + * shuts down/ restarts. + * + * Unlike virDomainFDAssociate, this does not require a domain to exist + * at the time of storing the FDs. + * + * Returns 0 on success, -1 on error. + * + * Since: 12.0.0 + */
This looks identical to as long as you are about to use the FD with a VM object:
Looking at the usage in the code it seems to be in a place which has @domain available, so no new API should be required for this usage.
/** * virDomainFDAssociate: * @domain: a domain object * @name: name for the file descriptor group * @nfds: number of fds in @fds * @fds: file descriptors to associate with domain * @flags: optional flags; bitwise-OR of supported virDomainFDAssociateFlags * * Associate the FDs in @fd with @domain under @name. The FDs are associated as * long as the connection used to associated exists and are disposed of * afterwards. FD may still be kept open by the hypervisor for as long as it's * needed. * * Security labelling (e.g. via the selinux) may be applied on the passed FDs * when required for usage by the VM. By default libvirt does not restore the * seclabels on the FDs afterwards to avoid keeping it open unnecessarily. * * Restoring of the security label can be requested by passing either * VIR_DOMAIN_FD_ASSOCIATE_SECLABEL_RESTORE for a best-effort attempt to restore * the security label after use. * Requesting the restore of security label will require that the file * descriptors are kept open for the whole time they are used by the hypervisor, * or other additional overhead. * * In certain cases usage of the fd group would imply read-only access. Passing * VIR_DOMAIN_FD_ASSOCIATE_SECLABEL_WRITABLE in @flags ensures that a writable * security label is picked in case when the file represented by the fds may * be used in write mode. * * Returns 0 on success, -1 on error. * * Since: 9.0.0 */ int virDomainFDAssociate(virDomainPtr domain, const char *name, unsigned int nfds, int *fds, unsigned int flags)
+ int + virDomainFDStore(virConnectPtr conn, + const char *name, + unsigned int nfds, + int *fds) + { + int rc; + + VIR_DEBUG("conn=%p, name='%s', nfds=%u, fds=%p", + conn, name, nfds, fds);
On 14 Jan 2026, at 5:54 PM, Peter Krempa <pkrempa@redhat.com> wrote:
!-------------------------------------------------------------------| CAUTION: External Email
|-------------------------------------------------------------------!
On Wed, Jan 14, 2026 at 13:18:51 +0100, Peter Krempa via Devel wrote:
On Wed, Jan 14, 2026 at 10:42:02 +0000, Tejus GK wrote:
This API allows callers to pass and store file descriptors inside libvirt using a hash table, with a provided "name" serving as the key. The stored file descriptors can be queried and reused later, enabling use cases such as FD-based live migration.
Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- include/libvirt/libvirt-domain.h | 9 +++++ src/driver-hypervisor.h | 7 ++++ src/libvirt-domain.c | 62 +++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 +++ src/qemu/qemu_conf.h | 4 ++ src/qemu/qemu_driver.c | 41 +++++++++++++++++++ src/remote/remote_daemon_dispatch.c | 36 +++++++++++++++++ src/remote/remote_driver.c | 22 ++++++++++ src/remote/remote_protocol.x | 12 +++++- 9 files changed, 197 insertions(+), 1 deletion(-)
[...]
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 74c70a0a43..7e194f7193 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14269,3 +14269,65 @@ virDomainDelThrottleGroup(virDomainPtr dom, virDispatchError(dom->conn); return -1; } + + +/** + * virDomainFDStore: + * @conn: a connection object + * @name: name for the file descriptor group + * @nfds: number of fds in @fds + * @fds: file descriptors to store + * + * Store the FDs in @fds with @conn under @name. The FDs persist unitl + * explicitly consumed (e.g. during a live migration) or until libvirt + * shuts down/ restarts. + * + * Unlike virDomainFDAssociate, this does not require a domain to exist + * at the time of storing the FDs. + * + * Returns 0 on success, -1 on error. + * + * Since: 12.0.0 + */
This looks identical to as long as you are about to use the FD with a VM object:
Looking at the usage in the code it seems to be in a place which has @domain available, so no new API should be required for this usage.
Hi Peter, Thank you for the review! I agree that the API I’ve introduced is quite similar to virDomainFDAssociate. But it can’t be reused for the client FD-based live migration. The reason is that virDomainFDAssociate requires a @domain object, meaning a domain up and running at the time of the API call. However, in the live-migration use case, on the destination host, the domain doesn’t exist prior to the migration being triggered; hence, the client won’t be able to pass an FD to the VM, which doesn’t exist at that point. Hence, virDomainFDStore is there to store FDs associated with the “name” key, rather than the domain. Regards, Tejus
On Wed, Jan 14, 2026 at 13:46:53 +0000, Tejus GK wrote:
On 14 Jan 2026, at 5:54 PM, Peter Krempa <pkrempa@redhat.com> wrote:
!-------------------------------------------------------------------| CAUTION: External Email
|-------------------------------------------------------------------!
On Wed, Jan 14, 2026 at 13:18:51 +0100, Peter Krempa via Devel wrote:
On Wed, Jan 14, 2026 at 10:42:02 +0000, Tejus GK wrote:
This API allows callers to pass and store file descriptors inside libvirt using a hash table, with a provided "name" serving as the key. The stored file descriptors can be queried and reused later, enabling use cases such as FD-based live migration.
Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- include/libvirt/libvirt-domain.h | 9 +++++ src/driver-hypervisor.h | 7 ++++ src/libvirt-domain.c | 62 +++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 +++ src/qemu/qemu_conf.h | 4 ++ src/qemu/qemu_driver.c | 41 +++++++++++++++++++ src/remote/remote_daemon_dispatch.c | 36 +++++++++++++++++ src/remote/remote_driver.c | 22 ++++++++++ src/remote/remote_protocol.x | 12 +++++- 9 files changed, 197 insertions(+), 1 deletion(-)
[...]
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 74c70a0a43..7e194f7193 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14269,3 +14269,65 @@ virDomainDelThrottleGroup(virDomainPtr dom, virDispatchError(dom->conn); return -1; } + + +/** + * virDomainFDStore: + * @conn: a connection object + * @name: name for the file descriptor group + * @nfds: number of fds in @fds + * @fds: file descriptors to store + * + * Store the FDs in @fds with @conn under @name. The FDs persist unitl + * explicitly consumed (e.g. during a live migration) or until libvirt + * shuts down/ restarts. + * + * Unlike virDomainFDAssociate, this does not require a domain to exist + * at the time of storing the FDs. + * + * Returns 0 on success, -1 on error. + * + * Since: 12.0.0 + */
This looks identical to as long as you are about to use the FD with a VM object:
Looking at the usage in the code it seems to be in a place which has @domain available, so no new API should be required for this usage.
Hi Peter,
Thank you for the review! I agree that the API I’ve introduced is quite similar to virDomainFDAssociate. But it can’t be reused for the client FD-based live migration. The reason is that virDomainFDAssociate requires a @domain object, meaning a domain up and running at the time of the API call. However, in the live-migration use case, on the destination host, the domain doesn’t exist prior to the migration being triggered; hence, the client won’t be able to pass an FD to the VM, which doesn’t exist at that point.
Good point.
Hence, virDomainFDStore is there to store FDs associated with the “name” key, rather than the domain.
Okay but that will make things much more complex due to our ACL stuff. Allowing to pass any FD means that any user (even those who can't see the VM) could pass a FD and theoretically siphon data off from it. I don't think we want to have a 'global' FD pool to pass. In case of migration I'd either suggest to define the VM (with possibly a dummy definition) just to have the domain object available. Alternatively we should have a migration API allowing FD passing.
On 14 Jan 2026, at 9:00 PM, Peter Krempa <pkrempa@redhat.com> wrote:
!-------------------------------------------------------------------| CAUTION: External Email
|-------------------------------------------------------------------!
On Wed, Jan 14, 2026 at 13:46:53 +0000, Tejus GK wrote:
On 14 Jan 2026, at 5:54 PM, Peter Krempa <pkrempa@redhat.com> wrote:
!-------------------------------------------------------------------| CAUTION: External Email
|-------------------------------------------------------------------!
On Wed, Jan 14, 2026 at 13:18:51 +0100, Peter Krempa via Devel wrote:
On Wed, Jan 14, 2026 at 10:42:02 +0000, Tejus GK wrote:
This API allows callers to pass and store file descriptors inside libvirt using a hash table, with a provided "name" serving as the key. The stored file descriptors can be queried and reused later, enabling use cases such as FD-based live migration.
Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- include/libvirt/libvirt-domain.h | 9 +++++ src/driver-hypervisor.h | 7 ++++ src/libvirt-domain.c | 62 +++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 +++ src/qemu/qemu_conf.h | 4 ++ src/qemu/qemu_driver.c | 41 +++++++++++++++++++ src/remote/remote_daemon_dispatch.c | 36 +++++++++++++++++ src/remote/remote_driver.c | 22 ++++++++++ src/remote/remote_protocol.x | 12 +++++- 9 files changed, 197 insertions(+), 1 deletion(-)
[...]
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 74c70a0a43..7e194f7193 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -14269,3 +14269,65 @@ virDomainDelThrottleGroup(virDomainPtr dom, virDispatchError(dom->conn); return -1; } + + +/** + * virDomainFDStore: + * @conn: a connection object + * @name: name for the file descriptor group + * @nfds: number of fds in @fds + * @fds: file descriptors to store + * + * Store the FDs in @fds with @conn under @name. The FDs persist unitl + * explicitly consumed (e.g. during a live migration) or until libvirt + * shuts down/ restarts. + * + * Unlike virDomainFDAssociate, this does not require a domain to exist + * at the time of storing the FDs. + * + * Returns 0 on success, -1 on error.
+ *
+ * Since: 12.0.0 + */
This looks identical to as long as you are about to use the FD with a VM object:
Looking at the usage in the code it seems to be in a place which has @domain available, so no new API should be required for this usage.
Hi Peter,
Thank you for the review! I agree that the API I’ve introduced is quite similar to virDomainFDAssociate. But it can’t be reused for the client FD-based live migration. The reason is that virDomainFDAssociate requires a @domain object, meaning a domain up and running at the time of the API call. However, in the live-migration use case, on the destination host, the domain doesn’t exist prior to the migration being triggered; hence, the client won’t be able to pass an FD to the VM, which doesn’t exist at that point.
Good point.
Hence, virDomainFDStore is there to store FDs associated with the “name” key, rather than the domain.
Okay but that will make things much more complex due to our ACL stuff.
Allowing to pass any FD means that any user (even those who can't see the VM) could pass a FD and theoretically siphon data off from it.
I don't think we want to have a 'global' FD pool to pass. In case of migration I'd either suggest to define the VM (with possibly a dummy definition) just to have the domain object available. Alternatively we should have a migration API allowing FD passing.
Hi Peter, Thank you for the suggestion! I wanted to clarify that my design of maintaining a global pool of FDs, was so to seamlessly support the use case for managed P2P migrations.
I'd either suggest to define the VM (with possibly a dummy definition) Is this workflow supported in P2P migrations? Also, we would have to at some point of time, blow up the VM definition to match the one on the source, does libvirt have support for that?
Alternatively we should have a migration API allowing FD passing. Hmm, even if we go ahead and implement that, I’m wondering how to make it work for P2P managed migrations. Since, source libvirt won’t be able to pass on FDs to the destination host. Thoughts?
Regards, Tejus
On Mon, Jan 19, 2026 at 14:08:29 +0000, Tejus GK wrote:
On 14 Jan 2026, at 9:00 PM, Peter Krempa <pkrempa@redhat.com> wrote:
!-------------------------------------------------------------------| CAUTION: External Email
|-------------------------------------------------------------------!
Oh no! [...]
Hence, virDomainFDStore is there to store FDs associated with the “name” key, rather than the domain.
Okay but that will make things much more complex due to our ACL stuff.
Allowing to pass any FD means that any user (even those who can't see the VM) could pass a FD and theoretically siphon data off from it.
I don't think we want to have a 'global' FD pool to pass. In case of migration I'd either suggest to define the VM (with possibly a dummy definition) just to have the domain object available. Alternatively we should have a migration API allowing FD passing.
Hi Peter, Thank you for the suggestion! I wanted to clarify that my design of maintaining a global pool of FDs, was so to seamlessly support the use case for managed P2P migrations.
I'd either suggest to define the VM (with possibly a dummy definition) Is this workflow supported in P2P migrations? Also, we would have to at some point of time, blow up the VM definition to match the one on the source, does libvirt have support for that?
Yes, ...
Alternatively we should have a migration API allowing FD passing. Hmm, even if we go ahead and implement that, I’m wondering how to make it work for P2P managed migrations. Since, source libvirt won’t be able to pass on FDs to the destination host. Thoughts?
you need to open another new client connection on the destination and pass the FDs. In fact that's requrired since FD passing works only locally and the remote daemon is thus unable to do that. For now you can test it for FD-passed disks, where prior to migration you open a new connection and pass the FDs then kick off the migration This is a complex deployment where you do need to set up stuff on the destination manually so IMO this is a reasonable trade-off.
On 20 Jan 2026, at 8:45 PM, Peter Krempa <pkrempa@redhat.com> wrote:
!-------------------------------------------------------------------| CAUTION: External Email
|-------------------------------------------------------------------!
On Mon, Jan 19, 2026 at 14:08:29 +0000, Tejus GK wrote:
On 14 Jan 2026, at 9:00 PM, Peter Krempa <pkrempa@redhat.com> wrote:
!-------------------------------------------------------------------| CAUTION: External Email
|-------------------------------------------------------------------!
Oh no!
Hah!
Hence, virDomainFDStore is there to store FDs associated with the “name” key, rather than the domain.
Okay but that will make things much more complex due to our ACL stuff.
Allowing to pass any FD means that any user (even those who can't see the VM) could pass a FD and theoretically siphon data off from it.
I don't think we want to have a 'global' FD pool to pass. In case of migration I'd either suggest to define the VM (with possibly a dummy definition) just to have the domain object available. Alternatively we should have a migration API allowing FD passing.
Hi Peter, Thank you for the suggestion! I wanted to clarify that my design of maintaining a global pool of FDs, was so to seamlessly support the use case for managed P2P migrations.
I'd either suggest to define the VM (with possibly a dummy definition) Is this workflow supported in P2P migrations? Also, we would have to at some point of time, blow up the VM definition to match the one on the source, does libvirt have support for that?
Yes, ...
Alternatively we should have a migration API allowing FD passing. Hmm, even if we go ahead and implement that, I’m wondering how to make it work for P2P managed migrations. Since, source libvirt won’t be able to pass on FDs to the destination host. Thoughts?
you need to open another new client connection on the destination and pass the FDs. In fact that's requrired since FD passing works only locally and the remote daemon is thus unable to do that.
For now you can test it for FD-passed disks, where prior to migration you open a new connection and pass the FDs then kick off the migration
This is a complex deployment where you do need to set up stuff on the destination manually so IMO this is a reasonable trade-off.
Hi Peter, thanks for the dummy VM suggestion, I gave it a spin, and it seems to work! I’ll rework the patch-series keeping the approach in mind, and share it on the list soon. Regards, Tejus
If "fd" is the choosen protocol for the live migration, the destination VM should start up with -incoming as "fd", this patch adds support for the same. Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- src/qemu/qemu_migration.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 6dd022163b..0cd417c15d 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2427,7 +2427,8 @@ qemuMigrationDstGetURI(const char *migrateFrom, { char *uri = NULL; - if (STREQ(migrateFrom, "stdio")) + if (STREQ(migrateFrom, "stdio") || + STREQ(migrateFrom, "fd")) uri = g_strdup_printf("fd:%d", migrateFd); else uri = g_strdup(migrateFrom); -- 2.43.7
QEMU already supports migration using a file descriptor that is passed to QEMU. As of now, libvirt, uses this path via the tunnelled migration approach, where libvirt creates a set of fds from a pipe, passes those fds to QEMU, and migrates the VM. This patch introduces supports for fd based live migrations in libvirt, where the fds are opened and given to libvirt by the client. Clients are expected to pass the FDs on both source and destination libvirt, via the virDomainFDStore API, with the "fd" key being the domain UUID. If the URI of the migration has been set to "fd", libvirt will try to look for an FD passed to it via the client meant for migrating the VM, and will pass on that FD to QEMU and trigger the migration. Signed-off-by: Tejus GK <tejus.gk@nutanix.com> --- src/qemu/qemu_migration.c | 114 +++++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 0cd417c15d..66e1236e6c 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -38,6 +38,7 @@ #include "qemu_security.h" #include "qemu_slirp.h" #include "qemu_block.h" +#include "qemu_fd.h" #include "qemu_tpm.h" #include "qemu_vhost_user.h" @@ -3137,6 +3138,8 @@ qemuMigrationDstPrepare(virQEMUDriver *driver, if (tunnel) { migrateFrom = g_strdup("stdio"); + } else if (g_strcmp0(protocol, "fd") == 0) { + migrateFrom = g_strdup("fd"); } else if (g_strcmp0(protocol, "unix") == 0) { migrateFrom = g_strdup_printf("%s:%s", protocol, listenAddress); } else { @@ -3301,6 +3304,8 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, unsigned int startFlags; bool relabel = false; bool tunnel = !!st; + bool useDestFD = STREQ_NULLABLE(protocol, "fd"); + int destfd = -1; int ret = -1; int rv; @@ -3319,6 +3324,37 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, virPipe(dataFD) < 0) goto error; + if (useDestFD) { + qemuFDTuple *fdtuple = NULL; + const char *fdName = vm->def->uuid; + + VIR_WITH_MUTEX_LOCK_GUARD(&driver->lock) { + fdtuple = g_hash_table_lookup(driver->domainFDs, fdName); + if (!fdtuple) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("no file descriptor stored for migration of domain '%1$s'"), + fdName); + goto error; + } + + if (fdtuple->nfds != 1) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("expected a single file descriptor for migration of domain '%1$s'"), + fdName); + goto error; + } + + if ((destfd = dup(fdtuple->fds[0])) < 0) { + virReportSystemError(errno, + _("failed to duplicate migration FD for domain '%1$s'"), + fdName); + goto error; + } + + g_hash_table_remove(driver->domainFDs, fdName); + } + } + startFlags = VIR_QEMU_PROCESS_START_AUTODESTROY; if (qemuProcessInit(driver, vm, mig->cpu, VIR_ASYNC_JOB_MIGRATION_IN, @@ -3328,7 +3364,7 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, if (!(incoming = qemuMigrationDstPrepare(driver, vm, tunnel, protocol, listenAddress, port, - &dataFD[0]))) + useDestFD ? destfd : dataFD[0]))) goto error; qemuMigrationDstPrepareDiskSeclabels(vm, migrate_disks, flags); @@ -3437,6 +3473,7 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, cleanup: qemuProcessIncomingDefFree(incoming); + VIR_FORCE_CLOSE(destfd); VIR_FORCE_CLOSE(dataFD[0]); VIR_FORCE_CLOSE(dataFD[1]); virObjectEventStateQueue(driver->domainEventState, event); @@ -3974,7 +4011,8 @@ qemuMigrationDstPrepareDirect(virQEMUDriver *driver, if (STRNEQ(uri->scheme, "tcp") && STRNEQ(uri->scheme, "rdma") && - STRNEQ(uri->scheme, "unix")) { + STRNEQ(uri->scheme, "unix") && + STRNEQ(uri->scheme, "fd")) { virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, _("unsupported scheme %1$s in migration URI %2$s"), uri->scheme, uri_in); @@ -3984,6 +4022,9 @@ qemuMigrationDstPrepareDirect(virQEMUDriver *driver, if (STREQ(uri->scheme, "unix")) { autoPort = false; listenAddress = uri->path; + } else if (STREQ(uri->scheme, "fd")) { + autoPort = false; + listenAddress = NULL; } else { if (uri->server == NULL) { virReportError(VIR_ERR_INVALID_ARG, @@ -5411,6 +5452,75 @@ qemuMigrationSrcPerformNative(virQEMUDriver *driver, spec.destType = MIGRATION_DEST_CONNECT_SOCKET; spec.dest.socket.path = uribits->path; + } else if (STREQ(uribits->scheme, "fd")) { + + if (flags & VIR_MIGRATE_PARALLEL) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("FD-based migration is not supported with multi-fd migrations")); + return -1; + } + + if (flags & VIR_MIGRATE_POSTCOPY) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("FD-based migration is not supported with post-copy migration")); + return -1; + } + if (flags & VIR_MIGRATE_POSTCOPY_RESUME) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("FD-based migration is not supported with post-copy resume")); + return -1; + } + if (flags & VIR_MIGRATE_ZEROCOPY) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("FD-based migration is not supported with zero-copy migration")); + return -1; + } + + if (flags & VIR_MIGRATE_TLS) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("FD-based migration is not supported with TLS migration")); + return -1; + } + + qemuFDTuple *fdtuple = NULL; + const char *fdName = vm->def->uuid; + int srcfd = -1; + + VIR_WITH_MUTEX_LOCK_GUARD(&driver->lock) { + fdtuple = g_hash_table_lookup(driver->domainFDs, fdName); + if (!fdtuple) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("no file descriptor stored for migration of domain '%1$s'"), + fdName); + return -1; + } + + if (fdtuple->nfds != 1) { + virReportError(VIR_ERR_OPERATION_INVALID, + _("expected a single file descriptor for migration of domain '%1$s'"), + fdName); + return -1; + } + + if ((srcfd = dup(fdtuple->fds[0])) < 0) { + virReportSystemError(errno, + _("failed to duplicate migration FD for domain '%1$s'"), + fdName); + return -1; + } + + g_hash_table_remove(driver->domainFDs, fdName); + } + + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, srcfd) < 0) { + VIR_FORCE_CLOSE(srcfd); + return -1; + } + + spec.destType = MIGRATION_DEST_FD; + spec.dest.fd.qemu = srcfd; + spec.dest.fd.local = -1; + } else { /* RDMA, multi-fd, and postcopy-preempt migration require QEMU to * connect to the destination itself. -- 2.43.7
participants (2)
-
Peter Krempa -
Tejus GK