[libvirt RFC v2 00/11] multifd save prototype
This is the multifd save prototype in its first semi-functional state, (save direction only), with surely a lot of rough edges as I am not very familiar with the libvirt code base yet or its rules. It seems to work for me, but take a look if this is something realistic to pursue further, whether this could be acceptable for libvirt. Claudio Fontana (11): libvirt: introduce virDomainSaveParametersFlags public API remote: Add RPC support for the virDomainSaveParametersFlags API qemu: add a stub for virDomainSaveParametersFlags API qemu: initial implementation for the virDomainSaveParametersFlags API iohelper: move runIO function to a separate module runio: add arguments to extend use beyond just stdin and stdout multifd-helper: new helper for parallel saves qemu: wire up saveimage code with the multifd helper qemu: implement qemuMigrationSrcToFilesMultiFd tools: add parallel parameter to virsh save command qemu: add migration parameter multifd-compression docs/manpages/virsh.rst | 23 ++- include/libvirt/libvirt-domain.h | 9 + src/driver-hypervisor.h | 7 + src/libvirt-domain.c | 51 ++++++ src/libvirt_private.syms | 1 + src/libvirt_public.syms | 5 + src/qemu/qemu_capabilities.c | 3 + src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_driver.c | 66 +++++++- src/qemu/qemu_migration.c | 135 ++++++++++----- src/qemu/qemu_migration.h | 7 + src/qemu/qemu_migration_params.c | 71 +++++--- src/qemu/qemu_migration_params.h | 15 ++ src/qemu/qemu_saveimage.c | 281 ++++++++++++++++++++++++++----- src/qemu/qemu_saveimage.h | 9 + src/qemu/qemu_snapshot.c | 2 +- src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 +- src/remote_protocol-structs | 9 + src/util/iohelper.c | 180 +------------------- src/util/meson.build | 15 ++ src/util/multifd-helper.c | 205 ++++++++++++++++++++++ src/util/runio.c | 214 +++++++++++++++++++++++ src/util/runio.h | 38 +++++ src/util/virthread.c | 5 + src/util/virthread.h | 1 + tools/virsh-domain.c | 49 +++++- 27 files changed, 1108 insertions(+), 312 deletions(-) create mode 100644 src/util/multifd-helper.c create mode 100644 src/util/runio.c create mode 100644 src/util/runio.h -- 2.34.1
add new API in order to be able to extend parameters to the domain save operation. We will use it to fit the existing arguments of VirDomainSaveFlags, and then add parallel saves functionality. Signed-off-by: Claudio Fontana <cfontana@suse.de> --- include/libvirt/libvirt-domain.h | 9 ++++++ src/driver-hypervisor.h | 7 +++++ src/libvirt-domain.c | 51 ++++++++++++++++++++++++++++++++ src/libvirt_public.syms | 5 ++++ 4 files changed, 72 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 2d5718301e..f3a4daf034 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -1270,6 +1270,7 @@ typedef enum { VIR_DOMAIN_SAVE_RUNNING = 1 << 1, /* Favor running over paused */ VIR_DOMAIN_SAVE_PAUSED = 1 << 2, /* Favor paused over running */ VIR_DOMAIN_SAVE_RESET_NVRAM = 1 << 3, /* Re-initialize NVRAM from template */ + VIR_DOMAIN_SAVE_PARALLEL = 1 << 4, /* Parallel Save/Restore to multiple files */ } virDomainSaveRestoreFlags; int virDomainSave (virDomainPtr domain, @@ -1278,6 +1279,10 @@ int virDomainSaveFlags (virDomainPtr domain, const char *to, const char *dxml, unsigned int flags); +int virDomainSaveParametersFlags (virDomainPtr domain, + virTypedParameterPtr params, + int nparams, + unsigned int flags); int virDomainRestore (virConnectPtr conn, const char *from); int virDomainRestoreFlags (virConnectPtr conn, @@ -1285,6 +1290,10 @@ int virDomainRestoreFlags (virConnectPtr conn, const char *dxml, unsigned int flags); +# define VIR_SAVE_PARAM_TO "to" +# define VIR_SAVE_PARAM_DXML "dxml" +# define VIR_SAVE_PARAM_PARALLEL_CONNECTIONS "parallel.connections" + /* See below for virDomainSaveImageXMLFlags */ char * virDomainSaveImageGetXMLDesc (virConnectPtr conn, const char *file, diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h index 4423eb0885..a4e1d21e76 100644 --- a/src/driver-hypervisor.h +++ b/src/driver-hypervisor.h @@ -240,6 +240,12 @@ typedef int const char *dxml, unsigned int flags); +typedef int +(*virDrvDomainSaveParametersFlags)(virDomainPtr domain, + virTypedParameterPtr params, + int nparams, + unsigned int flags); + typedef int (*virDrvDomainRestore)(virConnectPtr conn, const char *from); @@ -1489,6 +1495,7 @@ struct _virHypervisorDriver { virDrvDomainGetControlInfo domainGetControlInfo; virDrvDomainSave domainSave; virDrvDomainSaveFlags domainSaveFlags; + virDrvDomainSaveParametersFlags domainSaveParametersFlags; virDrvDomainRestore domainRestore; virDrvDomainRestoreFlags domainRestoreFlags; virDrvDomainSaveImageGetXMLDesc domainSaveImageGetXMLDesc; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index a197618673..1ab1580f8a 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -953,6 +953,57 @@ virDomainSaveFlags(virDomainPtr domain, const char *to, return -1; } +/** + * virDomainSaveParametersFlags: + * @domain: a domain object + * @params: save parameters + * @nparams: number of save parameters + * @flags: bitwise-OR of virDomainSaveRestoreFlags + * + * This method extends virDomainSaveFlags by adding parameters to Save. + * + * If @flags includes VIR_DOMAIN_SAVE_PARALLEL, then libvirt will + * attempt to trigger a parallel transfer to multiple files, + * where the number of extra files is determined by the parameter + * VIR_SAVE_PARAM_PARALLEL_CONNECTIONS. + * + * Returns 0 in case of success and -1 in case of failure. + */ +int +virDomainSaveParametersFlags(virDomainPtr domain, + virTypedParameterPtr params, int nparams, + unsigned int flags) +{ + virConnectPtr conn; + + VIR_DOMAIN_DEBUG(domain, "params=%p, nparams=%d, flags=0x%x", + params, nparams, flags); + VIR_TYPED_PARAMS_DEBUG(params, nparams); + + virResetLastError(); + + virCheckDomainReturn(domain, -1); + conn = domain->conn; + + virCheckReadOnlyGoto(conn->flags, error); + + VIR_EXCLUSIVE_FLAGS_GOTO(VIR_DOMAIN_SAVE_RUNNING, + VIR_DOMAIN_SAVE_PAUSED, + error); + + if (conn->driver->domainSaveParametersFlags) { + if (conn->driver->domainSaveParametersFlags(domain, params, nparams, flags) < 0) + goto error; + return 0; + } + + virReportUnsupportedError(); + + error: + virDispatchError(domain->conn); + return -1; +} + /** * virDomainRestore: diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index f93692c427..eb3a7afb75 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -916,4 +916,9 @@ LIBVIRT_8.0.0 { virDomainSetLaunchSecurityState; } LIBVIRT_7.8.0; +LIBVIRT_8.3.0 { + global: + virDomainSaveParametersFlags; +} LIBVIRT_8.0.0; + # .... define new API here using predicted next version number .... -- 2.34.1
Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/remote/remote_driver.c | 1 + src/remote/remote_protocol.x | 17 ++++++++++++++++- src/remote_protocol-structs | 9 +++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c index 7e7a21fcab..1fc5d41971 100644 --- a/src/remote/remote_driver.c +++ b/src/remote/remote_driver.c @@ -8446,6 +8446,7 @@ static virHypervisorDriver hypervisor_driver = { .domainGetControlInfo = remoteDomainGetControlInfo, /* 0.9.3 */ .domainSave = remoteDomainSave, /* 0.3.0 */ .domainSaveFlags = remoteDomainSaveFlags, /* 0.9.4 */ + .domainSaveParametersFlags = remoteDomainSaveParametersFlags, /* 8.3.0 */ .domainRestore = remoteDomainRestore, /* 0.3.0 */ .domainRestoreFlags = remoteDomainRestoreFlags, /* 0.9.4 */ .domainSaveImageGetXMLDesc = remoteDomainSaveImageGetXMLDesc, /* 0.9.4 */ diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x index 4f13cef662..c2ae5c5748 100644 --- a/src/remote/remote_protocol.x +++ b/src/remote/remote_protocol.x @@ -230,6 +230,9 @@ const REMOTE_NODE_MEMORY_PARAMETERS_MAX = 64; /* Upper limit on migrate parameters */ const REMOTE_DOMAIN_MIGRATE_PARAM_LIST_MAX = 64; +/* Upper limit on save/restore parameters */ +const REMOTE_DOMAIN_SAVE_PARAMS_MAX = 64; + /* Upper limit on number of job stats */ const REMOTE_DOMAIN_JOB_STATS_MAX = 64; @@ -3227,6 +3230,12 @@ struct remote_domain_migrate_confirm3_params_args { int cancelled; }; +struct remote_domain_save_parameters_flags_args { + remote_nonnull_domain dom; + remote_typed_param params<REMOTE_DOMAIN_SAVE_PARAMS_MAX>; + unsigned int flags; +}; + /* The device removed event is the last event where we have to support * dual forms for back-compat to older clients; all future events can * use just the modern form with callbackID. */ @@ -6920,5 +6929,11 @@ enum remote_procedure { * @generate: both * @acl: domain:write */ - REMOTE_PROC_DOMAIN_SET_LAUNCH_SECURITY_STATE = 439 + REMOTE_PROC_DOMAIN_SET_LAUNCH_SECURITY_STATE = 439, + + /** + * @generate: both + * @acl: domain:hibernate + */ + REMOTE_PROC_DOMAIN_SAVE_PARAMETERS_FLAGS = 440 }; diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs index d88176781d..89eadeb644 100644 --- a/src/remote_protocol-structs +++ b/src/remote_protocol-structs @@ -563,6 +563,14 @@ struct remote_domain_save_flags_args { remote_string dxml; u_int flags; }; +struct remote_domain_save_parameters_flags_args { + remote_nonnull_domain dom; + struct { + u_int params_len; + remote_typed_param * params_val; + } params; + u_int flags; +}; struct remote_domain_restore_args { remote_nonnull_string from; }; @@ -3689,4 +3697,5 @@ enum remote_procedure { REMOTE_PROC_NETWORK_CREATE_XML_FLAGS = 437, REMOTE_PROC_DOMAIN_EVENT_MEMORY_DEVICE_SIZE_CHANGE = 438, REMOTE_PROC_DOMAIN_SET_LAUNCH_SECURITY_STATE = 439, + REMOTE_PROC_DOMAIN_SAVE_PARAMETERS_FLAGS = 440, }; -- 2.34.1
Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/qemu/qemu_driver.c | 66 ++++++++++++++++++++++++++++++++++++--- src/qemu/qemu_saveimage.c | 2 ++ src/qemu/qemu_saveimage.h | 1 + src/qemu/qemu_snapshot.c | 2 +- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 77012eb527..cfc68e3135 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -2631,7 +2631,7 @@ static int qemuDomainSaveInternal(virQEMUDriver *driver, virDomainObj *vm, const char *path, int compressed, virCommand *compressor, - const char *xmlin, unsigned int flags) + const char *xmlin, int nconn, unsigned int flags) { g_autofree char *xml = NULL; bool was_running = false; @@ -2712,7 +2712,7 @@ qemuDomainSaveInternal(virQEMUDriver *driver, xml = NULL; ret = qemuSaveImageCreate(driver, vm, path, data, compressor, - flags, VIR_ASYNC_JOB_SAVE); + nconn, flags, VIR_ASYNC_JOB_SAVE); if (ret < 0) goto endjob; @@ -2781,7 +2781,7 @@ qemuDomainSaveFlags(virDomainPtr dom, const char *path, const char *dxml, goto cleanup; ret = qemuDomainSaveInternal(driver, vm, path, compressed, - compressor, dxml, flags); + compressor, dxml, 0, flags); cleanup: virDomainObjEndAPI(&vm); @@ -2794,6 +2794,63 @@ qemuDomainSave(virDomainPtr dom, const char *path) return qemuDomainSaveFlags(dom, path, NULL, 0); } +static int +qemuDomainSaveParametersFlags(virDomainPtr dom, + virTypedParameterPtr params, int nparams, + unsigned int flags) +{ + const char *to = NULL; + const char *dxml = NULL; + virQEMUDriver *driver = dom->conn->privateData; + int compressed; + g_autoptr(virCommand) compressor = NULL; + int ret = -1; + int nconn = 0; + virDomainObj *vm = NULL; + g_autoptr(virQEMUDriverConfig) cfg = NULL; + + virCheckFlags(VIR_DOMAIN_SAVE_BYPASS_CACHE | + VIR_DOMAIN_SAVE_RUNNING | + VIR_DOMAIN_SAVE_PAUSED | + VIR_DOMAIN_SAVE_PARALLEL, -1); + + if (virTypedParamsValidate(params, nparams, + VIR_SAVE_PARAM_TO, VIR_TYPED_PARAM_STRING, + VIR_SAVE_PARAM_DXML, VIR_TYPED_PARAM_STRING, + VIR_SAVE_PARAM_PARALLEL_CONNECTIONS, VIR_TYPED_PARAM_INT, + NULL) < 0) + return -1; + + if (virTypedParamsGetString(params, nparams, VIR_SAVE_PARAM_TO, &to) < 0) + return -1; + if (virTypedParamsGetString(params, nparams, VIR_SAVE_PARAM_DXML, &dxml) < 0) + return -1; + if (virTypedParamsGetInt(params, nparams, VIR_SAVE_PARAM_PARALLEL_CONNECTIONS, &nconn) < 0) + return -1; + + cfg = virQEMUDriverGetConfig(driver); + if ((compressed = qemuSaveImageGetCompressionProgram(cfg->saveImageFormat, + &compressor, + "save", false)) < 0) + goto cleanup; + + if (!(vm = qemuDomainObjFromDomain(dom))) + goto cleanup; + + if (virDomainSaveFlagsEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjCheckActive(vm) < 0) + goto cleanup; + + ret = qemuDomainSaveInternal(driver, vm, to, compressed, + compressor, dxml, nconn, flags); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + static char * qemuDomainManagedSavePath(virQEMUDriver *driver, virDomainObj *vm) { @@ -2844,7 +2901,7 @@ qemuDomainManagedSave(virDomainPtr dom, unsigned int flags) VIR_INFO("Saving state of domain '%s' to '%s'", vm->def->name, name); ret = qemuDomainSaveInternal(driver, vm, name, compressed, - compressor, NULL, flags); + compressor, NULL, 0, flags); if (ret == 0) vm->hasManagedSave = true; @@ -20826,6 +20883,7 @@ static virHypervisorDriver qemuHypervisorDriver = { .domainGetControlInfo = qemuDomainGetControlInfo, /* 0.9.3 */ .domainSave = qemuDomainSave, /* 0.2.0 */ .domainSaveFlags = qemuDomainSaveFlags, /* 0.9.4 */ + .domainSaveParametersFlags = qemuDomainSaveParametersFlags, /* 8.3.0 */ .domainRestore = qemuDomainRestore, /* 0.2.0 */ .domainRestoreFlags = qemuDomainRestoreFlags, /* 0.9.4 */ .domainSaveImageGetXMLDesc = qemuDomainSaveImageGetXMLDesc, /* 0.9.4 */ diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 4fd4c5cfcd..6e7f067be2 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -258,6 +258,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, const char *path, virQEMUSaveData *data, virCommand *compressor, + int nconn, unsigned int flags, virDomainAsyncJob asyncJob) { @@ -269,6 +270,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, virFileWrapperFd *wrapperFd = NULL; unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; + nconn = nconn; /* unused */ /* Obtain the file handle. */ if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) { wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; diff --git a/src/qemu/qemu_saveimage.h b/src/qemu/qemu_saveimage.h index 391cd55ed0..b3d5c02fd6 100644 --- a/src/qemu/qemu_saveimage.h +++ b/src/qemu/qemu_saveimage.h @@ -96,6 +96,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, const char *path, virQEMUSaveData *data, virCommand *compressor, + int nconn, unsigned int flags, virDomainAsyncJob asyncJob); diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c index b62fab7bb3..f4eeb9276c 100644 --- a/src/qemu/qemu_snapshot.c +++ b/src/qemu/qemu_snapshot.c @@ -1457,7 +1457,7 @@ qemuSnapshotCreateActiveExternal(virQEMUDriver *driver, memory_existing = virFileExists(snapdef->memorysnapshotfile); if ((ret = qemuSaveImageCreate(driver, vm, snapdef->memorysnapshotfile, - data, compressor, 0, + data, compressor, 0, 0, VIR_ASYNC_JOB_SNAPSHOT)) < 0) goto cleanup; -- 2.34.1
make the qemuDomainSaveParametersFlags minimally functional by preparing the files for the multifd channels, but stop short of using them, relying on normal migration to file for now. Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/qemu/qemu_saveimage.c | 256 +++++++++++++++++++++++++++++++------- src/qemu/qemu_saveimage.h | 8 ++ 2 files changed, 220 insertions(+), 44 deletions(-) diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 6e7f067be2..6a65d72fb3 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -248,6 +248,192 @@ qemuSaveImageGetCompressionCommand(virQEMUSaveFormat compression) return ret; } +/* + * virQEMUSaveFdInit: initialize a virQEMUSaveFd + * + * @saveFd: the structure to initialize + * @base: the main file name + * @idx: 0 for the main file, >0 for multifd channels. + * @user: uid + * @group: gid + * @flags: the general flags + * + * Returns -1 on error, 0 on success, + * and in both cases virQEMUSaveFdFini must be called to free resources. + */ +static int virQEMUSaveFdInit(virQEMUSaveFd *saveFd, const char *base, int idx, + uid_t user, gid_t group, unsigned int flags) +{ + unsigned int directFlag = 0; + unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; + + if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { + directFlag = virFileDirectFdFlag(); + wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; + } + if (idx > 0) { + saveFd->path = g_strdup_printf("%s.%d", base, idx); + } else { + saveFd->path = g_strdup(base); + } + saveFd->wrapper = NULL; + saveFd->fd = virQEMUFileOpenAs(user, group, false, saveFd->path, + O_WRONLY | O_TRUNC | O_CREAT | directFlag, + &saveFd->need_unlink); + if (saveFd->fd < 0) + return -1; + + if (idx > 0) { + /* no wrapper required for the multifd channels */ + } else { + saveFd->wrapper = virFileWrapperFdNew(&saveFd->fd, saveFd->path, wrapperFlags); + if (!saveFd->wrapper) + return -1; + } + return 0; +} + +/* + * virQEMUSaveFdClose: close a virQEMUSaveFd descriptor with normal close. + * + * @saveFd: the saveFd structure with the file descriptors to close. + * @vm: the virDomainObj (necessary to release lock) + * + * If saveFd is NULL, the function will return success. + * + * Returns -1 on error, 0 on success. + */ +static int virQEMUSaveFdClose(virQEMUSaveFd *saveFd, virDomainObj *vm) +{ + if (!saveFd) + return 0; + + if (VIR_CLOSE(saveFd->fd) < 0) { + virReportSystemError(errno, _("unable to close %s"), saveFd->path); + return -1; + } + if (qemuDomainFileWrapperFDClose(vm, saveFd->wrapper) < 0) + return -1; + + return 0; +} + +/* + * virQEMUSaveFdFini: finalize a virQEMUSaveFd + * + * @saveFd: the saveFd structure containing the resources to free. + * @vm: the virDomainObj (necessary to release lock) + * @ret: the current operation result (< 0 is failure) + * + * If saveFd is NULL, the return value will be unchanged. + * + * Returns ret, or -1 if an error is detected. + */ +static int virQEMUSaveFdFini(virQEMUSaveFd *saveFd, virDomainObj *vm, int ret) +{ + if (!saveFd) + return ret; + VIR_FORCE_CLOSE(saveFd->fd); + if (qemuDomainFileWrapperFDClose(vm, saveFd->wrapper) < 0) { + ret = -1; + } + if (ret < 0 && saveFd->need_unlink && saveFd->path) { + unlink(saveFd->path); + } + if (saveFd->wrapper) { + virFileWrapperFdFree(saveFd->wrapper); + saveFd->wrapper = NULL; + } + + g_free(saveFd->path); + saveFd->path = NULL; + return ret; +} + +/* + * qemuSaveImageFreeMultiFd: free all multifd virQEMUSaveFds. + * @multiFd: the array of saveFds + * @vm: the virDomainObj, to release lock + * @nconn: number of multifd channels + * @ret: the current operation result (< 0 is failure) + * + * If multiFd is NULL, the return value will be unchanged. + * + * Returns ret, or -1 if an error is detected. + */ +static int qemuSaveImageFreeMultiFd(virQEMUSaveFd *multiFd, virDomainObj *vm, int nconn, int ret) +{ + int i; + + if (!multiFd) + return ret; + + for (i = 0; i < nconn; i++) { + ret = virQEMUSaveFdFini(&multiFd[i], vm, ret); + } + /* + * do it again to unlink all in the error case, + * if error happened in the middle of previous loop. + */ + for (i = 0; i < nconn; i++) { + ret = virQEMUSaveFdFini(&multiFd[i], vm, ret); + } + g_free(multiFd); + return ret; +} + +/* + * qemuSaveImageCloseMultiFd: perform normal close on all multifd virQEMUSaveFds. + * If multiFd is NULL, the function will return success. + * + * Returns -1 on error, 0 on success. + */ + +static int qemuSaveImageCloseMultiFd(virQEMUSaveFd *multiFd, int nconn, virDomainObj *vm) +{ + int i; + + if (!multiFd) + return 0; + + for (i = 0; i < nconn; i++) { + if (virQEMUSaveFdClose(&multiFd[i], vm) < 0) { + return -1; + } + } + return 0; +} + +/* + * qemuSaveImageCreateMultiFd: allocate and initialize all multifd virQEMUSaveFds. + * + * Returns the new array of virQEMUSaveFds, or NULL on error. + */ + +static virQEMUSaveFd * +qemuSaveImageCreateMultiFd(virQEMUDriver *driver, virDomainObj *vm, + const char *path, uid_t user, gid_t group, + int nconn, unsigned int flags) +{ + virQEMUSaveFd *multiFd = g_new0(virQEMUSaveFd, nconn); + int i; + + for (i = 0; i < nconn; i++) { + virQEMUSaveFd *m = &multiFd[i]; + if (virQEMUSaveFdInit(m, path, i + 1, user, group, flags) < 0 || + qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, m->fd) < 0) { + + virQEMUSaveFdFini(m, vm, -1); + goto error; + } + } + return multiFd; + + error: + qemuSaveImageFreeMultiFd(multiFd, vm, nconn, -1); + return NULL; +} + /* Helper function to execute a migration to file with a correct save header * the caller needs to make sure that the processors are stopped and do all other @@ -263,42 +449,35 @@ qemuSaveImageCreate(virQEMUDriver *driver, virDomainAsyncJob asyncJob) { g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); - bool needUnlink = false; + virQEMUSaveFd saveFd; + virQEMUSaveFd *multiFd = NULL; int ret = -1; - int fd = -1; - int directFlag = 0; - virFileWrapperFd *wrapperFd = NULL; - unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; - - nconn = nconn; /* unused */ - /* Obtain the file handle. */ - if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) { - wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; - directFlag = virFileDirectFdFlag(); - if (directFlag < 0) { - virReportError(VIR_ERR_OPERATION_FAILED, "%s", - _("bypass cache unsupported by this system")); - goto cleanup; - } - } - fd = virQEMUFileOpenAs(cfg->user, cfg->group, false, path, - O_WRONLY | O_TRUNC | O_CREAT | directFlag, - &needUnlink); - if (fd < 0) + if (virQEMUSaveFdInit(&saveFd, path, 0, cfg->user, cfg->group, flags) < 0) { goto cleanup; - - if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, fd) < 0) + } + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, saveFd.fd) < 0) goto cleanup; - if (!(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) + if (virQEMUSaveDataWrite(data, saveFd.fd, saveFd.path) < 0) goto cleanup; - if (virQEMUSaveDataWrite(data, fd, path) < 0) - goto cleanup; + if (flags & VIR_DOMAIN_SAVE_PARALLEL) { + /* Perform parallel multifd migration to files (main fd + channels) */ + if (!(multiFd = qemuSaveImageCreateMultiFd(driver, vm, saveFd.path, cfg->user, cfg->group, nconn, flags))) + goto cleanup; + /* still using single fd migration for now */ + if (qemuMigrationSrcToFile(driver, vm, saveFd.fd, compressor, asyncJob) < 0) + goto cleanup; + if (qemuSaveImageCloseMultiFd(multiFd, nconn, vm) < 0) + goto cleanup; + } else { + /* Perform non-parallel migration to file */ + if (qemuMigrationSrcToFile(driver, vm, saveFd.fd, compressor, asyncJob) < 0) + goto cleanup; + } - /* Perform the migration */ - if (qemuMigrationSrcToFile(driver, vm, fd, compressor, asyncJob) < 0) + if (virQEMUSaveFdClose(&saveFd, vm) < 0) goto cleanup; /* Touch up file header to mark image complete. */ @@ -307,29 +486,18 @@ qemuSaveImageCreate(virQEMUDriver *driver, * up to seek backwards on wrapperFd. The reopened fd will * trigger a single page of file system cache pollution, but * that's acceptable. */ - if (VIR_CLOSE(fd) < 0) { - virReportSystemError(errno, _("unable to close %s"), path); - goto cleanup; - } - - if (qemuDomainFileWrapperFDClose(vm, wrapperFd) < 0) - goto cleanup; - if ((fd = qemuDomainOpenFile(cfg, vm->def, path, O_WRONLY, NULL)) < 0 || - virQEMUSaveDataFinish(data, &fd, path) < 0) + if ((saveFd.fd = qemuDomainOpenFile(cfg, vm->def, saveFd.path, O_WRONLY, NULL)) < 0 || + virQEMUSaveDataFinish(data, &saveFd.fd, saveFd.path) < 0) goto cleanup; ret = 0; - cleanup: - VIR_FORCE_CLOSE(fd); - if (qemuDomainFileWrapperFDClose(vm, wrapperFd) < 0) - ret = -1; - virFileWrapperFdFree(wrapperFd); - if (ret < 0 && needUnlink) - unlink(path); + cleanup: + ret = qemuSaveImageFreeMultiFd(multiFd, vm, nconn, ret); + ret = virQEMUSaveFdFini(&saveFd, vm, ret); return ret; } diff --git a/src/qemu/qemu_saveimage.h b/src/qemu/qemu_saveimage.h index b3d5c02fd6..24dcd213a5 100644 --- a/src/qemu/qemu_saveimage.h +++ b/src/qemu/qemu_saveimage.h @@ -54,6 +54,14 @@ struct _virQEMUSaveData { }; +typedef struct _virQEMUSaveFd virQEMUSaveFd; +struct _virQEMUSaveFd { + char *path; + int fd; + bool need_unlink; + virFileWrapperFd *wrapper; +}; + virDomainDef * qemuSaveImageUpdateDef(virQEMUDriver *driver, virDomainDef *def, -- 2.34.1
where it can be reused by other helpers. No changes other than the move. Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/util/iohelper.c | 178 +---------------------------------- src/util/meson.build | 2 + src/util/runio.c | 214 +++++++++++++++++++++++++++++++++++++++++++ src/util/runio.h | 23 +++++ 4 files changed, 240 insertions(+), 177 deletions(-) create mode 100644 src/util/runio.c create mode 100644 src/util/runio.h diff --git a/src/util/iohelper.c b/src/util/iohelper.c index 1584321839..5a0098542e 100644 --- a/src/util/iohelper.c +++ b/src/util/iohelper.c @@ -38,183 +38,7 @@ #include "virrandom.h" #include "virstring.h" #include "virgettext.h" - -#define VIR_FROM_THIS VIR_FROM_STORAGE - -#ifndef O_DIRECT -# define O_DIRECT 0 -#endif - -struct runIOParams { - bool isBlockDev; - bool isDirect; - bool isWrite; - int fdin; - const char *fdinname; - int fdout; - const char *fdoutname; -}; - -/** - * runIOCopy: execute the IO copy based on the passed parameters - * @p: the IO parameters - * - * Execute the copy based on the passed parameters. - * - * Returns: size transfered, or < 0 on error. - */ - -static off_t -runIOCopy(const struct runIOParams p) -{ - g_autofree void *base = NULL; /* Location to be freed */ - char *buf = NULL; /* Aligned location within base */ - size_t buflen = 1024*1024; - intptr_t alignMask = 64*1024 - 1; - off_t total = 0; - -#if WITH_POSIX_MEMALIGN - if (posix_memalign(&base, alignMask + 1, buflen)) - abort(); - buf = base; -#else - buf = g_new0(char, buflen + alignMask); - base = buf; - buf = (char *) (((intptr_t) base + alignMask) & ~alignMask); -#endif - - while (1) { - ssize_t got; - - /* If we read with O_DIRECT from file we can't use saferead as - * it can lead to unaligned read after reading last bytes. - * If we write with O_DIRECT use should use saferead so that - * writes will be aligned. - * In other cases using saferead reduces number of syscalls. - */ - if (!p.isWrite && p.isDirect) { - if ((got = read(p.fdin, buf, buflen)) < 0 && - errno == EINTR) - continue; - } else { - got = saferead(p.fdin, buf, buflen); - } - - if (got < 0) { - virReportSystemError(errno, _("Unable to read %s"), p.fdinname); - return -2; - } - if (got == 0) - break; - - total += got; - - /* handle last write size align in direct case */ - if (got < buflen && p.isDirect && p.isWrite) { - ssize_t aligned_got = (got + alignMask) & ~alignMask; - - memset(buf + got, 0, aligned_got - got); - - if (safewrite(p.fdout, buf, aligned_got) < 0) { - virReportSystemError(errno, _("Unable to write %s"), p.fdoutname); - return -3; - } - - if (!p.isBlockDev && ftruncate(p.fdout, total) < 0) { - virReportSystemError(errno, _("Unable to truncate %s"), p.fdoutname); - return -4; - } - - break; - } - - if (safewrite(p.fdout, buf, got) < 0) { - virReportSystemError(errno, _("Unable to write %s"), p.fdoutname); - return -3; - } - } - return total; -} - -static int -runIO(const char *path, int fd, int oflags) -{ - int ret = -1; - off_t total = 0; - struct stat sb; - struct runIOParams p; - - if (fstat(fd, &sb) < 0) { - virReportSystemError(errno, - _("Unable to access file descriptor %d path %s"), - fd, path); - goto cleanup; - } - p.isBlockDev = S_ISBLK(sb.st_mode); - p.isDirect = O_DIRECT && (oflags & O_DIRECT); - - switch (oflags & O_ACCMODE) { - case O_RDONLY: - p.isWrite = false; - p.fdin = fd; - p.fdinname = path; - p.fdout = STDOUT_FILENO; - p.fdoutname = "stdout"; - break; - case O_WRONLY: - p.isWrite = true; - p.fdin = STDIN_FILENO; - p.fdinname = "stdin"; - p.fdout = fd; - p.fdoutname = path; - break; - - case O_RDWR: - default: - virReportSystemError(EINVAL, - _("Unable to process file with flags %d"), - (oflags & O_ACCMODE)); - goto cleanup; - } - /* To make the implementation simpler, we give up on any - * attempt to use O_DIRECT in a non-trivial manner. */ - if (!p.isBlockDev && p.isDirect) { - off_t off; - if (p.isWrite) { - if ((off = lseek(fd, 0, SEEK_END)) != 0) { - virReportSystemError(off < 0 ? errno : EINVAL, "%s", - _("O_DIRECT write needs empty seekable file")); - goto cleanup; - } - } else if ((off = lseek(fd, 0, SEEK_CUR)) != 0) { - virReportSystemError(off < 0 ? errno : EINVAL, "%s", - _("O_DIRECT read needs entire seekable file")); - goto cleanup; - } - } - total = runIOCopy(p); - if (total < 0) - goto cleanup; - - /* Ensure all data is written */ - if (virFileDataSync(p.fdout) < 0) { - if (errno != EINVAL && errno != EROFS) { - /* fdatasync() may fail on some special FDs, e.g. pipes */ - virReportSystemError(errno, _("unable to fsync %s"), p.fdoutname); - goto cleanup; - } - } - - ret = 0; - - cleanup: - if (VIR_CLOSE(fd) < 0 && - ret == 0) { - virReportSystemError(errno, _("Unable to close %s"), path); - ret = -1; - } - return ret; -} +#include "runio.h" static const char *program_name; diff --git a/src/util/meson.build b/src/util/meson.build index 24350a3e67..58001a1699 100644 --- a/src/util/meson.build +++ b/src/util/meson.build @@ -175,6 +175,8 @@ keycode_dep = declare_dependency( io_helper_sources = [ 'iohelper.c', + 'runio.c', + 'runio.h', ] virt_util_lib = static_library( diff --git a/src/util/runio.c b/src/util/runio.c new file mode 100644 index 0000000000..a7b902af7e --- /dev/null +++ b/src/util/runio.c @@ -0,0 +1,214 @@ +/* + * runio.c: I/O copy function + * + * Copyright (C) 2011-2012 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + */ + +#include <config.h> + +#include <unistd.h> +#include <fcntl.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include "virthread.h" +#include "virfile.h" +#include "viralloc.h" +#include "virerror.h" +#include "virrandom.h" +#include "virstring.h" +#include "virgettext.h" +#include "runio.h" + +#define VIR_FROM_THIS VIR_FROM_STORAGE + +#ifndef O_DIRECT +# define O_DIRECT 0 +#endif + +struct runIOParams { + bool isBlockDev; + bool isDirect; + bool isWrite; + int fdin; + const char *fdinname; + int fdout; + const char *fdoutname; +}; + +/** + * runIOCopy: execute the IO copy based on the passed parameters + * @p: the IO parameters + * + * Execute the copy based on the passed parameters. + * + * Returns: size transfered, or < 0 on error. + */ + +static off_t +runIOCopy(const struct runIOParams p) +{ + g_autofree void *base = NULL; /* Location to be freed */ + char *buf = NULL; /* Aligned location within base */ + size_t buflen = 1024*1024; + intptr_t alignMask = 64*1024 - 1; + off_t total = 0; + +#if WITH_POSIX_MEMALIGN + if (posix_memalign(&base, alignMask + 1, buflen)) + abort(); + buf = base; +#else + buf = g_new0(char, buflen + alignMask); + base = buf; + buf = (char *) (((intptr_t) base + alignMask) & ~alignMask); +#endif + + while (1) { + ssize_t got; + + /* If we read with O_DIRECT from file we can't use saferead as + * it can lead to unaligned read after reading last bytes. + * If we write with O_DIRECT use should use saferead so that + * writes will be aligned. + * In other cases using saferead reduces number of syscalls. + */ + if (!p.isWrite && p.isDirect) { + if ((got = read(p.fdin, buf, buflen)) < 0 && + errno == EINTR) + continue; + } else { + got = saferead(p.fdin, buf, buflen); + } + + if (got < 0) { + virReportSystemError(errno, _("Unable to read %s"), p.fdinname); + return -2; + } + if (got == 0) + break; + + total += got; + + /* handle last write size align in direct case */ + if (got < buflen && p.isDirect && p.isWrite) { + ssize_t aligned_got = (got + alignMask) & ~alignMask; + + memset(buf + got, 0, aligned_got - got); + + if (safewrite(p.fdout, buf, aligned_got) < 0) { + virReportSystemError(errno, _("Unable to write %s"), p.fdoutname); + return -3; + } + + if (!p.isBlockDev && ftruncate(p.fdout, total) < 0) { + virReportSystemError(errno, _("Unable to truncate %s"), p.fdoutname); + return -4; + } + + break; + } + + if (safewrite(p.fdout, buf, got) < 0) { + virReportSystemError(errno, _("Unable to write %s"), p.fdoutname); + return -3; + } + } + return total; +} + + +off_t +runIO(const char *path, int fd, int oflags) +{ + int ret = -1; + off_t total = 0; + struct stat sb; + struct runIOParams p; + + if (fstat(fd, &sb) < 0) { + virReportSystemError(errno, + _("Unable to access file descriptor %d path %s"), + fd, path); + goto cleanup; + } + p.isBlockDev = S_ISBLK(sb.st_mode); + p.isDirect = O_DIRECT && (oflags & O_DIRECT); + + switch (oflags & O_ACCMODE) { + case O_RDONLY: + p.isWrite = false; + p.fdin = fd; + p.fdinname = path; + p.fdout = STDOUT_FILENO; + p.fdoutname = "stdout"; + break; + case O_WRONLY: + p.isWrite = true; + p.fdin = STDIN_FILENO; + p.fdinname = "stdin"; + p.fdout = fd; + p.fdoutname = path; + break; + + case O_RDWR: + default: + virReportSystemError(EINVAL, + _("Unable to process file with flags %d"), + (oflags & O_ACCMODE)); + goto cleanup; + } + /* To make the implementation simpler, we give up on any + * attempt to use O_DIRECT in a non-trivial manner. */ + if (!p.isBlockDev && p.isDirect) { + off_t off; + if (p.isWrite) { + if ((off = lseek(fd, 0, SEEK_END)) != 0) { + virReportSystemError(off < 0 ? errno : EINVAL, "%s", + _("O_DIRECT write needs empty seekable file")); + goto cleanup; + } + } else if ((off = lseek(fd, 0, SEEK_CUR)) != 0) { + virReportSystemError(off < 0 ? errno : EINVAL, "%s", + _("O_DIRECT read needs entire seekable file")); + goto cleanup; + } + } + total = runIOCopy(p); + if (total < 0) + goto cleanup; + + /* Ensure all data is written */ + if (virFileDataSync(p.fdout) < 0) { + if (errno != EINVAL && errno != EROFS) { + /* fdatasync() may fail on some special FDs, e.g. pipes */ + virReportSystemError(errno, _("unable to fsync %s"), p.fdoutname); + goto cleanup; + } + } + + ret = 0; + + cleanup: + if (VIR_CLOSE(fd) < 0 && + ret == 0) { + virReportSystemError(errno, _("Unable to close %s"), path); + ret = -1; + } + return ret; +} diff --git a/src/util/runio.h b/src/util/runio.h new file mode 100644 index 0000000000..beb58606c9 --- /dev/null +++ b/src/util/runio.h @@ -0,0 +1,23 @@ +/* + * runio.h: I/O copy function + * + * Copyright (C) 2011-2012 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + */ + +#pragma once + +off_t runIO(const char *path, int fd, int oflags); -- 2.34.1
add arguments to runio to allow read/write from/to arbitrary file descriptors, as opposed to just stdin and stdout. Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/util/iohelper.c | 2 +- src/util/runio.c | 6 +++--- src/util/runio.h | 17 ++++++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/util/iohelper.c b/src/util/iohelper.c index 5a0098542e..93674c1e2f 100644 --- a/src/util/iohelper.c +++ b/src/util/iohelper.c @@ -96,7 +96,7 @@ main(int argc, char **argv) usage(EXIT_FAILURE); } - if (fd < 0 || runIO(path, fd, oflags) < 0) + if (fd < 0 || runIO(path, fd, oflags, STDIN_FILENO, STDOUT_FILENO) < 0) goto error; return 0; diff --git a/src/util/runio.c b/src/util/runio.c index a7b902af7e..ffd5141f00 100644 --- a/src/util/runio.c +++ b/src/util/runio.c @@ -134,7 +134,7 @@ runIOCopy(const struct runIOParams p) off_t -runIO(const char *path, int fd, int oflags) +runIO(const char *path, int fd, int oflags, int in_fd, int out_fd) { int ret = -1; off_t total = 0; @@ -155,12 +155,12 @@ runIO(const char *path, int fd, int oflags) p.isWrite = false; p.fdin = fd; p.fdinname = path; - p.fdout = STDOUT_FILENO; + p.fdout = out_fd; p.fdoutname = "stdout"; break; case O_WRONLY: p.isWrite = true; - p.fdin = STDIN_FILENO; + p.fdin = in_fd; p.fdinname = "stdin"; p.fdout = fd; p.fdoutname = path; diff --git a/src/util/runio.h b/src/util/runio.h index beb58606c9..66df588881 100644 --- a/src/util/runio.h +++ b/src/util/runio.h @@ -20,4 +20,19 @@ #pragma once -off_t runIO(const char *path, int fd, int oflags); +/* + * runIO: copy unidirectionally all data to/from a file descriptor. + * + * @path: the pathname corresponding to FD. + * @fd: the file descriptor to read from or write to. + * @oflags: the file status flags of FD. + * + * If the oflags indicate O_RDONLY, then the direction will be from FD, + * and @out_fd indicates the file to write to. + * + * If the oflags indicate O_WRONLY, then the direction will be to FD, + * and @in_fd indicates the file to read from. + * + * Returns the number of bytes transferred, or < 0 on error. + */ +off_t runIO(const char *path, int fd, int oflags, int in_fd, int out_fd); -- 2.34.1
this helper listens on a unix socket which is provided to QEMU for multifd migration to/from files. The file descriptors are passed as command line parameters. Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/libvirt_private.syms | 1 + src/util/meson.build | 13 +++ src/util/multifd-helper.c | 205 ++++++++++++++++++++++++++++++++++++++ src/util/virthread.c | 5 + src/util/virthread.h | 1 + 5 files changed, 225 insertions(+) create mode 100644 src/util/multifd-helper.c diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 8a3e5f7f7c..cd279b8713 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -3427,6 +3427,7 @@ virThreadCreateFull; virThreadID; virThreadIsSelf; virThreadJoin; +virThreadJoinRet; virThreadMaxName; virThreadSelf; virThreadSelfID; diff --git a/src/util/meson.build b/src/util/meson.build index 58001a1699..8ea74ff9e8 100644 --- a/src/util/meson.build +++ b/src/util/meson.build @@ -179,6 +179,12 @@ io_helper_sources = [ 'runio.h', ] +multifd_helper_sources = [ + 'multifd-helper.c', + 'runio.c', + 'runio.h', +] + virt_util_lib = static_library( 'virt_util', [ @@ -216,6 +222,13 @@ if conf.has('WITH_LIBVIRTD') dtrace_gen_headers, ], } + virt_helpers += { + 'name': 'libvirt_multifd_helper', + 'sources': [ + files(multifd_helper_sources), + dtrace_gen_headers, + ], + } endif util_inc_dir = include_directories('.') diff --git a/src/util/multifd-helper.c b/src/util/multifd-helper.c new file mode 100644 index 0000000000..fab28bdb79 --- /dev/null +++ b/src/util/multifd-helper.c @@ -0,0 +1,205 @@ +/* + * multifd-helper.c: listens on Unix socket to perform I/O to multiple files + * + * Copyright (C) 2022 SUSE LLC + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + * + * This has been written to support QEMU multifd migration to file, + * allowing better use of cpu resources to speed up the save/restore. + */ + +#include <config.h> + +#include <unistd.h> +#include <fcntl.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/socket.h> +#include <sys/un.h> + +#include "virthread.h" +#include "virfile.h" +#include "virerror.h" +#include "virstring.h" +#include "virgettext.h" +#include "runio.h" + +#define VIR_FROM_THIS VIR_FROM_STORAGE + +typedef struct _multiFdConnData multiFdConnData; +struct _multiFdConnData { + int clientfd; + int filefd; + int oflags; + const char *path; + virThread tid; + + off_t total; +}; + +typedef struct _multiFdThreadArgs multiFdThreadArgs; +struct _multiFdThreadArgs { + int nchannels; + multiFdConnData *conn; /* contains main fd + nchannels */ + const char *sun_path; /* unix socket name to use for the server */ + + off_t total; +}; + +static void clientThreadFunc(void *a) +{ + multiFdConnData *c = a; + c->total = runIO(c->path, c->filefd, c->oflags, c->clientfd, c->clientfd); +} + +static void serverThreadFunc(void *a) +{ + struct sockaddr_un serv_addr; + multiFdThreadArgs *args = a; + int i; + int sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + const char buf[1] = {'R'}; + + args->total = -1; + + if (sockfd < 0) { + virReportSystemError(errno, "%s", _("serverThread: socket() failed")); + return; + } + /* initialize socket structure */ + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sun_family = AF_UNIX; + strncpy(serv_addr.sun_path, args->sun_path, sizeof(serv_addr.sun_path) - 1); + unlink(args->sun_path); + if (bind(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + virReportSystemError(errno, "%s", _("serverThread: bind() failed")); + goto cleanup; + } + if (listen(sockfd, args->nchannels + 1) < 0) { + virReportSystemError(errno, "%s", _("serverThread: listen() failed")); + goto cleanup; + } + + /* signal that the server is ready */ + if (safewrite(1, &buf, 1) != 1) { + virReportSystemError(errno, "%s", _("serverThread: safewrite failed")); + goto cleanup; + } + + for (i = 0; i < args->nchannels + 1; i++) { + /* Wait for incoming connection. */ + multiFdConnData *c = &args->conn[i]; + if ((c->clientfd = accept(sockfd, NULL, NULL)) < 0) { + virReportSystemError(errno, "%s", _("serverThread: accept() failed")); + goto cleanup; + } + if (virThreadCreate(&c->tid, true, &clientThreadFunc, c) < 0) { + virReportSystemError(errno, "%s", _("serverThread: client thread creation failed")); + goto cleanup; + } + } + + args->total = 0; + for (i = 0; i < args->nchannels + 1; i++) { + multiFdConnData *c = &args->conn[i]; + if (virThreadJoinRet(&args->conn[i].tid) < 0) { + args->total = -1; + } else if (args->total >= 0) { + args->total += c->total; + } + if (VIR_CLOSE(c->clientfd) < 0) { + args->total = -1; + } + } + + cleanup: + for (i = 0; i < args->nchannels + 1; i++) { + multiFdConnData *c = &args->conn[i]; + VIR_FORCE_CLOSE(c->clientfd); + } + if (VIR_CLOSE(sockfd) < 0) + args->total = -1; +} + +static const char *program_name; + +G_GNUC_NORETURN static void +usage(int status) +{ + if (status) { + fprintf(stderr, _("%s: try --help for more details"), program_name); + } else { + fprintf(stderr, _("Usage: %s UNIX_SOCNAME N MAINFD FD0 FD1 ... FDn"), program_name); + } + exit(status); +} + +int +main(int argc, char **argv) +{ + virThread tid; + multiFdThreadArgs args = { 0 }; + int i; + + program_name = argv[0]; + + if (virGettextInitialize() < 0 || + virErrorInitialize() < 0) { + fprintf(stderr, _("%s: initialization failed"), program_name); + exit(EXIT_FAILURE); + } + + if (argc > 1 && STREQ(argv[1], "--help")) + usage(EXIT_SUCCESS); + if (argc < 4) + usage(EXIT_FAILURE); + + args.sun_path = argv[1]; + if (virStrToLong_i(argv[2], NULL, 10, &args.nchannels) < 0) + fprintf(stderr, _("%s: malformed number of channels N %s"), program_name, argv[2]); + + if (argc < 4 + args.nchannels) + usage(EXIT_FAILURE); + + args.conn = g_new0(multiFdConnData, args.nchannels + 1); + + for (i = 3; i < 3 + args.nchannels + 1; i++) { + multiFdConnData *c = &args.conn[i - 3]; + + if (virStrToLong_i(argv[i], NULL, 10, &c->filefd) < 0) { + fprintf(stderr, _("%s: malformed FD %s"), program_name, argv[i]); + usage(EXIT_FAILURE); + } +#ifndef F_GETFL +#error "multifd-helper requires F_GETFL parameter of fcntl" +#endif + c->oflags = fcntl(c->filefd, F_GETFL); + } + + if (virThreadCreate(&tid, true, serverThreadFunc, &args) < 0) { + virReportSystemError(errno, _("%s: failed to create server thread"), program_name); + exit(EXIT_FAILURE); + } + + if (virThreadJoinRet(&tid) < 0) + exit(EXIT_FAILURE); + + if (args.total < 0) + exit(EXIT_FAILURE); + + exit(EXIT_SUCCESS); +} diff --git a/src/util/virthread.c b/src/util/virthread.c index 5422bb74fd..0f6c6a68fa 100644 --- a/src/util/virthread.c +++ b/src/util/virthread.c @@ -348,6 +348,11 @@ void virThreadJoin(virThread *thread) pthread_join(thread->thread, NULL); } +int virThreadJoinRet(virThread *thread) +{ + return pthread_join(thread->thread, NULL); +} + void virThreadCancel(virThread *thread) { pthread_cancel(thread->thread); diff --git a/src/util/virthread.h b/src/util/virthread.h index 23abe0b6c9..5cecb9bd8a 100644 --- a/src/util/virthread.h +++ b/src/util/virthread.h @@ -89,6 +89,7 @@ int virThreadCreateFull(virThread *thread, void virThreadSelf(virThread *thread); bool virThreadIsSelf(virThread *thread); void virThreadJoin(virThread *thread); +int virThreadJoinRet(virThread *thread); size_t virThreadMaxName(void); -- 2.34.1
Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/qemu/qemu_saveimage.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 6a65d72fb3..bb23788a5e 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -17,6 +17,7 @@ */ #include <config.h> +#include <configmake.h> #include "qemu_saveimage.h" #include "qemu_domain.h" @@ -412,7 +413,8 @@ static int qemuSaveImageCloseMultiFd(virQEMUSaveFd *multiFd, int nconn, virDomai static virQEMUSaveFd * qemuSaveImageCreateMultiFd(virQEMUDriver *driver, virDomainObj *vm, - const char *path, uid_t user, gid_t group, + virCommand *cmd, const char *path, + uid_t user, gid_t group, int nconn, unsigned int flags) { virQEMUSaveFd *multiFd = g_new0(virQEMUSaveFd, nconn); @@ -426,6 +428,8 @@ qemuSaveImageCreateMultiFd(virQEMUDriver *driver, virDomainObj *vm, virQEMUSaveFdFini(m, vm, -1); goto error; } + virCommandAddArgFormat(cmd, "%d", m->fd); + virCommandPassFD(cmd, m->fd, 0); } return multiFd; @@ -463,8 +467,30 @@ qemuSaveImageCreate(virQEMUDriver *driver, goto cleanup; if (flags & VIR_DOMAIN_SAVE_PARALLEL) { + g_autoptr(virCommand) cmd = NULL; + g_autofree char *helper_path = NULL; + qemuDomainObjPrivate *priv = vm->privateData; + g_autofree char *sun_path = g_strdup_printf("%s/save-multifd.sock", priv->libDir); + char buf[1]; + int helper_out = -1; + if (!(helper_path = virFileFindResource("libvirt_multifd_helper", + abs_top_builddir "/src", + LIBEXECDIR))) + goto cleanup; + cmd = virCommandNewArgList(helper_path, sun_path, NULL); + virCommandAddArgFormat(cmd, "%d", nconn); + virCommandAddArgFormat(cmd, "%d", saveFd.fd); + virCommandPassFD(cmd, saveFd.fd, 0); + virCommandSetOutputFD(cmd, &helper_out); /* should create pipe automagically */ + /* Perform parallel multifd migration to files (main fd + channels) */ - if (!(multiFd = qemuSaveImageCreateMultiFd(driver, vm, saveFd.path, cfg->user, cfg->group, nconn, flags))) + if (!(multiFd = qemuSaveImageCreateMultiFd(driver, vm, cmd, saveFd.path, cfg->user, cfg->group, nconn, flags))) + goto cleanup; + if (virCommandRunAsync(cmd, NULL) < 0) + goto cleanup; + if (saferead(helper_out, &buf, 1) != 1 || buf[0] != 'R') + goto cleanup; + if (chown(sun_path, cfg->user, cfg->group) < 0) goto cleanup; /* still using single fd migration for now */ if (qemuMigrationSrcToFile(driver, vm, saveFd.fd, compressor, asyncJob) < 0) -- 2.34.1
implement a function similar to qemuMigrationSrcToFile that migrates to multiple files using QEMU multifd. Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/qemu/qemu_capabilities.c | 1 + src/qemu/qemu_migration.c | 129 ++++++++++++++++++++----------- src/qemu/qemu_migration.h | 7 ++ src/qemu/qemu_migration_params.c | 22 ++++++ src/qemu/qemu_migration_params.h | 9 +++ src/qemu/qemu_saveimage.c | 3 +- 6 files changed, 124 insertions(+), 47 deletions(-) diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 6b4ed08499..01b570807b 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -1240,6 +1240,7 @@ struct virQEMUCapsStringFlags virQEMUCapsCommands[] = { struct virQEMUCapsStringFlags virQEMUCapsMigration[] = { { "rdma-pin-all", QEMU_CAPS_MIGRATE_RDMA }, + { "multifd", QEMU_MIGRATION_CAP_MULTIFD }, }; /* Use virQEMUCapsQMPSchemaQueries for querying parameters of events */ diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 3e653543c6..9ce24c8cfc 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -5897,13 +5897,14 @@ qemuMigrationDstFinish(virQEMUDriver *driver, return dom; } - /* Helper function called while vm is active. */ -int -qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, - int fd, - virCommand *compressor, - virDomainAsyncJob asyncJob) +static int +qemuMigrationSrcToFileAux(virQEMUDriver *driver, virDomainObj *vm, + int fd, + virCommand *compressor, + virDomainAsyncJob asyncJob, + const char *sun_path, + int nchannels) { qemuDomainObjPrivate *priv = vm->privateData; bool bwParam = virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_PARAM_BANDWIDTH); @@ -5914,24 +5915,24 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, char *errbuf = NULL; virErrorPtr orig_err = NULL; g_autoptr(qemuMigrationParams) migParams = NULL; + bool needParams = (bwParam || sun_path); if (qemuMigrationSetDBusVMState(driver, vm) < 0) return -1; + if (sun_path && !virQEMUCapsGet(priv->qemuCaps, QEMU_MIGRATION_CAP_MULTIFD)) + return -1; + /* Increase migration bandwidth to unlimited since target is a file. * Failure to change migration speed is not fatal. */ - if (bwParam) { - if (!(migParams = qemuMigrationParamsNew())) - return -1; + if (needParams && !((migParams = qemuMigrationParamsNew()))) + return -1; + if (bwParam) { if (qemuMigrationParamsSetULL(migParams, QEMU_MIGRATION_PARAM_MAX_BANDWIDTH, QEMU_DOMAIN_MIG_BANDWIDTH_MAX * 1024 * 1024) < 0) return -1; - - if (qemuMigrationParamsApply(driver, vm, asyncJob, migParams) < 0) - return -1; - priv->migMaxBandwidth = QEMU_DOMAIN_MIG_BANDWIDTH_MAX; } else { if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) == 0) { @@ -5942,6 +5943,17 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, } } + if (sun_path) { + qemuMigrationParamsSetCap(migParams, QEMU_MIGRATION_CAP_MULTIFD); + if (qemuMigrationParamsSetInt(migParams, + QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS, + nchannels) < 0) + return -1; + } + + if (needParams && qemuMigrationParamsApply(driver, vm, asyncJob, migParams) < 0) + return -1; + if (!virDomainObjIsActive(vm)) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("guest unexpectedly quit")); @@ -5949,45 +5961,53 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, return -1; } - if (compressor && virPipe(pipeFD) < 0) + if (!sun_path && compressor && virPipe(pipeFD) < 0) return -1; - /* All right! We can use fd migration, which means that qemu - * doesn't have to open() the file, so while we still have to - * grant SELinux access, we can do it on fd and avoid cleanup - * later, as well as skip futzing with cgroup. */ - if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, - compressor ? pipeFD[1] : fd) < 0) - goto cleanup; - if (qemuDomainObjEnterMonitorAsync(driver, vm, asyncJob) < 0) goto cleanup; - if (!compressor) { - rc = qemuMonitorMigrateToFd(priv->mon, - QEMU_MONITOR_MIGRATE_BACKGROUND, - fd); + if (sun_path) { + rc = qemuMonitorMigrateToSocket(priv->mon, + QEMU_MONITOR_MIGRATE_BACKGROUND, + sun_path); } else { - virCommandSetInputFD(compressor, pipeFD[0]); - virCommandSetOutputFD(compressor, &fd); - virCommandSetErrorBuffer(compressor, &errbuf); - virCommandDoAsyncIO(compressor); - if (virSetCloseExec(pipeFD[1]) < 0) { - virReportSystemError(errno, "%s", - _("Unable to set cloexec flag")); - qemuDomainObjExitMonitor(vm); - goto cleanup; - } - if (virCommandRunAsync(compressor, NULL) < 0) { - qemuDomainObjExitMonitor(vm); + /* + * All right! We can use fd migration, which means that qemu + * doesn't have to open() the file, so while we still have to + * grant SELinux access, we can do it on fd and avoid cleanup + * later, as well as skip futzing with cgroup. + */ + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, + compressor ? pipeFD[1] : fd) < 0) goto cleanup; + + if (!compressor) { + rc = qemuMonitorMigrateToFd(priv->mon, + QEMU_MONITOR_MIGRATE_BACKGROUND, + fd); + } else { + virCommandSetInputFD(compressor, pipeFD[0]); + virCommandSetOutputFD(compressor, &fd); + virCommandSetErrorBuffer(compressor, &errbuf); + virCommandDoAsyncIO(compressor); + if (virSetCloseExec(pipeFD[1]) < 0) { + virReportSystemError(errno, "%s", + _("Unable to set cloexec flag")); + qemuDomainObjExitMonitor(vm); + goto cleanup; + } + if (virCommandRunAsync(compressor, NULL) < 0) { + qemuDomainObjExitMonitor(vm); + goto cleanup; + } + rc = qemuMonitorMigrateToFd(priv->mon, + QEMU_MONITOR_MIGRATE_BACKGROUND, + pipeFD[1]); + if (VIR_CLOSE(pipeFD[0]) < 0 || + VIR_CLOSE(pipeFD[1]) < 0) + VIR_WARN("failed to close intermediate pipe"); } - rc = qemuMonitorMigrateToFd(priv->mon, - QEMU_MONITOR_MIGRATE_BACKGROUND, - pipeFD[1]); - if (VIR_CLOSE(pipeFD[0]) < 0 || - VIR_CLOSE(pipeFD[1]) < 0) - VIR_WARN("failed to close intermediate pipe"); } qemuDomainObjExitMonitor(vm); if (rc < 0) @@ -6008,7 +6028,7 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, goto cleanup; } - if (compressor && virCommandWait(compressor, NULL) < 0) + if (!sun_path && compressor && virCommandWait(compressor, NULL) < 0) goto cleanup; qemuDomainEventEmitJobCompleted(driver, vm); @@ -6047,6 +6067,25 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, return ret; } +int +qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, + int fd, + virCommand *compressor, + virDomainAsyncJob asyncJob) +{ + return qemuMigrationSrcToFileAux(driver, vm, fd, compressor, + asyncJob, NULL, -1); +} + +int +qemuMigrationSrcToFilesMultiFd(virQEMUDriver *driver, virDomainObj *vm, + virDomainAsyncJob asyncJob, + const char *sun_path, + int nchannels) +{ + return qemuMigrationSrcToFileAux(driver, vm, -1, NULL, + asyncJob, sun_path, nchannels); +} int qemuMigrationSrcCancel(virQEMUDriver *driver, diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index a8afa66119..ddc8e65489 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -213,6 +213,13 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainAsyncJob asyncJob) ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT; +int +qemuMigrationSrcToFilesMultiFd(virQEMUDriver *driver, virDomainObj *vm, + virDomainAsyncJob asyncJob, + const char *sun_path, + int nchannels) + ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT; + int qemuMigrationSrcCancel(virQEMUDriver *driver, virDomainObj *vm); diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index df2384b213..36174a66d8 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -1109,6 +1109,28 @@ qemuMigrationParamsFetch(virQEMUDriver *driver, } +void +qemuMigrationParamsSetCap(qemuMigrationParams *migParams, + virQEMUCapsFlags flag) +{ + ignore_value(virBitmapSetBit(migParams->caps, flag)); +} + + +int +qemuMigrationParamsSetInt(qemuMigrationParams *migParams, + qemuMigrationParam param, + int value) +{ + if (qemuMigrationParamsCheckType(param, QEMU_MIGRATION_PARAM_TYPE_INT) < 0) + return -1; + + migParams->params[param].value.i = value; + migParams->params[param].set = true; + return 0; +} + + int qemuMigrationParamsSetULL(qemuMigrationParams *migParams, qemuMigrationParam param, diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 4a8815e776..99af73b4a4 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -123,6 +123,15 @@ qemuMigrationParamsFetch(virQEMUDriver *driver, int asyncJob, qemuMigrationParams **migParams); +void +qemuMigrationParamsSetCap(qemuMigrationParams *migParams, + virQEMUCapsFlags flag); + +int +qemuMigrationParamsSetInt(qemuMigrationParams *migParams, + qemuMigrationParam param, + int value); + int qemuMigrationParamsSetULL(qemuMigrationParams *migParams, qemuMigrationParam param, diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index bb23788a5e..9147acd70a 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -492,8 +492,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, goto cleanup; if (chown(sun_path, cfg->user, cfg->group) < 0) goto cleanup; - /* still using single fd migration for now */ - if (qemuMigrationSrcToFile(driver, vm, saveFd.fd, compressor, asyncJob) < 0) + if (qemuMigrationSrcToFilesMultiFd(driver, vm, asyncJob, sun_path, nconn) < 0) goto cleanup; if (qemuSaveImageCloseMultiFd(multiFd, nconn, vm) < 0) goto cleanup; -- 2.34.1
Signed-off-by: Claudio Fontana <cfontana@suse.de> --- docs/manpages/virsh.rst | 23 ++++++++++++++----- tools/virsh-domain.c | 49 +++++++++++++++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index d2e6528533..8d077b3cc5 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -3791,15 +3791,18 @@ save :: save domain state-file [--bypass-cache] [--xml file] + [--parallel] [--parallel-connections connections] [{--running | --paused}] [--verbose] -Saves a running domain (RAM, but not disk state) to a state file so that -it can be restored -later. Once saved, the domain will no longer be running on the -system, thus the memory allocated for the domain will be free for -other domains to use. ``virsh restore`` restores from this state file. +Saves a paused or running domain (RAM, but not disk state) to one or more +state files, so that it can be restored later. +Once saved, the domain will no longer be running on the system, +thus the memory allocated for the domain will be free for +other domains to use. ``virsh restore`` restores from state file/s. + If *--bypass-cache* is specified, the save will avoid the file system -cache, although this may slow down the operation. +cache; depending on the specific scenario this may slow down or speed up +the operation. The progress may be monitored using ``domjobinfo`` virsh command and canceled with ``domjobabort`` command (sent by another virsh instance). Another option @@ -3821,6 +3824,14 @@ based on the state the domain was in when the save was done; passing either the *--running* or *--paused* flag will allow overriding which state the ``restore`` should use. +*--parallel* option will cause the save data to be sent over multiple +parallel connections to multiple files. The main save file is specified +with ``state-file``, and a number of additional connections can be +set using *--parallel-connections*, which will save to files named +``state-file``.1 , ``state-file``.2 ... up to ``connections``. + +Parallel connections may help in speeding up the save operation. + Domain saved state files assume that disk images will be unchanged between the creation and restore point. For a more complete system restore point, where the disk state is saved alongside the memory diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index d5fd8be7c3..8e7b0a7f5d 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -4164,6 +4164,14 @@ static const vshCmdOptDef opts_save[] = { .type = VSH_OT_BOOL, .help = N_("avoid file system cache when saving") }, + {.name = "parallel", + .type = VSH_OT_BOOL, + .help = N_("enable parallel save to files") + }, + {.name = "parallel-connections", + .type = VSH_OT_INT, + .help = N_("number of connections/files for parallel save") + }, {.name = "xml", .type = VSH_OT_STRING, .completer = virshCompletePathLocalExisting, @@ -4193,6 +4201,11 @@ doSave(void *opaque) g_autoptr(virshDomain) dom = NULL; const char *name = NULL; const char *to = NULL; + virTypedParameterPtr params = NULL; + int nparams = 0; + int maxparams = 0; + int intOpt = 0; + int rv = -1; unsigned int flags = 0; const char *xmlfile = NULL; g_autofree char *xml = NULL; @@ -4206,29 +4219,46 @@ doSave(void *opaque) goto out_sig; #endif /* !WIN32 */ - if (vshCommandOptStringReq(ctl, cmd, "file", &to) < 0) + if ((rv = vshCommandOptStringReq(ctl, cmd, "file", &to)) < 0) { goto out; - + } else { + if (virTypedParamsAddString(¶ms, &nparams, &maxparams, + VIR_SAVE_PARAM_TO, to) < 0) + goto out; + } if (vshCommandOptBool(cmd, "bypass-cache")) flags |= VIR_DOMAIN_SAVE_BYPASS_CACHE; + if (vshCommandOptBool(cmd, "parallel")) + flags |= VIR_DOMAIN_SAVE_PARALLEL; + if ((rv = vshCommandOptInt(ctl, cmd, "parallel-connections", &intOpt)) < 0) { + goto out; + } else if (rv > 0) { + if (virTypedParamsAddInt(¶ms, &nparams, &maxparams, + VIR_SAVE_PARAM_PARALLEL_CONNECTIONS, intOpt) < 0) + goto out; + } if (vshCommandOptBool(cmd, "running")) flags |= VIR_DOMAIN_SAVE_RUNNING; if (vshCommandOptBool(cmd, "paused")) flags |= VIR_DOMAIN_SAVE_PAUSED; - if (vshCommandOptStringReq(ctl, cmd, "xml", &xmlfile) < 0) + if ((rv = vshCommandOptStringReq(ctl, cmd, "xml", &xmlfile)) < 0) goto out; if (!(dom = virshCommandOptDomain(ctl, cmd, &name))) goto out; - if (xmlfile && - virFileReadAll(xmlfile, VSH_MAX_XML_FILE, &xml) < 0) { - vshReportError(ctl); - goto out; + if (xmlfile) { + if (virFileReadAll(xmlfile, VSH_MAX_XML_FILE, &xml) < 0) { + vshReportError(ctl); + goto out; + } else if (virTypedParamsAddString(¶ms, &nparams, &maxparams, + VIR_SAVE_PARAM_DXML, xml) < 0) + goto out; } - - if (flags || xml) { + if (flags & VIR_DOMAIN_SAVE_PARALLEL) { + rc = virDomainSaveParametersFlags(dom, params, nparams, flags); + } else if (flags || xml) { rc = virDomainSaveFlags(dom, to, xml, flags); } else { rc = virDomainSave(dom, to); @@ -4242,6 +4272,7 @@ doSave(void *opaque) data->ret = 0; out: + virTypedParamsFree(params, nparams); #ifndef WIN32 pthread_sigmask(SIG_SETMASK, &oldsigmask, NULL); out_sig: -- 2.34.1
use zstd which is the only really interesting one. Signed-off-by: Claudio Fontana <cfontana@suse.de> --- src/qemu/qemu_capabilities.c | 2 ++ src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_migration.c | 6 ++++ src/qemu/qemu_migration_params.c | 49 ++++++++++++++++---------------- src/qemu/qemu_migration_params.h | 6 ++++ 5 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 01b570807b..9ad34969f7 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -669,6 +669,7 @@ VIR_ENUM_IMPL(virQEMUCaps, /* 425 */ "blockdev.nbd.tls-hostname", /* QEMU_CAPS_BLOCKDEV_NBD_TLS_HOSTNAME */ "memory-backend-file.prealloc-threads", /* QEMU_CAPS_MEMORY_BACKEND_PREALLOC_THREADS */ + "migration-param.multifd-compression", /* QEMU_CAPS_MIGRATION_PARAM_MULTIFD_COMPRESSION */ ); @@ -1616,6 +1617,7 @@ static struct virQEMUCapsStringFlags virQEMUCapsQMPSchemaQueries[] = { { "migrate-set-parameters/arg-type/downtime-limit", QEMU_CAPS_MIGRATION_PARAM_DOWNTIME }, { "migrate-set-parameters/arg-type/xbzrle-cache-size", QEMU_CAPS_MIGRATION_PARAM_XBZRLE_CACHE_SIZE }, { "migrate-set-parameters/arg-type/block-bitmap-mapping/bitmaps/transform", QEMU_CAPS_MIGRATION_PARAM_BLOCK_BITMAP_MAPPING }, + { "migrate-set-parameters/arg-type/multifd-compression", QEMU_CAPS_MIGRATION_PARAM_MULTIFD_COMPRESSION }, { "nbd-server-start/arg-type/tls-creds", QEMU_CAPS_NBD_TLS }, { "nbd-server-add/arg-type/bitmap", QEMU_CAPS_NBD_BITMAP }, { "netdev_add/arg-type/+vhost-vdpa", QEMU_CAPS_NETDEV_VHOST_VDPA }, diff --git a/src/qemu/qemu_capabilities.h b/src/qemu/qemu_capabilities.h index 948029d60d..2b0de4113c 100644 --- a/src/qemu/qemu_capabilities.h +++ b/src/qemu/qemu_capabilities.h @@ -644,6 +644,7 @@ typedef enum { /* virQEMUCapsFlags grouping marker for syntax-check */ /* 425 */ QEMU_CAPS_BLOCKDEV_NBD_TLS_HOSTNAME, /* tls hostname can be overriden for NBD clients */ QEMU_CAPS_MEMORY_BACKEND_PREALLOC_THREADS, /* -object memory-backend-*.prealloc-threads */ + QEMU_CAPS_MIGRATION_PARAM_MULTIFD_COMPRESSION, /* multifd-compression in migrate-set-parameters */ QEMU_CAPS_LAST /* this must always be the last item */ } virQEMUCapsFlags; diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 9ce24c8cfc..65a3343622 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -5949,6 +5949,12 @@ qemuMigrationSrcToFileAux(virQEMUDriver *driver, virDomainObj *vm, QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS, nchannels) < 0) return -1; + if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATION_PARAM_MULTIFD_COMPRESSION)) { + if (qemuMigrationParamsSetString(migParams, + QEMU_MIGRATION_PARAM_MULTIFD_COMPRESSION, + "zstd") < 0) + return -1; + } } if (needParams && qemuMigrationParamsApply(driver, vm, asyncJob, migParams) < 0) diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 36174a66d8..f6b9dc337d 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -115,6 +115,7 @@ VIR_ENUM_IMPL(qemuMigrationParam, "xbzrle-cache-size", "max-postcopy-bandwidth", "multifd-channels", + "multifd-compression", ); typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem; @@ -234,6 +235,7 @@ static const qemuMigrationParamType qemuMigrationParamTypes[] = { [QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE] = QEMU_MIGRATION_PARAM_TYPE_ULL, [QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH] = QEMU_MIGRATION_PARAM_TYPE_ULL, [QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS] = QEMU_MIGRATION_PARAM_TYPE_INT, + [QEMU_MIGRATION_PARAM_MULTIFD_COMPRESSION] = QEMU_MIGRATION_PARAM_TYPE_STRING, }; G_STATIC_ASSERT(G_N_ELEMENTS(qemuMigrationParamTypes) == QEMU_MIGRATION_PARAM_LAST); @@ -898,31 +900,6 @@ qemuMigrationParamsApply(virQEMUDriver *driver, } -/** - * qemuMigrationParamsSetString: - * @migrParams: migration parameter object - * @param: parameter to set - * @value: new value - * - * Enables and sets the migration parameter @param in @migrParams. Returns 0 on - * success and -1 on error. Libvirt error is reported. - */ -static int -qemuMigrationParamsSetString(qemuMigrationParams *migParams, - qemuMigrationParam param, - const char *value) -{ - if (qemuMigrationParamsCheckType(param, QEMU_MIGRATION_PARAM_TYPE_STRING) < 0) - return -1; - - migParams->params[param].value.s = g_strdup(value); - - migParams->params[param].set = true; - - return 0; -} - - /* qemuMigrationParamsEnableTLS * @driver: pointer to qemu driver * @vm: domain object @@ -1144,6 +1121,28 @@ qemuMigrationParamsSetULL(qemuMigrationParams *migParams, return 0; } +/** + * qemuMigrationParamsSetString: + * @migrParams: migration parameter object + * @param: parameter to set + * @value: new value + * + * Enables and sets the migration parameter @param in @migrParams. Returns 0 on + * success and -1 on error. Libvirt error is reported. + */ +int +qemuMigrationParamsSetString(qemuMigrationParams *migParams, + qemuMigrationParam param, + const char *value) +{ + if (qemuMigrationParamsCheckType(param, QEMU_MIGRATION_PARAM_TYPE_STRING) < 0) + return -1; + + migParams->params[param].value.s = g_strdup(value); + migParams->params[param].set = true; + return 0; +} + /** * Returns -1 on error, diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 99af73b4a4..23a4e0c8a2 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -60,6 +60,7 @@ typedef enum { QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE, QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH, QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS, + QEMU_MIGRATION_PARAM_MULTIFD_COMPRESSION, QEMU_MIGRATION_PARAM_LAST } qemuMigrationParam; @@ -137,6 +138,11 @@ qemuMigrationParamsSetULL(qemuMigrationParams *migParams, qemuMigrationParam param, unsigned long long value); +int +qemuMigrationParamsSetString(qemuMigrationParams *migParams, + qemuMigrationParam param, + const char *value); + int qemuMigrationParamsGetULL(qemuMigrationParams *migParams, qemuMigrationParam param, -- 2.34.1
participants (1)
-
Claudio Fontana