[PATCH V2 00/20] qemu: support mapped-ram+directio+mulitfd
I'm claiming this to be V2 of a series to support QEMU's mapped-ram stream format [1] and migration capability, even though the approach to control the feature is different than V1. To control mapped-ram in V1, I bumped the save format version and added a 'features' field to the save image header. In V2, I use the save_image_format setting in qemu.conf, extending it with a 'sparse' option to enable mapped-ram. IMO, the latter is nicer approach that fits better with existing save options. V2 leaves the "raw" save image format as default. save_image_format must be set to "sparse" to enable use of mapped-ram. Like V1, this version of the series also includes patches that leverage mapped-ram to add support for parallel save/restore. In one of the previous threads on this topic we discussed per-VM control of save image format by adding a 'format' parameter to virDomainSaveParams. I can do that as a followup series if folks agree its useful. Changes in V2 * Use save_image_format setting to control mapped-ram instead of introducing a new field to the header and bumping the save image version * Rename save parameter parallel-connections to parallel-channels * Address other minor comments from V1 [1] https://lists.libvirt.org/archives/list/devel@lists.libvirt.org/message/MNBH... Claudio Fontana (3): include: Define constants for parallel save/restore tools: add parallel parameter to virsh save command tools: add parallel parameter to virsh restore command Jim Fehlig (17): lib: virDomain{Save,Restore}Params: Ensure absolute path qemu_fd: Add function to retrieve fdset ID qemu: Add function to check capability in migration params qemu: Add function to get bool value from migration params qemu: Add mapped-ram migration capability qemu: Add function to get migration params for save qemu_saveimage: add "sparse" to supported save image formats qemu: Add helper function for creating save image fd qemu: Move declaration of virQEMUSaveFormat to header file qemu: Add support for mapped-ram on save qemu: Decompose qemuSaveImageOpen qemu: Move creation of qemuProcessIncomingDef struct qemu: Apply migration parameters in qemuMigrationDstRun qemu: Add support for mapped-ram on restore qemu: Support O_DIRECT with mapped-ram on save qemu: Support O_DIRECT with mapped-ram on restore qemu: Add support for parallel save and restore docs/manpages/virsh.rst | 21 ++- include/libvirt/libvirt-domain.h | 11 ++ src/libvirt-domain.c | 95 +++++++++++--- src/qemu/qemu.conf.in | 9 +- src/qemu/qemu_driver.c | 108 +++++++++++----- src/qemu/qemu_fd.c | 18 +++ src/qemu/qemu_fd.h | 3 + src/qemu/qemu_migration.c | 194 +++++++++++++++++++--------- src/qemu/qemu_migration.h | 10 +- src/qemu/qemu_migration_params.c | 92 ++++++++++++++ src/qemu/qemu_migration_params.h | 17 +++ src/qemu/qemu_monitor.c | 40 ++++++ src/qemu/qemu_monitor.h | 5 + src/qemu/qemu_process.c | 117 ++++++++++++----- src/qemu/qemu_process.h | 19 ++- src/qemu/qemu_saveimage.c | 211 ++++++++++++++++++------------- src/qemu/qemu_saveimage.h | 39 +++++- src/qemu/qemu_snapshot.c | 29 +++-- tools/virsh-domain.c | 79 ++++++++++-- 19 files changed, 856 insertions(+), 261 deletions(-) -- 2.43.0
When invoking virDomainSaveParams with a relative path, the image is saved to the daemon's CWD. Similarly, when providing virDomainRestoreParams with a relative path, it attempts to restore from the daemon's CWD. In most configurations, the daemon's CWD is set to '/'. Ensure a relative path is converted to absolute before invoking the driver domain{Save,Restore}Params functions. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/libvirt-domain.c | 89 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index e8e5379672..3fb091f240 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -1020,6 +1020,11 @@ virDomainSaveParams(virDomainPtr domain, unsigned int flags) { virConnectPtr conn; + virTypedParameterPtr params_copy = NULL; + int nparams_copy = 0; + const char *to = NULL; + g_autofree char *absolute_to = NULL; + int ret = -1; VIR_DOMAIN_DEBUG(domain, "params=%p, nparams=%d, flags=0x%x", params, nparams, flags); @@ -1030,23 +1035,46 @@ virDomainSaveParams(virDomainPtr domain, virCheckDomainReturn(domain, -1); conn = domain->conn; - virCheckReadOnlyGoto(conn->flags, error); + virCheckReadOnlyGoto(conn->flags, done); VIR_EXCLUSIVE_FLAGS_GOTO(VIR_DOMAIN_SAVE_RUNNING, VIR_DOMAIN_SAVE_PAUSED, - error); + done); + + /* We must absolutize the file path as the save is done out of process */ + virTypedParamsCopy(¶ms_copy, params, nparams); + nparams_copy = nparams; + if (virTypedParamsGetString(params_copy, nparams_copy, + VIR_DOMAIN_SAVE_PARAM_FILE, &to) < 0) + goto done; + + if (to) { + if (!(absolute_to = g_canonicalize_filename(to, NULL))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("could not build absolute output file path")); + goto done; + } + + if (virTypedParamsReplaceString(¶ms_copy, &nparams_copy, + VIR_DOMAIN_SAVE_PARAM_FILE, + absolute_to) < 0) + goto done; + } if (conn->driver->domainSaveParams) { - if (conn->driver->domainSaveParams(domain, params, nparams, flags) < 0) - goto error; - return 0; + if (conn->driver->domainSaveParams(domain, params_copy, nparams_copy, flags) < 0) + goto done; + ret = 0; + } else { + virReportUnsupportedError(); } - virReportUnsupportedError(); + done: + if (ret < 0) + virDispatchError(domain->conn); + virTypedParamsFree(params_copy, nparams_copy); - error: - virDispatchError(domain->conn); - return -1; + return ret; } @@ -1203,6 +1231,12 @@ virDomainRestoreParams(virConnectPtr conn, virTypedParameterPtr params, int nparams, unsigned int flags) { + virTypedParameterPtr params_copy = NULL; + int nparams_copy = 0; + const char *from = NULL; + g_autofree char *absolute_from = NULL; + int ret = -1; + VIR_DEBUG("conn=%p, params=%p, nparams=%d, flags=0x%x", conn, params, nparams, flags); VIR_TYPED_PARAMS_DEBUG(params, nparams); @@ -1210,23 +1244,46 @@ virDomainRestoreParams(virConnectPtr conn, virResetLastError(); virCheckConnectReturn(conn, -1); - virCheckReadOnlyGoto(conn->flags, error); + virCheckReadOnlyGoto(conn->flags, done); VIR_EXCLUSIVE_FLAGS_GOTO(VIR_DOMAIN_SAVE_RUNNING, VIR_DOMAIN_SAVE_PAUSED, - error); + done); if (conn->driver->domainRestoreParams) { + /* We must absolutize the file path as the save is done out of process */ + virTypedParamsCopy(¶ms_copy, params, nparams); + nparams_copy = nparams; + if (virTypedParamsGetString(params_copy, nparams_copy, + VIR_DOMAIN_SAVE_PARAM_FILE, &from) < 0) + goto done; + + if (from) { + if (!(absolute_from = g_canonicalize_filename(from, NULL))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("could not build absolute output file path")); + goto done; + } + + if (virTypedParamsReplaceString(¶ms_copy, &nparams_copy, + VIR_DOMAIN_SAVE_PARAM_FILE, + absolute_from) < 0) + goto done; + } + if (conn->driver->domainRestoreParams(conn, params, nparams, flags) < 0) - goto error; - return 0; + goto done; + ret = 0; } virReportUnsupportedError(); - error: - virDispatchError(conn); - return -1; + done: + if (ret < 0) + virDispatchError(conn); + virTypedParamsFree(params_copy, nparams_copy); + + return ret; } -- 2.43.0
Add new function qemuFDPassGetId() for retrieving the fdset ID of provided qemuFDPass object. Signed-off-by: Jim Fehlig <jfehlig@suse.com> Reviewed-by: Daniel P. Berrangé <berrange@redhat.com> --- src/qemu/qemu_fd.c | 18 ++++++++++++++++++ src/qemu/qemu_fd.h | 3 +++ 2 files changed, 21 insertions(+) diff --git a/src/qemu/qemu_fd.c b/src/qemu/qemu_fd.c index e847056573..3ae4a87a20 100644 --- a/src/qemu/qemu_fd.c +++ b/src/qemu/qemu_fd.c @@ -276,6 +276,24 @@ qemuFDPassGetPath(qemuFDPass *fdpass) } +/** + * qemuFDPassGetId: + * @fdpass: The fd passing helper struct + * @id: An out parameter for providing the fdset ID + * + * Returns 0 on success, -1 on error. + */ +int +qemuFDPassGetId(qemuFDPass *fdpass, unsigned int *id) +{ + if (!fdpass) + return -1; + + *id = fdpass->fdSetID; + return 0; +} + + struct _qemuFDPassDirect { int fd; char *name; diff --git a/src/qemu/qemu_fd.h b/src/qemu/qemu_fd.h index cd0ff2c690..dfcd5fb754 100644 --- a/src/qemu/qemu_fd.h +++ b/src/qemu/qemu_fd.h @@ -58,6 +58,9 @@ qemuFDPassTransferMonitorRollback(qemuFDPass *fdpass, const char * qemuFDPassGetPath(qemuFDPass *fdpass); +int +qemuFDPassGetId(qemuFDPass *fdpass, unsigned int *id); + typedef struct _qemuFDPassDirect qemuFDPassDirect; -- 2.43.0
Add new function qemuMigrationParamsCapEnabled() to check if a capability is set in the caller-provided migration parameters. Signed-off-by: Jim Fehlig <jfehlig@suse.com> Reviewed-by: Daniel P. Berrangé <berrange@redhat.com> --- src/qemu/qemu_migration_params.c | 16 ++++++++++++++++ src/qemu/qemu_migration_params.h | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 98822012cc..c3c9120c22 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -1303,6 +1303,22 @@ qemuMigrationParamsGetULL(qemuMigrationParams *migParams, } +/** + * Returns true if @cap is enabled in @migParams, false otherwise. + */ +bool +qemuMigrationParamsCapEnabled(qemuMigrationParams *migParams, + qemuMigrationCapability cap) +{ + bool enabled = false; + + if (migParams) + ignore_value(virBitmapGetBit(migParams->caps, cap, &enabled)); + + return enabled; +} + + /** * qemuMigrationParamsCheck: * diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index df67f1fb92..020071e8ef 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -136,6 +136,10 @@ qemuMigrationParamsGetULL(qemuMigrationParams *migParams, qemuMigrationParam param, unsigned long long *value); +bool +qemuMigrationParamsCapEnabled(qemuMigrationParams *migParams, + qemuMigrationCapability cap); + void qemuMigrationParamsSetBlockDirtyBitmapMapping(qemuMigrationParams *migParams, virJSONValue **params); -- 2.43.0
Signed-off-by: Jim Fehlig <jfehlig@suse.com> Reviewed-by: Daniel P. Berrangé <berrange@redhat.com> --- src/qemu/qemu_migration_params.c | 18 ++++++++++++++++++ src/qemu/qemu_migration_params.h | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index c3c9120c22..daa52269f4 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -1303,6 +1303,24 @@ qemuMigrationParamsGetULL(qemuMigrationParams *migParams, } +int +qemuMigrationParamsGetBool(qemuMigrationParams *migParams, + qemuMigrationParam param, + bool *value) +{ + if (!migParams || !value) + return 0; + + if (qemuMigrationParamsCheckType(param, QEMU_MIGRATION_PARAM_TYPE_BOOL) < 0) + return -1; + + *value = migParams->params[param].set ? + migParams->params[param].value.b : false; + + return 0; +} + + /** * Returns true if @cap is enabled in @migParams, false otherwise. */ diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 020071e8ef..64b2dde59b 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -136,6 +136,11 @@ qemuMigrationParamsGetULL(qemuMigrationParams *migParams, qemuMigrationParam param, unsigned long long *value); +int +qemuMigrationParamsGetBool(qemuMigrationParams *migParams, + qemuMigrationParam param, + bool *value); + bool qemuMigrationParamsCapEnabled(qemuMigrationParams *migParams, qemuMigrationCapability cap); -- 2.43.0
Add the mapped-ram migration capability introduced in QEMU 9.0. Reviewed-by: Daniel P. Berrangé <berrange@redhat.com> Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_migration_params.c | 1 + src/qemu/qemu_migration_params.h | 1 + 2 files changed, 2 insertions(+) diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index daa52269f4..03937df2d3 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -106,6 +106,7 @@ VIR_ENUM_IMPL(qemuMigrationCapability, "zero-copy-send", "postcopy-preempt", "switchover-ack", + "mapped-ram", ); diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 64b2dde59b..91a1965a74 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -42,6 +42,7 @@ typedef enum { QEMU_MIGRATION_CAP_ZERO_COPY_SEND, QEMU_MIGRATION_CAP_POSTCOPY_PREEMPT, QEMU_MIGRATION_CAP_SWITCHOVER_ACK, + QEMU_MIGRATION_CAP_MAPPED_RAM, QEMU_MIGRATION_CAP_LAST } qemuMigrationCapability; -- 2.43.0
Introduce qemuMigrationParamsForSave() to create a qemuMigrationParams object initialized with appropriate migration capabilities and parameters for a save operation. Note that mapped-ram capability also requires the multifd capability. For now, the number of multifd channels is set to 1. Future work to support parallel save/restore can set the number of channels to a user-specified value. Signed-off-by: Jim Fehlig <jfehlig@suse.com> Reviewed-by: Daniel P. Berrangé <berrange@redhat.com> --- src/qemu/qemu_migration.c | 2 +- src/qemu/qemu_migration_params.c | 21 +++++++++++++++++++++ src/qemu/qemu_migration_params.h | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 1582a738a3..d46a6e2e2a 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -7086,7 +7086,7 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, /* Increase migration bandwidth to unlimited since target is a file. * Failure to change migration speed is not fatal. */ - if (!(migParams = qemuMigrationParamsNew())) + if (!(migParams = qemuMigrationParamsForSave(false))) return -1; if (qemuMigrationParamsSetULL(migParams, diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 03937df2d3..0faeb6b0c1 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -783,6 +783,27 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, } +qemuMigrationParams * +qemuMigrationParamsForSave(bool sparse) +{ + g_autoptr(qemuMigrationParams) saveParams = NULL; + + if (!(saveParams = qemuMigrationParamsNew())) + return NULL; + + if (sparse) { + if (virBitmapSetBit(saveParams->caps, QEMU_MIGRATION_CAP_MAPPED_RAM) < 0) + return NULL; + if (virBitmapSetBit(saveParams->caps, QEMU_MIGRATION_CAP_MULTIFD) < 0) + return NULL; + saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i = 1; + saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set = true; + } + + return g_steal_pointer(&saveParams); +} + + int qemuMigrationParamsDump(qemuMigrationParams *migParams, virTypedParameterPtr *params, diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 91a1965a74..bb24af4ef9 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -86,6 +86,9 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, unsigned int flags, qemuMigrationParty party); +qemuMigrationParams * +qemuMigrationParamsForSave(bool sparse); + int qemuMigrationParamsDump(qemuMigrationParams *migParams, virTypedParameterPtr *params, -- 2.43.0
Extend the list of formats to include "sparse", which uses QEMU's mapped-ram stream format [1] to write guest memory blocks at fixed offsets in the save image file. [1] https://gitlab.com/qemu-project/qemu/-/blob/master/docs/devel/migration/mapp... Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu.conf.in | 9 ++++++++- src/qemu/qemu_saveimage.c | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu.conf.in b/src/qemu/qemu.conf.in index d853136f10..ab9725e40d 100644 --- a/src/qemu/qemu.conf.in +++ b/src/qemu/qemu.conf.in @@ -588,11 +588,18 @@ # disk space when saving large memory guests. Various compression formats are # available for specifying a save image compressed by the named algorithm. # Supported compression formats are "zstd", "lzop", "gzip", "bzip2", and "xz". +# The "sparse" format uses QEMU's mapped-ram stream format to write guest memory +# blocks at fixed offsets in the save image file. # save_image_format can be used to select the desired save format. "raw" is # the traditional format used by libvirt and is also the default. The # compression formats can be used to save disk space, although this typically -# results in longer save and restore times. +# results in longer save and restore times. The "sparse" format results in a +# save image file that is roughly the logical size of the guest's memory, +# although on-disk size is a function of guest memory usage. The "sparse" +# format is useful when a predictable maximum save image file size is +# needed. The other formats can result in a save image file much larger +# than guest memory if the guest runs a memory intensive workload. # # save_image_format is used with 'virsh save' or 'virsh managedsave'. It is # an error if the specified save_image_format is not valid, or cannot be diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 69617e07eb..ff5b9e72eb 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -48,6 +48,7 @@ typedef enum { QEMU_SAVE_FORMAT_XZ = 3, QEMU_SAVE_FORMAT_LZOP = 4, QEMU_SAVE_FORMAT_ZSTD = 5, + QEMU_SAVE_FORMAT_SPARSE = 6, /* Note: add new members only at the end. These values are used in the on-disk format. Do not change or re-use numbers. */ @@ -64,6 +65,7 @@ VIR_ENUM_IMPL(qemuSaveFormat, "xz", "lzop", "zstd", + "sparse", ); static inline void -- 2.43.0
Move the code in qemuSaveImageCreate that opens, labels, and wraps the save image fd to a helper function, providing more flexibility for upcoming mapped-ram support. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_saveimage.c | 65 +++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index ff5b9e72eb..65464fd1e1 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -361,6 +361,50 @@ qemuSaveImageDecompressionStop(virCommand *cmd, } +static int +qemuSaveImageCreateFd(virQEMUDriver *driver, + virDomainObj *vm, + const char *path, + virFileWrapperFd *wrapperFd, + bool *needUnlink, + unsigned int flags) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + int ret = -1; + VIR_AUTOCLOSE fd = -1; + int directFlag = 0; + unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; + + if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { + wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; + directFlag = virFileDirectFdFlag(); + if (directFlag < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("bypass cache unsupported by this system")); + return -1; + } + } + + fd = virQEMUFileOpenAs(cfg->user, cfg->group, false, path, + O_WRONLY | O_TRUNC | O_CREAT | directFlag, + needUnlink); + + if (fd < 0) + return -1; + + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, fd) < 0) + return -1; + + if (!(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) + return -1; + + ret = fd; + fd = -1; + + return ret; +} + + /* Helper function to execute a migration to file with a correct save header * the caller needs to make sure that the processors are stopped and do all other * actions besides saving memory */ @@ -377,33 +421,14 @@ qemuSaveImageCreate(virQEMUDriver *driver, bool needUnlink = false; int ret = -1; int fd = -1; - int directFlag = 0; virFileWrapperFd *wrapperFd = NULL; - unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; /* Obtain the file handle. */ - if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) { - wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; - directFlag = virFileDirectFdFlag(); - if (directFlag < 0) { - virReportError(VIR_ERR_OPERATION_FAILED, "%s", - _("bypass cache unsupported by this system")); - goto cleanup; - } - } + fd = qemuSaveImageCreateFd(driver, vm, path, wrapperFd, &needUnlink, flags); - fd = virQEMUFileOpenAs(cfg->user, cfg->group, false, path, - O_WRONLY | O_TRUNC | O_CREAT | directFlag, - &needUnlink); if (fd < 0) goto cleanup; - if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, fd) < 0) - goto cleanup; - - if (!(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) - goto cleanup; - if (virQEMUSaveDataWrite(data, fd, path) < 0) goto cleanup; -- 2.43.0
Allow use of the enum outside of qemu_saveimage. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_saveimage.c | 20 -------------------- src/qemu/qemu_saveimage.h | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 65464fd1e1..e655be60d2 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -37,26 +37,6 @@ VIR_LOG_INIT("qemu.qemu_saveimage"); -typedef enum { - QEMU_SAVE_FORMAT_RAW = 0, - QEMU_SAVE_FORMAT_GZIP = 1, - QEMU_SAVE_FORMAT_BZIP2 = 2, - /* - * Deprecated by xz and never used as part of a release - * QEMU_SAVE_FORMAT_LZMA - */ - QEMU_SAVE_FORMAT_XZ = 3, - QEMU_SAVE_FORMAT_LZOP = 4, - QEMU_SAVE_FORMAT_ZSTD = 5, - QEMU_SAVE_FORMAT_SPARSE = 6, - /* Note: add new members only at the end. - These values are used in the on-disk format. - Do not change or re-use numbers. */ - - QEMU_SAVE_FORMAT_LAST -} virQEMUSaveFormat; - -VIR_ENUM_DECL(qemuSaveFormat); VIR_ENUM_IMPL(qemuSaveFormat, QEMU_SAVE_FORMAT_LAST, "raw", diff --git a/src/qemu/qemu_saveimage.h b/src/qemu/qemu_saveimage.h index 0e58dd14b6..fc875f9a0c 100644 --- a/src/qemu/qemu_saveimage.h +++ b/src/qemu/qemu_saveimage.h @@ -32,6 +32,26 @@ G_STATIC_ASSERT(sizeof(QEMU_SAVE_MAGIC) == sizeof(QEMU_SAVE_PARTIAL)); +typedef enum { + QEMU_SAVE_FORMAT_RAW = 0, + QEMU_SAVE_FORMAT_GZIP = 1, + QEMU_SAVE_FORMAT_BZIP2 = 2, + /* + * Deprecated by xz and never used as part of a release + * QEMU_SAVE_FORMAT_LZMA + */ + QEMU_SAVE_FORMAT_XZ = 3, + QEMU_SAVE_FORMAT_LZOP = 4, + QEMU_SAVE_FORMAT_ZSTD = 5, + QEMU_SAVE_FORMAT_SPARSE = 6, + /* Note: add new members only at the end. + These values are used in the on-disk format. + Do not change or re-use numbers. */ + + QEMU_SAVE_FORMAT_LAST +} virQEMUSaveFormat; +VIR_ENUM_DECL(qemuSaveFormat); + typedef struct _virQEMUSaveHeader virQEMUSaveHeader; struct _virQEMUSaveHeader { char magic[sizeof(QEMU_SAVE_MAGIC)-1]; -- 2.43.0
Introduce support for QEMU's new mapped-ram stream format [1]. mapped-ram is enabled by default if the underlying QEMU advertises the mapped-ram migration capability. It can be disabled by changing the 'save_image_version' setting in qemu.conf to version '2'. To use mapped-ram with QEMU: - The 'mapped-ram' migration capability must be set to true - The 'multifd' migration capability must be set to true and the 'multifd-channels' migration parameter must set to 1 - QEMU must be provided an fdset containing the migration fd - The 'migrate' qmp command is invoked with a URI referencing the fdset and an offset where to start reading or writing the data stream, e.g. {"execute":"migrate", "arguments":{"detach":true,"resume":false, "uri":"file:/dev/fdset/0,offset=0x11921"}} The mapped-ram stream, in conjunction with direct IO and multifd support provided by subsequent patches, can significantly improve the time required to save VM memory state. The following tables compare mapped-ram with the existing, sequential save stream. In all cases, the save and restore operations are to/from a block device comprised of two NVMe disks in RAID0 configuration with xfs (~8600MiB/s). The values in the 'save time' and 'restore time' columns were scraped from the 'real' time reported by time(1). The 'Size' and 'Blocks' columns were provided by the corresponding outputs of stat(1). VM: 32G RAM, 1 vcpu, idle (shortly after boot) | save | restore | | time | time | Size | Blocks -----------------------+---------+---------+--------------+-------- legacy | 6.193s | 4.399s | 985744812 | 1925288 -----------------------+---------+---------+--------------+-------- mapped-ram | 5.109s | 1.176s | 34368554354 | 1774472 -----------------------+---------+---------+--------------+-------- legacy + direct IO | 5.725s | 4.512s | 985765251 | 1925328 -----------------------+---------+---------+--------------+-------- mapped-ram + direct IO | 4.627s | 1.490s | 34368554354 | 1774304 -----------------------+---------+---------+--------------+-------- mapped-ram + direct IO | | | | + multifd-channels=8 | 4.421s | 0.845s | 34368554318 | 1774312 ------------------------------------------------------------------- VM: 32G RAM, 30G dirty, 1 vcpu in tight loop dirtying memory | save | restore | | time | time | Size | Blocks -----------------------+---------+---------+--------------+--------- legacy | 25.800s | 14.332s | 33154309983 | 64754512 -----------------------+---------+---------+--------------+--------- mapped-ram | 18.742s | 15.027s | 34368559228 | 64617160 -----------------------+---------+---------+--------------+--------- legacy + direct IO | 13.115s | 18.050s | 33154310496 | 64754520 -----------------------+---------+---------+--------------+--------- mapped-ram + direct IO | 13.623s | 15.959s | 34368557392 | 64662040 -----------------------+-------- +---------+--------------+--------- mapped-ram + direct IO | | | | + multifd-channels=8 | 6.994s | 6.470s | 34368554980 | 64665776 -------------------------------------------------------------------- As can be seen from the tables, one caveat of mapped-ram is the logical file size of a saved image is basically equivalent to the VM memory size. Note however that mapped-ram typically uses fewer blocks on disk. Another caveat of mapped-ram is the requirement for a seekable file descriptor, which currently makes it incompatible with libvirt's support for save image compression. Also note the mapped-ram stream is incompatible with the existing stream format, hence mapped-ram cannot be used to restore an image saved with the existing format and vice versa. [1] https://gitlab.com/qemu-project/qemu/-/blob/master/docs/devel/migration/mapp... Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_driver.c | 23 ++++++- src/qemu/qemu_migration.c | 139 ++++++++++++++++++++++++++------------ src/qemu/qemu_migration.h | 4 +- src/qemu/qemu_monitor.c | 37 ++++++++++ src/qemu/qemu_monitor.h | 4 ++ src/qemu/qemu_saveimage.c | 33 +++++---- src/qemu/qemu_saveimage.h | 1 + src/qemu/qemu_snapshot.c | 14 +++- 8 files changed, 193 insertions(+), 62 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index d2eddbd9ae..716eddba09 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -2620,6 +2620,7 @@ qemuDomainSaveInternal(virQEMUDriver *driver, qemuDomainObjPrivate *priv = vm->privateData; virQEMUSaveData *data = NULL; g_autoptr(qemuDomainSaveCookie) cookie = NULL; + g_autoptr(qemuMigrationParams) saveParams = NULL; if (virDomainObjBeginAsyncJob(vm, VIR_ASYNC_JOB_SAVE, VIR_DOMAIN_JOB_OPERATION_SAVE, flags) < 0) @@ -2628,6 +2629,14 @@ qemuDomainSaveInternal(virQEMUDriver *driver, if (!qemuMigrationSrcIsAllowed(vm, false, VIR_ASYNC_JOB_SAVE, 0)) goto endjob; + if (format == QEMU_SAVE_FORMAT_SPARSE && + !qemuMigrationCapsGet(vm, QEMU_MIGRATION_CAP_MAPPED_RAM)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, + _("save image format %1$s is not supported by this QEMU binary"), + qemuSaveFormatTypeToString(format)); + goto endjob; + } + if (!virDomainObjIsActive(vm)) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("guest unexpectedly quit")); @@ -2691,8 +2700,11 @@ qemuDomainSaveInternal(virQEMUDriver *driver, goto endjob; xml = NULL; + if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE))) + goto endjob; + ret = qemuSaveImageCreate(driver, vm, path, data, compressor, - flags, VIR_ASYNC_JOB_SAVE); + saveParams, flags, VIR_ASYNC_JOB_SAVE); if (ret < 0) goto endjob; @@ -3126,6 +3138,8 @@ doCoreDump(virQEMUDriver *driver, memory_dump_format) < 0) goto cleanup; } else { + g_autoptr(qemuMigrationParams) dump_params = NULL; + if (dumpformat != VIR_DOMAIN_CORE_DUMP_FORMAT_RAW) { virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", _("kdump-compressed format is only supported with memory-only dump")); @@ -3135,8 +3149,11 @@ doCoreDump(virQEMUDriver *driver, if (!qemuMigrationSrcIsAllowed(vm, false, VIR_ASYNC_JOB_DUMP, 0)) goto cleanup; - if (qemuMigrationSrcToFile(driver, vm, fd, compressor, - VIR_ASYNC_JOB_DUMP) < 0) + if (!(dump_params = qemuMigrationParamsNew())) + goto cleanup; + + if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, + dump_params, dump_flags, VIR_ASYNC_JOB_DUMP) < 0) goto cleanup; } diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index d46a6e2e2a..6a378128d1 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -7065,46 +7065,17 @@ qemuMigrationProcessUnattended(virQEMUDriver *driver, } -/* Helper function called while vm is active. */ -int -qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, - int fd, - virCommand *compressor, - virDomainAsyncJob asyncJob) +static int +qemuMigrationSrcToLegacyFile(virQEMUDriver *driver, + virDomainObj *vm, + int fd, + virCommand *compressor, + virDomainAsyncJob asyncJob) { qemuDomainObjPrivate *priv = vm->privateData; - int rc; int ret = -1; int pipeFD[2] = { -1, -1 }; - unsigned long saveMigBandwidth = priv->migMaxBandwidth; char *errbuf = NULL; - virErrorPtr orig_err = NULL; - g_autoptr(qemuMigrationParams) migParams = NULL; - - if (qemuMigrationSetDBusVMState(driver, vm) < 0) - return -1; - - /* Increase migration bandwidth to unlimited since target is a file. - * Failure to change migration speed is not fatal. */ - if (!(migParams = qemuMigrationParamsForSave(false))) - return -1; - - if (qemuMigrationParamsSetULL(migParams, - QEMU_MIGRATION_PARAM_MAX_BANDWIDTH, - QEMU_DOMAIN_MIG_BANDWIDTH_MAX * 1024 * 1024) < 0) - return -1; - - if (qemuMigrationParamsApply(vm, asyncJob, migParams, 0) < 0) - return -1; - - priv->migMaxBandwidth = QEMU_DOMAIN_MIG_BANDWIDTH_MAX; - - if (!virDomainObjIsActive(vm)) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("guest unexpectedly quit")); - /* nothing to tear down */ - return -1; - } if (compressor && virPipe(pipeFD) < 0) return -1; @@ -7121,7 +7092,7 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, goto cleanup; if (!compressor) { - rc = qemuMonitorMigrateToFd(priv->mon, 0, fd); + ret = qemuMonitorMigrateToFd(priv->mon, 0, fd); } else { virCommandSetInputFD(compressor, pipeFD[0]); virCommandSetOutputFD(compressor, &fd); @@ -7137,12 +7108,99 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, qemuDomainObjExitMonitor(vm); goto cleanup; } - rc = qemuMonitorMigrateToFd(priv->mon, 0, pipeFD[1]); + ret = qemuMonitorMigrateToFd(priv->mon, 0, pipeFD[1]); if (VIR_CLOSE(pipeFD[0]) < 0 || VIR_CLOSE(pipeFD[1]) < 0) VIR_WARN("failed to close intermediate pipe"); } qemuDomainObjExitMonitor(vm); + + cleanup: + VIR_FORCE_CLOSE(pipeFD[0]); + VIR_FORCE_CLOSE(pipeFD[1]); + + if (errbuf) { + VIR_DEBUG("Compression binary stderr: %s", NULLSTR(errbuf)); + VIR_FREE(errbuf); + } + + return ret; +} + + +static int +qemuMigrationSrcToMappedFile(virQEMUDriver *driver, + virDomainObj *vm, + int *fd, + unsigned int flags, + virDomainAsyncJob asyncJob) +{ + int ret; + + /* mapped-ram does not support directIO */ + if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("bypass cache unsupported by this system")); + return -1; + } + + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, *fd) < 0) + return -1; + + if (qemuDomainObjEnterMonitorAsync(vm, asyncJob) < 0) + return -1; + + ret = qemuMonitorMigrateToFdSet(vm, 0, fd); + qemuDomainObjExitMonitor(vm); + return ret; +} + + +/* Helper function called while vm is active. */ +int +qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, + int *fd, + virCommand *compressor, + qemuMigrationParams *migParams, + unsigned int flags, + virDomainAsyncJob asyncJob) +{ + qemuDomainObjPrivate *priv = vm->privateData; + int rc; + int ret = -1; + unsigned long saveMigBandwidth = priv->migMaxBandwidth; + virErrorPtr orig_err = NULL; + + if (qemuMigrationSetDBusVMState(driver, vm) < 0) + return -1; + + /* Increase migration bandwidth to unlimited since target is a file. + * Failure to change migration speed is not fatal. */ + if (migParams && + qemuMigrationParamsSetULL(migParams, + QEMU_MIGRATION_PARAM_MAX_BANDWIDTH, + QEMU_DOMAIN_MIG_BANDWIDTH_MAX * 1024 * 1024) < 0) + return -1; + + if (qemuMigrationParamsApply(vm, asyncJob, migParams, 0) < 0) + return -1; + + priv->migMaxBandwidth = QEMU_DOMAIN_MIG_BANDWIDTH_MAX; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("guest unexpectedly quit")); + /* nothing to tear down */ + return -1; + } + + if (migParams && + qemuMigrationParamsCapEnabled(migParams, QEMU_MIGRATION_CAP_MAPPED_RAM)) { + rc = qemuMigrationSrcToMappedFile(driver, vm, fd, flags, asyncJob); + } else { + rc = qemuMigrationSrcToLegacyFile(driver, vm, *fd, compressor, asyncJob); + } + if (rc < 0) goto cleanup; @@ -7178,13 +7236,6 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, priv->migMaxBandwidth = saveMigBandwidth; } - VIR_FORCE_CLOSE(pipeFD[0]); - VIR_FORCE_CLOSE(pipeFD[1]); - if (errbuf) { - VIR_DEBUG("Compression binary stderr: %s", NULLSTR(errbuf)); - VIR_FREE(errbuf); - } - virErrorRestore(&orig_err); return ret; diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 4b7ef9688a..627d0cb3fb 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -237,8 +237,10 @@ qemuMigrationSrcIsAllowed(virDomainObj *vm, int qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, - int fd, + int *fd, virCommand *compressor, + qemuMigrationParams *migParams, + unsigned int flags, virDomainAsyncJob asyncJob) ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT; diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index ec2f166785..d88e1b1ecd 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -2230,6 +2230,43 @@ qemuMonitorMigrateToFd(qemuMonitor *mon, } +int +qemuMonitorMigrateToFdSet(virDomainObj *vm, + unsigned int flags, + int *fd) +{ + qemuDomainObjPrivate *priv = vm->privateData; + qemuMonitor *mon = priv->mon; + off_t offset; + g_autoptr(qemuFDPass) fdPassMigrate = NULL; + unsigned int setId; + g_autofree char *uri = NULL; + int ret; + + VIR_DEBUG("fd=%d flags=0x%x", *fd, flags); + + QEMU_CHECK_MONITOR(mon); + + if ((offset = lseek(*fd, 0, SEEK_CUR)) == -1) { + virReportSystemError(errno, + "%s", _("failed to seek on file descriptor")); + return -1; + } + + fdPassMigrate = qemuFDPassNew("migrate", priv); + qemuFDPassAddFD(fdPassMigrate, fd, "-fd"); + qemuFDPassTransferMonitor(fdPassMigrate, mon); + + if (qemuFDPassGetId(fdPassMigrate, &setId) < 0) + return -1; + + uri = g_strdup_printf("file:/dev/fdset/%u,offset=%#lx", setId, offset); + ret = qemuMonitorJSONMigrate(mon, flags, uri); + + return ret; +} + + int qemuMonitorMigrateToHost(qemuMonitor *mon, unsigned int flags, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index c74892c4dc..7f54203fe3 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -855,6 +855,10 @@ int qemuMonitorMigrateToFd(qemuMonitor *mon, unsigned int flags, int fd); +int qemuMonitorMigrateToFdSet(virDomainObj *vm, + unsigned int flags, + int *fd); + int qemuMonitorMigrateToHost(qemuMonitor *mon, unsigned int flags, const char *protocol, diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index e655be60d2..245bbf9dfc 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -346,6 +346,7 @@ qemuSaveImageCreateFd(virQEMUDriver *driver, virDomainObj *vm, const char *path, virFileWrapperFd *wrapperFd, + bool sparse, bool *needUnlink, unsigned int flags) { @@ -355,7 +356,7 @@ qemuSaveImageCreateFd(virQEMUDriver *driver, int directFlag = 0; unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; - if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { + if (!sparse && flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; directFlag = virFileDirectFdFlag(); if (directFlag < 0) { @@ -375,7 +376,7 @@ qemuSaveImageCreateFd(virQEMUDriver *driver, if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, fd) < 0) return -1; - if (!(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) + if (!sparse && !(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) return -1; ret = fd; @@ -394,6 +395,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, const char *path, virQEMUSaveData *data, virCommand *compressor, + qemuMigrationParams *saveParams, unsigned int flags, virDomainAsyncJob asyncJob) { @@ -402,9 +404,10 @@ qemuSaveImageCreate(virQEMUDriver *driver, int ret = -1; int fd = -1; virFileWrapperFd *wrapperFd = NULL; + bool sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; /* Obtain the file handle. */ - fd = qemuSaveImageCreateFd(driver, vm, path, wrapperFd, &needUnlink, flags); + fd = qemuSaveImageCreateFd(driver, vm, path, wrapperFd, sparse, &needUnlink, flags); if (fd < 0) goto cleanup; @@ -413,7 +416,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, goto cleanup; /* Perform the migration */ - if (qemuMigrationSrcToFile(driver, vm, fd, compressor, asyncJob) < 0) + if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, saveParams, flags, asyncJob) < 0) goto cleanup; /* Touch up file header to mark image complete. */ @@ -421,14 +424,18 @@ qemuSaveImageCreate(virQEMUDriver *driver, /* Reopen the file to touch up the header, since we aren't set * up to seek backwards on wrapperFd. The reopened fd will * trigger a single page of file system cache pollution, but - * that's acceptable. */ - if (VIR_CLOSE(fd) < 0) { - virReportSystemError(errno, _("unable to close %1$s"), path); - goto cleanup; - } + * that's acceptable. + * If using mapped-ram, the fd was passed to qemu, so no need + * to close it. */ + if (!sparse) { + if (VIR_CLOSE(fd) < 0) { + virReportSystemError(errno, _("unable to close %1$s"), path); + goto cleanup; + } - if (qemuDomainFileWrapperFDClose(vm, wrapperFd) < 0) - goto cleanup; + if (qemuDomainFileWrapperFDClose(vm, wrapperFd) < 0) + goto cleanup; + } if ((fd = qemuDomainOpenFile(cfg, vm->def, path, O_WRONLY, NULL)) < 0 || virQEMUSaveDataFinish(data, &fd, path) < 0) @@ -485,8 +492,8 @@ qemuSaveImageGetCompressionProgram(const char *imageFormat, if ((ret = qemuSaveFormatTypeFromString(imageFormat)) < 0) goto error; - if (ret == QEMU_SAVE_FORMAT_RAW) - return QEMU_SAVE_FORMAT_RAW; + if (ret == QEMU_SAVE_FORMAT_RAW || ret == QEMU_SAVE_FORMAT_SPARSE) + return ret; if (!(prog = virFindFileInPath(imageFormat))) goto error; diff --git a/src/qemu/qemu_saveimage.h b/src/qemu/qemu_saveimage.h index fc875f9a0c..481e280d75 100644 --- a/src/qemu/qemu_saveimage.h +++ b/src/qemu/qemu_saveimage.h @@ -129,6 +129,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, const char *path, virQEMUSaveData *data, virCommand *compressor, + qemuMigrationParams *saveParams, unsigned int flags, virDomainAsyncJob asyncJob); diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c index 80cd54bf33..07883d67fa 100644 --- a/src/qemu/qemu_snapshot.c +++ b/src/qemu/qemu_snapshot.c @@ -1631,6 +1631,8 @@ qemuSnapshotCreateActiveExternal(virQEMUDriver *driver, /* do the memory snapshot if necessary */ if (memory) { + g_autoptr(qemuMigrationParams) snap_params = NULL; + /* check if migration is possible */ if (!qemuMigrationSrcIsAllowed(vm, false, VIR_ASYNC_JOB_SNAPSHOT, 0)) goto cleanup; @@ -1648,6 +1650,13 @@ qemuSnapshotCreateActiveExternal(virQEMUDriver *driver, "snapshot", false)) < 0) goto cleanup; + if (format == QEMU_SAVE_FORMAT_SPARSE) { + virReportError(VIR_ERR_OPERATION_FAILED, + _("Snapshots do not support image format %1$s"), + qemuSaveFormatTypeToString(format)); + goto cleanup; + } + if (!(xml = qemuDomainDefFormatLive(driver, priv->qemuCaps, vm->def, priv->origCPU, true, true)) || @@ -1662,8 +1671,11 @@ qemuSnapshotCreateActiveExternal(virQEMUDriver *driver, memory_existing = virFileExists(snapdef->memorysnapshotfile); + if (!(snap_params = qemuMigrationParamsNew())) + goto cleanup; + if ((ret = qemuSaveImageCreate(driver, vm, snapdef->memorysnapshotfile, - data, compressor, 0, + data, compressor, snap_params, 0, VIR_ASYNC_JOB_SNAPSHOT)) < 0) goto cleanup; -- 2.43.0
Split the reading of libvirt's save image metadata from the opening of the fd that will be passed to QEMU. This provides flexibility for an upcoming patch adding mapped-ram support for restore. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_driver.c | 37 ++++++++-------- src/qemu/qemu_saveimage.c | 89 ++++++++++++++++++++++++--------------- src/qemu/qemu_saveimage.h | 16 ++++--- src/qemu/qemu_snapshot.c | 9 ++-- 4 files changed, 89 insertions(+), 62 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 716eddba09..5afc2ea846 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -5792,9 +5792,12 @@ qemuDomainRestoreInternal(virConnectPtr conn, if (flags & VIR_DOMAIN_SAVE_RESET_NVRAM) reset_nvram = true; - fd = qemuSaveImageOpen(driver, NULL, path, &def, &data, + if (qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, false) < 0) + goto cleanup; + + fd = qemuSaveImageOpen(driver, path, (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) != 0, - &wrapperFd, false, false); + &wrapperFd, false); if (fd < 0) goto cleanup; @@ -5923,15 +5926,11 @@ qemuDomainSaveImageGetXMLDesc(virConnectPtr conn, const char *path, virQEMUDriver *driver = conn->privateData; char *ret = NULL; g_autoptr(virDomainDef) def = NULL; - int fd = -1; virQEMUSaveData *data = NULL; virCheckFlags(VIR_DOMAIN_SAVE_IMAGE_XML_SECURE, NULL); - fd = qemuSaveImageOpen(driver, NULL, path, &def, &data, - false, NULL, false, false); - - if (fd < 0) + if (qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, false) < 0) goto cleanup; if (virDomainSaveImageGetXMLDescEnsureACL(conn, def) < 0) @@ -5941,7 +5940,6 @@ qemuDomainSaveImageGetXMLDesc(virConnectPtr conn, const char *path, cleanup: virQEMUSaveDataFree(data); - VIR_FORCE_CLOSE(fd); return ret; } @@ -5965,8 +5963,10 @@ qemuDomainSaveImageDefineXML(virConnectPtr conn, const char *path, else if (flags & VIR_DOMAIN_SAVE_PAUSED) state = 0; - fd = qemuSaveImageOpen(driver, NULL, path, &def, &data, - false, NULL, true, false); + if (qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, false) < 0) + goto cleanup; + + fd = qemuSaveImageOpen(driver, path, 0, NULL, false); if (fd < 0) goto cleanup; @@ -6024,7 +6024,6 @@ qemuDomainManagedSaveGetXMLDesc(virDomainPtr dom, unsigned int flags) g_autofree char *path = NULL; char *ret = NULL; g_autoptr(virDomainDef) def = NULL; - int fd = -1; virQEMUSaveData *data = NULL; qemuDomainObjPrivate *priv; @@ -6046,15 +6045,14 @@ qemuDomainManagedSaveGetXMLDesc(virDomainPtr dom, unsigned int flags) goto cleanup; } - if ((fd = qemuSaveImageOpen(driver, priv->qemuCaps, path, &def, &data, - false, NULL, false, false)) < 0) + if (qemuSaveImageGetMetadata(driver, priv->qemuCaps, path, + &def, &data, false) < 0) goto cleanup; ret = qemuDomainDefFormatXML(driver, priv->qemuCaps, def, flags); cleanup: virQEMUSaveDataFree(data); - VIR_FORCE_CLOSE(fd); virDomainObjEndAPI(&vm); return ret; } @@ -6110,14 +6108,17 @@ qemuDomainObjRestore(virConnectPtr conn, virQEMUSaveData *data = NULL; virFileWrapperFd *wrapperFd = NULL; - fd = qemuSaveImageOpen(driver, NULL, path, &def, &data, - bypass_cache, &wrapperFd, false, true); - if (fd < 0) { - if (fd == -3) + ret = qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, true); + if (ret < 0) { + if (ret == -3) ret = 1; goto cleanup; } + fd = qemuSaveImageOpen(driver, path, bypass_cache, &wrapperFd, false); + if (fd < 0) + goto cleanup; + if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) { int hookret; diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 245bbf9dfc..3a71c23c01 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -536,58 +536,35 @@ qemuSaveImageGetCompressionProgram(const char *imageFormat, /** - * qemuSaveImageOpen: + * qemuSaveImageGetMetadata: * @driver: qemu driver data * @qemuCaps: pointer to qemuCaps if the domain is running or NULL * @path: path of the save image * @ret_def: returns domain definition created from the XML stored in the image * @ret_data: returns structure filled with data from the image header - * @bypass_cache: bypass cache when opening the file - * @wrapperFd: returns the file wrapper structure - * @open_write: open the file for writing (for updates) * @unlink_corrupt: remove the image file if it is corrupted * - * Returns the opened fd of the save image file and fills the appropriate fields - * on success. On error returns -1 on most failures, -3 if corrupt image was - * unlinked (no error raised). + * Open the save image file, read libvirt's save image metadata, and populate + * the @ret_def and @ret_data structures. Returns 0 on success and -1 on most + * failures. Returns -3 if corrupt image was unlinked (no error raised). */ int -qemuSaveImageOpen(virQEMUDriver *driver, - virQEMUCaps *qemuCaps, - const char *path, - virDomainDef **ret_def, - virQEMUSaveData **ret_data, - bool bypass_cache, - virFileWrapperFd **wrapperFd, - bool open_write, - bool unlink_corrupt) +qemuSaveImageGetMetadata(virQEMUDriver *driver, + virQEMUCaps *qemuCaps, + const char *path, + virDomainDef **ret_def, + virQEMUSaveData **ret_data, + bool unlink_corrupt) { g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); VIR_AUTOCLOSE fd = -1; - int ret = -1; g_autoptr(virQEMUSaveData) data = NULL; virQEMUSaveHeader *header; g_autoptr(virDomainDef) def = NULL; - int oflags = open_write ? O_RDWR : O_RDONLY; size_t xml_len; size_t cookie_len; - if (bypass_cache) { - int directFlag = virFileDirectFdFlag(); - if (directFlag < 0) { - virReportError(VIR_ERR_OPERATION_FAILED, "%s", - _("bypass cache unsupported by this system")); - return -1; - } - oflags |= directFlag; - } - - if ((fd = qemuDomainOpenFile(cfg, NULL, path, oflags, NULL)) < 0) - return -1; - - if (bypass_cache && - !(*wrapperFd = virFileWrapperFdNew(&fd, path, - VIR_FILE_WRAPPER_BYPASS_CACHE))) + if ((fd = qemuDomainOpenFile(cfg, NULL, path, O_RDONLY, NULL)) < 0) return -1; data = g_new0(virQEMUSaveData, 1); @@ -685,6 +662,50 @@ qemuSaveImageOpen(virQEMUDriver *driver, *ret_def = g_steal_pointer(&def); *ret_data = g_steal_pointer(&data); + return 0; +} + + +/** + * qemuSaveImageOpen: + * @driver: qemu driver data + * @path: path of the save image + * @bypass_cache: bypass cache when opening the file + * @wrapperFd: returns the file wrapper structure + * @open_write: open the file for writing (for updates) + * + * Returns the opened fd of the save image file on success, -1 on failure. + */ +int +qemuSaveImageOpen(virQEMUDriver *driver, + const char *path, + bool bypass_cache, + virFileWrapperFd **wrapperFd, + bool open_write) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + VIR_AUTOCLOSE fd = -1; + int ret = -1; + int oflags = open_write ? O_RDWR : O_RDONLY; + + if (bypass_cache) { + int directFlag = virFileDirectFdFlag(); + if (directFlag < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("bypass cache unsupported by this system")); + return -1; + } + oflags |= directFlag; + } + + if ((fd = qemuDomainOpenFile(cfg, NULL, path, oflags, NULL)) < 0) + return -1; + + if (bypass_cache && + !(*wrapperFd = virFileWrapperFdNew(&fd, path, + VIR_FILE_WRAPPER_BYPASS_CACHE))) + return -1; + ret = fd; fd = -1; diff --git a/src/qemu/qemu_saveimage.h b/src/qemu/qemu_saveimage.h index 481e280d75..7340925274 100644 --- a/src/qemu/qemu_saveimage.h +++ b/src/qemu/qemu_saveimage.h @@ -89,17 +89,21 @@ qemuSaveImageStartVM(virConnectPtr conn, virDomainAsyncJob asyncJob) ATTRIBUTE_NONNULL(4) ATTRIBUTE_NONNULL(5) ATTRIBUTE_NONNULL(6); +int +qemuSaveImageGetMetadata(virQEMUDriver *driver, + virQEMUCaps *qemuCaps, + const char *path, + virDomainDef **ret_def, + virQEMUSaveData **ret_data, + bool unlink_corrupt) + ATTRIBUTE_NONNULL(4) ATTRIBUTE_NONNULL(5); + int qemuSaveImageOpen(virQEMUDriver *driver, - virQEMUCaps *qemuCaps, const char *path, - virDomainDef **ret_def, - virQEMUSaveData **ret_data, bool bypass_cache, virFileWrapperFd **wrapperFd, - bool open_write, - bool unlink_corrupt) - ATTRIBUTE_NONNULL(3) ATTRIBUTE_NONNULL(4); + bool open_write); int qemuSaveImageGetCompressionProgram(const char *imageFormat, diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c index 07883d67fa..5991455a4e 100644 --- a/src/qemu/qemu_snapshot.c +++ b/src/qemu/qemu_snapshot.c @@ -2389,11 +2389,12 @@ qemuSnapshotRevertExternalPrepare(virDomainObj *vm, g_autoptr(virDomainDef) savedef = NULL; memdata->path = snapdef->memorysnapshotfile; - memdata->fd = qemuSaveImageOpen(driver, NULL, memdata->path, - &savedef, &memdata->data, - false, NULL, - false, false); + if (qemuSaveImageGetMetadata(driver, NULL, memdata->path, &savedef, + &memdata->data, false) < 0) + return -1; + memdata->fd = qemuSaveImageOpen(driver, memdata->path, + false, NULL, false); if (memdata->fd < 0) return -1; -- 2.43.0
On 1/22/25 16:16, Jim Fehlig wrote:
Split the reading of libvirt's save image metadata from the opening of the fd that will be passed to QEMU. This provides flexibility for an upcoming patch adding mapped-ram support for restore.
This patch may be useful on it's own. I've included it, along with another small fix, for consideration outside of this series https://lists.libvirt.org/archives/list/devel@lists.libvirt.org/thread/DATZF... Regards, Jim
Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_driver.c | 37 ++++++++-------- src/qemu/qemu_saveimage.c | 89 ++++++++++++++++++++++++--------------- src/qemu/qemu_saveimage.h | 16 ++++--- src/qemu/qemu_snapshot.c | 9 ++-- 4 files changed, 89 insertions(+), 62 deletions(-)
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 716eddba09..5afc2ea846 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -5792,9 +5792,12 @@ qemuDomainRestoreInternal(virConnectPtr conn, if (flags & VIR_DOMAIN_SAVE_RESET_NVRAM) reset_nvram = true;
- fd = qemuSaveImageOpen(driver, NULL, path, &def, &data, + if (qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, false) < 0) + goto cleanup; + + fd = qemuSaveImageOpen(driver, path, (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) != 0, - &wrapperFd, false, false); + &wrapperFd, false); if (fd < 0) goto cleanup;
@@ -5923,15 +5926,11 @@ qemuDomainSaveImageGetXMLDesc(virConnectPtr conn, const char *path, virQEMUDriver *driver = conn->privateData; char *ret = NULL; g_autoptr(virDomainDef) def = NULL; - int fd = -1; virQEMUSaveData *data = NULL;
virCheckFlags(VIR_DOMAIN_SAVE_IMAGE_XML_SECURE, NULL);
- fd = qemuSaveImageOpen(driver, NULL, path, &def, &data, - false, NULL, false, false); - - if (fd < 0) + if (qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, false) < 0) goto cleanup;
if (virDomainSaveImageGetXMLDescEnsureACL(conn, def) < 0) @@ -5941,7 +5940,6 @@ qemuDomainSaveImageGetXMLDesc(virConnectPtr conn, const char *path,
cleanup: virQEMUSaveDataFree(data); - VIR_FORCE_CLOSE(fd); return ret; }
@@ -5965,8 +5963,10 @@ qemuDomainSaveImageDefineXML(virConnectPtr conn, const char *path, else if (flags & VIR_DOMAIN_SAVE_PAUSED) state = 0;
- fd = qemuSaveImageOpen(driver, NULL, path, &def, &data, - false, NULL, true, false); + if (qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, false) < 0) + goto cleanup; + + fd = qemuSaveImageOpen(driver, path, 0, NULL, false);
if (fd < 0) goto cleanup; @@ -6024,7 +6024,6 @@ qemuDomainManagedSaveGetXMLDesc(virDomainPtr dom, unsigned int flags) g_autofree char *path = NULL; char *ret = NULL; g_autoptr(virDomainDef) def = NULL; - int fd = -1; virQEMUSaveData *data = NULL; qemuDomainObjPrivate *priv;
@@ -6046,15 +6045,14 @@ qemuDomainManagedSaveGetXMLDesc(virDomainPtr dom, unsigned int flags) goto cleanup; }
- if ((fd = qemuSaveImageOpen(driver, priv->qemuCaps, path, &def, &data, - false, NULL, false, false)) < 0) + if (qemuSaveImageGetMetadata(driver, priv->qemuCaps, path, + &def, &data, false) < 0) goto cleanup;
ret = qemuDomainDefFormatXML(driver, priv->qemuCaps, def, flags);
cleanup: virQEMUSaveDataFree(data); - VIR_FORCE_CLOSE(fd); virDomainObjEndAPI(&vm); return ret; } @@ -6110,14 +6108,17 @@ qemuDomainObjRestore(virConnectPtr conn, virQEMUSaveData *data = NULL; virFileWrapperFd *wrapperFd = NULL;
- fd = qemuSaveImageOpen(driver, NULL, path, &def, &data, - bypass_cache, &wrapperFd, false, true); - if (fd < 0) { - if (fd == -3) + ret = qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, true); + if (ret < 0) { + if (ret == -3) ret = 1; goto cleanup; }
+ fd = qemuSaveImageOpen(driver, path, bypass_cache, &wrapperFd, false); + if (fd < 0) + goto cleanup; + if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) { int hookret;
diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 245bbf9dfc..3a71c23c01 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -536,58 +536,35 @@ qemuSaveImageGetCompressionProgram(const char *imageFormat,
/** - * qemuSaveImageOpen: + * qemuSaveImageGetMetadata: * @driver: qemu driver data * @qemuCaps: pointer to qemuCaps if the domain is running or NULL * @path: path of the save image * @ret_def: returns domain definition created from the XML stored in the image * @ret_data: returns structure filled with data from the image header - * @bypass_cache: bypass cache when opening the file - * @wrapperFd: returns the file wrapper structure - * @open_write: open the file for writing (for updates) * @unlink_corrupt: remove the image file if it is corrupted * - * Returns the opened fd of the save image file and fills the appropriate fields - * on success. On error returns -1 on most failures, -3 if corrupt image was - * unlinked (no error raised). + * Open the save image file, read libvirt's save image metadata, and populate + * the @ret_def and @ret_data structures. Returns 0 on success and -1 on most + * failures. Returns -3 if corrupt image was unlinked (no error raised). */ int -qemuSaveImageOpen(virQEMUDriver *driver, - virQEMUCaps *qemuCaps, - const char *path, - virDomainDef **ret_def, - virQEMUSaveData **ret_data, - bool bypass_cache, - virFileWrapperFd **wrapperFd, - bool open_write, - bool unlink_corrupt) +qemuSaveImageGetMetadata(virQEMUDriver *driver, + virQEMUCaps *qemuCaps, + const char *path, + virDomainDef **ret_def, + virQEMUSaveData **ret_data, + bool unlink_corrupt) { g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); VIR_AUTOCLOSE fd = -1; - int ret = -1; g_autoptr(virQEMUSaveData) data = NULL; virQEMUSaveHeader *header; g_autoptr(virDomainDef) def = NULL; - int oflags = open_write ? O_RDWR : O_RDONLY; size_t xml_len; size_t cookie_len;
- if (bypass_cache) { - int directFlag = virFileDirectFdFlag(); - if (directFlag < 0) { - virReportError(VIR_ERR_OPERATION_FAILED, "%s", - _("bypass cache unsupported by this system")); - return -1; - } - oflags |= directFlag; - } - - if ((fd = qemuDomainOpenFile(cfg, NULL, path, oflags, NULL)) < 0) - return -1; - - if (bypass_cache && - !(*wrapperFd = virFileWrapperFdNew(&fd, path, - VIR_FILE_WRAPPER_BYPASS_CACHE))) + if ((fd = qemuDomainOpenFile(cfg, NULL, path, O_RDONLY, NULL)) < 0) return -1;
data = g_new0(virQEMUSaveData, 1); @@ -685,6 +662,50 @@ qemuSaveImageOpen(virQEMUDriver *driver, *ret_def = g_steal_pointer(&def); *ret_data = g_steal_pointer(&data);
+ return 0; +} + + +/** + * qemuSaveImageOpen: + * @driver: qemu driver data + * @path: path of the save image + * @bypass_cache: bypass cache when opening the file + * @wrapperFd: returns the file wrapper structure + * @open_write: open the file for writing (for updates) + * + * Returns the opened fd of the save image file on success, -1 on failure. + */ +int +qemuSaveImageOpen(virQEMUDriver *driver, + const char *path, + bool bypass_cache, + virFileWrapperFd **wrapperFd, + bool open_write) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + VIR_AUTOCLOSE fd = -1; + int ret = -1; + int oflags = open_write ? O_RDWR : O_RDONLY; + + if (bypass_cache) { + int directFlag = virFileDirectFdFlag(); + if (directFlag < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("bypass cache unsupported by this system")); + return -1; + } + oflags |= directFlag; + } + + if ((fd = qemuDomainOpenFile(cfg, NULL, path, oflags, NULL)) < 0) + return -1; + + if (bypass_cache && + !(*wrapperFd = virFileWrapperFdNew(&fd, path, + VIR_FILE_WRAPPER_BYPASS_CACHE))) + return -1; + ret = fd; fd = -1;
diff --git a/src/qemu/qemu_saveimage.h b/src/qemu/qemu_saveimage.h index 481e280d75..7340925274 100644 --- a/src/qemu/qemu_saveimage.h +++ b/src/qemu/qemu_saveimage.h @@ -89,17 +89,21 @@ qemuSaveImageStartVM(virConnectPtr conn, virDomainAsyncJob asyncJob) ATTRIBUTE_NONNULL(4) ATTRIBUTE_NONNULL(5) ATTRIBUTE_NONNULL(6);
+int +qemuSaveImageGetMetadata(virQEMUDriver *driver, + virQEMUCaps *qemuCaps, + const char *path, + virDomainDef **ret_def, + virQEMUSaveData **ret_data, + bool unlink_corrupt) + ATTRIBUTE_NONNULL(4) ATTRIBUTE_NONNULL(5); + int qemuSaveImageOpen(virQEMUDriver *driver, - virQEMUCaps *qemuCaps, const char *path, - virDomainDef **ret_def, - virQEMUSaveData **ret_data, bool bypass_cache, virFileWrapperFd **wrapperFd, - bool open_write, - bool unlink_corrupt) - ATTRIBUTE_NONNULL(3) ATTRIBUTE_NONNULL(4); + bool open_write);
int qemuSaveImageGetCompressionProgram(const char *imageFormat, diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c index 07883d67fa..5991455a4e 100644 --- a/src/qemu/qemu_snapshot.c +++ b/src/qemu/qemu_snapshot.c @@ -2389,11 +2389,12 @@ qemuSnapshotRevertExternalPrepare(virDomainObj *vm, g_autoptr(virDomainDef) savedef = NULL;
memdata->path = snapdef->memorysnapshotfile; - memdata->fd = qemuSaveImageOpen(driver, NULL, memdata->path, - &savedef, &memdata->data, - false, NULL, - false, false); + if (qemuSaveImageGetMetadata(driver, NULL, memdata->path, &savedef, + &memdata->data, false) < 0) + return -1;
+ memdata->fd = qemuSaveImageOpen(driver, memdata->path, + false, NULL, false); if (memdata->fd < 0) return -1;
qemuProcessStartWithMemoryState() is the only caller of qemuProcessStart() that uses the qemuProcessIncomingDef struct. Move creation of the struct to qemuProcessStartWithMemoryState(). Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_process.c | 44 ++++++++++++++++++++--------------------- src/qemu/qemu_process.h | 2 +- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 6db48b0e7b..1f00c0016d 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -8295,7 +8295,7 @@ qemuProcessStart(virConnectPtr conn, virDomainObj *vm, virCPUDef *updatedCPU, virDomainAsyncJob asyncJob, - const char *migrateFrom, + qemuProcessIncomingDef *incoming, int migrateFd, const char *migratePath, virDomainMomentObj *snapshot, @@ -8303,7 +8303,6 @@ qemuProcessStart(virConnectPtr conn, unsigned int flags) { qemuDomainObjPrivate *priv = vm->privateData; - qemuProcessIncomingDef *incoming = NULL; unsigned int stopFlags; bool relabel = false; bool relabelSavedState = false; @@ -8311,11 +8310,11 @@ qemuProcessStart(virConnectPtr conn, int rv; VIR_DEBUG("conn=%p driver=%p vm=%p name=%s id=%d asyncJob=%s " - "migrateFrom=%s migrateFd=%d migratePath=%s " + "incoming=%p migrateFd=%d migratePath=%s " "snapshot=%p vmop=%d flags=0x%x", conn, driver, vm, vm->def->name, vm->def->id, virDomainAsyncJobTypeToString(asyncJob), - NULLSTR(migrateFrom), migrateFd, NULLSTR(migratePath), + incoming, migrateFd, NULLSTR(migratePath), snapshot, vmop, flags); virCheckFlagsGoto(VIR_QEMU_PROCESS_START_COLD | @@ -8324,20 +8323,13 @@ qemuProcessStart(virConnectPtr conn, VIR_QEMU_PROCESS_START_GEN_VMID | VIR_QEMU_PROCESS_START_RESET_NVRAM, cleanup); - if (!migrateFrom && !snapshot) + if (!incoming && !snapshot) flags |= VIR_QEMU_PROCESS_START_NEW; if (qemuProcessInit(driver, vm, updatedCPU, - asyncJob, !!migrateFrom, flags) < 0) + asyncJob, !!incoming, flags) < 0) goto cleanup; - if (migrateFrom) { - incoming = qemuProcessIncomingDefNew(priv->qemuCaps, NULL, migrateFrom, - migrateFd, migratePath); - if (!incoming) - goto stop; - } - if (qemuProcessPrepareDomain(driver, vm, flags) < 0) goto stop; @@ -8390,14 +8382,13 @@ qemuProcessStart(virConnectPtr conn, qemuSecurityRestoreSavedStateLabel(driver->securityManager, vm->def, migratePath) < 0) VIR_WARN("failed to restore save state label on %s", migratePath); - qemuProcessIncomingDefFree(incoming); return ret; stop: stopFlags = 0; if (!relabel) stopFlags |= VIR_QEMU_PROCESS_STOP_NO_RELABEL; - if (migrateFrom) + if (incoming) stopFlags |= VIR_QEMU_PROCESS_STOP_MIGRATED; if (priv->mon) qemuMonitorSetDomainLog(priv->mon, NULL, NULL, NULL); @@ -8451,20 +8442,24 @@ qemuProcessStartWithMemoryState(virConnectPtr conn, VIR_AUTOCLOSE intermediatefd = -1; g_autoptr(virCommand) cmd = NULL; g_autofree char *errbuf = NULL; - const char *migrateFrom = NULL; + qemuProcessIncomingDef *incoming = NULL; int rc = 0; + int ret = -1; + + incoming = qemuProcessIncomingDefNew(priv->qemuCaps, NULL, "stdio", + *fd, path); + if (!incoming) + return -1; if (data) { if (virSaveCookieParseString(data->cookie, (virObject **)&cookie, virDomainXMLOptionGetSaveCookie(driver->xmlopt)) < 0) - return -1; + goto cleanup; if (qemuSaveImageDecompressionStart(data, fd, &intermediatefd, &errbuf, &cmd) < 0) { - return -1; + goto cleanup; } - - migrateFrom = "stdio"; } /* No cookie means libvirt which saved the domain was too old to mess up @@ -8477,7 +8472,7 @@ qemuProcessStartWithMemoryState(virConnectPtr conn, priv->disableSlirp = true; if (qemuProcessStart(conn, driver, vm, cookie ? cookie->cpu : NULL, - asyncJob, migrateFrom, *fd, path, snapshot, + asyncJob, incoming, *fd, path, snapshot, VIR_NETDEV_VPORT_PROFILE_OP_RESTORE, start_flags) == 0) *started = true; @@ -8489,14 +8484,17 @@ qemuProcessStartWithMemoryState(virConnectPtr conn, virDomainAuditStart(vm, reason, *started); if (!*started || rc < 0) - return -1; + goto cleanup; /* qemuProcessStart doesn't unset the qemu error reporting infrastructure * in case of migration (which is used in this case) so we need to reset it * so that the handle to virtlogd is not held open unnecessarily */ qemuMonitorSetDomainLog(qemuDomainGetMonitor(vm), NULL, NULL, NULL); + ret = 0; - return 0; + cleanup: + qemuProcessIncomingDefFree(incoming); + return ret; } diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index 12781673c5..1dd6ae23c8 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -83,7 +83,7 @@ int qemuProcessStart(virConnectPtr conn, virDomainObj *vm, virCPUDef *updatedCPU, virDomainAsyncJob asyncJob, - const char *migrateFrom, + qemuProcessIncomingDef *incoming, int stdin_fd, const char *stdin_path, virDomainMomentObj *snapshot, -- 2.43.0
Similar to qemuMigrationSrcRun, apply migration parameters in qemuMigrationDstRun. This allows callers to create customized migration parameters, but delegates their application to the function performing the migration. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_migration.c | 16 ++++++++++------ src/qemu/qemu_migration.h | 5 ++++- src/qemu/qemu_process.c | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 6a378128d1..b2f17f8006 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2376,7 +2376,10 @@ qemuMigrationDstGetURI(const char *migrateFrom, int qemuMigrationDstRun(virDomainObj *vm, const char *uri, - virDomainAsyncJob asyncJob) + virDomainAsyncJob asyncJob, + qemuMigrationParams *migParams, + unsigned int flags) + { virTristateBool exitOnError = VIR_TRISTATE_BOOL_ABSENT; qemuDomainObjPrivate *priv = vm->privateData; @@ -2384,6 +2387,10 @@ qemuMigrationDstRun(virDomainObj *vm, VIR_DEBUG("Setting up incoming migration with URI %s", uri); + if (migParams && qemuMigrationParamsApply(vm, asyncJob, + migParams, flags) < 0) + return -1; + /* Ask QEMU not to exit on failure during incoming migration (if supported) * so that we can properly check and report error during Finish phase. */ @@ -3317,10 +3324,6 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, goto error; } - if (qemuMigrationParamsApply(vm, VIR_ASYNC_JOB_MIGRATION_IN, - migParams, flags) < 0) - goto error; - if (mig->nbd && flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC)) { const char *nbdTLSAlias = NULL; @@ -3352,7 +3355,8 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, } if (qemuMigrationDstRun(vm, incoming->uri, - VIR_ASYNC_JOB_MIGRATION_IN) < 0) + VIR_ASYNC_JOB_MIGRATION_IN, + migParams, flags) < 0) goto error; if (qemuProcessFinishStartup(driver, vm, VIR_ASYNC_JOB_MIGRATION_IN, diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 627d0cb3fb..b1f894e501 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -282,7 +282,10 @@ qemuMigrationDstGetURI(const char *migrateFrom, int qemuMigrationDstRun(virDomainObj *vm, const char *uri, - virDomainAsyncJob asyncJob); + virDomainAsyncJob asyncJob, + qemuMigrationParams *migParams, + unsigned int flags); + void qemuMigrationSrcPostcopyFailed(virDomainObj *vm); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 1f00c0016d..8d51da77c5 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -8352,7 +8352,7 @@ qemuProcessStart(virConnectPtr conn, relabel = true; if (incoming) { - if (qemuMigrationDstRun(vm, incoming->uri, asyncJob) < 0) + if (qemuMigrationDstRun(vm, incoming->uri, asyncJob, NULL, 0) < 0) goto stop; } else { /* Refresh state of devices from QEMU. During migration this happens -- 2.43.0
Add support for the mapped-ram migration capability on restore. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_driver.c | 26 +++++++++++++----- src/qemu/qemu_migration.c | 12 ++++---- src/qemu/qemu_process.c | 58 +++++++++++++++++++++++++++++++-------- src/qemu/qemu_process.h | 15 ++++++---- src/qemu/qemu_saveimage.c | 12 +++++--- src/qemu/qemu_saveimage.h | 2 ++ src/qemu/qemu_snapshot.c | 8 +++--- 7 files changed, 95 insertions(+), 38 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 5afc2ea846..ad58ec92f1 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -1614,7 +1614,7 @@ static virDomainPtr qemuDomainCreateXML(virConnectPtr conn, } if (qemuProcessStart(conn, driver, vm, NULL, VIR_ASYNC_JOB_START, - NULL, -1, NULL, NULL, + NULL, -1, NULL, NULL, NULL, VIR_NETDEV_VPORT_PROFILE_OP_CREATE, start_flags) < 0) { virDomainAuditStart(vm, "booted", false); @@ -5783,6 +5783,8 @@ qemuDomainRestoreInternal(virConnectPtr conn, virFileWrapperFd *wrapperFd = NULL; bool hook_taint = false; bool reset_nvram = false; + bool sparse = false; + g_autoptr(qemuMigrationParams) restoreParams = NULL; virCheckFlags(VIR_DOMAIN_SAVE_BYPASS_CACHE | VIR_DOMAIN_SAVE_RUNNING | @@ -5795,9 +5797,13 @@ qemuDomainRestoreInternal(virConnectPtr conn, if (qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, false) < 0) goto cleanup; + sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; + if (!(restoreParams = qemuMigrationParamsForSave(sparse))) + goto cleanup; + fd = qemuSaveImageOpen(driver, path, (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) != 0, - &wrapperFd, false); + sparse, &wrapperFd, false); if (fd < 0) goto cleanup; @@ -5851,7 +5857,7 @@ qemuDomainRestoreInternal(virConnectPtr conn, if (qemuProcessBeginJob(vm, VIR_DOMAIN_JOB_OPERATION_RESTORE, flags) < 0) goto cleanup; - ret = qemuSaveImageStartVM(conn, driver, vm, &fd, data, path, + ret = qemuSaveImageStartVM(conn, driver, vm, &fd, data, path, restoreParams, false, reset_nvram, VIR_ASYNC_JOB_START); qemuProcessEndJob(vm); @@ -5966,7 +5972,7 @@ qemuDomainSaveImageDefineXML(virConnectPtr conn, const char *path, if (qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, false) < 0) goto cleanup; - fd = qemuSaveImageOpen(driver, path, 0, NULL, false); + fd = qemuSaveImageOpen(driver, path, 0, false, NULL, false); if (fd < 0) goto cleanup; @@ -6107,6 +6113,8 @@ qemuDomainObjRestore(virConnectPtr conn, g_autofree char *xmlout = NULL; virQEMUSaveData *data = NULL; virFileWrapperFd *wrapperFd = NULL; + bool sparse = false; + g_autoptr(qemuMigrationParams) restoreParams = NULL; ret = qemuSaveImageGetMetadata(driver, NULL, path, &def, &data, true); if (ret < 0) { @@ -6115,7 +6123,11 @@ qemuDomainObjRestore(virConnectPtr conn, goto cleanup; } - fd = qemuSaveImageOpen(driver, path, bypass_cache, &wrapperFd, false); + sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; + if (!(restoreParams = qemuMigrationParamsForSave(sparse))) + return -1; + + fd = qemuSaveImageOpen(driver, path, bypass_cache, sparse, &wrapperFd, false); if (fd < 0) goto cleanup; @@ -6157,7 +6169,7 @@ qemuDomainObjRestore(virConnectPtr conn, virDomainObjAssignDef(vm, &def, true, NULL); - ret = qemuSaveImageStartVM(conn, driver, vm, &fd, data, path, + ret = qemuSaveImageStartVM(conn, driver, vm, &fd, data, path, restoreParams, start_paused, reset_nvram, asyncJob); cleanup: @@ -6363,7 +6375,7 @@ qemuDomainObjStart(virConnectPtr conn, } ret = qemuProcessStart(conn, driver, vm, NULL, asyncJob, - NULL, -1, NULL, NULL, + NULL, -1, NULL, NULL, NULL, VIR_NETDEV_VPORT_PROFILE_OP_CREATE, start_flags); virDomainAuditStart(vm, "booted", ret >= 0); if (ret >= 0) { diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index b2f17f8006..8af6d1d3e2 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -3064,9 +3064,8 @@ qemuMigrationDstPrepare(virDomainObj *vm, const char *protocol, const char *listenAddress, unsigned short port, - int fd) + int *fd) { - qemuDomainObjPrivate *priv = vm->privateData; g_autofree char *migrateFrom = NULL; if (tunnel) { @@ -3120,8 +3119,9 @@ qemuMigrationDstPrepare(virDomainObj *vm, migrateFrom = g_strdup_printf(incFormat, protocol, listenAddress, port); } - return qemuProcessIncomingDefNew(priv->qemuCaps, listenAddress, - migrateFrom, fd, NULL); + return qemuProcessIncomingDefNew(vm, listenAddress, + migrateFrom, fd, + NULL, NULL); } @@ -3263,7 +3263,7 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, if (!(incoming = qemuMigrationDstPrepare(vm, tunnel, protocol, listenAddress, port, - dataFD[0]))) + &dataFD[0]))) goto error; qemuMigrationDstPrepareDiskSeclabels(vm, migrate_disks, flags); @@ -3634,7 +3634,7 @@ qemuMigrationDstPrepareResume(virQEMUDriver *driver, priv->origname = g_strdup(origname); if (!(incoming = qemuMigrationDstPrepare(vm, false, protocol, - listenAddress, port, -1))) + listenAddress, port, NULL))) goto cleanup; if (qemuDomainObjEnterMonitorAsync(vm, VIR_ASYNC_JOB_MIGRATION_IN) < 0) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 8d51da77c5..e5ad6f0528 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -4804,6 +4804,7 @@ qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc) g_free(inc->address); g_free(inc->uri); + qemuFDPassFree(inc->fdPassMigrate); g_free(inc); } @@ -4817,26 +4818,54 @@ qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc) * qemuProcessIncomingDefFree will NOT close it. */ qemuProcessIncomingDef * -qemuProcessIncomingDefNew(virQEMUCaps *qemuCaps, +qemuProcessIncomingDefNew(virDomainObj *vm, const char *listenAddress, const char *migrateFrom, - int fd, - const char *path) + int *fd, + const char *path, + virQEMUSaveData *data) { + qemuDomainObjPrivate *priv = vm->privateData; qemuProcessIncomingDef *inc = NULL; - if (qemuMigrationDstCheckProtocol(qemuCaps, migrateFrom) < 0) + if (qemuMigrationDstCheckProtocol(priv->qemuCaps, migrateFrom) < 0) return NULL; inc = g_new0(qemuProcessIncomingDef, 1); inc->address = g_strdup(listenAddress); - inc->uri = qemuMigrationDstGetURI(migrateFrom, fd); + if (data) { + size_t offset = sizeof(virQEMUSaveHeader) + data->header.data_len; + + if (data->header.format == QEMU_SAVE_FORMAT_SPARSE) { + unsigned int fdsetId; + + inc->fdPassMigrate = qemuFDPassNew("migrate", priv); + qemuFDPassAddFD(inc->fdPassMigrate, fd, "-fd"); + qemuFDPassGetId(inc->fdPassMigrate, &fdsetId); + inc->uri = g_strdup_printf("file:/dev/fdset/%u,offset=%#lx", fdsetId, offset); + } else { + g_autofree char *tmp = g_new(char, offset); + + /* HACK: + * We provide qemu the offset in case of mapped-ram, but must set + * the file offset for the legacy save format. Unfortunately we + * can't lseek when the fd is wrapped by virFileWrapperFd, so + * we do a needless read instead. + */ + if (saferead(*fd, tmp, offset) != offset) + VIR_DEBUG("failed to read from save file"); + inc->uri = qemuMigrationDstGetURI(migrateFrom, *fd); + } + } else { + inc->uri = qemuMigrationDstGetURI(migrateFrom, *fd); + } + if (!inc->uri) goto error; - inc->fd = fd; + inc->fd = *fd; inc->path = path; return inc; @@ -7879,8 +7908,11 @@ qemuProcessLaunch(virConnectPtr conn, &nnicindexes, &nicindexes))) goto cleanup; - if (incoming && incoming->fd != -1) - virCommandPassFD(cmd, incoming->fd, 0); + if (incoming) { + if (incoming->fd != -1) + virCommandPassFD(cmd, incoming->fd, 0); + qemuFDPassTransferCommand(incoming->fdPassMigrate, cmd); + } /* now that we know it is about to start call the hook if present */ if (qemuProcessStartHook(driver, vm, @@ -8299,6 +8331,7 @@ qemuProcessStart(virConnectPtr conn, int migrateFd, const char *migratePath, virDomainMomentObj *snapshot, + qemuMigrationParams *migParams, virNetDevVPortProfileOp vmop, unsigned int flags) { @@ -8352,7 +8385,7 @@ qemuProcessStart(virConnectPtr conn, relabel = true; if (incoming) { - if (qemuMigrationDstRun(vm, incoming->uri, asyncJob, NULL, 0) < 0) + if (qemuMigrationDstRun(vm, incoming->uri, asyncJob, migParams, 0) < 0) goto stop; } else { /* Refresh state of devices from QEMU. During migration this happens @@ -8406,6 +8439,7 @@ qemuProcessStart(virConnectPtr conn, * @path: path to memory state file * @snapshot: internal snapshot to load when starting QEMU process or NULL * @data: data from memory state file or NULL + * @migParams: Migration params to use on restore or NULL * @asyncJob: type of asynchronous job * @start_flags: flags to start QEMU process with * @reason: audit log reason @@ -8432,6 +8466,7 @@ qemuProcessStartWithMemoryState(virConnectPtr conn, const char *path, virDomainMomentObj *snapshot, virQEMUSaveData *data, + qemuMigrationParams *migParams, virDomainAsyncJob asyncJob, unsigned int start_flags, const char *reason, @@ -8446,8 +8481,7 @@ qemuProcessStartWithMemoryState(virConnectPtr conn, int rc = 0; int ret = -1; - incoming = qemuProcessIncomingDefNew(priv->qemuCaps, NULL, "stdio", - *fd, path); + incoming = qemuProcessIncomingDefNew(vm, NULL, "stdio", fd, path, data); if (!incoming) return -1; @@ -8473,7 +8507,7 @@ qemuProcessStartWithMemoryState(virConnectPtr conn, if (qemuProcessStart(conn, driver, vm, cookie ? cookie->cpu : NULL, asyncJob, incoming, *fd, path, snapshot, - VIR_NETDEV_VPORT_PROFILE_OP_RESTORE, + migParams, VIR_NETDEV_VPORT_PROFILE_OP_RESTORE, start_flags) == 0) *started = true; diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index 1dd6ae23c8..a6258b075c 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -53,14 +53,17 @@ struct _qemuProcessIncomingDef { char *address; /* address where QEMU is supposed to listen */ char *uri; /* used when calling migrate-incoming QMP command */ int fd; /* for fd:N URI */ + qemuFDPass *fdPassMigrate; /* for file:/dev/fdset/n,offset=x URI */ const char *path; /* path associated with fd */ }; -qemuProcessIncomingDef *qemuProcessIncomingDefNew(virQEMUCaps *qemuCaps, - const char *listenAddress, - const char *migrateFrom, - int fd, - const char *path); +qemuProcessIncomingDef *qemuProcessIncomingDefNew(virDomainObj *vm, + const char *listenAddress, + const char *migrateFrom, + int *fd, + const char *path, + virQEMUSaveData *data); + void qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc); int qemuProcessBeginJob(virDomainObj *vm, @@ -87,6 +90,7 @@ int qemuProcessStart(virConnectPtr conn, int stdin_fd, const char *stdin_path, virDomainMomentObj *snapshot, + qemuMigrationParams *migParams, virNetDevVPortProfileOp vmop, unsigned int flags); @@ -97,6 +101,7 @@ int qemuProcessStartWithMemoryState(virConnectPtr conn, const char *path, virDomainMomentObj *snapshot, virQEMUSaveData *data, + qemuMigrationParams *migParams, virDomainAsyncJob asyncJob, unsigned int start_flags, const char *reason, diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 3a71c23c01..3e89f88301 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -264,7 +264,8 @@ qemuSaveImageDecompressionStart(virQEMUSaveData *data, if (header->version != 2) return 0; - if (header->format == QEMU_SAVE_FORMAT_RAW) + if (header->format == QEMU_SAVE_FORMAT_RAW || + header->format == QEMU_SAVE_FORMAT_SPARSE) return 0; if (!(cmd = qemuSaveImageGetCompressionCommand(header->format))) @@ -671,6 +672,7 @@ qemuSaveImageGetMetadata(virQEMUDriver *driver, * @driver: qemu driver data * @path: path of the save image * @bypass_cache: bypass cache when opening the file + * @sparse: Image contains mapped-ram save format * @wrapperFd: returns the file wrapper structure * @open_write: open the file for writing (for updates) * @@ -680,6 +682,7 @@ int qemuSaveImageOpen(virQEMUDriver *driver, const char *path, bool bypass_cache, + bool sparse, virFileWrapperFd **wrapperFd, bool open_write) { @@ -701,7 +704,7 @@ qemuSaveImageOpen(virQEMUDriver *driver, if ((fd = qemuDomainOpenFile(cfg, NULL, path, oflags, NULL)) < 0) return -1; - if (bypass_cache && + if (!sparse && bypass_cache && !(*wrapperFd = virFileWrapperFdNew(&fd, path, VIR_FILE_WRAPPER_BYPASS_CACHE))) return -1; @@ -720,6 +723,7 @@ qemuSaveImageStartVM(virConnectPtr conn, int *fd, virQEMUSaveData *data, const char *path, + qemuMigrationParams *restoreParams, bool start_paused, bool reset_nvram, virDomainAsyncJob asyncJob) @@ -736,8 +740,8 @@ qemuSaveImageStartVM(virConnectPtr conn, start_flags |= VIR_QEMU_PROCESS_START_RESET_NVRAM; if (qemuProcessStartWithMemoryState(conn, driver, vm, fd, path, NULL, data, - asyncJob, start_flags, "restored", - &started) < 0) { + restoreParams, asyncJob, start_flags, + "restored", &started) < 0) { goto cleanup; } diff --git a/src/qemu/qemu_saveimage.h b/src/qemu/qemu_saveimage.h index 7340925274..320728145a 100644 --- a/src/qemu/qemu_saveimage.h +++ b/src/qemu/qemu_saveimage.h @@ -84,6 +84,7 @@ qemuSaveImageStartVM(virConnectPtr conn, int *fd, virQEMUSaveData *data, const char *path, + qemuMigrationParams *restoreParams, bool start_paused, bool reset_nvram, virDomainAsyncJob asyncJob) @@ -102,6 +103,7 @@ int qemuSaveImageOpen(virQEMUDriver *driver, const char *path, bool bypass_cache, + bool sparse, virFileWrapperFd **wrapperFd, bool open_write); diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c index 5991455a4e..cc304add88 100644 --- a/src/qemu/qemu_snapshot.c +++ b/src/qemu/qemu_snapshot.c @@ -2394,7 +2394,7 @@ qemuSnapshotRevertExternalPrepare(virDomainObj *vm, return -1; memdata->fd = qemuSaveImageOpen(driver, memdata->path, - false, NULL, false); + false, false, NULL, false); if (memdata->fd < 0) return -1; @@ -2634,7 +2634,7 @@ qemuSnapshotRevertActive(virDomainObj *vm, if (qemuProcessStartWithMemoryState(snapshot->domain->conn, driver, vm, &memdata.fd, memdata.path, loadSnap, - memdata.data, VIR_ASYNC_JOB_SNAPSHOT, + memdata.data, NULL, VIR_ASYNC_JOB_SNAPSHOT, start_flags, "from-snapshot", &started) < 0) { if (started) { @@ -2787,7 +2787,7 @@ qemuSnapshotRevertInactive(virDomainObj *vm, rc = qemuProcessStart(snapshot->domain->conn, driver, vm, NULL, VIR_ASYNC_JOB_SNAPSHOT, NULL, -1, NULL, NULL, - VIR_NETDEV_VPORT_PROFILE_OP_CREATE, + NULL, VIR_NETDEV_VPORT_PROFILE_OP_CREATE, start_flags); virDomainAuditStart(vm, "from-snapshot", rc >= 0); if (rc < 0) { @@ -3264,7 +3264,7 @@ qemuSnapshotDeleteExternalPrepare(virDomainObj *vm, if (!virDomainObjIsActive(vm)) { if (qemuProcessStart(NULL, driver, vm, NULL, VIR_ASYNC_JOB_SNAPSHOT, - NULL, -1, NULL, NULL, + NULL, -1, NULL, NULL, NULL, VIR_NETDEV_VPORT_PROFILE_OP_CREATE, VIR_QEMU_PROCESS_START_PAUSED) < 0) { return -1; -- 2.43.0
When using the mapped-ram migration capability, direct IO is enabled by setting the "direct-io" migration parameter to "true" and passing QEMU an additional fd with O_DIRECT set. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_driver.c | 10 ++++++---- src/qemu/qemu_migration.c | 32 ++++++++++++++++++++++++++------ src/qemu/qemu_migration.h | 1 + src/qemu/qemu_migration_params.c | 11 ++++++++++- src/qemu/qemu_migration_params.h | 3 ++- src/qemu/qemu_monitor.c | 7 +++++-- src/qemu/qemu_monitor.h | 3 ++- src/qemu/qemu_saveimage.c | 2 +- 8 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index ad58ec92f1..803c7be3f7 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -2700,7 +2700,8 @@ qemuDomainSaveInternal(virQEMUDriver *driver, goto endjob; xml = NULL; - if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE))) + if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE, + flags))) goto endjob; ret = qemuSaveImageCreate(driver, vm, path, data, compressor, @@ -3152,7 +3153,7 @@ doCoreDump(virQEMUDriver *driver, if (!(dump_params = qemuMigrationParamsNew())) goto cleanup; - if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, + if (qemuMigrationSrcToFile(driver, vm, path, &fd, compressor, dump_params, dump_flags, VIR_ASYNC_JOB_DUMP) < 0) goto cleanup; } @@ -5798,7 +5799,7 @@ qemuDomainRestoreInternal(virConnectPtr conn, goto cleanup; sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; - if (!(restoreParams = qemuMigrationParamsForSave(sparse))) + if (!(restoreParams = qemuMigrationParamsForSave(sparse, flags))) goto cleanup; fd = qemuSaveImageOpen(driver, path, @@ -6124,7 +6125,8 @@ qemuDomainObjRestore(virConnectPtr conn, } sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; - if (!(restoreParams = qemuMigrationParamsForSave(sparse))) + if (!(restoreParams = qemuMigrationParamsForSave(sparse, + bypass_cache ? VIR_DOMAIN_SAVE_BYPASS_CACHE : 0))) return -1; fd = qemuSaveImageOpen(driver, path, bypass_cache, sparse, &wrapperFd, false); diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 8af6d1d3e2..61a14dc9d6 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -7135,17 +7135,36 @@ qemuMigrationSrcToLegacyFile(virQEMUDriver *driver, static int qemuMigrationSrcToMappedFile(virQEMUDriver *driver, virDomainObj *vm, + const char *path, int *fd, unsigned int flags, virDomainAsyncJob asyncJob) { + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + VIR_AUTOCLOSE directFd = -1; + int directFlag = 0; + bool needUnlink = false; int ret; - /* mapped-ram does not support directIO */ + /* When using directio with mapped-ram, qemu needs two fds. One with + * O_DIRECT set writing the memory, and another without it set for + * writing small bits of unaligned state. */ if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) { - virReportError(VIR_ERR_OPERATION_FAILED, "%s", - _("bypass cache unsupported by this system")); - return -1; + directFlag = virFileDirectFdFlag(); + if (directFlag < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("bypass cache unsupported by this system")); + return -1; + } + directFd = virQEMUFileOpenAs(cfg->user, cfg->group, false, path, + O_WRONLY | directFlag, &needUnlink); + + if (directFd < 0) + return -1; + + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, directFd) < 0) + return -1; + } if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, *fd) < 0) @@ -7154,7 +7173,7 @@ qemuMigrationSrcToMappedFile(virQEMUDriver *driver, if (qemuDomainObjEnterMonitorAsync(vm, asyncJob) < 0) return -1; - ret = qemuMonitorMigrateToFdSet(vm, 0, fd); + ret = qemuMonitorMigrateToFdSet(vm, 0, fd, &directFd); qemuDomainObjExitMonitor(vm); return ret; } @@ -7163,6 +7182,7 @@ qemuMigrationSrcToMappedFile(virQEMUDriver *driver, /* Helper function called while vm is active. */ int qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, + const char *path, int *fd, virCommand *compressor, qemuMigrationParams *migParams, @@ -7200,7 +7220,7 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, if (migParams && qemuMigrationParamsCapEnabled(migParams, QEMU_MIGRATION_CAP_MAPPED_RAM)) { - rc = qemuMigrationSrcToMappedFile(driver, vm, fd, flags, asyncJob); + rc = qemuMigrationSrcToMappedFile(driver, vm, path, fd, flags, asyncJob); } else { rc = qemuMigrationSrcToLegacyFile(driver, vm, *fd, compressor, asyncJob); } diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index b1f894e501..13ce9d33f2 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -237,6 +237,7 @@ qemuMigrationSrcIsAllowed(virDomainObj *vm, int qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, + const char *path, int *fd, virCommand *compressor, qemuMigrationParams *migParams, diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 0faeb6b0c1..79b47333f9 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -129,6 +129,7 @@ VIR_ENUM_IMPL(qemuMigrationParam, "multifd-compression", "multifd-zlib-level", "multifd-zstd-level", + "direct-io", ); typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem; @@ -319,6 +320,9 @@ static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = { [QEMU_MIGRATION_PARAM_MULTIFD_ZSTD_LEVEL] = { .type = QEMU_MIGRATION_PARAM_TYPE_INT, }, + [QEMU_MIGRATION_PARAM_DIRECT_IO] = { + .type = QEMU_MIGRATION_PARAM_TYPE_BOOL, + }, }; G_STATIC_ASSERT(G_N_ELEMENTS(qemuMigrationParamInfo) == QEMU_MIGRATION_PARAM_LAST); @@ -784,7 +788,7 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, qemuMigrationParams * -qemuMigrationParamsForSave(bool sparse) +qemuMigrationParamsForSave(bool sparse, unsigned int flags) { g_autoptr(qemuMigrationParams) saveParams = NULL; @@ -798,6 +802,11 @@ qemuMigrationParamsForSave(bool sparse) return NULL; saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i = 1; saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set = true; + + if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { + saveParams->params[QEMU_MIGRATION_PARAM_DIRECT_IO].value.b = true; + saveParams->params[QEMU_MIGRATION_PARAM_DIRECT_IO].set = true; + } } return g_steal_pointer(&saveParams); diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index bb24af4ef9..b326aa5bc1 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -65,6 +65,7 @@ typedef enum { QEMU_MIGRATION_PARAM_MULTIFD_COMPRESSION, QEMU_MIGRATION_PARAM_MULTIFD_ZLIB_LEVEL, QEMU_MIGRATION_PARAM_MULTIFD_ZSTD_LEVEL, + QEMU_MIGRATION_PARAM_DIRECT_IO, QEMU_MIGRATION_PARAM_LAST } qemuMigrationParam; @@ -87,7 +88,7 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, qemuMigrationParty party); qemuMigrationParams * -qemuMigrationParamsForSave(bool sparse); +qemuMigrationParamsForSave(bool sparse, unsigned int flags); int qemuMigrationParamsDump(qemuMigrationParams *migParams, diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index d88e1b1ecd..385d103ad3 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -2233,7 +2233,8 @@ qemuMonitorMigrateToFd(qemuMonitor *mon, int qemuMonitorMigrateToFdSet(virDomainObj *vm, unsigned int flags, - int *fd) + int *fd, + int *directFd) { qemuDomainObjPrivate *priv = vm->privateData; qemuMonitor *mon = priv->mon; @@ -2243,7 +2244,7 @@ qemuMonitorMigrateToFdSet(virDomainObj *vm, g_autofree char *uri = NULL; int ret; - VIR_DEBUG("fd=%d flags=0x%x", *fd, flags); + VIR_DEBUG("fd=%d directFd=%d flags=0x%x", *fd, *directFd, flags); QEMU_CHECK_MONITOR(mon); @@ -2255,6 +2256,8 @@ qemuMonitorMigrateToFdSet(virDomainObj *vm, fdPassMigrate = qemuFDPassNew("migrate", priv); qemuFDPassAddFD(fdPassMigrate, fd, "-fd"); + if (*directFd != -1) + qemuFDPassAddFD(fdPassMigrate, directFd, "-directio-fd"); qemuFDPassTransferMonitor(fdPassMigrate, mon); if (qemuFDPassGetId(fdPassMigrate, &setId) < 0) diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 7f54203fe3..79dbe5fcd7 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -857,7 +857,8 @@ int qemuMonitorMigrateToFd(qemuMonitor *mon, int qemuMonitorMigrateToFdSet(virDomainObj *vm, unsigned int flags, - int *fd); + int *fd, + int *directFd); int qemuMonitorMigrateToHost(qemuMonitor *mon, unsigned int flags, diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 3e89f88301..d417762749 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -417,7 +417,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, goto cleanup; /* Perform the migration */ - if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, saveParams, flags, asyncJob) < 0) + if (qemuMigrationSrcToFile(driver, vm, path, &fd, compressor, saveParams, flags, asyncJob) < 0) goto cleanup; /* Touch up file header to mark image complete. */ -- 2.43.0
When using the mapped-ram migration capability, direct IO is enabled by setting the "direct-io" migration parameter to "true" and passing QEMU an additional fd with O_DIRECT set. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_migration.c | 11 ++++++----- src/qemu/qemu_process.c | 27 +++++++++++++++++++++++---- src/qemu/qemu_process.h | 6 ++++-- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 61a14dc9d6..84d7710462 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -3059,7 +3059,8 @@ qemuMigrationDstPrepareCleanup(virQEMUDriver *driver, } static qemuProcessIncomingDef * -qemuMigrationDstPrepare(virDomainObj *vm, +qemuMigrationDstPrepare(virQEMUDriver *driver, + virDomainObj *vm, bool tunnel, const char *protocol, const char *listenAddress, @@ -3119,9 +3120,9 @@ qemuMigrationDstPrepare(virDomainObj *vm, migrateFrom = g_strdup_printf(incFormat, protocol, listenAddress, port); } - return qemuProcessIncomingDefNew(vm, listenAddress, + return qemuProcessIncomingDefNew(driver, vm, listenAddress, migrateFrom, fd, - NULL, NULL); + NULL, NULL, NULL); } @@ -3261,7 +3262,7 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, goto error; stopProcess = true; - if (!(incoming = qemuMigrationDstPrepare(vm, tunnel, protocol, + if (!(incoming = qemuMigrationDstPrepare(driver, vm, tunnel, protocol, listenAddress, port, &dataFD[0]))) goto error; @@ -3633,7 +3634,7 @@ qemuMigrationDstPrepareResume(virQEMUDriver *driver, priv->origname = g_strdup(origname); - if (!(incoming = qemuMigrationDstPrepare(vm, false, protocol, + if (!(incoming = qemuMigrationDstPrepare(driver, vm, false, protocol, listenAddress, port, NULL))) goto cleanup; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index e5ad6f0528..ad926650f7 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -4818,13 +4818,16 @@ qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc) * qemuProcessIncomingDefFree will NOT close it. */ qemuProcessIncomingDef * -qemuProcessIncomingDefNew(virDomainObj *vm, +qemuProcessIncomingDefNew(virQEMUDriver *driver, + virDomainObj *vm, const char *listenAddress, const char *migrateFrom, int *fd, const char *path, - virQEMUSaveData *data) + virQEMUSaveData *data, + qemuMigrationParams *migParams) { + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); qemuDomainObjPrivate *priv = vm->privateData; qemuProcessIncomingDef *inc = NULL; @@ -4839,10 +4842,26 @@ qemuProcessIncomingDefNew(virDomainObj *vm, size_t offset = sizeof(virQEMUSaveHeader) + data->header.data_len; if (data->header.format == QEMU_SAVE_FORMAT_SPARSE) { + bool directio = false; unsigned int fdsetId; inc->fdPassMigrate = qemuFDPassNew("migrate", priv); - qemuFDPassAddFD(inc->fdPassMigrate, fd, "-fd"); + /* When using directio with mapped-ram, qemu needs an fd without + * O_DIRECT set for reading small bits of unaligned state. */ + if (qemuMigrationParamsGetBool(migParams, QEMU_MIGRATION_PARAM_DIRECT_IO, &directio) < 0) + goto error; + + if (directio) { + VIR_AUTOCLOSE bufferedFd = -1; + + if ((bufferedFd = qemuDomainOpenFile(cfg, NULL, path, O_RDONLY, NULL)) < 0) + goto error; + + qemuFDPassAddFD(inc->fdPassMigrate, &bufferedFd, "-buffered-fd"); + qemuFDPassAddFD(inc->fdPassMigrate, fd, "direct-io-fd"); + } else { + qemuFDPassAddFD(inc->fdPassMigrate, fd, "-buffered-fd"); + } qemuFDPassGetId(inc->fdPassMigrate, &fdsetId); inc->uri = g_strdup_printf("file:/dev/fdset/%u,offset=%#lx", fdsetId, offset); } else { @@ -8481,7 +8500,7 @@ qemuProcessStartWithMemoryState(virConnectPtr conn, int rc = 0; int ret = -1; - incoming = qemuProcessIncomingDefNew(vm, NULL, "stdio", fd, path, data); + incoming = qemuProcessIncomingDefNew(driver, vm, NULL, "stdio", fd, path, data, migParams); if (!incoming) return -1; diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index a6258b075c..3c70835ac1 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -57,12 +57,14 @@ struct _qemuProcessIncomingDef { const char *path; /* path associated with fd */ }; -qemuProcessIncomingDef *qemuProcessIncomingDefNew(virDomainObj *vm, +qemuProcessIncomingDef *qemuProcessIncomingDefNew(virQEMUDriver *driver, + virDomainObj *vm, const char *listenAddress, const char *migrateFrom, int *fd, const char *path, - virQEMUSaveData *data); + virQEMUSaveData *data, + qemuMigrationParams *migParams); void qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc); -- 2.43.0
From: Claudio Fontana <cfontana@suse.de> Add a new VIR_DOMAIN_SAVE_PARALLEL flag to the save and restore APIs, which can be used to specify the use of multiple, parallel channels for saving a domain. The number of parallel channels can be set using the VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS typed parameter. Signed-off-by: Claudio Fontana <cfontana@suse.de> Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- include/libvirt/libvirt-domain.h | 11 +++++++++++ src/libvirt-domain.c | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 2a4b81f4df..00dd13509b 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -1637,6 +1637,7 @@ typedef enum { VIR_DOMAIN_SAVE_RUNNING = 1 << 1, /* Favor running over paused (Since: 0.9.5) */ VIR_DOMAIN_SAVE_PAUSED = 1 << 2, /* Favor paused over running (Since: 0.9.5) */ VIR_DOMAIN_SAVE_RESET_NVRAM = 1 << 3, /* Re-initialize NVRAM from template (Since: 8.1.0) */ + VIR_DOMAIN_SAVE_PARALLEL = 1 << 4, /* Save and restore using parallel channels (Since: 10.6.0) */ } virDomainSaveRestoreFlags; int virDomainSave (virDomainPtr domain, @@ -1682,6 +1683,16 @@ int virDomainRestoreParams (virConnectPtr conn, */ # define VIR_DOMAIN_SAVE_PARAM_DXML "dxml" +/** + * VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS: + * + * an optional parameter used to specify the number of IO channels to use + * during parallel save. As VIR_TYPED_PARAM_INT. + * + * Since: 11.1.0 + */ +# define VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS "parallel.channels" + /* See below for virDomainSaveImageXMLFlags */ char * virDomainSaveImageGetXMLDesc (virConnectPtr conn, const char *file, diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 3fb091f240..7b95748b37 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -1010,6 +1010,9 @@ virDomainSaveFlags(virDomainPtr domain, const char *to, * If VIR_DOMAIN_SAVE_PARAM_FILE is not provided then a managed save is * performed (see virDomainManagedSave). * + * See VIR_DOMAIN_SAVE_PARAM_* for detailed description of accepted save + * parameters. + * * Returns 0 in case of success and -1 in case of failure. * * Since: 8.4.0 @@ -1222,6 +1225,9 @@ virDomainRestoreFlags(virConnectPtr conn, const char *from, const char *dxml, * now, VIR_DOMAIN_SAVE_PARAM_FILE is required but this requirement may * be lifted in the future. * + * See VIR_DOMAIN_SAVE_PARAM_* for detailed description of accepted + * restore parameters. + * * Returns 0 in case of success and -1 in case of failure. * * Since: 8.4.0 -- 2.43.0
Add support for parallel save and restore by mapping libvirt's "parallel-connections" parameter to QEMU's "multifd-channels" migration parameter. Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- src/qemu/qemu_driver.c | 32 +++++++++++++++++++++----------- src/qemu/qemu_migration_params.c | 31 +++++++++++++++++++++++++++++-- src/qemu/qemu_migration_params.h | 5 ++++- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 803c7be3f7..4d8f5f08c3 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -2611,6 +2611,8 @@ qemuDomainSaveInternal(virQEMUDriver *driver, int format, virCommand *compressor, const char *xmlin, + virTypedParameterPtr params, + int nparams, unsigned int flags) { g_autofree char *xml = NULL; @@ -2700,7 +2702,8 @@ qemuDomainSaveInternal(virQEMUDriver *driver, goto endjob; xml = NULL; - if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE, + if (!(saveParams = qemuMigrationParamsForSave(params, nparams, + format == QEMU_SAVE_FORMAT_SPARSE, flags))) goto endjob; @@ -2787,7 +2790,7 @@ qemuDomainManagedSaveHelper(virQEMUDriver *driver, VIR_INFO("Saving state of domain '%s' to '%s'", vm->def->name, path); if (qemuDomainSaveInternal(driver, vm, path, format, - compressor, dxml, flags) < 0) + compressor, dxml, NULL, 0, flags) < 0) return -1; vm->hasManagedSave = true; @@ -2826,7 +2829,7 @@ qemuDomainSaveFlags(virDomainPtr dom, const char *path, const char *dxml, goto cleanup; ret = qemuDomainSaveInternal(driver, vm, path, format, - compressor, dxml, flags); + compressor, dxml, NULL, 0, flags); cleanup: virDomainObjEndAPI(&vm); @@ -2856,13 +2859,16 @@ qemuDomainSaveParams(virDomainPtr dom, virCheckFlags(VIR_DOMAIN_SAVE_BYPASS_CACHE | VIR_DOMAIN_SAVE_RUNNING | - VIR_DOMAIN_SAVE_PAUSED, -1); + VIR_DOMAIN_SAVE_PAUSED | + VIR_DOMAIN_SAVE_PARALLEL, -1); if (virTypedParamsValidate(params, nparams, VIR_DOMAIN_SAVE_PARAM_FILE, VIR_TYPED_PARAM_STRING, VIR_DOMAIN_SAVE_PARAM_DXML, VIR_TYPED_PARAM_STRING, + VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS, + VIR_TYPED_PARAM_INT, NULL) < 0) return -1; @@ -2894,7 +2900,7 @@ qemuDomainSaveParams(virDomainPtr dom, goto cleanup; ret = qemuDomainSaveInternal(driver, vm, to, format, - compressor, dxml, flags); + compressor, dxml, params, nparams, flags); cleanup: virDomainObjEndAPI(&vm); @@ -5769,6 +5775,8 @@ static int qemuDomainRestoreInternal(virConnectPtr conn, const char *path, const char *dxml, + virTypedParameterPtr params, + int nparams, unsigned int flags, int (*ensureACL)(virConnectPtr, virDomainDef *)) { @@ -5790,7 +5798,8 @@ qemuDomainRestoreInternal(virConnectPtr conn, virCheckFlags(VIR_DOMAIN_SAVE_BYPASS_CACHE | VIR_DOMAIN_SAVE_RUNNING | VIR_DOMAIN_SAVE_PAUSED | - VIR_DOMAIN_SAVE_RESET_NVRAM, -1); + VIR_DOMAIN_SAVE_RESET_NVRAM | + VIR_DOMAIN_SAVE_PARALLEL, -1); if (flags & VIR_DOMAIN_SAVE_RESET_NVRAM) reset_nvram = true; @@ -5799,7 +5808,7 @@ qemuDomainRestoreInternal(virConnectPtr conn, goto cleanup; sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; - if (!(restoreParams = qemuMigrationParamsForSave(sparse, flags))) + if (!(restoreParams = qemuMigrationParamsForSave(params, nparams, sparse, flags))) goto cleanup; fd = qemuSaveImageOpen(driver, path, @@ -5881,7 +5890,7 @@ qemuDomainRestoreFlags(virConnectPtr conn, const char *dxml, unsigned int flags) { - return qemuDomainRestoreInternal(conn, path, dxml, flags, + return qemuDomainRestoreInternal(conn, path, dxml, NULL, 0, flags, virDomainRestoreFlagsEnsureACL); } @@ -5889,7 +5898,7 @@ static int qemuDomainRestore(virConnectPtr conn, const char *path) { - return qemuDomainRestoreInternal(conn, path, NULL, 0, + return qemuDomainRestoreInternal(conn, path, NULL, NULL, 0, 0, virDomainRestoreEnsureACL); } @@ -5905,6 +5914,7 @@ qemuDomainRestoreParams(virConnectPtr conn, if (virTypedParamsValidate(params, nparams, VIR_DOMAIN_SAVE_PARAM_FILE, VIR_TYPED_PARAM_STRING, VIR_DOMAIN_SAVE_PARAM_DXML, VIR_TYPED_PARAM_STRING, + VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS, VIR_TYPED_PARAM_INT, NULL) < 0) return -1; @@ -5921,7 +5931,7 @@ qemuDomainRestoreParams(virConnectPtr conn, return -1; } - ret = qemuDomainRestoreInternal(conn, path, dxml, flags, + ret = qemuDomainRestoreInternal(conn, path, dxml, params, nparams, flags, virDomainRestoreParamsEnsureACL); return ret; } @@ -6125,7 +6135,7 @@ qemuDomainObjRestore(virConnectPtr conn, } sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; - if (!(restoreParams = qemuMigrationParamsForSave(sparse, + if (!(restoreParams = qemuMigrationParamsForSave(NULL, 0, sparse, bypass_cache ? VIR_DOMAIN_SAVE_BYPASS_CACHE : 0))) return -1; diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 79b47333f9..420bf2fdbe 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -788,10 +788,19 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, qemuMigrationParams * -qemuMigrationParamsForSave(bool sparse, unsigned int flags) +qemuMigrationParamsForSave(virTypedParameterPtr params, + int nparams, + bool sparse, + unsigned int flags) { g_autoptr(qemuMigrationParams) saveParams = NULL; + if (flags & VIR_DOMAIN_SAVE_PARALLEL && !sparse) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("Parallel save is only supported with the 'sparse' save image format")); + return NULL; + } + if (!(saveParams = qemuMigrationParamsNew())) return NULL; @@ -800,7 +809,25 @@ qemuMigrationParamsForSave(bool sparse, unsigned int flags) return NULL; if (virBitmapSetBit(saveParams->caps, QEMU_MIGRATION_CAP_MULTIFD) < 0) return NULL; - saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i = 1; + + if (flags & VIR_DOMAIN_SAVE_PARALLEL) { + int nchannels; + + if (params && virTypedParamsGetInt(params, nparams, + VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS, + &nchannels) < 0) + return NULL; + + if (nchannels < 1) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("number of parallel save channels cannot be less than 1")); + return NULL; + } + + saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i = nchannels; + } else { + saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i = 1; + } saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set = true; if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index b326aa5bc1..b6998d67a2 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -88,7 +88,10 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, qemuMigrationParty party); qemuMigrationParams * -qemuMigrationParamsForSave(bool sparse, unsigned int flags); +qemuMigrationParamsForSave(virTypedParameterPtr params, + int nparams, + bool sparse, + unsigned int flags); int qemuMigrationParamsDump(qemuMigrationParams *migParams, -- 2.43.0
From: Claudio Fontana <cfontana@suse.de> Signed-off-by: Claudio Fontana <cfontana@suse.de> Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- docs/manpages/virsh.rst | 12 +++++++++++- tools/virsh-domain.c | 41 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index e801037c04..669f27d2a6 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -3984,6 +3984,7 @@ save :: save domain state-file [--bypass-cache] [--xml file] + [--parallel] [--parallel-channels channels] [{--running | --paused}] [--verbose] Saves a running domain (RAM, but not disk state) to a state file so that @@ -3991,8 +3992,11 @@ it can be restored later. Once saved, the domain will no longer be running on the system, thus the memory allocated for the domain will be free for other domains to use. ``virsh restore`` restores from this state file. + If *--bypass-cache* is specified, the save will avoid the file system -cache, although this may slow down the operation. +cache. Depending on the specific scenario this may slow down or speed up +the operation. + The progress may be monitored using ``domjobinfo`` virsh command and canceled with ``domjobabort`` command (sent by another virsh instance). Another option @@ -4014,6 +4018,12 @@ based on the state the domain was in when the save was done; passing either the *--running* or *--paused* flag will allow overriding which state the ``restore`` should use. +*--parallel* option will cause the save data to be written to file +over multiple parallel IO channels. The number of channels can be +specified using *--parallel-channels*. Using parallel IO channels +requires the use of ``sparse`` image save format. Parallel save may +significantly reduce the time required to save large memory domains. + Domain saved state files assume that disk images will be unchanged between the creation and restore point. For a more complete system restore point, where the disk state is saved alongside the memory diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 546db955a9..99acb45079 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -4145,6 +4145,14 @@ static const vshCmdOptDef opts_save[] = { .type = VSH_OT_BOOL, .help = N_("avoid file system cache when saving") }, + {.name = "parallel", + .type = VSH_OT_BOOL, + .help = N_("enable parallel save") + }, + {.name = "parallel-channels", + .type = VSH_OT_INT, + .help = N_("number of extra IO channels to use for parallel save") + }, {.name = "xml", .type = VSH_OT_STRING, .unwanted_positional = true, @@ -4175,6 +4183,11 @@ doSave(void *opaque) g_autoptr(virshDomain) dom = NULL; const char *name = NULL; const char *to = NULL; + virTypedParameterPtr params = NULL; + int nparams = 0; + int maxparams = 0; + int nchannels = 0; + int rv = -1; unsigned int flags = 0; const char *xmlfile = NULL; g_autofree char *xml = NULL; @@ -4188,15 +4201,29 @@ doSave(void *opaque) goto out_sig; #endif /* !WIN32 */ - if (vshCommandOptString(ctl, cmd, "file", &to) < 0) - goto out; - if (vshCommandOptBool(cmd, "bypass-cache")) flags |= VIR_DOMAIN_SAVE_BYPASS_CACHE; if (vshCommandOptBool(cmd, "running")) flags |= VIR_DOMAIN_SAVE_RUNNING; if (vshCommandOptBool(cmd, "paused")) flags |= VIR_DOMAIN_SAVE_PAUSED; + if (vshCommandOptBool(cmd, "parallel")) + flags |= VIR_DOMAIN_SAVE_PARALLEL; + + if (vshCommandOptString(ctl, cmd, "file", &to) < 0) + goto out; + if (to && + virTypedParamsAddString(¶ms, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_FILE, to) < 0) + goto out; + + if ((rv = vshCommandOptInt(ctl, cmd, "parallel-channels", &nchannels)) < 0) { + goto out; + } else if (rv > 0) { + if (virTypedParamsAddInt(¶ms, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS, nchannels) < 0) + goto out; + } if (vshCommandOptString(ctl, cmd, "xml", &xmlfile) < 0) goto out; @@ -4209,9 +4236,13 @@ doSave(void *opaque) vshReportError(ctl); goto out; } + if (xml && + virTypedParamsAddString(¶ms, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_DXML, xml) < 0) + goto out; if (flags || xml) { - rc = virDomainSaveFlags(dom, to, xml, flags); + rc = virDomainSaveParams(dom, params, nparams, flags); } else { rc = virDomainSave(dom, to); } @@ -4224,6 +4255,8 @@ doSave(void *opaque) data->ret = 0; out: + virTypedParamsFree(params, nparams); + #ifndef WIN32 pthread_sigmask(SIG_SETMASK, &oldsigmask, NULL); out_sig: -- 2.43.0
From: Claudio Fontana <cfontana@suse.de> Signed-off-by: Claudio Fontana <cfontana@suse.de> Signed-off-by: Jim Fehlig <jfehlig@suse.com> --- docs/manpages/virsh.rst | 9 +++++++-- tools/virsh-domain.c | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index 669f27d2a6..619ba02a31 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -3935,12 +3935,13 @@ restore :: restore state-file [--bypass-cache] [--xml file] - [{--running | --paused}] [--reset-nvram] + [{--running | --paused}] [--reset-nvram] [--parallel] [--parallel-channels] Restores a domain from a ``virsh save`` state file. See *save* for more info. If *--bypass-cache* is specified, the restore will avoid the file system -cache, although this may slow down the operation. +cache. Depending on the specific scenario this may slow down or speed up +the operation. *--xml* ``file`` is usually omitted, but can be used to supply an alternative XML file for use on the restored guest with changes only @@ -3956,6 +3957,10 @@ domain should be started in. If *--reset-nvram* is specified, any existing NVRAM file will be deleted and re-initialized from its pristine template. +*--parallel* option will cause the save data to be loaded using the number +of parallel IO channels specified with *--parallel-channels*. Parallel +channels will help speed up large restore operations. + ``Note``: To avoid corrupting file system contents within the domain, you should not reuse the saved state file for a second ``restore`` unless you have also reverted all storage volumes back to the same contents as when diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 99acb45079..1b22698afb 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -5280,6 +5280,14 @@ static const vshCmdOptDef opts_restore[] = { .type = VSH_OT_BOOL, .help = N_("avoid file system cache when restoring") }, + {.name = "parallel", + .type = VSH_OT_BOOL, + .help = N_("enable parallel restore") + }, + {.name = "parallel-channels", + .type = VSH_OT_INT, + .help = N_("number of IO channels to use for parallel restore") + }, {.name = "xml", .type = VSH_OT_STRING, .unwanted_positional = true, @@ -5309,13 +5317,16 @@ cmdRestore(vshControl *ctl, const vshCmd *cmd) const char *xmlfile = NULL; g_autofree char *xml = NULL; virshControl *priv = ctl->privData; + virTypedParameterPtr params = NULL; + int nparams = 0; + int maxparams = 0; + int nchannels = 0; int rc; - if (vshCommandOptString(ctl, cmd, "file", &from) < 0) - return false; - if (vshCommandOptBool(cmd, "bypass-cache")) flags |= VIR_DOMAIN_SAVE_BYPASS_CACHE; + if (vshCommandOptBool(cmd, "parallel")) + flags |= VIR_DOMAIN_SAVE_PARALLEL; if (vshCommandOptBool(cmd, "running")) flags |= VIR_DOMAIN_SAVE_RUNNING; if (vshCommandOptBool(cmd, "paused")) @@ -5323,15 +5334,34 @@ cmdRestore(vshControl *ctl, const vshCmd *cmd) if (vshCommandOptBool(cmd, "reset-nvram")) flags |= VIR_DOMAIN_SAVE_RESET_NVRAM; + if (vshCommandOptString(ctl, cmd, "file", &from) < 0) + return false; + if (from && + virTypedParamsAddString(¶ms, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_FILE, from) < 0) + return false; + if (vshCommandOptString(ctl, cmd, "xml", &xmlfile) < 0) return false; if (xmlfile && virFileReadAll(xmlfile, VSH_MAX_XML_FILE, &xml) < 0) return false; + if (xml && + virTypedParamsAddString(¶ms, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_DXML, xml) < 0) + return false; + + if ((rc = vshCommandOptInt(ctl, cmd, "parallel-channels", &nchannels)) < 0) { + return false; + } else if (rc > 0) { + if (virTypedParamsAddInt(¶ms, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS, nchannels) < 0) + return false; + } if (flags || xml) { - rc = virDomainRestoreFlags(priv->conn, from, xml, flags); + rc = virDomainRestoreParams(priv->conn, params, nparams, flags); } else { rc = virDomainRestore(priv->conn, from); } -- 2.43.0
participants (1)
-
Jim Fehlig