Add new function qemuFDPassNewFromMonitor to get an fdset previously passed to qemu, based on the 'prefix' provided when the qemuFDPass object was initially created. Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; --- src/qemu/qemu_fd.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_fd.h | 4 ++++ 2 files changed, 50 insertions(+) diff --git a/src/qemu/qemu_fd.c b/src/qemu/qemu_fd.c index e847056573..333f9b128e 100644 --- a/src/qemu/qemu_fd.c +++ b/src/qemu/qemu_fd.c @@ -115,6 +115,52 @@ qemuFDPassNewPassed(unsigned int fdSetID) } +/** + * qemuFDPassNewFromMonitor: + * @prefix: Prefix of an FDset which was already passed to qemu + * @mon: monitor object + * + * Query the monitor for already passed FDsets and return a new qemuFDPass + * object if one is found to contain an fd with associated @prefix. Returns + * NULL on failure or if no matching FDset is found. Similar to + * qemuFDPassNewPassed, this is useful along with qemuFDPassTransferMonitorRollback + * when only knowing the qemuFDPass prefix. + */ +qemuFDPass * +qemuFDPassNewFromMonitor(const char *prefix, qemuMonitor *mon) +{ + g_autoptr(qemuMonitorFdsets) fdsets = NULL; + qemuFDPass *fdpass = NULL; + size_t i; + + VIR_DEBUG("prefix = %s", prefix); + + if (qemuMonitorQueryFdsets(mon, &fdsets) < 0) + return NULL; + + for (i = 0; i < fdsets->nfdsets; i++) { + qemuMonitorFdsetInfo fdset = fdsets->fdsets[i]; + size_t j; + + for (j = 0; j < fdset.nfds; j++) { + qemuMonitorFdsetFdInfo fdinfo = fdset.fds[j]; + + VIR_DEBUG("fdinfo opaque = %s", fdinfo.opaque); + if (STRPREFIX(prefix, fdinfo.opaque)) { + fdpass = g_new0(qemuFDPass, 1); + + fdpass->fdSetID = fdset.id; + fdpass->prefix = g_strdup(prefix); + fdpass->path = g_strdup_printf("/dev/fdset/%u", fdset.id); + fdpass->passed = true; + } + } + } + + return fdpass; +} + + /** * qemuFDPassIsPassed: * @fdpass: The fd passing helper struct diff --git a/src/qemu/qemu_fd.h b/src/qemu/qemu_fd.h index cd0ff2c690..d873e110a8 100644 --- a/src/qemu/qemu_fd.h +++ b/src/qemu/qemu_fd.h @@ -34,6 +34,10 @@ qemuFDPassNew(const char *prefix, qemuFDPass * qemuFDPassNewPassed(unsigned int fdSetID); +qemuFDPass * +qemuFDPassNewFromMonitor(const char *prefix, qemuMonitor *mon) + ATTRIBUTE_NONNULL(1); + bool qemuFDPassIsPassed(qemuFDPass *fdpass, unsigned *id); -- 2.43.0

Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; Reviewed-by: Daniel P. Berrangé <berrange(a)redhat.com&gt; --- src/qemu/qemu_migration_params.c | 18 ++++++++++++++++++ src/qemu/qemu_migration_params.h | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 252437e5b2..939d9f01e4 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -1312,6 +1312,24 @@ qemuMigrationParamsGetULL(qemuMigrationParams *migParams, } +int +qemuMigrationParamsGetBool(qemuMigrationParams *migParams, + qemuMigrationParam param, + bool *value) +{ + if (!migParams || !value) + return 0; + + if (qemuMigrationParamsCheckType(param, QEMU_MIGRATION_PARAM_TYPE_BOOL) < 0) + return -1; + + *value = migParams->params[param].set ? + migParams->params[param].value.b : false; + + return 0; +} + + /** * Returns true if @cap is enabled in @migParams, false otherwise. */ diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index da62ca5b8f..fb2cfd9fad 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -137,6 +137,11 @@ qemuMigrationParamsGetULL(qemuMigrationParams *migParams, qemuMigrationParam param, unsigned long long *value); +int +qemuMigrationParamsGetBool(qemuMigrationParams *migParams, + qemuMigrationParam param, + bool *value); + bool qemuMigrationParamsCapEnabled(qemuMigrationParams *migParams, qemuMigrationCapability cap); -- 2.43.0

Introduce qemuMigrationParamsForSave() to create a qemuMigrationParams object initialized with appropriate migration capabilities and parameters for a save operation. Note that mapped-ram capability also requires the multifd capability. For now, the number of multifd channels is set to 1. Future work to support parallel save/restore can set the number of channels to a user-specified value. Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; Reviewed-by: Daniel P. Berrangé <berrange(a)redhat.com&gt; --- src/qemu/qemu_migration.c | 2 +- src/qemu/qemu_migration_params.c | 21 +++++++++++++++++++++ src/qemu/qemu_migration_params.h | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 53bbbee629..4efc03fc00 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -7090,7 +7090,7 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, /* Increase migration bandwidth to unlimited since target is a file. * Failure to change migration speed is not fatal. */ - if (!(migParams = qemuMigrationParamsNew())) + if (!(migParams = qemuMigrationParamsForSave(false))) return -1; if (qemuMigrationParamsSetULL(migParams, diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index 4fc9199f1c..d48cdd5506 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -792,6 +792,27 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, } +qemuMigrationParams * +qemuMigrationParamsForSave(bool sparse) +{ + g_autoptr(qemuMigrationParams) saveParams = NULL; + + if (!(saveParams = qemuMigrationParamsNew())) + return NULL; + + if (sparse) { + if (virBitmapSetBit(saveParams->caps, QEMU_MIGRATION_CAP_MAPPED_RAM) < 0) + return NULL; + if (virBitmapSetBit(saveParams->caps, QEMU_MIGRATION_CAP_MULTIFD) < 0) + return NULL; + saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i = 1; + saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set = true; + } + + return g_steal_pointer(&saveParams); +} + + int qemuMigrationParamsDump(qemuMigrationParams *migParams, virTypedParameterPtr *params, diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 2379950ce9..88a1bc1a66 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -87,6 +87,9 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, unsigned int flags, qemuMigrationParty party); +qemuMigrationParams * +qemuMigrationParamsForSave(bool sparse); + int qemuMigrationParamsDump(qemuMigrationParams *migParams, virTypedParameterPtr *params, -- 2.43.0

Move the code in qemuSaveImageCreate that opens, labels, and wraps the save image fd to a helper function, providing more flexibility for upcoming mapped-ram support. Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; --- src/qemu/qemu_saveimage.c | 65 +++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 6888528729..4304530610 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -445,6 +445,50 @@ qemuSaveImageDecompressionStop(virCommand *cmd, } +static int +qemuSaveImageCreateFd(virQEMUDriver *driver, + virDomainObj *vm, + const char *path, + virFileWrapperFd *wrapperFd, + bool *needUnlink, + unsigned int flags) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + int ret = -1; + VIR_AUTOCLOSE fd = -1; + int directFlag = 0; + unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; + + if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { + wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; + directFlag = virFileDirectFdFlag(); + if (directFlag < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("bypass cache unsupported by this system")); + return -1; + } + } + + fd = virQEMUFileOpenAs(cfg->user, cfg->group, false, path, + O_WRONLY | O_TRUNC | O_CREAT | directFlag, + needUnlink); + + if (fd < 0) + return -1; + + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, fd) < 0) + return -1; + + if (!(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) + return -1; + + ret = fd; + fd = -1; + + return ret; +} + + /* Helper function to execute a migration to file with a correct save header * the caller needs to make sure that the processors are stopped and do all other * actions besides saving memory */ @@ -461,33 +505,14 @@ qemuSaveImageCreate(virQEMUDriver *driver, bool needUnlink = false; int ret = -1; int fd = -1; - int directFlag = 0; virFileWrapperFd *wrapperFd = NULL; - unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; /* Obtain the file handle. */ - if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) { - wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; - directFlag = virFileDirectFdFlag(); - if (directFlag < 0) { - virReportError(VIR_ERR_OPERATION_FAILED, "%s", - _("bypass cache unsupported by this system")); - goto cleanup; - } - } + fd = qemuSaveImageCreateFd(driver, vm, path, wrapperFd, &needUnlink, flags); - fd = virQEMUFileOpenAs(cfg->user, cfg->group, false, path, - O_WRONLY | O_TRUNC | O_CREAT | directFlag, - &needUnlink); if (fd < 0) goto cleanup; - if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, fd) < 0) - goto cleanup; - - if (!(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) - goto cleanup; - if (virQEMUSaveDataWrite(data, fd, path) < 0) goto cleanup; -- 2.43.0

Introduce support for QEMU's new mapped-ram stream format [1]. mapped-ram can be enabled by setting the 'save_image_format' setting in qemu.conf to 'sparse'. To use mapped-ram with QEMU: - The 'mapped-ram' migration capability must be set to true - The 'multifd' migration capability must be set to true and the 'multifd-channels' migration parameter must set to 1 - QEMU must be provided an fdset containing the migration fd - The 'migrate' qmp command is invoked with a URI referencing the fdset and an offset where to start reading or writing the data stream, e.g. {"execute":"migrate", "arguments":{"detach":true,"resume":false, "uri":"file:/dev/fdset/0,offset=0x11921"}} The mapped-ram stream, in conjunction with direct IO and multifd support provided by subsequent patches, can significantly improve the time required to save VM memory state. The following tables compare mapped-ram with the existing, sequential save stream. In all cases, the save and restore operations are to/from a block device comprised of two NVMe disks in RAID0 configuration with xfs (~8600MiB/s). The values in the 'save time' and 'restore time' columns were scraped from the 'real' time reported by time(1). The 'Size' and 'Blocks' columns were provided by the corresponding outputs of stat(1). VM: 32G RAM, 1 vcpu, idle (shortly after boot) | save | restore | | time | time | Size | Blocks -----------------------+---------+---------+--------------+-------- legacy | 6.193s | 4.399s | 985744812 | 1925288 -----------------------+---------+---------+--------------+-------- mapped-ram | 5.109s | 1.176s | 34368554354 | 1774472 -----------------------+---------+---------+--------------+-------- legacy + direct IO | 5.725s | 4.512s | 985765251 | 1925328 -----------------------+---------+---------+--------------+-------- mapped-ram + direct IO | 4.627s | 1.490s | 34368554354 | 1774304 -----------------------+---------+---------+--------------+-------- mapped-ram + direct IO | | | | + multifd-channels=8 | 4.421s | 0.845s | 34368554318 | 1774312 ------------------------------------------------------------------- VM: 32G RAM, 30G dirty, 1 vcpu in tight loop dirtying memory | save | restore | | time | time | Size | Blocks -----------------------+---------+---------+--------------+--------- legacy | 25.800s | 14.332s | 33154309983 | 64754512 -----------------------+---------+---------+--------------+--------- mapped-ram | 18.742s | 15.027s | 34368559228 | 64617160 -----------------------+---------+---------+--------------+--------- legacy + direct IO | 13.115s | 18.050s | 33154310496 | 64754520 -----------------------+---------+---------+--------------+--------- mapped-ram + direct IO | 13.623s | 15.959s | 34368557392 | 64662040 -----------------------+-------- +---------+--------------+--------- mapped-ram + direct IO | | | | + multifd-channels=8 | 6.994s | 6.470s | 34368554980 | 64665776 -------------------------------------------------------------------- As can be seen from the tables, one caveat of mapped-ram is the logical file size of a saved image is basically equivalent to the VM memory size. Note however that mapped-ram typically uses fewer blocks on disk, hence the name 'sparse' for 'save_image_format'. Also note the mapped-ram stream is incompatible with the existing stream format, hence mapped-ram cannot be used to restore an image saved with the existing format and vice versa. [1] https://gitlab.com/qemu-project/qemu/-/blob/master/docs/devel/migration/m... Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; --- src/qemu/qemu_driver.c | 23 +++++- src/qemu/qemu_migration.c | 149 ++++++++++++++++++++++++++------------ src/qemu/qemu_migration.h | 4 +- src/qemu/qemu_monitor.c | 34 +++++++++ src/qemu/qemu_monitor.h | 4 + src/qemu/qemu_saveimage.c | 33 +++++---- src/qemu/qemu_saveimage.h | 1 + src/qemu/qemu_snapshot.c | 14 +++- 8 files changed, 199 insertions(+), 63 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 78bfaa5b3a..f77516a4f4 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -2620,6 +2620,7 @@ qemuDomainSaveInternal(virQEMUDriver *driver, qemuDomainObjPrivate *priv = vm->privateData; virQEMUSaveData *data = NULL; g_autoptr(qemuDomainSaveCookie) cookie = NULL; + g_autoptr(qemuMigrationParams) saveParams = NULL; if (virDomainObjBeginAsyncJob(vm, VIR_ASYNC_JOB_SAVE, VIR_DOMAIN_JOB_OPERATION_SAVE, flags) < 0) @@ -2628,6 +2629,14 @@ qemuDomainSaveInternal(virQEMUDriver *driver, if (!qemuMigrationSrcIsAllowed(vm, false, VIR_ASYNC_JOB_SAVE, 0)) goto endjob; + if (format == QEMU_SAVE_FORMAT_SPARSE && + !qemuMigrationCapsGet(vm, QEMU_MIGRATION_CAP_MAPPED_RAM)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, + _("save image format %1$s is not supported by this QEMU binary"), + qemuSaveFormatTypeToString(format)); + goto endjob; + } + if (!virDomainObjIsActive(vm)) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("guest unexpectedly quit")); @@ -2691,8 +2700,11 @@ qemuDomainSaveInternal(virQEMUDriver *driver, goto endjob; xml = NULL; + if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE))) + goto endjob; + ret = qemuSaveImageCreate(driver, vm, path, data, compressor, - flags, VIR_ASYNC_JOB_SAVE); + saveParams, flags, VIR_ASYNC_JOB_SAVE); if (ret < 0) goto endjob; @@ -3126,6 +3138,8 @@ doCoreDump(virQEMUDriver *driver, memory_dump_format) < 0) goto cleanup; } else { + g_autoptr(qemuMigrationParams) dump_params = NULL; + if (dumpformat != VIR_DOMAIN_CORE_DUMP_FORMAT_RAW) { virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", _("kdump-compressed format is only supported with memory-only dump")); @@ -3135,8 +3149,11 @@ doCoreDump(virQEMUDriver *driver, if (!qemuMigrationSrcIsAllowed(vm, false, VIR_ASYNC_JOB_DUMP, 0)) goto cleanup; - if (qemuMigrationSrcToFile(driver, vm, fd, compressor, - VIR_ASYNC_JOB_DUMP) < 0) + if (!(dump_params = qemuMigrationParamsNew())) + goto cleanup; + + if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, + dump_params, dump_flags, VIR_ASYNC_JOB_DUMP) < 0) goto cleanup; } diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 4efc03fc00..1e3bee3781 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -7069,46 +7069,17 @@ qemuMigrationProcessUnattended(virQEMUDriver *driver, } -/* Helper function called while vm is active. */ -int -qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, - int fd, - virCommand *compressor, - virDomainAsyncJob asyncJob) +static int +qemuMigrationSrcToLegacyFile(virQEMUDriver *driver, + virDomainObj *vm, + int fd, + virCommand *compressor, + virDomainAsyncJob asyncJob) { qemuDomainObjPrivate *priv = vm->privateData; - int rc; int ret = -1; int pipeFD[2] = { -1, -1 }; - unsigned long saveMigBandwidth = priv->migMaxBandwidth; char *errbuf = NULL; - virErrorPtr orig_err = NULL; - g_autoptr(qemuMigrationParams) migParams = NULL; - - if (qemuMigrationSetDBusVMState(driver, vm) < 0) - return -1; - - /* Increase migration bandwidth to unlimited since target is a file. - * Failure to change migration speed is not fatal. */ - if (!(migParams = qemuMigrationParamsForSave(false))) - return -1; - - if (qemuMigrationParamsSetULL(migParams, - QEMU_MIGRATION_PARAM_MAX_BANDWIDTH, - QEMU_DOMAIN_MIG_BANDWIDTH_MAX * 1024 * 1024) < 0) - return -1; - - if (qemuMigrationParamsApply(vm, asyncJob, migParams, 0) < 0) - return -1; - - priv->migMaxBandwidth = QEMU_DOMAIN_MIG_BANDWIDTH_MAX; - - if (!virDomainObjIsActive(vm)) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("guest unexpectedly quit")); - /* nothing to tear down */ - return -1; - } if (compressor && virPipe(pipeFD) < 0) return -1; @@ -7125,7 +7096,7 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, goto cleanup; if (!compressor) { - rc = qemuMonitorMigrateToFd(priv->mon, 0, fd); + ret = qemuMonitorMigrateToFd(priv->mon, 0, fd); } else { virCommandSetInputFD(compressor, pipeFD[0]); virCommandSetOutputFD(compressor, &fd); @@ -7141,12 +7112,98 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, qemuDomainObjExitMonitor(vm); goto cleanup; } - rc = qemuMonitorMigrateToFd(priv->mon, 0, pipeFD[1]); + ret = qemuMonitorMigrateToFd(priv->mon, 0, pipeFD[1]); if (VIR_CLOSE(pipeFD[0]) < 0 || VIR_CLOSE(pipeFD[1]) < 0) VIR_WARN("failed to close intermediate pipe"); } qemuDomainObjExitMonitor(vm); + + cleanup: + VIR_FORCE_CLOSE(pipeFD[0]); + VIR_FORCE_CLOSE(pipeFD[1]); + + if (errbuf) { + VIR_DEBUG("Compression binary stderr: %s", NULLSTR(errbuf)); + VIR_FREE(errbuf); + } + + return ret; +} + + +static int +qemuMigrationSrcToSparseFile(virQEMUDriver *driver, + virDomainObj *vm, + int *fd, + unsigned int flags, + virDomainAsyncJob asyncJob) +{ + int ret; + + /* mapped-ram does not support directIO */ + if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("bypass cache unsupported by this system")); + return -1; + } + + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, *fd) < 0) + return -1; + + if (qemuDomainObjEnterMonitorAsync(vm, asyncJob) < 0) + return -1; + + ret = qemuMonitorMigrateToFdSet(vm, 0, fd); + qemuDomainObjExitMonitor(vm); + return ret; +} + + +/* Helper function called while vm is active. */ +int +qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, + int *fd, + virCommand *compressor, + qemuMigrationParams *migParams, + unsigned int flags, + virDomainAsyncJob asyncJob) +{ + qemuDomainObjPrivate *priv = vm->privateData; + int rc; + int ret = -1; + unsigned long saveMigBandwidth = priv->migMaxBandwidth; + virErrorPtr orig_err = NULL; + + if (qemuMigrationSetDBusVMState(driver, vm) < 0) + return -1; + + /* Increase migration bandwidth to unlimited since target is a file. + * Failure to change migration speed is not fatal. */ + if (migParams && + qemuMigrationParamsSetULL(migParams, + QEMU_MIGRATION_PARAM_MAX_BANDWIDTH, + QEMU_DOMAIN_MIG_BANDWIDTH_MAX * 1024 * 1024) < 0) + return -1; + + if (qemuMigrationParamsApply(vm, asyncJob, migParams, 0) < 0) + return -1; + + priv->migMaxBandwidth = QEMU_DOMAIN_MIG_BANDWIDTH_MAX; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("guest unexpectedly quit")); + /* nothing to tear down */ + return -1; + } + + if (migParams && + qemuMigrationParamsCapEnabled(migParams, QEMU_MIGRATION_CAP_MAPPED_RAM)) + rc = qemuMigrationSrcToSparseFile(driver, vm, fd, flags, asyncJob); + else + rc = qemuMigrationSrcToLegacyFile(driver, vm, *fd, compressor, asyncJob); + if (rc < 0) goto cleanup; @@ -7172,8 +7229,17 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, if (ret < 0 && !orig_err) virErrorPreserveLast(&orig_err); - /* Restore max migration bandwidth */ + /* Remove fdset passed to qemu and restore max migration bandwidth */ if (qemuDomainObjIsActive(vm)) { + if (qemuDomainObjEnterMonitorAsync(vm, asyncJob) == 0) { + qemuFDPass *fdPass = + qemuFDPassNewFromMonitor("libvirt-outgoing-migrate", priv->mon); + + if (fdPass) + qemuFDPassTransferMonitorRollback(fdPass, priv->mon); + qemuDomainObjExitMonitor(vm); + } + if (qemuMigrationParamsSetULL(migParams, QEMU_MIGRATION_PARAM_MAX_BANDWIDTH, saveMigBandwidth * 1024 * 1024) == 0) @@ -7182,13 +7248,6 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, priv->migMaxBandwidth = saveMigBandwidth; } - VIR_FORCE_CLOSE(pipeFD[0]); - VIR_FORCE_CLOSE(pipeFD[1]); - if (errbuf) { - VIR_DEBUG("Compression binary stderr: %s", NULLSTR(errbuf)); - VIR_FREE(errbuf); - } - virErrorRestore(&orig_err); return ret; diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index efe1b9e88a..9fa007b949 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -238,8 +238,10 @@ qemuMigrationSrcIsAllowed(virDomainObj *vm, int qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, - int fd, + int *fd, virCommand *compressor, + qemuMigrationParams *migParams, + unsigned int flags, virDomainAsyncJob asyncJob) ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT; diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index 830ecbad1c..e2043c0120 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -2232,6 +2232,40 @@ qemuMonitorMigrateToFd(qemuMonitor *mon, } +int +qemuMonitorMigrateToFdSet(virDomainObj *vm, + unsigned int flags, + int *fd) +{ + qemuDomainObjPrivate *priv = vm->privateData; + qemuMonitor *mon = priv->mon; + off_t offset; + g_autoptr(qemuFDPass) fdPassMigrate = NULL; + g_autofree char *uri = NULL; + int ret; + + VIR_DEBUG("fd=%d flags=0x%x", *fd, flags); + + QEMU_CHECK_MONITOR(mon); + + if ((offset = lseek(*fd, 0, SEEK_CUR)) == -1) { + virReportSystemError(errno, + "%s", _("failed to seek on file descriptor")); + return -1; + } + + fdPassMigrate = qemuFDPassNew("libvirt-outgoing-migrate", priv); + qemuFDPassAddFD(fdPassMigrate, fd, "-fd"); + qemuFDPassTransferMonitor(fdPassMigrate, mon); + + uri = g_strdup_printf("file:%s,offset=%#lx", + qemuFDPassGetPath(fdPassMigrate), offset); + ret = qemuMonitorJSONMigrate(mon, flags, uri); + + return ret; +} + + int qemuMonitorMigrateToHost(qemuMonitor *mon, unsigned int flags, diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 072f452e79..6da380aa65 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -859,6 +859,10 @@ int qemuMonitorMigrateToFd(qemuMonitor *mon, unsigned int flags, int fd); +int qemuMonitorMigrateToFdSet(virDomainObj *vm, + unsigned int flags, + int *fd); + int qemuMonitorMigrateToHost(qemuMonitor *mon, unsigned int flags, const char *protocol, diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 59237cc46a..0ffbe03f24 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -430,6 +430,7 @@ qemuSaveImageCreateFd(virQEMUDriver *driver, virDomainObj *vm, const char *path, virFileWrapperFd *wrapperFd, + bool sparse, bool *needUnlink, unsigned int flags) { @@ -439,7 +440,7 @@ qemuSaveImageCreateFd(virQEMUDriver *driver, int directFlag = 0; unsigned int wrapperFlags = VIR_FILE_WRAPPER_NON_BLOCKING; - if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { + if (!sparse && flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { wrapperFlags |= VIR_FILE_WRAPPER_BYPASS_CACHE; directFlag = virFileDirectFdFlag(); if (directFlag < 0) { @@ -459,7 +460,7 @@ qemuSaveImageCreateFd(virQEMUDriver *driver, if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, fd) < 0) return -1; - if (!(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) + if (!sparse && !(wrapperFd = virFileWrapperFdNew(&fd, path, wrapperFlags))) return -1; ret = fd; @@ -478,6 +479,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, const char *path, virQEMUSaveData *data, virCommand *compressor, + qemuMigrationParams *saveParams, unsigned int flags, virDomainAsyncJob asyncJob) { @@ -486,9 +488,10 @@ qemuSaveImageCreate(virQEMUDriver *driver, int ret = -1; int fd = -1; virFileWrapperFd *wrapperFd = NULL; + bool sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; /* Obtain the file handle. */ - fd = qemuSaveImageCreateFd(driver, vm, path, wrapperFd, &needUnlink, flags); + fd = qemuSaveImageCreateFd(driver, vm, path, wrapperFd, sparse, &needUnlink, flags); if (fd < 0) goto cleanup; @@ -497,7 +500,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, goto cleanup; /* Perform the migration */ - if (qemuMigrationSrcToFile(driver, vm, fd, compressor, asyncJob) < 0) + if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, saveParams, flags, asyncJob) < 0) goto cleanup; /* Touch up file header to mark image complete. */ @@ -505,14 +508,18 @@ qemuSaveImageCreate(virQEMUDriver *driver, /* Reopen the file to touch up the header, since we aren't set * up to seek backwards on wrapperFd. The reopened fd will * trigger a single page of file system cache pollution, but - * that's acceptable. */ - if (VIR_CLOSE(fd) < 0) { - virReportSystemError(errno, _("unable to close %1$s"), path); - goto cleanup; - } + * that's acceptable. + * If using mapped-ram, the fd was passed to qemu, so no need + * to close it. */ + if (!sparse) { + if (VIR_CLOSE(fd) < 0) { + virReportSystemError(errno, _("unable to close %1$s"), path); + goto cleanup; + } - if (qemuDomainFileWrapperFDClose(vm, wrapperFd) < 0) - goto cleanup; + if (qemuDomainFileWrapperFDClose(vm, wrapperFd) < 0) + goto cleanup; + } if ((fd = qemuDomainOpenFile(cfg, vm->def, path, O_WRONLY, NULL)) < 0 || virQEMUSaveDataFinish(data, &fd, path) < 0) @@ -569,8 +576,8 @@ qemuSaveImageGetCompressionProgram(const char *imageFormat, if ((ret = qemuSaveFormatTypeFromString(imageFormat)) < 0) goto error; - if (ret == QEMU_SAVE_FORMAT_RAW) - return QEMU_SAVE_FORMAT_RAW; + if (ret == QEMU_SAVE_FORMAT_RAW || ret == QEMU_SAVE_FORMAT_SPARSE) + return ret; if (!(prog = virFindFileInPath(imageFormat))) goto error; diff --git a/src/qemu/qemu_saveimage.h b/src/qemu/qemu_saveimage.h index a5b55c6b10..2b3d839e5b 100644 --- a/src/qemu/qemu_saveimage.h +++ b/src/qemu/qemu_saveimage.h @@ -138,6 +138,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, const char *path, virQEMUSaveData *data, virCommand *compressor, + qemuMigrationParams *saveParams, unsigned int flags, virDomainAsyncJob asyncJob); diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c index ed140dd41c..3088b28716 100644 --- a/src/qemu/qemu_snapshot.c +++ b/src/qemu/qemu_snapshot.c @@ -1644,6 +1644,8 @@ qemuSnapshotCreateActiveExternal(virQEMUDriver *driver, /* do the memory snapshot if necessary */ if (memory) { + g_autoptr(qemuMigrationParams) snap_params = NULL; + /* check if migration is possible */ if (!qemuMigrationSrcIsAllowed(vm, false, VIR_ASYNC_JOB_SNAPSHOT, 0)) goto cleanup; @@ -1661,6 +1663,13 @@ qemuSnapshotCreateActiveExternal(virQEMUDriver *driver, "snapshot", false)) < 0) goto cleanup; + if (format == QEMU_SAVE_FORMAT_SPARSE) { + virReportError(VIR_ERR_OPERATION_FAILED, + _("Snapshots do not support image format %1$s"), + qemuSaveFormatTypeToString(format)); + goto cleanup; + } + if (!(xml = qemuDomainDefFormatLive(driver, priv->qemuCaps, vm->def, priv->origCPU, true, true)) || @@ -1675,8 +1684,11 @@ qemuSnapshotCreateActiveExternal(virQEMUDriver *driver, memory_existing = virFileExists(snapdef->memorysnapshotfile); + if (!(snap_params = qemuMigrationParamsNew())) + goto cleanup; + if ((ret = qemuSaveImageCreate(driver, vm, snapdef->memorysnapshotfile, - data, compressor, 0, + data, compressor, snap_params, 0, VIR_ASYNC_JOB_SNAPSHOT)) < 0) goto cleanup; -- 2.43.0

Similar to qemuMigrationSrcRun, apply migration parameters in qemuMigrationDstRun. This allows callers to create customized migration parameters, but delegates their application to the function performing the migration. Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; --- src/qemu/qemu_migration.c | 16 ++++++++++------ src/qemu/qemu_migration.h | 5 ++++- src/qemu/qemu_process.c | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 1e3bee3781..bd46143717 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2380,7 +2380,10 @@ qemuMigrationDstGetURI(const char *migrateFrom, int qemuMigrationDstRun(virDomainObj *vm, const char *uri, - virDomainAsyncJob asyncJob) + virDomainAsyncJob asyncJob, + qemuMigrationParams *migParams, + unsigned int flags) + { virTristateBool exitOnError = VIR_TRISTATE_BOOL_ABSENT; qemuDomainObjPrivate *priv = vm->privateData; @@ -2388,6 +2391,10 @@ qemuMigrationDstRun(virDomainObj *vm, VIR_DEBUG("Setting up incoming migration with URI %s", uri); + if (migParams && qemuMigrationParamsApply(vm, asyncJob, + migParams, flags) < 0) + return -1; + /* Ask QEMU not to exit on failure during incoming migration (if supported) * so that we can properly check and report error during Finish phase. */ @@ -3321,10 +3328,6 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, goto error; } - if (qemuMigrationParamsApply(vm, VIR_ASYNC_JOB_MIGRATION_IN, - migParams, flags) < 0) - goto error; - if (mig->nbd && flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC)) { const char *nbdTLSAlias = NULL; @@ -3356,7 +3359,8 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver, } if (qemuMigrationDstRun(vm, incoming->uri, - VIR_ASYNC_JOB_MIGRATION_IN) < 0) + VIR_ASYNC_JOB_MIGRATION_IN, + migParams, flags) < 0) goto error; if (qemuProcessFinishStartup(driver, vm, VIR_ASYNC_JOB_MIGRATION_IN, diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 9fa007b949..71a9974753 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -283,7 +283,10 @@ qemuMigrationDstGetURI(const char *migrateFrom, int qemuMigrationDstRun(virDomainObj *vm, const char *uri, - virDomainAsyncJob asyncJob); + virDomainAsyncJob asyncJob, + qemuMigrationParams *migParams, + unsigned int flags); + void qemuMigrationSrcPostcopyFailed(virDomainObj *vm); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 524b23fea8..063033cb95 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -8372,7 +8372,7 @@ qemuProcessStart(virConnectPtr conn, relabel = true; if (incoming) { - if (qemuMigrationDstRun(vm, incoming->uri, asyncJob) < 0) + if (qemuMigrationDstRun(vm, incoming->uri, asyncJob, NULL, 0) < 0) goto stop; } else { /* Refresh state of devices from QEMU. During migration this happens -- 2.43.0

When using the mapped-ram migration capability, direct IO is enabled by setting the "direct-io" migration parameter to "true" and passing QEMU an additional fd with O_DIRECT set. Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; --- src/qemu/qemu_driver.c | 10 ++++++---- src/qemu/qemu_migration.c | 32 ++++++++++++++++++++++++++------ src/qemu/qemu_migration.h | 1 + src/qemu/qemu_migration_params.c | 11 ++++++++++- src/qemu/qemu_migration_params.h | 3 ++- src/qemu/qemu_monitor.c | 7 +++++-- src/qemu/qemu_monitor.h | 3 ++- src/qemu/qemu_saveimage.c | 2 +- 8 files changed, 53 insertions(+), 16 deletions(-) diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 0f363849ba..ad15689eb8 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -2700,7 +2700,8 @@ qemuDomainSaveInternal(virQEMUDriver *driver, goto endjob; xml = NULL; - if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE))) + if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE, + flags))) goto endjob; ret = qemuSaveImageCreate(driver, vm, path, data, compressor, @@ -3152,7 +3153,7 @@ doCoreDump(virQEMUDriver *driver, if (!(dump_params = qemuMigrationParamsNew())) goto cleanup; - if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, + if (qemuMigrationSrcToFile(driver, vm, path, &fd, compressor, dump_params, dump_flags, VIR_ASYNC_JOB_DUMP) < 0) goto cleanup; } @@ -5798,7 +5799,7 @@ qemuDomainRestoreInternal(virConnectPtr conn, goto cleanup; sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; - if (!(restoreParams = qemuMigrationParamsForSave(sparse))) + if (!(restoreParams = qemuMigrationParamsForSave(sparse, flags))) goto cleanup; fd = qemuSaveImageOpen(driver, path, @@ -6132,7 +6133,8 @@ qemuDomainObjRestore(virConnectPtr conn, } sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE; - if (!(restoreParams = qemuMigrationParamsForSave(sparse))) + if (!(restoreParams = qemuMigrationParamsForSave(sparse, + bypass_cache ? VIR_DOMAIN_SAVE_BYPASS_CACHE : 0))) return -1; fd = qemuSaveImageOpen(driver, path, bypass_cache, sparse, &wrapperFd, false); diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 9665b523af..a5a80230b2 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -7139,17 +7139,36 @@ qemuMigrationSrcToLegacyFile(virQEMUDriver *driver, static int qemuMigrationSrcToSparseFile(virQEMUDriver *driver, virDomainObj *vm, + const char *path, int *fd, unsigned int flags, virDomainAsyncJob asyncJob) { + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + VIR_AUTOCLOSE directFd = -1; + int directFlag = 0; + bool needUnlink = false; int ret; - /* mapped-ram does not support directIO */ + /* When using directio with mapped-ram, qemu needs two fds. One with + * O_DIRECT set writing the memory, and another without it set for + * writing small bits of unaligned state. */ if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) { - virReportError(VIR_ERR_OPERATION_FAILED, "%s", - _("bypass cache unsupported by this system")); - return -1; + directFlag = virFileDirectFdFlag(); + if (directFlag < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("bypass cache unsupported by this system")); + return -1; + } + directFd = virQEMUFileOpenAs(cfg->user, cfg->group, false, path, + O_WRONLY | directFlag, &needUnlink); + + if (directFd < 0) + return -1; + + if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, directFd) < 0) + return -1; + } if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, *fd) < 0) @@ -7158,7 +7177,7 @@ qemuMigrationSrcToSparseFile(virQEMUDriver *driver, if (qemuDomainObjEnterMonitorAsync(vm, asyncJob) < 0) return -1; - ret = qemuMonitorMigrateToFdSet(vm, 0, fd); + ret = qemuMonitorMigrateToFdSet(vm, 0, fd, &directFd); qemuDomainObjExitMonitor(vm); return ret; } @@ -7167,6 +7186,7 @@ qemuMigrationSrcToSparseFile(virQEMUDriver *driver, /* Helper function called while vm is active. */ int qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, + const char *path, int *fd, virCommand *compressor, qemuMigrationParams *migParams, @@ -7204,7 +7224,7 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, if (migParams && qemuMigrationParamsCapEnabled(migParams, QEMU_MIGRATION_CAP_MAPPED_RAM)) - rc = qemuMigrationSrcToSparseFile(driver, vm, fd, flags, asyncJob); + rc = qemuMigrationSrcToSparseFile(driver, vm, path, fd, flags, asyncJob); else rc = qemuMigrationSrcToLegacyFile(driver, vm, *fd, compressor, asyncJob); diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 71a9974753..beb888160a 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -238,6 +238,7 @@ qemuMigrationSrcIsAllowed(virDomainObj *vm, int qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm, + const char *path, int *fd, virCommand *compressor, qemuMigrationParams *migParams, diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c index d48cdd5506..16e93f27d6 100644 --- a/src/qemu/qemu_migration_params.c +++ b/src/qemu/qemu_migration_params.c @@ -130,6 +130,7 @@ VIR_ENUM_IMPL(qemuMigrationParam, "multifd-zlib-level", "multifd-zstd-level", "avail-switchover-bandwidth", + "direct-io", ); typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem; @@ -328,6 +329,9 @@ static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = { [QEMU_MIGRATION_PARAM_AVAIL_SWITCHOVER_BANDWIDTH] = { .type = QEMU_MIGRATION_PARAM_TYPE_ULL, }, + [QEMU_MIGRATION_PARAM_DIRECT_IO] = { + .type = QEMU_MIGRATION_PARAM_TYPE_BOOL, + }, }; G_STATIC_ASSERT(G_N_ELEMENTS(qemuMigrationParamInfo) == QEMU_MIGRATION_PARAM_LAST); @@ -793,7 +797,7 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, qemuMigrationParams * -qemuMigrationParamsForSave(bool sparse) +qemuMigrationParamsForSave(bool sparse, unsigned int flags) { g_autoptr(qemuMigrationParams) saveParams = NULL; @@ -807,6 +811,11 @@ qemuMigrationParamsForSave(bool sparse) return NULL; saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i = 1; saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set = true; + + if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) { + saveParams->params[QEMU_MIGRATION_PARAM_DIRECT_IO].value.b = true; + saveParams->params[QEMU_MIGRATION_PARAM_DIRECT_IO].set = true; + } } return g_steal_pointer(&saveParams); diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h index 88a1bc1a66..87822332d3 100644 --- a/src/qemu/qemu_migration_params.h +++ b/src/qemu/qemu_migration_params.h @@ -66,6 +66,7 @@ typedef enum { QEMU_MIGRATION_PARAM_MULTIFD_ZLIB_LEVEL, QEMU_MIGRATION_PARAM_MULTIFD_ZSTD_LEVEL, QEMU_MIGRATION_PARAM_AVAIL_SWITCHOVER_BANDWIDTH, + QEMU_MIGRATION_PARAM_DIRECT_IO, QEMU_MIGRATION_PARAM_LAST } qemuMigrationParam; @@ -88,7 +89,7 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params, qemuMigrationParty party); qemuMigrationParams * -qemuMigrationParamsForSave(bool sparse); +qemuMigrationParamsForSave(bool sparse, unsigned int flags); int qemuMigrationParamsDump(qemuMigrationParams *migParams, diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c index e2043c0120..1e80af8a58 100644 --- a/src/qemu/qemu_monitor.c +++ b/src/qemu/qemu_monitor.c @@ -2235,7 +2235,8 @@ qemuMonitorMigrateToFd(qemuMonitor *mon, int qemuMonitorMigrateToFdSet(virDomainObj *vm, unsigned int flags, - int *fd) + int *fd, + int *directFd) { qemuDomainObjPrivate *priv = vm->privateData; qemuMonitor *mon = priv->mon; @@ -2244,7 +2245,7 @@ qemuMonitorMigrateToFdSet(virDomainObj *vm, g_autofree char *uri = NULL; int ret; - VIR_DEBUG("fd=%d flags=0x%x", *fd, flags); + VIR_DEBUG("fd=%d directFd=%d flags=0x%x", *fd, *directFd, flags); QEMU_CHECK_MONITOR(mon); @@ -2256,6 +2257,8 @@ qemuMonitorMigrateToFdSet(virDomainObj *vm, fdPassMigrate = qemuFDPassNew("libvirt-outgoing-migrate", priv); qemuFDPassAddFD(fdPassMigrate, fd, "-fd"); + if (*directFd != -1) + qemuFDPassAddFD(fdPassMigrate, directFd, "-directio-fd"); qemuFDPassTransferMonitor(fdPassMigrate, mon); uri = g_strdup_printf("file:%s,offset=%#lx", diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h index 6da380aa65..dfb0e8abb6 100644 --- a/src/qemu/qemu_monitor.h +++ b/src/qemu/qemu_monitor.h @@ -861,7 +861,8 @@ int qemuMonitorMigrateToFd(qemuMonitor *mon, int qemuMonitorMigrateToFdSet(virDomainObj *vm, unsigned int flags, - int *fd); + int *fd, + int *directFd); int qemuMonitorMigrateToHost(qemuMonitor *mon, unsigned int flags, diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c index 3f20ffd58c..28d668231e 100644 --- a/src/qemu/qemu_saveimage.c +++ b/src/qemu/qemu_saveimage.c @@ -501,7 +501,7 @@ qemuSaveImageCreate(virQEMUDriver *driver, goto cleanup; /* Perform the migration */ - if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, saveParams, flags, asyncJob) < 0) + if (qemuMigrationSrcToFile(driver, vm, path, &fd, compressor, saveParams, flags, asyncJob) < 0) goto cleanup; /* Touch up file header to mark image complete. */ -- 2.43.0

From: Claudio Fontana <cfontana(a)suse.de&gt; Add a new VIR_DOMAIN_SAVE_PARALLEL flag to the save and restore APIs, which can be used to specify the use of multiple, parallel channels for saving and restoring a domain. The number of parallel channels can be set using the VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS typed parameter. Signed-off-by: Claudio Fontana <cfontana(a)suse.de&gt; Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; --- include/libvirt/libvirt-domain.h | 11 +++++++++++ src/libvirt-domain.c | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index f5420bca6e..0e94a4bd09 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -1655,6 +1655,7 @@ typedef enum { VIR_DOMAIN_SAVE_RUNNING = 1 << 1, /* Favor running over paused (Since: 0.9.5) */ VIR_DOMAIN_SAVE_PAUSED = 1 << 2, /* Favor paused over running (Since: 0.9.5) */ VIR_DOMAIN_SAVE_RESET_NVRAM = 1 << 3, /* Re-initialize NVRAM from template (Since: 8.1.0) */ + VIR_DOMAIN_SAVE_PARALLEL = 1 << 4, /* Save and restore using parallel channels (Since: 10.6.0) */ } virDomainSaveRestoreFlags; int virDomainSave (virDomainPtr domain, @@ -1700,6 +1701,16 @@ int virDomainRestoreParams (virConnectPtr conn, */ # define VIR_DOMAIN_SAVE_PARAM_DXML "dxml" +/** + * VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS: + * + * an optional parameter used to specify the number of IO channels to use + * during parallel save. As VIR_TYPED_PARAM_INT. + * + * Since: 11.1.0 + */ +# define VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS "parallel.channels" + /* See below for virDomainSaveImageXMLFlags */ char * virDomainSaveImageGetXMLDesc (virConnectPtr conn, const char *file, diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index cf68b0520b..d75ee25580 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -1010,6 +1010,9 @@ virDomainSaveFlags(virDomainPtr domain, const char *to, * If VIR_DOMAIN_SAVE_PARAM_FILE is not provided then a managed save is * performed (see virDomainManagedSave). * + * See VIR_DOMAIN_SAVE_PARAM_* for detailed description of accepted save + * parameters. + * * Returns 0 in case of success and -1 in case of failure. * * Since: 8.4.0 @@ -1222,6 +1225,9 @@ virDomainRestoreFlags(virConnectPtr conn, const char *from, const char *dxml, * now, VIR_DOMAIN_SAVE_PARAM_FILE is required but this requirement may * be lifted in the future. * + * See VIR_DOMAIN_SAVE_PARAM_* for detailed description of accepted + * restore parameters. + * * Returns 0 in case of success and -1 in case of failure. * * Since: 8.4.0 -- 2.43.0

From: Claudio Fontana <cfontana(a)suse.de&gt; Signed-off-by: Claudio Fontana <cfontana(a)suse.de&gt; Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; --- docs/manpages/virsh.rst | 12 +++++++++++- tools/virsh-domain.c | 42 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/manpages/virsh.rst b/docs/manpages/virsh.rst index 0eb1d6ea93..959393be13 100644 --- a/docs/manpages/virsh.rst +++ b/docs/manpages/virsh.rst @@ -3997,6 +3997,7 @@ save :: save domain state-file [--bypass-cache] [--xml file] + [--parallel] [--parallel-channels channels] [{--running | --paused}] [--verbose] Saves a running domain (RAM, but not disk state) to a state file so that @@ -4004,8 +4005,11 @@ it can be restored later. Once saved, the domain will no longer be running on the system, thus the memory allocated for the domain will be free for other domains to use. ``virsh restore`` restores from this state file. + If *--bypass-cache* is specified, the save will avoid the file system -cache, although this may slow down the operation. +cache. Depending on the specific scenario this may slow down or speed up +the operation. + The progress may be monitored using ``domjobinfo`` virsh command and canceled with ``domjobabort`` command (sent by another virsh instance). Another option @@ -4027,6 +4031,12 @@ based on the state the domain was in when the save was done; passing either the *--running* or *--paused* flag will allow overriding which state the ``restore`` should use. +*--parallel* option will cause the save data to be written to file +over multiple parallel IO channels. The number of channels can be +specified using *--parallel-channels*. Using parallel IO channels +requires the use of ``sparse`` image save format. Parallel save may +significantly reduce the time required to save large memory domains. + Domain saved state files assume that disk images will be unchanged between the creation and restore point. For a more complete system restore point, where the disk state is saved alongside the memory diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index f3da2f903f..3b765a95b5 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -4145,6 +4145,14 @@ static const vshCmdOptDef opts_save[] = { .type = VSH_OT_BOOL, .help = N_("avoid file system cache when saving") }, + {.name = "parallel", + .type = VSH_OT_BOOL, + .help = N_("enable parallel save") + }, + {.name = "parallel-channels", + .type = VSH_OT_INT, + .help = N_("number of extra IO channels to use for parallel save") + }, {.name = "xml", .type = VSH_OT_STRING, .unwanted_positional = true, @@ -4175,6 +4183,11 @@ doSave(void *opaque) g_autoptr(virshDomain) dom = NULL; const char *name = NULL; const char *to = NULL; + virTypedParameterPtr params = NULL; + int nparams = 0; + int maxparams = 0; + int nchannels = 1; + int rv = -1; unsigned int flags = 0; const char *xmlfile = NULL; g_autofree char *xml = NULL; @@ -4188,15 +4201,30 @@ doSave(void *opaque) goto out_sig; #endif /* !WIN32 */ - if (vshCommandOptString(ctl, cmd, "file", &to) < 0) - goto out; - if (vshCommandOptBool(cmd, "bypass-cache")) flags |= VIR_DOMAIN_SAVE_BYPASS_CACHE; if (vshCommandOptBool(cmd, "running")) flags |= VIR_DOMAIN_SAVE_RUNNING; if (vshCommandOptBool(cmd, "paused")) flags |= VIR_DOMAIN_SAVE_PAUSED; + if (vshCommandOptBool(cmd, "parallel")) + flags |= VIR_DOMAIN_SAVE_PARALLEL; + + if (vshCommandOptString(ctl, cmd, "file", &to) < 0) + goto out; + if (to && + virTypedParamsAddString(&params, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_FILE, to) < 0) + goto out; + + if (flags & VIR_DOMAIN_SAVE_PARALLEL) { + if ((rv = vshCommandOptInt(ctl, cmd, "parallel-channels", &nchannels)) < 0) + goto out; + + if (virTypedParamsAddInt(&params, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_PARALLEL_CHANNELS, nchannels) < 0) + goto out; + } if (vshCommandOptString(ctl, cmd, "xml", &xmlfile) < 0) goto out; @@ -4209,9 +4237,13 @@ doSave(void *opaque) vshReportError(ctl); goto out; } + if (xml && + virTypedParamsAddString(&params, &nparams, &maxparams, + VIR_DOMAIN_SAVE_PARAM_DXML, xml) < 0) + goto out; if (flags || xml) { - rc = virDomainSaveFlags(dom, to, xml, flags); + rc = virDomainSaveParams(dom, params, nparams, flags); } else { rc = virDomainSave(dom, to); } @@ -4224,6 +4256,8 @@ doSave(void *opaque) data->ret = 0; out: + virTypedParamsFree(params, nparams); + #ifndef WIN32 pthread_sigmask(SIG_SETMASK, &oldsigmask, NULL); out_sig: -- 2.43.0