[PATCH 0/5] implement suspend/resume support
This series implements suspend and resume support for the bhyve driver. Suspend and resume is still considered an experimental feature in bhyve, mainly because the suspend files format is not finalized yet. Therefore, it's not enabled by default and requires a rebuild to enable it. Luckily, it's easy to detect this feature in runtime, so capabilities guard from using it when it's not supported. I've been testing this change manually for a several weeks, and also extended TCK tests. Related MRs: https://gitlab.com/libvirt/libvirt-tck/-/merge_requests/95 https://gitlab.com/libvirt/libvirt-tck/-/merge_requests/96 Roman Bogorodskiy (5): bhyve: capabilities: detect suspend/resume support bhyve: implement suspend/resume support bhyve: validate suspend status and clean up snapshot files bhyve: forbid shutdown and reboot for suspended domains bhyve: fix domain autodestroy src/bhyve/bhyve_capabilities.c | 3 + src/bhyve/bhyve_capabilities.h | 1 + src/bhyve/bhyve_command.c | 22 ++- src/bhyve/bhyve_command.h | 5 + src/bhyve/bhyve_conf.c | 3 + src/bhyve/bhyve_driver.c | 114 +++++++++++++++- src/bhyve/bhyve_process.c | 240 +++++++++++++++++++++++++++++---- src/bhyve/bhyve_process.h | 6 + src/bhyve/bhyve_utils.h | 1 + tests/bhyvexml2argvtest.c | 2 +- 10 files changed, 370 insertions(+), 27 deletions(-) -- 2.56.0
bhyve implements suspend using the "bhyvectl --suspend" command and resuming using the "-r" flag for "bhyve". Even though this feature was there for quite a while, it is still not compiled in by default. Thus, it is important to check this capability. Only check the "bhyvectl" part as it's safe to assume that the bhyve part check result will be the same. Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com> --- src/bhyve/bhyve_capabilities.c | 3 +++ src/bhyve/bhyve_capabilities.h | 1 + 2 files changed, 4 insertions(+) diff --git a/src/bhyve/bhyve_capabilities.c b/src/bhyve/bhyve_capabilities.c index 42da2a7c47..70c1c31260 100644 --- a/src/bhyve/bhyve_capabilities.c +++ b/src/bhyve/bhyve_capabilities.c @@ -425,6 +425,9 @@ bhyveProbeCapsBhyvectl(virBitmap *caps) if (strstr(help, "--get-vm-pid") != NULL) virBhyveCapsSet(caps, BHYVE_CAP_GET_VMPID); + if (strstr(help, "--suspend=") != NULL) + virBhyveCapsSet(caps, BHYVE_CAP_SUSPEND_RESUME); + return 0; } diff --git a/src/bhyve/bhyve_capabilities.h b/src/bhyve/bhyve_capabilities.h index 90e604c41b..25700c2e7c 100644 --- a/src/bhyve/bhyve_capabilities.h +++ b/src/bhyve/bhyve_capabilities.h @@ -62,6 +62,7 @@ typedef enum { BHYVE_CAP_MONITOR, BHYVE_CAP_GET_VMPID, BHYVE_CAP_LPC_TPM, + BHYVE_CAP_SUSPEND_RESUME, BHYVE_CAP_LAST, } virBhyveCapsFlags; -- 2.56.0
bhyve VMs are suspended using the "--suspend" argument to the "bhyvectl" tool: bhyvectl --suspend=<snapshot_path> --vm=<vm_name> --rundir=<rundir> Where <snapshot_path> is the base filename for the snapshot files, <vm_name> is a VM name, and <rundir> points to a directory with the control socket. Resume uses the same "bhyve" command as a normal VM startup, but adds "-r" with a path to the <snapshot_path>. For non-UEFI VMs, it also requires running "bhyveload". Add the bhyveDomainSuspend() which implements domain suspend. Add bhyveDomainResume() which implements domain resume, using virBhyveProcessResume() as a wrapper to virBhyveProcessStartImpl(). virBhyveProcessStartImpl() now accepts the "resume" argument which controls whether the "-r" resume argument should be added. Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com> --- src/bhyve/bhyve_command.c | 22 +++++++- src/bhyve/bhyve_command.h | 5 ++ src/bhyve/bhyve_conf.c | 3 ++ src/bhyve/bhyve_driver.c | 102 +++++++++++++++++++++++++++++++++++++- src/bhyve/bhyve_process.c | 68 ++++++++++++++++++++----- src/bhyve/bhyve_process.h | 6 +++ src/bhyve/bhyve_utils.h | 1 + tests/bhyvexml2argvtest.c | 2 +- 8 files changed, 193 insertions(+), 16 deletions(-) diff --git a/src/bhyve/bhyve_command.c b/src/bhyve/bhyve_command.c index be66d26c50..f1a6d9878f 100644 --- a/src/bhyve/bhyve_command.c +++ b/src/bhyve/bhyve_command.c @@ -1147,7 +1147,7 @@ bhyveBuildSysinfoArgStr(virSysinfoDef *sysinfo, virCommand * virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def, - bool dryRun) + bool resume, bool dryRun) { /* * /usr/sbin/bhyve -c 2 -m 256 -AI -H -P \ @@ -1164,6 +1164,12 @@ virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def, unsigned nvcpus = virDomainDefGetVcpus(def); size_t ncells = virDomainNumaGetNodeCount(def->numa); + if (resume) { + virCommandAddArg(cmd, "-r"); + virCommandAddArgFormat(cmd, "%s/%s/snapshot", driver->config->snapshotDir, + def->name); + } + /* Monitor mode */ if (!driver->privileged) { virCommandAddArg(cmd, "-M"); @@ -1427,6 +1433,20 @@ virBhyveProcessBuildResetCmd(struct _bhyveConn *driver G_GNUC_UNUSED, return cmd; } +virCommand * +virBhyveProcessBuildSuspendCmd(struct _bhyveConn *driver, + virDomainDef *def) +{ + virCommand *cmd = virCommandNew("bhyvectl"); + + virCommandAddArgFormat(cmd, "--suspend=%s/%s/snapshot", + driver->config->snapshotDir, def->name); + virCommandAddArgPair(cmd, "--vm", def->name); + virCommandAddArgPair(cmd, "--rundir", driver->config->bhyveRunDir); + + return cmd; +} + virCommand * virBhyveProcessBuildGetVMPidCmd(struct _bhyveConn *driver, virDomainDef *def) diff --git a/src/bhyve/bhyve_command.h b/src/bhyve/bhyve_command.h index 2ee3f3a607..32a6b27ca3 100644 --- a/src/bhyve/bhyve_command.h +++ b/src/bhyve/bhyve_command.h @@ -31,6 +31,7 @@ virCommand *virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def, + bool resume, bool dryRun); virCommand * @@ -44,5 +45,9 @@ virBhyveProcessBuildLoadCmd(struct _bhyveConn *driver, virDomainDef *def, virCommand * virBhyveProcessBuildResetCmd(struct _bhyveConn *driver, virDomainDef *def); +virCommand * +virBhyveProcessBuildSuspendCmd(struct _bhyveConn *driver, + virDomainDef *def); + virCommand * virBhyveProcessBuildGetVMPidCmd(struct _bhyveConn *driver, virDomainDef *def); diff --git a/src/bhyve/bhyve_conf.c b/src/bhyve/bhyve_conf.c index 68bd22f00a..c6895fe245 100644 --- a/src/bhyve/bhyve_conf.c +++ b/src/bhyve/bhyve_conf.c @@ -94,6 +94,8 @@ virBhyveDriverConfigNew(bool privileged) cfg->logDir = g_strdup_printf("%s/bhyve/log", cachedir); } + cfg->snapshotDir = g_build_filename(cfg->stateDir, "snapshot", NULL); + cfg->bhyveloadTimeout = 300; cfg->bhyveloadTimeoutKill = 15; @@ -153,6 +155,7 @@ virBhyveDriverConfigDispose(void *obj) g_free(cfg->logDir); g_free(cfg->nvramDir); g_free(cfg->stateDir); + g_free(cfg->snapshotDir); g_free(cfg->bhyveRunDir); g_free(cfg->ubootPath); diff --git a/src/bhyve/bhyve_driver.c b/src/bhyve/bhyve_driver.c index 47ed7d6963..bae1961036 100644 --- a/src/bhyve/bhyve_driver.c +++ b/src/bhyve/bhyve_driver.c @@ -797,7 +797,7 @@ bhyveConnectDomainXMLToNative(virConnectPtr conn, virBufferAddChar(&buf, '\n'); } - if (!(cmd = virBhyveProcessBuildBhyveCmd(privconn, def, true))) + if (!(cmd = virBhyveProcessBuildBhyveCmd(privconn, def, false, true))) return NULL; virCommandToStringBuf(cmd, &buf, false, false); @@ -1475,6 +1475,13 @@ bhyveStateInitialize(bool privileged, goto cleanup; } + if (g_mkdir_with_parents(cfg->snapshotDir, 0700) < 0) { + virReportSystemError(errno, + _("Failed to mkdir %1$s"), + cfg->snapshotDir); + goto cleanup; + } + if (g_mkdir_with_parents(cfg->nvramDir, 0777) < 0) { virReportSystemError(errno, _("Failed to mkdir %1$s"), @@ -3226,6 +3233,97 @@ bhyveDomainGetGuestInfo(virDomainPtr domain, } +static int +bhyveDomainSuspend(virDomainPtr domain) +{ + struct _bhyveConn *driver = domain->conn->privateData; + g_autoptr(virBhyveDriverConfig) cfg = virBhyveDriverGetConfig(driver); + virDomainObj *vm; + virObjectEvent *event = NULL; + int ret = -1; + + if (!(vm = bhyveDomObjFromDomain(domain))) + return -1; + + if (virDomainSuspendEnsureACL(domain->conn, vm->def) < 0) + goto cleanup; + + if (!virBhyveCapsGet(bhyveDriverGetBhyveCaps(driver), BHYVE_CAP_SUSPEND_RESUME)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("bhyve does not support suspend/resume")); + goto cleanup; + } + + if (virDomainObjCheckActive(vm) < 0) + goto cleanup; + + if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_PAUSED) { + if (virBhyveProcessSuspend(driver, vm) < 0) + goto cleanup; + + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_PAUSED); + } + + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) + goto cleanup; + + ret = 0; + + cleanup: + virObjectEventStateQueue(driver->domainEventState, event); + virDomainObjEndAPI(&vm); + return ret; +} + + +static int +bhyveDomainResume(virDomainPtr domain) +{ + struct _bhyveConn *driver = domain->conn->privateData; + g_autoptr(virBhyveDriverConfig) cfg = virBhyveDriverGetConfig(driver); + virDomainObj *vm; + virObjectEvent *event = NULL; + int ret = -1; + + if (!(vm = bhyveDomObjFromDomain(domain))) + return -1; + + if (virDomainResumeEnsureACL(domain->conn, vm->def) < 0) + goto cleanup; + + if (!virBhyveCapsGet(bhyveDriverGetBhyveCaps(driver), BHYVE_CAP_SUSPEND_RESUME)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("bhyve does not support suspend/resume")); + goto cleanup; + } + + if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_PAUSED) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cannot resume domain not in paused state")); + goto cleanup; + } + + if (virBhyveProcessResume(driver, vm) < 0) + goto cleanup; + + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_RESUMED, + VIR_DOMAIN_EVENT_RESUMED_UNPAUSED); + + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) + goto cleanup; + + ret = 0; + + cleanup: + virObjectEventStateQueue(driver->domainEventState, event); + virDomainObjEndAPI(&vm); + return ret; +} + + static int bhyveDomainReset(virDomainPtr domain, unsigned int flags) { @@ -3334,6 +3432,8 @@ static virHypervisorDriver bhyveHypervisorDriver = { .domainGetGuestInfo = bhyveDomainGetGuestInfo, /* 12.7.0 */ .domainSetLifecycleAction = bhyveDomainSetLifecycleAction, /* 12.8.0 */ .domainReset = bhyveDomainReset, /* 12.9.0 */ + .domainSuspend = bhyveDomainSuspend, /* 12.10.0 */ + .domainResume = bhyveDomainResume, /* 12.10.0 */ }; diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c index c27a5cf6ed..0cac9e841a 100644 --- a/src/bhyve/bhyve_process.c +++ b/src/bhyve/bhyve_process.c @@ -377,7 +377,8 @@ bhyveConnectAgent(struct _bhyveConn *driver G_GNUC_UNUSED, virDomainObj *vm) static int virBhyveProcessStartImpl(struct _bhyveConn *driver, virDomainObj *vm, - virDomainRunningReason reason) + virDomainRunningReason reason, + bool resume) { g_autofree char *devmap_file = NULL; g_autofree char *devicemap = NULL; @@ -421,7 +422,7 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver, goto cleanup; /* Call bhyve to start the VM */ - if (!(cmd = virBhyveProcessBuildBhyveCmd(driver, vm->def, false))) + if (!(cmd = virBhyveProcessBuildBhyveCmd(driver, vm->def, resume, false))) goto cleanup; virCommandSetOutputFD(cmd, &logfd); @@ -493,8 +494,11 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver, goto cleanup; } - /* /dev/vmm can appear before the monitor mode IPC socket is ready. */ - if (!driver->privileged && bhyveProcessWaitForVMPid(vm) < 0) + /* /dev/vmm can appear before the monitor mode IPC socket is ready. + * A loader also creates it before bhyve starts restoring + * a suspended VM. */ + if ((!driver->privileged || resume) && + bhyveProcessWaitForVMPid(vm) < 0) goto cleanup; vm->def->id = vm->pid; @@ -698,7 +702,7 @@ virBhyveProcessStart(bhyveConn *driver, if (bhyveProcessPrepareHost(driver, vm->def, flags) < 0) goto cleanup; - return virBhyveProcessStartImpl(driver, vm, reason); + return virBhyveProcessStartImpl(driver, vm, reason, false); cleanup: virErrorPreserveLast(&save_err); @@ -756,9 +760,10 @@ bhyveProcessRemoveIPCSocket(struct _bhyveConn *driver, static int virBhyveProcessStopImpl(struct _bhyveConn *driver, virDomainObj *vm, - virDomainShutoffReason reason, + int reason, bool forceCleanup, - bool restoreDef) + bool restoreDef, + bool suspend) { int ret = 0; size_t i = 0; @@ -787,6 +792,20 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, } } + if (suspend) { + g_autofree char *snapshot_dir = g_build_filename(driver->config->snapshotDir, + vm->def->name, NULL); + + if (g_mkdir_with_parents(snapshot_dir, 0700) < 0) { + virReportSystemError(errno, + _("Failed to mkdir %1$s"), snapshot_dir); + ret = -1; + if (!save_err) + virErrorPreserveLast(&save_err); + goto cleanup; + } + } + /* Destroy monitor before running the actual destroy command to prevent * it from detecting VM shutdown and entering this cleanup routine again */ if ((priv != NULL) && (priv->mon != NULL)) @@ -794,7 +813,10 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, virBhyveDomainObjStopWorker(vm); - cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); + if (suspend) + cmd = virBhyveProcessBuildSuspendCmd(driver, vm->def); + else + cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); if (virCommandRun(cmd, NULL) < 0) { /* Only failure of the actual destroy command warrants unsuccessful return code, * other failures are not considered critical */ @@ -832,7 +854,9 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, virCloseCallbacksDomainRemove(vm, NULL, bhyveProcessAutoDestroy); - virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, reason); + virDomainObjSetState(vm, + suspend ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_SHUTOFF, + reason); vm->pid = 0; vm->def->id = -1; @@ -842,7 +866,7 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, bhyveProcessRemoveDomainStatus(driver->config->stateDir, vm->def->name); bhyveProcessRemoveIPCSocket(driver, vm->def->name); - if (restoreDef) + if (restoreDef && !suspend) virDomainObjRemoveTransientDef(vm); cleanup: @@ -856,7 +880,16 @@ virBhyveProcessStop(struct _bhyveConn *driver, virDomainShutoffReason reason, bool forceCleanup) { - return virBhyveProcessStopImpl(driver, vm, reason, forceCleanup, true); + return virBhyveProcessStopImpl(driver, vm, reason, forceCleanup, true, + false); +} + +int +virBhyveProcessSuspend(struct _bhyveConn *driver, + virDomainObj *vm) +{ + return virBhyveProcessStopImpl(driver, vm, VIR_DOMAIN_PAUSED_USER, + false, false, true); } int @@ -885,10 +918,11 @@ virBhyveProcessRestart(struct _bhyveConn *driver, virDomainObj *vm) { if (virBhyveProcessStopImpl(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, - false, false) < 0) + false, false, false) < 0) return -1; - if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_BOOTED) < 0) + if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_BOOTED, + false) < 0) return -1; return 0; @@ -905,6 +939,14 @@ virBhyveProcessReset(struct _bhyveConn *driver, virDomainObj *vm) return 0; } +int +virBhyveProcessResume(struct _bhyveConn *driver, + virDomainObj *vm) +{ + return virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED, + true); +} + int virBhyveGetDomainTotalCpuStats(virDomainObj *vm, unsigned long long *cpustats) diff --git a/src/bhyve/bhyve_process.h b/src/bhyve/bhyve_process.h index 2f0ce21167..655121a150 100644 --- a/src/bhyve/bhyve_process.h +++ b/src/bhyve/bhyve_process.h @@ -47,12 +47,18 @@ int virBhyveProcessStop(struct _bhyveConn *driver, virDomainShutoffReason reason, bool forceCleanup); +int virBhyveProcessSuspend(struct _bhyveConn *driver, + virDomainObj *vm); + int virBhyveProcessRestart(struct _bhyveConn *driver, virDomainObj *vm); int virBhyveProcessReset(struct _bhyveConn *driver, virDomainObj *vm); +int virBhyveProcessResume(struct _bhyveConn *driver, + virDomainObj *vm); + int virBhyveProcessShutdown(virDomainObj *vm); pid_t virBhyveProcessGetBhyvePid(virDomainObj *vm); diff --git a/src/bhyve/bhyve_utils.h b/src/bhyve/bhyve_utils.h index 6d15035693..fce67dfdf3 100644 --- a/src/bhyve/bhyve_utils.h +++ b/src/bhyve/bhyve_utils.h @@ -42,6 +42,7 @@ struct _virBhyveDriverConfig { char *logDir; char *nvramDir; char *stateDir; + char *snapshotDir; char *bhyveRunDir; char *ubootPath; diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c index f5df8a0265..078d5faae1 100644 --- a/tests/bhyvexml2argvtest.c +++ b/tests/bhyvexml2argvtest.c @@ -70,7 +70,7 @@ static int testCompareXMLToArgvFiles(const char *xml, goto out; } - cmd = virBhyveProcessBuildBhyveCmd(&driver, vm->def, false); + cmd = virBhyveProcessBuildBhyveCmd(&driver, vm->def, false, false); if (vm->def->os.loader) ldcmd = virCommandNew("dummy"); else -- 2.56.0
Successful completion of bhyvectl --suspend alone is not sufficient to confirm suspension. Remove stale snapshot files before suspending and require the image, .kern, and .meta files to exist afterward. On failure, preserve the error, destroy the remaining VM instance, remove partial snapshot files, and mark the domain shut off with a failed reason. Keep successfully suspended domains active by retaining their domain ID. Allow destroying suspended domains without requiring a running process, and remove snapshot files after destruction or successful resume. Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com> --- src/bhyve/bhyve_process.c | 186 ++++++++++++++++++++++++++++++++++---- 1 file changed, 168 insertions(+), 18 deletions(-) diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c index 0cac9e841a..001ee59130 100644 --- a/src/bhyve/bhyve_process.c +++ b/src/bhyve/bhyve_process.c @@ -535,6 +535,106 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver, return ret; } + +static int +bhyveProcessRemoveSuspendFile(const char *path, + bool report) +{ + if (unlink(path) < 0 && errno != ENOENT) { + if (report) { + virReportSystemError(errno, + _("Unable to remove bhyve snapshot file '%1$s'"), + path); + } else { + VIR_WARN("Unable to remove bhyve snapshot file '%s': %s", + path, g_strerror(errno)); + } + return -1; + } + + return 0; +} + + +static int +bhyveProcessRemoveSuspendFiles(bhyveConn *driver, + virDomainDef *def, + bool report) +{ + g_autofree char *snapshot_dir = NULL; + g_autofree char *snapshot_path = NULL; + g_autofree char *kern_path = NULL; + g_autofree char *meta_path = NULL; + int ret = 0; + + snapshot_dir = g_build_filename(driver->config->snapshotDir, def->name, NULL); + snapshot_path = g_build_filename(snapshot_dir, "snapshot", NULL); + kern_path = g_strdup_printf("%s.kern", snapshot_path); + meta_path = g_strdup_printf("%s.meta", snapshot_path); + + if (bhyveProcessRemoveSuspendFile(snapshot_path, report) < 0) + ret = -1; + if (bhyveProcessRemoveSuspendFile(kern_path, report) < 0) + ret = -1; + if (bhyveProcessRemoveSuspendFile(meta_path, report) < 0) + ret = -1; + + if (rmdir(snapshot_dir) < 0 && errno != ENOENT) { + if (report) { + virReportSystemError(errno, + _("Unable to remove bhyve snapshot directory '%1$s'"), + snapshot_dir); + } else { + VIR_WARN("Unable to remove bhyve snapshot directory '%s': %s", + snapshot_dir, g_strerror(errno)); + } + ret = -1; + } + + return ret; +} + + +static int +bhyveProcessCheckSuspendFile(const char *path) +{ + struct stat sb; + + if (stat(path, &sb) < 0) { + virReportSystemError(errno, + _("Unable to stat bhyve snapshot file '%1$s'"), + path); + return -1; + } + + return 0; +} + + +static int +bhyveProcessCheckSuspend(bhyveConn *driver, + virDomainDef *def) +{ + g_autofree char *snapshot_path = NULL; + g_autofree char *kern_path = NULL; + g_autofree char *meta_path = NULL; + + /* bhyvectl stores guest memory in the requested file and guest device + * and vCPU state in files with the .kern and .meta suffixes */ + snapshot_path = g_build_filename(driver->config->snapshotDir, def->name, + "snapshot", NULL); + kern_path = g_strdup_printf("%s.kern", snapshot_path); + meta_path = g_strdup_printf("%s.meta", snapshot_path); + + if (bhyveProcessCheckSuspendFile(snapshot_path) < 0 || + bhyveProcessCheckSuspendFile(kern_path) < 0 || + bhyveProcessCheckSuspendFile(meta_path) < 0) + return -1; + + return 0; +} + + int bhyveProcessPrepareDomain(bhyveConn *driver, virDomainObj *vm, @@ -746,6 +846,7 @@ bhyveProcessRemoveIPCSocket(struct _bhyveConn *driver, * @reason: shutoff reason * @forceCleanup: boolean controlling cleanup * @restoreDef: whether to restore the persistent definition after cleanup + * @suspend: whether to suspend instead of shutoff * * Stops the domain and cleans up its resources. * It could be used whether as a direct call or as a cleanup routine. @@ -768,8 +869,11 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, int ret = 0; size_t i = 0; g_autoptr(virCommand) cmd = NULL; + g_autoptr(virCommand) cleanup_cmd = NULL; bhyveDomainObjPrivate *priv = vm->privateData; bool vm_started = false; + bool suspended = false; + bool suspendSucceeded = false; virErrorPtr save_err = NULL; virErrorPreserveLast(&save_err); @@ -777,13 +881,16 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, if (vm->pid != 0) vm_started = true; + if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED && vm->pid == 0) + suspended = true; + if (!forceCleanup) { if (!virDomainObjIsActive(vm)) { VIR_DEBUG("VM '%s' not active", vm->def->name); goto cleanup; } - if (!vm_started) { + if (!vm_started && !suspended) { virReportError(VIR_ERR_INTERNAL_ERROR, _("Invalid PID %1$d for VM"), (int)vm->pid); @@ -796,6 +903,13 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, g_autofree char *snapshot_dir = g_build_filename(driver->config->snapshotDir, vm->def->name, NULL); + if (bhyveProcessRemoveSuspendFiles(driver, vm->def, true) < 0) { + ret = -1; + if (!save_err) + virErrorPreserveLast(&save_err); + goto cleanup; + } + if (g_mkdir_with_parents(snapshot_dir, 0700) < 0) { virReportSystemError(errno, _("Failed to mkdir %1$s"), snapshot_dir); @@ -806,24 +920,50 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, } } - /* Destroy monitor before running the actual destroy command to prevent + /* Destroy monitor before running the actual stop command to prevent * it from detecting VM shutdown and entering this cleanup routine again */ if ((priv != NULL) && (priv->mon != NULL)) g_clear_pointer(&priv->mon, bhyveMonitorClose); virBhyveDomainObjStopWorker(vm); - if (suspend) - cmd = virBhyveProcessBuildSuspendCmd(driver, vm->def); - else - cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); - if (virCommandRun(cmd, NULL) < 0) { - /* Only failure of the actual destroy command warrants unsuccessful return code, - * other failures are not considered critical */ - ret = -1; - VIR_WARN("Failed to run the domain destroy command"); + if (!suspended) { + if (suspend) { + cmd = virBhyveProcessBuildSuspendCmd(driver, vm->def); + } else { + cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); + } + + if (virCommandRun(cmd, NULL) < 0) { + /* Only failure of the actual stop command warrants unsuccessful + * return code; other cleanup failures are not considered critical. */ + ret = -1; + VIR_WARN("Failed to run the domain %s command", + suspend ? "suspend" : "destroy"); + if (suspend && !save_err) + virErrorPreserveLast(&save_err); + } + + if (suspend && ret == 0) { + if (bhyveProcessCheckSuspend(driver, vm->def) < 0) { + ret = -1; + if (!save_err) + virErrorPreserveLast(&save_err); + } else { + suspendSucceeded = true; + } + } + + if (suspend && !suspendSucceeded) { + cleanup_cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); + if (virCommandRun(cleanup_cmd, NULL) < 0) + VIR_WARN("Failed to destroy domain after unsuccessful suspend"); + } } + if (suspended) + goto state_cleanup; + if (vm_started) bhyveProcessStopHook(driver, vm, VIR_HOOK_BHYVE_OP_STOPPED); @@ -852,13 +992,19 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, virFileRemove(channel->source->data.nix.path, 0, 0); } - virCloseCallbacksDomainRemove(vm, NULL, bhyveProcessAutoDestroy); + state_cleanup: + if (!suspendSucceeded) + virCloseCallbacksDomainRemove(vm, NULL, bhyveProcessAutoDestroy); + + if (suspended || (suspend && !suspendSucceeded)) + ignore_value(bhyveProcessRemoveSuspendFiles(driver, vm->def, false)); virDomainObjSetState(vm, - suspend ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_SHUTOFF, - reason); + suspendSucceeded ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_SHUTOFF, + suspend && !suspendSucceeded ? VIR_DOMAIN_SHUTOFF_FAILED : reason); vm->pid = 0; - vm->def->id = -1; + if (!suspendSucceeded) + vm->def->id = -1; if (vm_started) bhyveProcessStopHook(driver, vm, VIR_HOOK_BHYVE_OP_RELEASE); @@ -866,7 +1012,7 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, bhyveProcessRemoveDomainStatus(driver->config->stateDir, vm->def->name); bhyveProcessRemoveIPCSocket(driver, vm->def->name); - if (restoreDef && !suspend) + if (restoreDef && !suspendSucceeded) virDomainObjRemoveTransientDef(vm); cleanup: @@ -943,8 +1089,12 @@ int virBhyveProcessResume(struct _bhyveConn *driver, virDomainObj *vm) { - return virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED, - true); + if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED, + true) < 0) + return -1; + + ignore_value(bhyveProcessRemoveSuspendFiles(driver, vm->def, false)); + return 0; } int -- 2.56.0
The bhyveDomainShutdownFlags() and bhyveDomainReboot() already exit early if virDomainObjCheckActive(vm) < 0. With the suspended it's not enough, so make sure that the domain is actually running. Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com> --- src/bhyve/bhyve_driver.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/bhyve/bhyve_driver.c b/src/bhyve/bhyve_driver.c index bae1961036..05b20c0cf8 100644 --- a/src/bhyve/bhyve_driver.c +++ b/src/bhyve/bhyve_driver.c @@ -1137,6 +1137,12 @@ bhyveDomainShutdownFlags(virDomainPtr dom, unsigned int flags) if (virDomainObjCheckActive(vm) < 0) goto cleanup; + if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_RUNNING) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("domain is not running")); + goto cleanup; + } + agentForced = agentRequested && !signalRequested; if (useAgent) { ret = bhyveDomainShutdownFlagsAgent(vm, isReboot, agentForced); @@ -1202,6 +1208,12 @@ bhyveDomainReboot(virDomainPtr dom, unsigned int flags) if (virDomainObjCheckActive(vm) < 0) goto cleanup; + if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_RUNNING) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("domain is not running")); + goto cleanup; + } + agentForced = agentRequested && !signalRequested; if (useAgent) { ret = bhyveDomainRebootAgent(vm, isReboot, agentForced); -- 2.56.0
Currently, domain autodestroy does not work with the bhyve driver. The bhyveDomainCreateWithFlags() has: virCheckFlags(VIR_DOMAIN_START_AUTODESTROY, -1); And it passes this flag to virBhyveProcessStart() which in its turn calls bhyveProcessPrepareDomain() and bhyveProcessPrepareHost() passing these flags. However, both bhyveProcessPrepareDomain() and bhyveProcessPrepareHost() call virCheckFlags(0, -1), so the VIR_DOMAIN_START_AUTODESTROY flag is not accepted, thus autodestroy fails. Fix by accepting the VIR_DOMAIN_START_AUTODESTROY flag in both bhyveProcessPrepareDomain() and bhyveProcessPrepareHost(). Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com> --- src/bhyve/bhyve_process.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c index 001ee59130..122dc3de99 100644 --- a/src/bhyve/bhyve_process.c +++ b/src/bhyve/bhyve_process.c @@ -640,7 +640,7 @@ bhyveProcessPrepareDomain(bhyveConn *driver, virDomainObj *vm, unsigned int flags) { - virCheckFlags(0, -1); + virCheckFlags(VIR_BHYVE_PROCESS_START_AUTODESTROY, -1); if (bhyveFirmwareFillDomain(driver, vm->def, false) < 0) return -1; @@ -769,7 +769,7 @@ bhyveProcessPrepareHost(bhyveConn *driver, virDomainDef *def, unsigned int flags) { - virCheckFlags(0, -1); + virCheckFlags(VIR_BHYVE_PROCESS_START_AUTODESTROY, -1); if (bhyvePrepareNVRAM(driver, def) < 0) return -1; -- 2.56.0
participants (1)
-
Roman Bogorodskiy