The bhyve driver in unprivileged mode runs bhyve in monitor mode. In the monitor mode bhyve destroys the VM on close. When a domain is rebooted in this mode, libvirt bhyve monitor detects process exit and calls virBhyveProcessStop() to clean up resources. virBhyveProcessStop() tries to clean up resources and run `bhyvectl --destroy` which fails in this case because the VM is already destroyed. To prevent that, add a processExited argument to virBhyveProcessStopImpl(), and do not run destroy when it's not necessary. This issue is only relevant to a signal based reboot which is still simulated as SIGTERM initiated shutdown and a new boot. Other reboot paths, such as a normal reboot within a guest, or qga initiated reboot, are not affected. Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com> --- src/bhyve/bhyve_driver.c | 3 ++- src/bhyve/bhyve_monitor.c | 14 ++++++++------ src/bhyve/bhyve_process.c | 39 ++++++++++++++++++++++++++------------- src/bhyve/bhyve_process.h | 6 ++++-- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/bhyve/bhyve_driver.c b/src/bhyve/bhyve_driver.c index 47ed7d6963..5834392bd4 100644 --- a/src/bhyve/bhyve_driver.c +++ b/src/bhyve/bhyve_driver.c @@ -1034,7 +1034,8 @@ bhyveDomainDestroyFlags(virDomainPtr dom, unsigned int flags) if (virDomainObjCheckActive(vm) < 0) goto cleanup; - ret = virBhyveProcessStop(privconn, vm, VIR_DOMAIN_SHUTOFF_DESTROYED, false); + ret = virBhyveProcessStop(privconn, vm, VIR_DOMAIN_SHUTOFF_DESTROYED, + false, false); event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_STOPPED, VIR_DOMAIN_EVENT_STOPPED_DESTROYED); diff --git a/src/bhyve/bhyve_monitor.c b/src/bhyve/bhyve_monitor.c index 8740c856a8..3d49501e3d 100644 --- a/src/bhyve/bhyve_monitor.c +++ b/src/bhyve/bhyve_monitor.c @@ -165,32 +165,34 @@ bhyveMonitorIO(int watch, int kq, int events G_GNUC_UNUSED, void *opaque) virReportError(VIR_ERR_INTERNAL_ERROR, _("Guest %1$s got signal %2$d and crashed"), name, WTERMSIG(status)); - virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_CRASHED, false); + virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_CRASHED, false, true); } else if (WIFEXITED(status) || shutdownBySignal) { if ((!shutdownBySignal && WEXITSTATUS(status) == 0) || mon->reboot) { /* 0 - reboot */ if (vm->def->onReboot == VIR_DOMAIN_LIFECYCLE_ACTION_DESTROY || vm->def->onReboot == VIR_DOMAIN_LIFECYCLE_ACTION_PRESERVE) { VIR_INFO("Guest %s rebooted; domain on_reboot setting overridden, shutting down.", name); - virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, false); + virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, + false, true); } else { VIR_INFO("Guest %s rebooted; restarting domain.", name); - virBhyveProcessRestart(driver, vm); + virBhyveProcessRestart(driver, vm, true); } } else if (shutdownBySignal || WEXITSTATUS(status) < 3) { /* 1 - shutdown, 2 - halt, 3 - triple fault. others - error */ if (vm->def->onPoweroff == VIR_DOMAIN_LIFECYCLE_ACTION_RESTART || vm->def->onPoweroff == VIR_DOMAIN_LIFECYCLE_ACTION_RESTART_RENAME) { VIR_INFO("Guest %s shut itself down; domain on_poweroff setting overridden, attempting reboot.", name); - virBhyveProcessRestart(driver, vm); + virBhyveProcessRestart(driver, vm, true); } else { VIR_INFO("Guest %s shut itself down; destroying domain.", name); - virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, false); + virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, + false, true); } } else { VIR_INFO("Guest %s had an error and exited with status %d; destroying domain.", name, WEXITSTATUS(status)); - virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_UNKNOWN, false); + virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_UNKNOWN, false, true); } } diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c index c27a5cf6ed..3ee8461316 100644 --- a/src/bhyve/bhyve_process.c +++ b/src/bhyve/bhyve_process.c @@ -67,7 +67,7 @@ bhyveProcessAutoDestroy(virDomainObj *vm, bhyveDomainObjPrivate *priv = vm->privateData; struct _bhyveConn *driver = priv->driver; - virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_DESTROYED, false); + virBhyveProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_DESTROYED, false, false); if (!vm->persistent) virDomainObjListRemove(driver->domains, vm); @@ -526,7 +526,7 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver, if (ret < 0) ignore_value(virBhyveProcessStop(driver, vm, - VIR_DOMAIN_SHUTOFF_FAILED, true)); + VIR_DOMAIN_SHUTOFF_FAILED, true, false)); return ret; } @@ -742,6 +742,7 @@ bhyveProcessRemoveIPCSocket(struct _bhyveConn *driver, * @reason: shutoff reason * @forceCleanup: boolean controlling cleanup * @restoreDef: whether to restore the persistent definition after cleanup + * @processExited: whether the monitor has observed process exit * * Stops the domain and cleans up its resources. * It could be used whether as a direct call or as a cleanup routine. @@ -758,7 +759,8 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, virDomainObj *vm, virDomainShutoffReason reason, bool forceCleanup, - bool restoreDef) + bool restoreDef, + bool processExited) { int ret = 0; size_t i = 0; @@ -794,12 +796,20 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, virBhyveDomainObjStopWorker(vm); - cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); - if (virCommandRun(cmd, NULL) < 0) { - /* Only failure of the actual destroy command warrants unsuccessful return code, - * other failures are not considered critical */ - ret = -1; - VIR_WARN("Failed to run the domain destroy command"); + /* When bhyve runs in a monitor mode (i.e. unprivileged), it destroys + * the VM when its own monitor process exits. In this case the bhyve + * driver's monitor detects the exit and calls this function with + * processExited == true. In this case running `bhyvectl --destroy` + * is not needed. */ + if (driver->privileged || !processExited) { + cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); + if (virCommandRun(cmd, NULL) < 0) { + /* Only failure of the actual destroy command warrants + * unsuccessful return code; other failures are not + * considered critical. */ + ret = -1; + VIR_WARN("Failed to run the domain destroy command"); + } } if (vm_started) @@ -854,9 +864,11 @@ int virBhyveProcessStop(struct _bhyveConn *driver, virDomainObj *vm, virDomainShutoffReason reason, - bool forceCleanup) + bool forceCleanup, + bool processExited) { - return virBhyveProcessStopImpl(driver, vm, reason, forceCleanup, true); + return virBhyveProcessStopImpl(driver, vm, reason, forceCleanup, true, + processExited); } int @@ -882,10 +894,11 @@ virBhyveProcessShutdown(virDomainObj *vm) int virBhyveProcessRestart(struct _bhyveConn *driver, - virDomainObj *vm) + virDomainObj *vm, + bool processExited) { if (virBhyveProcessStopImpl(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, - false, false) < 0) + false, false, processExited) < 0) return -1; if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_BOOTED) < 0) diff --git a/src/bhyve/bhyve_process.h b/src/bhyve/bhyve_process.h index 2f0ce21167..e4f343dc32 100644 --- a/src/bhyve/bhyve_process.h +++ b/src/bhyve/bhyve_process.h @@ -45,10 +45,12 @@ int virBhyveProcessStart(bhyveConn *driver, int virBhyveProcessStop(struct _bhyveConn *driver, virDomainObj *vm, virDomainShutoffReason reason, - bool forceCleanup); + bool forceCleanup, + bool processExited); int virBhyveProcessRestart(struct _bhyveConn *driver, - virDomainObj *vm); + virDomainObj *vm, + bool processExited); int virBhyveProcessReset(struct _bhyveConn *driver, virDomainObj *vm); -- 2.56.0