Re: [PATCH rfcv3 07/11] qemu: add hard reboot in QEMU driver

From: Chenyi Qiang <chenyi.qiang(a)intel.com&gt; Add the new flag VIR_DOMAIN_REBOOT_HARD/VIR_DOMAIN_SHUTDOWN_HARD to carry out a hard reboot, which kills the QEMU process and creates a new one with the same definition.

Hard reboot will be the highest priority to check. If succeed, other reboot policy (i.e. agent and acpi) would be skipped. Otherwise, use the traditional way. To make the shutdown graceful, the workflow of hard reboot is similar to the existing fakeReboot procedure. - In qemuDomainObjPrivatePtr we have a bool hardReboot flag. - The virDomainReboot method sets this flag and then triggers a normal "system_powerdown" if we specify the reboot mode "--mode hard" - The QEMU process is started with '-no-shutdown' so that the guest CPUs pause when it powers off the guest. - When we receive the "SHUTDOWN" event from QEMU monitor, if hardReboot is not set, then check fakeReboot or the last choice to invoke qemuProcessKill command to shutdown normally. - If hardReboot was set, we spawn a background thread, which issues qemuProcessStop to kill the QEMU process and cleanup the working environment. Then issue qemuProcessStart to create a new QEMU process with the same domain definition. At last, issues 'cont' to start the CPUs again. The state transfer during the hardReboot is RUNNING->SHUTDOWN->SHUTOFF->PAUSED->RUNNING, This patch doesn't hide it as the states is not transient. Signed-off-by: Chenyi Qiang <chenyi.qiang(a)intel.com&gt; --- include/libvirt/libvirt-domain.h | 2 + src/qemu/qemu_domain.c | 18 +++++++ src/qemu/qemu_domain.h | 4 ++ src/qemu/qemu_driver.c | 76 +++++++++++++++++++++------ src/qemu/qemu_process.c | 88 +++++++++++++++++++++++++++++++- tools/virsh-domain.c | 8 ++- 6 files changed, 177 insertions(+), 19 deletions(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index a1902546bb..5cedbbd1fc 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -1520,6 +1520,7 @@ typedef enum { VIR_DOMAIN_SHUTDOWN_INITCTL = (1 << 2), /* Use initctl (Since: 1.0.1) */ VIR_DOMAIN_SHUTDOWN_SIGNAL = (1 << 3), /* Send a signal (Since: 1.0.1) */ VIR_DOMAIN_SHUTDOWN_PARAVIRT = (1 << 4), /* Use paravirt guest control (Since: 1.2.5) */ + VIR_DOMAIN_SHUTDOWN_HARD = (1 << 5), /* Powercycle control (Since: 8.6.0) */ } virDomainShutdownFlagValues; int virDomainShutdown (virDomainPtr domain); @@ -1538,6 +1539,7 @@ typedef enum { VIR_DOMAIN_REBOOT_INITCTL = (1 << 2), /* Use initctl (Since: 1.0.1) */ VIR_DOMAIN_REBOOT_SIGNAL = (1 << 3), /* Send a signal (Since: 1.0.1) */ VIR_DOMAIN_REBOOT_PARAVIRT = (1 << 4), /* Use paravirt guest control (Since: 1.2.5) */ + VIR_DOMAIN_REBOOT_HARD = (1 << 5), /* Powercycle control (Since: 8.6.0) */ } virDomainRebootFlagValues; int virDomainReboot (virDomainPtr domain, diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index ae19ce884b..b65d85f91b 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -2614,6 +2614,9 @@ qemuDomainObjPrivateXMLFormat(virBuffer *buf, if (priv->fakeReboot) virBufferAddLit(buf, "<fakereboot/>\n"); + if (priv->hardReboot) + virBufferAddLit(buf, "<hardreboot/>\n"); + if (priv->qemuDevices && *priv->qemuDevices) { char **tmp = priv->qemuDevices; virBufferAddLit(buf, "<devices>\n"); @@ -3305,6 +3308,8 @@ qemuDomainObjPrivateXMLParse(xmlXPathContextPtr ctxt, priv->fakeReboot = virXPathBoolean("boolean(./fakereboot)", ctxt) == 1; + priv->hardReboot = virXPathBoolean("boolean(./hardreboot)", ctxt) == 1; + if ((n = virXPathNodeSet("./devices/device", ctxt, &nodes)) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("failed to parse qemu device list")); @@ -7445,6 +7450,19 @@ qemuDomainSetFakeReboot(virDomainObj *vm, qemuDomainSaveStatus(vm); } +void +qemuDomainSetHardReboot(virDomainObj *vm, + bool value) +{ + qemuDomainObjPrivate *priv = vm->privateData; + + if (priv->hardReboot == value) + return; + + priv->hardReboot = value; + qemuDomainSaveStatus(vm); +} + static void qemuDomainCheckRemoveOptionalDisk(virQEMUDriver *driver, virDomainObj *vm, diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 1e56e50672..fd6058c5bc 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -137,6 +137,7 @@ struct _qemuDomainObjPrivate { * -no-reboot instead. */ virTristateBool allowReboot; + bool hardReboot; unsigned long migMaxBandwidth; char *origname; @@ -700,6 +701,9 @@ qemuDomainRemoveInactiveLocked(virQEMUDriver *driver, void qemuDomainSetFakeReboot(virDomainObj *vm, bool value); +void qemuDomainSetHardReboot(virDomainObj *vm, + bool value); + int qemuDomainCheckDiskStartupPolicy(virQEMUDriver *driver, virDomainObj *vm, size_t diskIndex, diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index d00d2a27c6..86e8efbfcb 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -1788,6 +1788,7 @@ qemuDomainShutdownFlagsAgent(virDomainObj *vm, goto endjob; qemuDomainSetFakeReboot(vm, false); + qemuDomainSetHardReboot(vm, false); agent = qemuDomainObjEnterAgent(vm); ret = qemuAgentShutdown(agent, agentFlag); qemuDomainObjExitAgent(vm, agent); @@ -1800,7 +1801,8 @@ qemuDomainShutdownFlagsAgent(virDomainObj *vm, static int qemuDomainShutdownFlagsMonitor(virDomainObj *vm, - bool isReboot) + bool isReboot, + bool hard) { int ret = -1; qemuDomainObjPrivate *priv; @@ -1816,7 +1818,17 @@ qemuDomainShutdownFlagsMonitor(virDomainObj *vm, goto endjob; } - qemuDomainSetFakeReboot(vm, isReboot); + if (hard) { + /* hard reboot control the reboot */ + VIR_DEBUG("Set hard reboot %d in shutdown monitor", isReboot); + qemuDomainSetHardReboot(vm, isReboot); + qemuDomainSetFakeReboot(vm, false); + } else { + /* fake reboot control the reboot */ + VIR_DEBUG("Set fake reboot %d in shutdown monitor", isReboot); + qemuDomainSetFakeReboot(vm, isReboot); + qemuDomainSetHardReboot(vm, false); + } qemuDomainObjEnterMonitor(vm); ret = qemuMonitorSystemPowerdown(priv->mon); qemuDomainObjExitMonitor(vm); @@ -1832,12 +1844,13 @@ static int qemuDomainShutdownFlags(virDomainPtr dom, unsigned int flags) virDomainObj *vm; int ret = -1; qemuDomainObjPrivate *priv; - bool useAgent = false, agentRequested, acpiRequested; + bool useAgent = false, agentRequested, acpiRequested, hardRequested; bool isReboot = false; bool agentForced; virCheckFlags(VIR_DOMAIN_SHUTDOWN_ACPI_POWER_BTN | - VIR_DOMAIN_SHUTDOWN_GUEST_AGENT, -1); + VIR_DOMAIN_SHUTDOWN_GUEST_AGENT | + VIR_DOMAIN_SHUTDOWN_HARD, -1); if (!(vm = qemuDomainObjFromDomain(dom))) goto cleanup; @@ -1851,14 +1864,23 @@ static int qemuDomainShutdownFlags(virDomainPtr dom, unsigned int flags) priv = vm->privateData; agentRequested = flags & VIR_DOMAIN_SHUTDOWN_GUEST_AGENT; acpiRequested = flags & VIR_DOMAIN_SHUTDOWN_ACPI_POWER_BTN; + hardRequested = flags & VIR_DOMAIN_SHUTDOWN_HARD; + + if (virDomainShutdownFlagsEnsureACL(dom->conn, vm->def, flags) < 0) + goto cleanup; + + /* Take hard reboot as the highest priority. + * if failed, consider the agent and acpi */ + if (hardRequested) + ret = qemuDomainShutdownFlagsMonitor(vm, isReboot, true); + + if (!ret) + goto cleanup; /* Prefer agent unless we were requested to not to. */ if (agentRequested || (!flags && priv->agent)) useAgent = true; - if (virDomainShutdownFlagsEnsureACL(dom->conn, vm->def, flags) < 0) - goto cleanup; - agentForced = agentRequested && !acpiRequested; if (useAgent) { ret = qemuDomainShutdownFlagsAgent(vm, isReboot, agentForced); @@ -1877,7 +1899,7 @@ static int qemuDomainShutdownFlags(virDomainPtr dom, unsigned int flags) goto cleanup; } - ret = qemuDomainShutdownFlagsMonitor(vm, isReboot); + ret = qemuDomainShutdownFlagsMonitor(vm, isReboot, false); } cleanup: @@ -1913,6 +1935,7 @@ qemuDomainRebootAgent(virDomainObj *vm, goto endjob; qemuDomainSetFakeReboot(vm, false); + qemuDomainSetHardReboot(vm, false); agent = qemuDomainObjEnterAgent(vm); ret = qemuAgentShutdown(agent, agentFlag); qemuDomainObjExitAgent(vm, agent); @@ -1925,7 +1948,8 @@ qemuDomainRebootAgent(virDomainObj *vm, static int qemuDomainRebootMonitor(virDomainObj *vm, - bool isReboot) + bool isReboot, + bool hard) { qemuDomainObjPrivate *priv = vm->privateData; int ret = -1; @@ -1936,7 +1960,15 @@ qemuDomainRebootMonitor(virDomainObj *vm, if (virDomainObjCheckActive(vm) < 0) goto endjob; - qemuDomainSetFakeReboot(vm, isReboot); + if (hard) { + VIR_DEBUG("Set hard reboot %d in reboot monitor", isReboot); + qemuDomainSetHardReboot(vm, isReboot); + qemuDomainSetFakeReboot(vm, false); + } else { + VIR_DEBUG("Set fake reboot %d in reboot monitor", isReboot); + qemuDomainSetFakeReboot(vm, isReboot); + qemuDomainSetHardReboot(vm, false); + } qemuDomainObjEnterMonitor(vm); ret = qemuMonitorSystemPowerdown(priv->mon); qemuDomainObjExitMonitor(vm); @@ -1953,12 +1985,13 @@ qemuDomainReboot(virDomainPtr dom, unsigned int flags) virDomainObj *vm; int ret = -1; qemuDomainObjPrivate *priv; - bool useAgent = false, agentRequested, acpiRequested; + bool useAgent = false, agentRequested, acpiRequested, hardRequested; bool isReboot = true; bool agentForced; virCheckFlags(VIR_DOMAIN_REBOOT_ACPI_POWER_BTN | - VIR_DOMAIN_REBOOT_GUEST_AGENT, -1); + VIR_DOMAIN_REBOOT_GUEST_AGENT | + VIR_DOMAIN_REBOOT_HARD, -1); if (!(vm = qemuDomainObjFromDomain(dom))) goto cleanup; @@ -1972,14 +2005,24 @@ qemuDomainReboot(virDomainPtr dom, unsigned int flags) priv = vm->privateData; agentRequested = flags & VIR_DOMAIN_REBOOT_GUEST_AGENT; acpiRequested = flags & VIR_DOMAIN_REBOOT_ACPI_POWER_BTN; + hardRequested = flags & VIR_DOMAIN_REBOOT_HARD; + + if (virDomainRebootEnsureACL(dom->conn, vm->def, flags) < 0) + goto cleanup; + + /* Take hard reboot as the highest priority. + * This is for TDX which is not allowed to warm reboot. + */ + if (hardRequested) + ret = qemuDomainRebootMonitor(vm, isReboot, true); + + if (!ret) + goto cleanup; /* Prefer agent unless we were requested to not to. */ if (agentRequested || (!flags && priv->agent)) useAgent = true; - if (virDomainRebootEnsureACL(dom->conn, vm->def, flags) < 0) - goto cleanup; - agentForced = agentRequested && !acpiRequested; if (useAgent) ret = qemuDomainRebootAgent(vm, isReboot, agentForced); @@ -1992,7 +2035,7 @@ qemuDomainReboot(virDomainPtr dom, unsigned int flags) */ if ((!useAgent) || (ret < 0 && (acpiRequested || !flags))) { - ret = qemuDomainRebootMonitor(vm, isReboot); + ret = qemuDomainRebootMonitor(vm, isReboot, false); } cleanup: @@ -2095,6 +2138,7 @@ qemuDomainDestroyFlags(virDomainPtr dom, } qemuDomainSetFakeReboot(vm, false); + qemuDomainSetHardReboot(vm, false); if (vm->job->asyncJob == VIR_ASYNC_JOB_MIGRATION_IN) stopFlags |= VIR_QEMU_PROCESS_STOP_MIGRATED; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index f27f6653f5..9fa679e408 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -502,13 +502,98 @@ qemuProcessFakeReboot(void *opaque) virDomainObjEndAPI(&vm); } +static void +qemuProcessHardReboot(void *opaque) +{ + virDomainObj *vm = opaque; + qemuDomainObjPrivate *priv = vm->privateData; + virQEMUDriver *driver = priv->driver; + unsigned int stopFlags = 0; + virObjectEvent *event = NULL; + virObjectEvent * event2 = NULL; + + VIR_DEBUG("Handle hard reboot: destroy phase"); + + virObjectLock(vm); + if (virDomainObjCheckActive(vm) < 0) + goto cleanup; + + if (qemuProcessBeginStopJob(vm, VIR_JOB_DESTROY, 0) < 0) + goto cleanup; + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + "%s", _("domain is not running")); + virDomainObjEndJob(vm); + goto cleanup; + } + + if (vm->job->asyncJob == VIR_ASYNC_JOB_MIGRATION_IN) + stopFlags |= VIR_QEMU_PROCESS_STOP_MIGRATED; + + qemuProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_DESTROYED, + VIR_ASYNC_JOB_NONE, stopFlags); + virDomainAuditStop(vm, "destroyed"); + + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_DESTROYED); + + virObjectEventStateQueue(driver->domainEventState, event); + /* skip remove inactive domain from active list */ + virDomainObjEndJob(vm); + + VIR_DEBUG("Handle hard reboot: boot phase"); + + if (qemuProcessBeginJob(vm, VIR_DOMAIN_JOB_OPERATION_START, + 0) < 0) { + qemuDomainRemoveInactive(driver, vm, 0, false); + goto cleanup; + } + + if (qemuProcessStart(NULL, driver, vm, NULL, VIR_ASYNC_JOB_START, + NULL, -1, NULL, NULL, + VIR_NETDEV_VPORT_PROFILE_OP_CREATE, + 0) < 0) { + virDomainAuditStart(vm, "booted", false); + qemuDomainRemoveInactive(driver, vm, 0, false); + qemuProcessEndJob(vm); + goto cleanup; + } + + virDomainAuditStart(vm, "booted", true); + event2 = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_STARTED, + VIR_DOMAIN_EVENT_STARTED_BOOTED); + virObjectEventStateQueue(driver->domainEventState, event2); + + qemuProcessEndJob(vm); + + cleanup: + virDomainObjEndAPI(&vm); +} void qemuProcessShutdownOrReboot(virDomainObj *vm) { qemuDomainObjPrivate *priv = vm->privateData; - if (priv->fakeReboot || + if (priv->hardReboot) { + g_autofree char *name = g_strdup_printf("hard reboot-%s", vm->def->name); + virThread th; + + virObjectRef(vm); + if (virThreadCreateFull(&th, + false, + qemuProcessHardReboot, + name, + false, + vm) < 0) { + VIR_ERROR(_("Failed to create hard reboot thread, killing domain")); + ignore_value(qemuProcessKill(vm, VIR_QEMU_PROCESS_KILL_NOWAIT)); + virObjectUnref(vm); + } + } else if (priv->fakeReboot || vm->def->onPoweroff == VIR_DOMAIN_LIFECYCLE_ACTION_RESTART) { g_autofree char *name = g_strdup_printf("reboot-%s", vm->def->name); virThread th; @@ -5673,6 +5758,7 @@ qemuProcessInit(virQEMUDriver *driver, } else { vm->def->id = qemuDriverAllocateID(driver); qemuDomainSetFakeReboot(vm, false); + qemuDomainSetFakeReboot(vm, false); virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, VIR_DOMAIN_PAUSED_STARTING_UP); if (g_atomic_int_add(&driver->nactive, 1) == 0 && driver->inhibitCallback) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index 66f933dead..21864c92cb 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -5915,7 +5915,7 @@ static const vshCmdOptDef opts_shutdown[] = { {.name = "mode", .type = VSH_OT_STRING, .completer = virshDomainShutdownModeCompleter, - .help = N_("shutdown mode: acpi|agent|initctl|signal|paravirt") + .help = N_("shutdown mode: acpi|agent|initctl|signal|paravirt|hard") }, {.name = NULL} }; @@ -5952,6 +5952,8 @@ cmdShutdown(vshControl *ctl, const vshCmd *cmd) flags |= VIR_DOMAIN_SHUTDOWN_SIGNAL; } else if (STREQ(mode, "paravirt")) { flags |= VIR_DOMAIN_SHUTDOWN_PARAVIRT; + } else if (STREQ(mode, "hard")) { + flags |= VIR_DOMAIN_SHUTDOWN_HARD; } else { vshError(ctl, _("Unknown mode %1$s value, expecting 'acpi', 'agent', 'initctl', 'signal', or 'paravirt'"), mode); @@ -5995,7 +5997,7 @@ static const vshCmdOptDef opts_reboot[] = { {.name = "mode", .type = VSH_OT_STRING, .completer = virshDomainShutdownModeCompleter, - .help = N_("shutdown mode: acpi|agent|initctl|signal|paravirt") + .help = N_("shutdown mode: acpi|agent|initctl|signal|paravirt|hard") }, {.name = NULL} }; @@ -6031,6 +6033,8 @@ cmdReboot(vshControl *ctl, const vshCmd *cmd) flags |= VIR_DOMAIN_REBOOT_SIGNAL; } else if (STREQ(mode, "paravirt")) { flags |= VIR_DOMAIN_REBOOT_PARAVIRT; + } else if (STREQ(mode, "hard")) { + flags |= VIR_DOMAIN_REBOOT_HARD; } else { vshError(ctl, _("Unknown mode %1$s value, expecting 'acpi', 'agent', 'initctl', 'signal' or 'paravirt'"), mode); -- 2.34.1