[libvirt] [PATCH] Keep track of guest paused state after disk IO / watchdog events

When a watchdog/IO error occurs, one of the possible actions that QEMU might take is to pause the guest. In this scenario libvirt needs to update its internal state for the VM, and emit a lifecycle event: VIR_DOMAIN_EVENT_SUSPENDED with a detail being one of: VIR_DOMAIN_EVENT_SUSPENDED_IOERROR VIR_DOMAIN_EVENT_SUSPENDED_WATCHDOG To future proof against possible QEMU support for multiple monitor consoles, this patch also hooks into the 'STOPPED' event in QEMU and emits a generic VIR_DOMAIN_EVENT_SUSPENDED_PAUSED event * include/libvirt/libvirt.h.in: Add VIR_DOMAIN_EVENT_SUSPENDED_IOERROR * src/qemu/qemu_driver.c: Update VM state to paused when IO error or watchdog events occurrs * src/qemu/qemu_monitor_json.c: Fix typo in disk IO event name --- include/libvirt/libvirt.h.in | 2 + src/qemu/qemu_driver.c | 83 ++++++++++++++++++++++++++++++++++++++---- src/qemu/qemu_monitor_json.c | 2 +- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in index 6d8552f..7cb483e 100644 --- a/include/libvirt/libvirt.h.in +++ b/include/libvirt/libvirt.h.in @@ -1384,6 +1384,8 @@ typedef enum { typedef enum { VIR_DOMAIN_EVENT_SUSPENDED_PAUSED = 0, /* Normal suspend due to admin pause */ VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED = 1, /* Suspended for offline migration */ + VIR_DOMAIN_EVENT_SUSPENDED_IOERROR = 2, /* Suspended due to a disk I/O error */ + VIR_DOMAIN_EVENT_SUSPENDED_WATCHDOG = 3, /* Suspended due to a watchdog firing */ } virDomainEventSuspendedDetailType; /** diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 9ee5da7..4291bc7 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -909,6 +909,38 @@ qemuHandleDomainReset(qemuMonitorPtr mon ATTRIBUTE_UNUSED, static int +qemuHandleDomainStop(qemuMonitorPtr mon ATTRIBUTE_UNUSED, + virDomainObjPtr vm) +{ + struct qemud_driver *driver = qemu_driver; + virDomainEventPtr event = NULL; + + virDomainObjLock(vm); + if (vm->state == VIR_DOMAIN_RUNNING) { + VIR_DEBUG("Transitioned guest %s to paused state due to unknown event", vm->def->name); + + vm->state = VIR_DOMAIN_PAUSED; + event = virDomainEventNewFromObj(vm, + VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_PAUSED); + + if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0) + VIR_WARN("Unable to save status on vm %s after IO error", vm->def->name); + } + virDomainObjUnlock(vm); + + if (event) { + qemuDriverLock(driver); + if (event) + qemuDomainEventQueue(driver, event); + qemuDriverUnlock(driver); + } + + return 0; +} + + +static int qemuHandleDomainRTCChange(qemuMonitorPtr mon ATTRIBUTE_UNUSED, virDomainObjPtr vm, long long offset) @@ -943,15 +975,32 @@ qemuHandleDomainWatchdog(qemuMonitorPtr mon ATTRIBUTE_UNUSED, int action) { struct qemud_driver *driver = qemu_driver; - virDomainEventPtr event; + virDomainEventPtr watchdogEvent = NULL; + virDomainEventPtr lifecycleEvent = NULL; virDomainObjLock(vm); - event = virDomainEventWatchdogNewFromObj(vm, action); + watchdogEvent = virDomainEventWatchdogNewFromObj(vm, action); + + if (action == VIR_DOMAIN_EVENT_WATCHDOG_PAUSE && + vm->state == VIR_DOMAIN_RUNNING) { + VIR_DEBUG("Transitioned guest %s to paused state due to watchdog", vm->def->name); + + vm->state = VIR_DOMAIN_PAUSED; + lifecycleEvent = virDomainEventNewFromObj(vm, + VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_WATCHDOG); + + if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0) + VIR_WARN("Unable to save status on vm %s after IO error", vm->def->name); + } virDomainObjUnlock(vm); - if (event) { + if (watchdogEvent || lifecycleEvent) { qemuDriverLock(driver); - qemuDomainEventQueue(driver, event); + if (watchdogEvent) + qemuDomainEventQueue(driver, watchdogEvent); + if (lifecycleEvent) + qemuDomainEventQueue(driver, lifecycleEvent); qemuDriverUnlock(driver); } @@ -966,7 +1015,8 @@ qemuHandleDomainIOError(qemuMonitorPtr mon ATTRIBUTE_UNUSED, int action) { struct qemud_driver *driver = qemu_driver; - virDomainEventPtr event; + virDomainEventPtr ioErrorEvent = NULL; + virDomainEventPtr lifecycleEvent = NULL; const char *srcPath; const char *devAlias; virDomainDiskDefPtr disk; @@ -982,12 +1032,28 @@ qemuHandleDomainIOError(qemuMonitorPtr mon ATTRIBUTE_UNUSED, devAlias = ""; } - event = virDomainEventIOErrorNewFromObj(vm, srcPath, devAlias, action); + ioErrorEvent = virDomainEventIOErrorNewFromObj(vm, srcPath, devAlias, action); + + if (action == VIR_DOMAIN_EVENT_IO_ERROR_PAUSE && + vm->state == VIR_DOMAIN_RUNNING) { + VIR_DEBUG("Transitioned guest %s to paused state due to IO error", vm->def->name); + + vm->state = VIR_DOMAIN_PAUSED; + lifecycleEvent = virDomainEventNewFromObj(vm, + VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_IOERROR); + + if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0) + VIR_WARN("Unable to save status on vm %s after IO error", vm->def->name); + } virDomainObjUnlock(vm); - if (event) { + if (ioErrorEvent || lifecycleEvent) { qemuDriverLock(driver); - qemuDomainEventQueue(driver, event); + if (ioErrorEvent) + qemuDomainEventQueue(driver, ioErrorEvent); + if (lifecycleEvent) + qemuDomainEventQueue(driver, lifecycleEvent); qemuDriverUnlock(driver); } @@ -1090,6 +1156,7 @@ no_memory: static qemuMonitorCallbacks monitorCallbacks = { .eofNotify = qemuHandleMonitorEOF, .diskSecretLookup = findVolumeQcowPassphrase, + .domainStop = qemuHandleDomainStop, .domainReset = qemuHandleDomainReset, .domainRTCChange = qemuHandleDomainRTCChange, .domainWatchdog = qemuHandleDomainWatchdog, diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c index 3901d46..eac3aca 100644 --- a/src/qemu/qemu_monitor_json.c +++ b/src/qemu/qemu_monitor_json.c @@ -66,7 +66,7 @@ struct { { "STOP", qemuMonitorJSONHandleStop, }, { "RTC_CHANGE", qemuMonitorJSONHandleRTCChange, }, { "WATCHDOG", qemuMonitorJSONHandleWatchdog, }, - { "DISK_IO_ERROR", qemuMonitorJSONHandleIOError, }, + { "BLOCK_IO_ERROR", qemuMonitorJSONHandleIOError, }, { "VNC_CONNECTED", qemuMonitorJSONHandleVNCConnect, }, { "VNC_INITIALIZED", qemuMonitorJSONHandleVNCInitialize, }, { "VNC_DISCONNECTED", qemuMonitorJSONHandleVNCDisconnect, }, -- 1.6.2.5

On Wed, Mar 31, 2010 at 01:14:13PM +0100, Daniel P. Berrange wrote:
When a watchdog/IO error occurs, one of the possible actions that QEMU might take is to pause the guest. In this scenario libvirt needs to update its internal state for the VM, and emit a lifecycle event:
VIR_DOMAIN_EVENT_SUSPENDED
with a detail being one of:
VIR_DOMAIN_EVENT_SUSPENDED_IOERROR VIR_DOMAIN_EVENT_SUSPENDED_WATCHDOG
To future proof against possible QEMU support for multiple monitor consoles, this patch also hooks into the 'STOPPED' event in QEMU and emits a generic VIR_DOMAIN_EVENT_SUSPENDED_PAUSED event
you mean if a command was emitted on one console stopping the guest so that the event is captured on other consoles. I wonder what's the limits here, it sounds impossible to garantee a coherent view in general in case of access on an extra console outside of libvirtd control. So you plan on doing some kind of best-effort to cover this new use case ?
* include/libvirt/libvirt.h.in: Add VIR_DOMAIN_EVENT_SUSPENDED_IOERROR * src/qemu/qemu_driver.c: Update VM state to paused when IO error or watchdog events occurrs * src/qemu/qemu_monitor_json.c: Fix typo in disk IO event name [...]
ACK, but I wonder how far we can go in that event reinjection process without risks, Daniel -- Daniel Veillard | libxml Gnome XML XSLT toolkit http://xmlsoft.org/ daniel@veillard.com | Rpmfind RPM search engine http://rpmfind.net/ http://veillard.com/ | virtualization library http://libvirt.org/

On Wed, Mar 31, 2010 at 03:44:17PM +0200, Daniel Veillard wrote:
On Wed, Mar 31, 2010 at 01:14:13PM +0100, Daniel P. Berrange wrote:
When a watchdog/IO error occurs, one of the possible actions that QEMU might take is to pause the guest. In this scenario libvirt needs to update its internal state for the VM, and emit a lifecycle event:
VIR_DOMAIN_EVENT_SUSPENDED
with a detail being one of:
VIR_DOMAIN_EVENT_SUSPENDED_IOERROR VIR_DOMAIN_EVENT_SUSPENDED_WATCHDOG
To future proof against possible QEMU support for multiple monitor consoles, this patch also hooks into the 'STOPPED' event in QEMU and emits a generic VIR_DOMAIN_EVENT_SUSPENDED_PAUSED event
you mean if a command was emitted on one console stopping the guest so that the event is captured on other consoles. I wonder what's the limits here, it sounds impossible to garantee a coherent view in general in case of access on an extra console outside of libvirtd control. So you plan on doing some kind of best-effort to cover this new use case ?
It is currently impossible to cope with this in general, but over time QEMU needs to provide events for any machine config changes so that apps can track them. The STOPPED state is a fairly important & easy one to deal with though. Daniel -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://deltacloud.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
participants (2)
-
Daniel P. Berrange
-
Daniel Veillard