The `--event-monitor` option in cloud-hypervisor outputs events to a specified file. This file can then be used to monitor VM lifecycle, other vmm events and trigger appropriate actions. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> Co-authored-by: Vineeth Pillai <viremana@linux.microsoft.com> --- src/ch/ch_monitor.c | 20 ++++++++++++++++---- src/ch/ch_monitor.h | 2 ++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index 3e49902791..21a9ee273e 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -540,7 +540,6 @@ virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg) { g_autoptr(virCHMonitor) mon = NULL; g_autoptr(virCommand) cmd = NULL; - const char *socketdir = cfg->stateDir; int socket_fd = 0; if (virCHMonitorInitialize() < 0) @@ -556,11 +555,13 @@ virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg) } /* prepare to launch Cloud-Hypervisor socket */ - mon->socketpath = g_strdup_printf("%s/%s-socket", socketdir, vm->def->name); - if (g_mkdir_with_parents(socketdir, 0777) < 0) { + mon->socketpath = g_strdup_printf("%s/%s-socket", cfg->stateDir, vm->def->name); + mon->eventmonitorpath = g_strdup_printf("%s/%s-event-monitor", + cfg->stateDir, vm->def->name); + if (g_mkdir_with_parents(cfg->stateDir, 0777) < 0) { virReportSystemError(errno, _("Cannot create socket directory '%1$s'"), - socketdir); + cfg->stateDir); return NULL; } @@ -585,6 +586,9 @@ virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg) virCommandAddArgFormat(cmd, "fd=%d", socket_fd); virCommandPassFD(cmd, socket_fd, VIR_COMMAND_PASS_FD_CLOSE_PARENT); + virCommandAddArg(cmd, "--event-monitor"); + virCommandAddArgFormat(cmd, "path=%s", mon->eventmonitorpath); + /* launch Cloud-Hypervisor socket */ if (virCommandRunAsync(cmd, &mon->pid) < 0) return NULL; @@ -629,6 +633,14 @@ void virCHMonitorClose(virCHMonitor *mon) g_free(mon->socketpath); } + if (virFileExists(mon->eventmonitorpath)) { + if (virFileRemove(mon->eventmonitorpath, -1, -1) < 0) { + VIR_WARN("Unable to remove CH event monitor file '%s'", + mon->eventmonitorpath); + } + g_free(mon->eventmonitorpath); + } + virObjectUnref(mon); } diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index b35f5ea027..2ef8706b99 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -95,6 +95,8 @@ struct _virCHMonitor { char *socketpath; + char *eventmonitorpath; + pid_t pid; virDomainObj *vm; -- 2.34.1

Use a FIFO(named pipe) for --event-monitor option in CH. Introduce a new thread, `virCHEventHandlerLoop`, to continuously monitor and handle events from cloud-hypervisor. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> Co-authored-by: Vineeth Pillai <viremana@linux.microsoft.com> --- po/POTFILES | 1 + src/ch/ch_events.c | 93 +++++++++++++++++++++++++++++++++++++++++++++ src/ch/ch_events.h | 26 +++++++++++++ src/ch/ch_monitor.c | 32 ++++++++++++++-- src/ch/ch_monitor.h | 3 ++ src/ch/meson.build | 2 + 6 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 src/ch/ch_events.c create mode 100644 src/ch/ch_events.h diff --git a/po/POTFILES b/po/POTFILES index 1ed4086d2c..d53307cec4 100644 --- a/po/POTFILES +++ b/po/POTFILES @@ -21,6 +21,7 @@ src/bhyve/bhyve_process.c src/ch/ch_conf.c src/ch/ch_domain.c src/ch/ch_driver.c +src/ch/ch_events.c src/ch/ch_interface.c src/ch/ch_monitor.c src/ch/ch_process.c diff --git a/src/ch/ch_events.c b/src/ch/ch_events.c new file mode 100644 index 0000000000..bb27f340e2 --- /dev/null +++ b/src/ch/ch_events.c @@ -0,0 +1,93 @@ +/* + * Copyright Microsoft Corp. 2024 + * + * ch_events.c: Handle Cloud-Hypervisor events + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + */ + +#include <config.h> + +#include <fcntl.h> + +#include "ch_events.h" +#include "virfile.h" +#include "virlog.h" + +VIR_LOG_INIT("ch.ch_events"); + +static void virCHEventHandlerLoop(void *data) +{ + virCHMonitor *mon = data; + virDomainObj *vm = NULL; + int event_monitor_fd; + + /* Obtain a vm reference */ + vm = virObjectRef(mon->vm); + + VIR_DEBUG("%s: Event handler loop thread starting", vm->def->name); + + while ((event_monitor_fd = open(mon->eventmonitorpath, O_RDONLY)) < 0) { + if (errno == EINTR) { + /* 100 milli seconds */ + g_usleep(100000); + continue; + } + /* Any other error should be a BUG(kernel/libc/libvirtd) + * (ENOMEM can happen on exceeding per-user limits) + */ + VIR_ERROR(_("%1$s: Failed to open the event monitor FIFO(%2$s) read end!"), + vm->def->name, mon->eventmonitorpath); + abort(); + } + VIR_DEBUG("%s: Opened the event monitor FIFO(%s)", vm->def->name, mon->eventmonitorpath); + + while (g_atomic_int_get(&mon->event_handler_stop) == 0) { + VIR_DEBUG("%s: Reading events from event monitor file", vm->def->name); + /* Read and process events here */ + } + + VIR_FORCE_CLOSE(event_monitor_fd); + virObjectUnref(vm); + + VIR_DEBUG("%s: Event handler loop thread exiting", vm->def->name); + return; +} + +int virCHStartEventHandler(virCHMonitor *mon) +{ + g_autofree char *name = NULL; + name = g_strdup_printf("ch-evt-%d", mon->pid); + + virObjectRef(mon); + if (virThreadCreateFull(&mon->event_handler_thread, + false, + virCHEventHandlerLoop, + name, + false, + mon) < 0) { + virObjectUnref(mon); + return -1; + } + virObjectUnref(mon); + + g_atomic_int_set(&mon->event_handler_stop, 0); + return 0; +} + +void virCHStopEventHandler(virCHMonitor *mon) +{ + g_atomic_int_set(&mon->event_handler_stop, 1); +} diff --git a/src/ch/ch_events.h b/src/ch/ch_events.h new file mode 100644 index 0000000000..4c8a48231d --- /dev/null +++ b/src/ch/ch_events.h @@ -0,0 +1,26 @@ +/* + * Copyright Microsoft Corp. 2024 + * + * ch_events.h: header file for handling Cloud-Hypervisor events + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + */ + +#pragma once + +#include "ch_monitor.h" + +int virCHStartEventHandler(virCHMonitor *mon); +void virCHStopEventHandler(virCHMonitor *mon); diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index 21a9ee273e..097e64d9d5 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -26,6 +26,7 @@ #include "datatypes.h" #include "ch_conf.h" +#include "ch_events.h" #include "ch_interface.h" #include "ch_monitor.h" #include "domain_interface.h" @@ -572,6 +573,27 @@ virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg) return NULL; } + /* Event monitor file to listen for VM state changes */ + mon->eventmonitorpath = g_strdup_printf("%s/%s-event-monitor-fifo", + cfg->stateDir, vm->def->name); + if (virFileExists(mon->eventmonitorpath)) { + VIR_WARN("Monitor file (%s) already exists, trying to delete!", + mon->eventmonitorpath); + if (virFileRemove(mon->eventmonitorpath, -1, -1) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to remove the file: %1$s"), + mon->eventmonitorpath); + return NULL; + } + } + + if (mkfifo(mon->eventmonitorpath, S_IWUSR | S_IRUSR) < 0 && + errno != EEXIST) { + virReportSystemError(errno, "%s", + _("Cannot create monitor FIFO")); + return NULL; + } + cmd = virCommandNew(vm->def->emulator); virCommandSetUmask(cmd, 0x002); socket_fd = chMonitorCreateSocket(mon->socketpath); @@ -593,12 +615,15 @@ virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg) if (virCommandRunAsync(cmd, &mon->pid) < 0) return NULL; - /* get a curl handle */ - mon->handle = curl_easy_init(); - /* now has its own reference */ mon->vm = virObjectRef(vm); + if (virCHStartEventHandler(mon) < 0) + return NULL; + + /* get a curl handle */ + mon->handle = curl_easy_init(); + return g_steal_pointer(&mon); } @@ -633,6 +658,7 @@ void virCHMonitorClose(virCHMonitor *mon) g_free(mon->socketpath); } + virCHStopEventHandler(mon); if (virFileExists(mon->eventmonitorpath)) { if (virFileRemove(mon->eventmonitorpath, -1, -1) < 0) { VIR_WARN("Unable to remove CH event monitor file '%s'", diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index 2ef8706b99..878a185f29 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -97,6 +97,9 @@ struct _virCHMonitor { char *eventmonitorpath; + virThread event_handler_thread; + int event_handler_stop; + pid_t pid; virDomainObj *vm; diff --git a/src/ch/meson.build b/src/ch/meson.build index 633966aac7..77bbb2d450 100644 --- a/src/ch/meson.build +++ b/src/ch/meson.build @@ -7,6 +7,8 @@ ch_driver_sources = [ 'ch_domain.h', 'ch_driver.c', 'ch_driver.h', + 'ch_events.c', + 'ch_events.h', 'ch_interface.c', 'ch_interface.h', 'ch_monitor.c', -- 2.34.1

Implement `chReadProcessEvents` and `chProcessEvents` to read events from event monitor FIFO file and parse them accordingly. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> Co-authored-by: Vineeth Pillai <viremana@linux.microsoft.com> --- src/ch/ch_events.c | 136 +++++++++++++++++++++++++++++++++++++++++++- src/ch/ch_events.h | 2 + src/ch/ch_monitor.h | 6 ++ 3 files changed, 143 insertions(+), 1 deletion(-) diff --git a/src/ch/ch_events.c b/src/ch/ch_events.c index bb27f340e2..d0ad5af0af 100644 --- a/src/ch/ch_events.c +++ b/src/ch/ch_events.c @@ -28,6 +28,136 @@ VIR_LOG_INIT("ch.ch_events"); +/** + * virCHProcessEvents: + * @mon: the CH monitor object + * + * Parse the events from the event buffer and process them + * Example event: + * { + * "timestamp": { + * "secs": 0, + * "nanos": 29228206 + * }, + * "source": "vm", + * "event": "booted", + * "properties": null + * } + * + * Returns: 0 on success, -1 on failure + */ +static int virCHProcessEvents(virCHMonitor *mon) +{ + virDomainObj *vm = mon->vm; + char *buf = mon->event_buffer.buffer; + ssize_t sz = mon->event_buffer.buf_fill_sz; + virJSONValue *obj = NULL; + int blocks = 0; + size_t i = 0; + char *json_start; + ssize_t start_index = -1; + ssize_t end_index = -1; + char tmp; + int ret = 0; + + while (i < sz) { + if (buf[i] == '{') { + blocks++; + if (blocks == 1) + start_index = i; + } else if (buf[i] == '}' && blocks > 0) { + blocks--; + if (blocks == 0) { + /* valid json document */ + end_index = i; + + /* temporarily null terminate the JSON doc */ + tmp = buf[end_index + 1]; + buf[end_index + 1] = '\0'; + json_start = buf + start_index; + + if ((obj = virJSONValueFromString(json_start))) { + /* Process the event string (obj) here */ + virJSONValueFree(obj); + } else { + VIR_WARN("%s: Invalid JSON event doc: %s", + vm->def->name, json_start); + ret = -1; + } + + /* replace the original character */ + buf[end_index + 1] = tmp; + start_index = -1; + } + } + + i++; + } + + if (start_index == -1) { + /* We have processed all the JSON docs in the buffer */ + mon->event_buffer.buf_fill_sz = 0; + } else if (start_index > 0) { + /* We have an incomplete JSON doc at the end of the buffer + * Move it to the start of the buffer + */ + mon->event_buffer.buf_fill_sz = sz - start_index; + memmove(buf, buf+start_index, mon->event_buffer.buf_fill_sz); + } + + return ret; +} + +static void virCHReadProcessEvents(virCHMonitor *mon, + int event_monitor_fd) +{ + /* Event json string must always terminate with null char. + * So, reserve one byte for '\0' at the end. + */ + size_t max_sz = CH_EVENT_BUFFER_SZ - 1; + char *buf = mon->event_buffer.buffer; + virDomainObj *vm = mon->vm; + bool incomplete = false; + size_t sz = 0; + + memset(buf, 0, max_sz); + do { + ssize_t ret; + + ret = read(event_monitor_fd, buf + sz, max_sz - sz); + if (ret == 0 || (ret < 0 && errno == EINTR)) { + g_usleep(G_USEC_PER_SEC); + continue; + } else if (ret < 0) { + /* We should never reach here. read(2) says possible errors + * are EINTR, EAGAIN, EBADF, EFAULT, EINVAL, EIO, EISDIR + * We handle EINTR gracefully. There is some serious issue + * if we encounter any of the other errors(either in our code + * or in the system). Better to bail out. + */ + VIR_ERROR(_("%1$s: Failed to read ch events!: %2$s"), + vm->def->name, g_strerror(errno)); + VIR_FORCE_CLOSE(event_monitor_fd); + abort(); + } + + sz += ret; + mon->event_buffer.buf_fill_sz = sz; + + if (virCHProcessEvents(mon) < 0) + VIR_WARN("%s: Failed to parse and process events", vm->def->name); + + if (mon->event_buffer.buf_fill_sz != 0) + incomplete = true; + else + incomplete = false; + sz = mon->event_buffer.buf_fill_sz; + + } while (virDomainObjIsActive(vm) && (sz < max_sz) && incomplete); + + return; +} + static void virCHEventHandlerLoop(void *data) { virCHMonitor *mon = data; @@ -54,11 +184,15 @@ static void virCHEventHandlerLoop(void *data) } VIR_DEBUG("%s: Opened the event monitor FIFO(%s)", vm->def->name, mon->eventmonitorpath); + mon->event_buffer.buffer = g_malloc_n(sizeof(char), CH_EVENT_BUFFER_SZ); + mon->event_buffer.buf_fill_sz = 0; + while (g_atomic_int_get(&mon->event_handler_stop) == 0) { VIR_DEBUG("%s: Reading events from event monitor file", vm->def->name); - /* Read and process events here */ + virCHReadProcessEvents(mon, event_monitor_fd); } + g_free(mon->event_buffer.buffer); VIR_FORCE_CLOSE(event_monitor_fd); virObjectUnref(vm); diff --git a/src/ch/ch_events.h b/src/ch/ch_events.h index 4c8a48231d..2e9cdf03bb 100644 --- a/src/ch/ch_events.h +++ b/src/ch/ch_events.h @@ -22,5 +22,7 @@ #include "ch_monitor.h" +#define CH_EVENT_BUFFER_SZ PIPE_BUF + int virCHStartEventHandler(virCHMonitor *mon); void virCHStopEventHandler(virCHMonitor *mon); diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index 878a185f29..d31b924d0e 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -99,6 +99,12 @@ struct _virCHMonitor { virThread event_handler_thread; int event_handler_stop; + struct { + /* Buffer to hold the data read from pipe */ + char *buffer; + /* Size of the data read from pipe into buffer */ + size_t buf_fill_sz; + } event_buffer; pid_t pid; -- 2.34.1

Implement `virCHProcessEvent` that maps event string to corresponding event type and take appropriate actions. As part of this, handle the shutdown event by correctly updating the domain state. this change also facilitates the handling of other VM lifecycle events, such as booting, rebooting, pause, resume, etc. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_events.c | 104 ++++++++++++++++++++++++++++++++++++++++++++- src/ch/ch_events.h | 26 ++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/src/ch/ch_events.c b/src/ch/ch_events.c index d0ad5af0af..39fbc52ed3 100644 --- a/src/ch/ch_events.c +++ b/src/ch/ch_events.c @@ -22,12 +22,110 @@ #include <fcntl.h> +#include "ch_domain.h" #include "ch_events.h" +#include "ch_process.h" #include "virfile.h" #include "virlog.h" VIR_LOG_INIT("ch.ch_events"); +VIR_ENUM_IMPL(virCHEvent, + VIR_CH_EVENT_LAST, + "vmm:starting", + "vmm:shutdown", + "vm:booting", + "vm:booted", + "vm:rebooting", + "vm:rebooted", + "vm:shutdown", + "vm:deleted", + "vm:pausing", + "vm:paused", + "vm:resuming", + "vm:resumed", + "vm:snapshotting", + "vm:snapshotted", + "vm:restoring", + "vm:restored", +); + +static int virCHEventStopProcess(virDomainObj *vm, + virDomainShutoffReason reason) +{ + virCHDriver *driver = ((virCHDomainObjPrivate *)vm->privateData)->driver; + + if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY)) + return -1; + virCHProcessStop(driver, vm, reason); + virDomainObjEndJob(vm); + + return 0; +} + +static int virCHProcessEvent(virCHMonitor *mon, + virJSONValue *eventJSON) +{ + const char *event; + const char *source; + virCHEvent ev; + g_autofree char *timestamp = NULL; + g_autofree char *full_event = NULL; + virDomainObj *vm = mon->vm; + int ret = 0; + + if (virJSONValueObjectHasKey(eventJSON, "source") == 0) { + VIR_WARN("%s: Invalid JSON from monitor, no source key", vm->def->name); + return -1; + } + if (virJSONValueObjectHasKey(eventJSON, "event") == 0) { + VIR_WARN("%s: Invalid JSON from monitor, no event key", vm->def->name); + return -1; + } + source = virJSONValueObjectGetString(eventJSON, "source"); + event = virJSONValueObjectGetString(eventJSON, "event"); + full_event = g_strdup_printf("%s:%s", source, event); + ev = virCHEventTypeFromString(full_event); + VIR_DEBUG("%s: Source: %s, Event: %s, ev: %d", vm->def->name, source, event, ev); + + switch (ev) { + case VIR_CH_EVENT_VMM_STARTING: + case VIR_CH_EVENT_VM_BOOTING: + case VIR_CH_EVENT_VM_BOOTED: + case VIR_CH_EVENT_VM_REBOOTING: + case VIR_CH_EVENT_VM_REBOOTED: + case VIR_CH_EVENT_VM_PAUSING: + case VIR_CH_EVENT_VM_PAUSED: + case VIR_CH_EVENT_VM_RESUMING: + case VIR_CH_EVENT_VM_RESUMED: + case VIR_CH_EVENT_VM_SNAPSHOTTING: + case VIR_CH_EVENT_VM_SNAPSHOTTED: + case VIR_CH_EVENT_VM_RESTORING: + case VIR_CH_EVENT_VM_RESTORED: + case VIR_CH_EVENT_VM_DELETED: + break; + case VIR_CH_EVENT_VMM_SHUTDOWN: + virObjectLock(vm); + if (virCHEventStopProcess(vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN)) { + VIR_WARN("Failed to mark the VM(%s) as SHUTDOWN!", + vm->def->name); + ret = -1; + } + virObjectUnlock(vm); + break; + case VIR_CH_EVENT_VM_SHUTDOWN: + virObjectLock(vm); + virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, VIR_DOMAIN_SHUTOFF_SHUTDOWN); + virObjectUnlock(vm); + break; + case VIR_CH_EVENT_LAST: + default: + VIR_WARN("%s: Unknown event: %s", vm->def->name, full_event); + } + + return ret; +} + /** * virCHProcessEvents: * @mon: the CH monitor object @@ -77,7 +175,11 @@ static int virCHProcessEvents(virCHMonitor *mon) json_start = buf + start_index; if ((obj = virJSONValueFromString(json_start))) { - /* Process the event string (obj) here */ + if (virCHProcessEvent(mon, obj) < 0) { + VIR_WARN("%s: Failed to process JSON event doc: %s", + vm->def->name, json_start); + ret = -1; + } virJSONValueFree(obj); } else { VIR_WARN("%s: Invalid JSON event doc: %s", diff --git a/src/ch/ch_events.h b/src/ch/ch_events.h index 2e9cdf03bb..3b360628f7 100644 --- a/src/ch/ch_events.h +++ b/src/ch/ch_events.h @@ -24,5 +24,31 @@ #define CH_EVENT_BUFFER_SZ PIPE_BUF +typedef enum { + /* source: vmm */ + VIR_CH_EVENT_VMM_STARTING = 0, + VIR_CH_EVENT_VMM_SHUTDOWN, + + /* source: vm */ + VIR_CH_EVENT_VM_BOOTING, + VIR_CH_EVENT_VM_BOOTED, + VIR_CH_EVENT_VM_REBOOTING, + VIR_CH_EVENT_VM_REBOOTED, + VIR_CH_EVENT_VM_SHUTDOWN, + VIR_CH_EVENT_VM_DELETED, + VIR_CH_EVENT_VM_PAUSING, + VIR_CH_EVENT_VM_PAUSED, + VIR_CH_EVENT_VM_RESUMING, + VIR_CH_EVENT_VM_RESUMED, + VIR_CH_EVENT_VM_SNAPSHOTTING, + VIR_CH_EVENT_VM_SNAPSHOTTED, + VIR_CH_EVENT_VM_RESTORING, + VIR_CH_EVENT_VM_RESTORED, + + VIR_CH_EVENT_LAST +} virCHEvent; + +VIR_ENUM_DECL(virCHEvent); + int virCHStartEventHandler(virCHMonitor *mon); void virCHStopEventHandler(virCHMonitor *mon); -- 2.34.1