[libvirt PATCH 0/2] util: avoid race in glib event loop which causes crashes
A while ago Cole reported that libvirt is exposing a long term bug in GLib's event loop that results in non-deterministic crashes: https://www.redhat.com/archives/libvir-list/2020-February/msg00422.html I kept forgetting to investigate it until now. Daniel P. Berrangé (2): util: keep track of full GSource object not source ID number util: avoid crash due to race in glib event loop code src/rpc/virnetclient.c | 27 ++++++----- src/util/vireventglib.c | 91 +++++++++++++++++++++++++----------- src/util/vireventglibwatch.c | 19 ++++---- src/util/vireventglibwatch.h | 13 +++--- 4 files changed, 97 insertions(+), 53 deletions(-) -- 2.24.1
The source ID number is an alternative way to identify a source that has been added to a GMainContext. Internally when a source ID is given, glib will lookup the corresponding GSource and use that. The use of a source ID is racy in some cases though, because it is invalid to continue to use an ID number after the GSource has been removed. It is thus safer to use the GSource object directly and have full control over the ref counting and thus cleanup. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- src/rpc/virnetclient.c | 27 +++++++------ src/util/vireventglib.c | 73 +++++++++++++++++++++++------------- src/util/vireventglibwatch.c | 19 ++++++---- src/util/vireventglibwatch.h | 13 ++++--- 4 files changed, 79 insertions(+), 53 deletions(-) diff --git a/src/rpc/virnetclient.c b/src/rpc/virnetclient.c index 441f1502a6..6031d9ab5e 100644 --- a/src/rpc/virnetclient.c +++ b/src/rpc/virnetclient.c @@ -826,6 +826,7 @@ virNetClientIOEventTLS(int fd, static gboolean virNetClientTLSHandshake(virNetClientPtr client) { + g_autoptr(GSource) source = NULL; GIOCondition ev; int ret; @@ -840,10 +841,10 @@ virNetClientTLSHandshake(virNetClientPtr client) else ev = G_IO_OUT; - virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock), - ev, - client->eventCtx, - virNetClientIOEventTLS, client, NULL); + source = virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock), + ev, + client->eventCtx, + virNetClientIOEventTLS, client, NULL); return TRUE; } @@ -882,6 +883,7 @@ int virNetClientSetTLSSession(virNetClientPtr client, int ret; char buf[1]; int len; + g_autoptr(GSource) source = NULL; #ifndef WIN32 sigset_t oldmask, blockedsigs; @@ -934,10 +936,10 @@ int virNetClientSetTLSSession(virNetClientPtr client, * etc. If we make the grade, it will send us a '\1' byte. */ - virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock), - G_IO_IN, - client->eventCtx, - virNetClientIOEventTLSConfirm, client, NULL); + source = virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock), + G_IO_IN, + client->eventCtx, + virNetClientIOEventTLSConfirm, client, NULL); #ifndef WIN32 /* Block SIGWINCH from interrupting poll in curses programs */ @@ -1617,6 +1619,7 @@ static int virNetClientIOEventLoop(virNetClientPtr client, #endif /* !WIN32 */ int timeout = -1; virNetMessagePtr msg = NULL; + g_autoptr(GSource) source = NULL; GIOCondition ev = 0; struct virNetClientIOEventData data = { .client = client, @@ -1651,10 +1654,10 @@ static int virNetClientIOEventLoop(virNetClientPtr client, if (client->nstreams) ev |= G_IO_IN; - virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock), - ev, - client->eventCtx, - virNetClientIOEventFD, &data, NULL); + source = virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock), + ev, + client->eventCtx, + virNetClientIOEventFD, &data, NULL); /* Release lock while poll'ing so other threads * can stuff themselves on the queue */ diff --git a/src/util/vireventglib.c b/src/util/vireventglib.c index 803332a6f8..6e334b3398 100644 --- a/src/util/vireventglib.c +++ b/src/util/vireventglib.c @@ -45,7 +45,7 @@ struct virEventGLibHandle int fd; int events; int removed; - guint source; + GSource *source; virEventHandleCallback cb; void *opaque; virFreeCallback ff; @@ -56,7 +56,7 @@ struct virEventGLibTimeout int timer; int interval; int removed; - guint source; + GSource *source; virEventTimeoutCallback cb; void *opaque; virFreeCallback ff; @@ -210,23 +210,25 @@ virEventGLibHandleUpdate(int watch, if (events == data->events) goto cleanup; - if (data->source != 0) { - VIR_DEBUG("Removed old handle watch=%d", data->source); - g_source_remove(data->source); + if (data->source != NULL) { + VIR_DEBUG("Removed old handle source=%p", data->source); + g_source_destroy(data->source); + g_source_unref(data->source); } data->source = virEventGLibAddSocketWatch( data->fd, cond, NULL, virEventGLibHandleDispatch, data, NULL); data->events = events; - VIR_DEBUG("Added new handle watch=%d", data->source); + VIR_DEBUG("Added new handle source=%p", data->source); } else { - if (data->source == 0) + if (data->source == NULL) goto cleanup; - VIR_DEBUG("Removed old handle watch=%d", data->source); - g_source_remove(data->source); - data->source = 0; + VIR_DEBUG("Removed old handle source=%p", data->source); + g_source_destroy(data->source); + g_source_unref(data->source); + data->source = NULL; data->events = 0; } @@ -272,9 +274,10 @@ virEventGLibHandleRemove(int watch) VIR_DEBUG("Remove handle data=%p watch=%d fd=%d", data, watch, data->fd); - if (data->source != 0) { - g_source_remove(data->source); - data->source = 0; + if (data->source != NULL) { + g_source_destroy(data->source); + g_source_unref(data->source); + data->source = NULL; data->events = 0; } @@ -309,6 +312,22 @@ virEventGLibTimeoutDispatch(void *opaque) return TRUE; } + +static GSource * +virEventGLibTimeoutCreate(int interval, + struct virEventGLibTimeout *data) +{ + GSource *source = g_timeout_source_new(interval); + + g_source_set_callback(source, + virEventGLibTimeoutDispatch, + data, NULL); + g_source_attach(source, NULL); + + return source; +} + + static int virEventGLibTimeoutAdd(int interval, virEventTimeoutCallback cb, @@ -327,9 +346,7 @@ virEventGLibTimeoutAdd(int interval, data->opaque = opaque; data->ff = ff; if (interval >= 0) - data->source = g_timeout_add(interval, - virEventGLibTimeoutDispatch, - data); + data->source = virEventGLibTimeoutCreate(interval, data); g_ptr_array_add(timeouts, data); @@ -390,19 +407,20 @@ virEventGLibTimeoutUpdate(int timer, VIR_DEBUG("Update timeout data=%p timer=%d interval=%d ms", data, timer, interval); if (interval >= 0) { - if (data->source != 0) - g_source_remove(data->source); + if (data->source != NULL) { + g_source_destroy(data->source); + g_source_unref(data->source); + } data->interval = interval; - data->source = g_timeout_add(data->interval, - virEventGLibTimeoutDispatch, - data); + data->source = virEventGLibTimeoutCreate(interval, data); } else { - if (data->source == 0) + if (data->source == NULL) goto cleanup; - g_source_remove(data->source); - data->source = 0; + g_source_destroy(data->source); + g_source_unref(data->source); + data->source = NULL; } cleanup: @@ -448,9 +466,10 @@ virEventGLibTimeoutRemove(int timer) VIR_DEBUG("Remove timeout data=%p timer=%d", data, timer); - if (data->source != 0) { - g_source_remove(data->source); - data->source = 0; + if (data->source != NULL) { + g_source_destroy(data->source); + g_source_unref(data->source); + data->source = NULL; } /* since the actual timeout deletion is done asynchronously, a timeoutUpdate call may diff --git a/src/util/vireventglibwatch.c b/src/util/vireventglibwatch.c index 178707f6b7..b7f3a8786a 100644 --- a/src/util/vireventglibwatch.c +++ b/src/util/vireventglibwatch.c @@ -233,17 +233,20 @@ GSource *virEventGLibCreateSocketWatch(int fd, #endif /* WIN32 */ -guint virEventGLibAddSocketWatch(int fd, - GIOCondition condition, - GMainContext *context, - virEventGLibSocketFunc func, - gpointer opaque, - GDestroyNotify notify) +GSource * +virEventGLibAddSocketWatch(int fd, + GIOCondition condition, + GMainContext *context, + virEventGLibSocketFunc func, + gpointer opaque, + GDestroyNotify notify) { - g_autoptr(GSource) source = NULL; + GSource *source = NULL; source = virEventGLibCreateSocketWatch(fd, condition); g_source_set_callback(source, (GSourceFunc)func, opaque, notify); - return g_source_attach(source, context); + g_source_attach(source, context); + + return source; } diff --git a/src/util/vireventglibwatch.h b/src/util/vireventglibwatch.h index 2f7e61cfba..f57be1f503 100644 --- a/src/util/vireventglibwatch.h +++ b/src/util/vireventglibwatch.h @@ -40,9 +40,10 @@ typedef gboolean (*virEventGLibSocketFunc)(int fd, GIOCondition condition, gpointer data); -guint virEventGLibAddSocketWatch(int fd, - GIOCondition condition, - GMainContext *context, - virEventGLibSocketFunc func, - gpointer opaque, - GDestroyNotify notify); +GSource *virEventGLibAddSocketWatch(int fd, + GIOCondition condition, + GMainContext *context, + virEventGLibSocketFunc func, + gpointer opaque, + GDestroyNotify notify) + G_GNUC_WARN_UNUSED_RESULT; -- 2.24.1
There is a fairly long standing race condition bug in glib which can hit if you call g_source_destroy or g_source_unref from a non-main thread: https://gitlab.gnome.org/GNOME/glib/-/merge_requests/1358 Unfortunately it is really common for libvirt to call g_source_destroy from a non-main thread. This glib bug is the cause of non-determinstic crashes in eventtest, and probably in libvirtd too. To work around the problem we need to ensure that we never release the last reference on a GSource from a non-main thread. The previous patch replaced our use of g_source_destroy with a pair of g_source_remove and g_source_unref. We can now delay the g_source_unref call by using a idle callback to invoke it from the main thread which avoids the race condition. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- src/util/vireventglib.c | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/util/vireventglib.c b/src/util/vireventglib.c index 6e334b3398..7e61b096ed 100644 --- a/src/util/vireventglib.c +++ b/src/util/vireventglib.c @@ -185,6 +185,24 @@ virEventGLibHandleFind(int watch) return NULL; } +/* + * If the last refernce to a GSource is released in a non-main + * thread we're exposed to a race condition that causes a + * crash: + * https://gitlab.gnome.org/GNOME/glib/-/merge_requests/1358 + * Thus we're using an idle func to release our ref + */ +static gboolean +virEventGLibSourceUnrefIdle(gpointer data) +{ + GSource *src = data; + + g_source_unref(src); + + return FALSE; +} + + static void virEventGLibHandleUpdate(int watch, int events) @@ -213,7 +231,7 @@ virEventGLibHandleUpdate(int watch, if (data->source != NULL) { VIR_DEBUG("Removed old handle source=%p", data->source); g_source_destroy(data->source); - g_source_unref(data->source); + g_idle_add(virEventGLibSourceUnrefIdle, data->source); } data->source = virEventGLibAddSocketWatch( @@ -227,7 +245,7 @@ virEventGLibHandleUpdate(int watch, VIR_DEBUG("Removed old handle source=%p", data->source); g_source_destroy(data->source); - g_source_unref(data->source); + g_idle_add(virEventGLibSourceUnrefIdle, data->source); data->source = NULL; data->events = 0; } @@ -276,7 +294,7 @@ virEventGLibHandleRemove(int watch) if (data->source != NULL) { g_source_destroy(data->source); - g_source_unref(data->source); + g_idle_add(virEventGLibSourceUnrefIdle, data->source); data->source = NULL; data->events = 0; } @@ -409,7 +427,7 @@ virEventGLibTimeoutUpdate(int timer, if (interval >= 0) { if (data->source != NULL) { g_source_destroy(data->source); - g_source_unref(data->source); + g_idle_add(virEventGLibSourceUnrefIdle, data->source); } data->interval = interval; @@ -419,7 +437,7 @@ virEventGLibTimeoutUpdate(int timer, goto cleanup; g_source_destroy(data->source); - g_source_unref(data->source); + g_idle_add(virEventGLibSourceUnrefIdle, data->source); data->source = NULL; } @@ -468,7 +486,7 @@ virEventGLibTimeoutRemove(int timer) if (data->source != NULL) { g_source_destroy(data->source); - g_source_unref(data->source); + g_idle_add(virEventGLibSourceUnrefIdle, data->source); data->source = NULL; } -- 2.24.1
On 7/29/20 6:10 PM, Daniel P. Berrangé wrote:
A while ago Cole reported that libvirt is exposing a long term bug in GLib's event loop that results in non-deterministic crashes:
https://www.redhat.com/archives/libvir-list/2020-February/msg00422.html
I kept forgetting to investigate it until now.
Daniel P. Berrangé (2): util: keep track of full GSource object not source ID number util: avoid crash due to race in glib event loop code
src/rpc/virnetclient.c | 27 ++++++----- src/util/vireventglib.c | 91 +++++++++++++++++++++++++----------- src/util/vireventglibwatch.c | 19 ++++---- src/util/vireventglibwatch.h | 13 +++--- 4 files changed, 97 insertions(+), 53 deletions(-)
Reviewed-by: Michal Privoznik <mprivozn@redhat.com> Michal
participants (2)
-
Daniel P. Berrangé -
Michal Privoznik