11:10 a.m.
A while ago Cole reported that libvirt is exposing a long
term bug in GLib's event loop that results in non-deterministic
crashes:
https://www.redhat.com/archives/libvir-list/2020-February/msg00422.html
I kept forgetting to investigate it until now.
Daniel P. Berrangé (2):
util: keep track of full GSource object not source ID number
util: avoid crash due to race in glib event loop code
src/rpc/virnetclient.c | 27 ++++++-----
src/util/vireventglib.c | 91 +++++++++++++++++++++++++-----------
src/util/vireventglibwatch.c | 19 ++++----
src/util/vireventglibwatch.h | 13 +++---
4 files changed, 97 insertions(+), 53 deletions(-)
--
2.24.1
11:10 a.m.
New subject: [libvirt PATCH 1/2] util: keep track of full GSource object not source ID number
The source ID number is an alternative way to identify a source that has
been added to a GMainContext. Internally when a source ID is given, glib
will lookup the corresponding GSource and use that. The use of a source
ID is racy in some cases though, because it is invalid to continue to
use an ID number after the GSource has been removed. It is thus safer
to use the GSource object directly and have full control over the ref
counting and thus cleanup.
Signed-off-by: Daniel P. Berrangé <berrange(a)redhat.com>
---
src/rpc/virnetclient.c | 27 +++++++------
src/util/vireventglib.c | 73 +++++++++++++++++++++++-------------
src/util/vireventglibwatch.c | 19 ++++++----
src/util/vireventglibwatch.h | 13 ++++---
4 files changed, 79 insertions(+), 53 deletions(-)
diff --git a/src/rpc/virnetclient.c b/src/rpc/virnetclient.c
index 441f1502a6..6031d9ab5e 100644
--- a/src/rpc/virnetclient.c
+++ b/src/rpc/virnetclient.c
@@ -826,6 +826,7 @@ virNetClientIOEventTLS(int fd,
static gboolean
virNetClientTLSHandshake(virNetClientPtr client)
{
+ g_autoptr(GSource) source = NULL;
GIOCondition ev;
int ret;
@@ -840,10 +841,10 @@ virNetClientTLSHandshake(virNetClientPtr client)
else
ev = G_IO_OUT;
- virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock),
- ev,
- client->eventCtx,
- virNetClientIOEventTLS, client, NULL);
+ source = virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock),
+ ev,
+ client->eventCtx,
+ virNetClientIOEventTLS, client, NULL);
return TRUE;
}
@@ -882,6 +883,7 @@ int virNetClientSetTLSSession(virNetClientPtr client,
int ret;
char buf[1];
int len;
+ g_autoptr(GSource) source = NULL;
#ifndef WIN32
sigset_t oldmask, blockedsigs;
@@ -934,10 +936,10 @@ int virNetClientSetTLSSession(virNetClientPtr client,
* etc. If we make the grade, it will send us a '\1' byte.
*/
- virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock),
- G_IO_IN,
- client->eventCtx,
- virNetClientIOEventTLSConfirm, client, NULL);
+ source = virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock),
+ G_IO_IN,
+ client->eventCtx,
+ virNetClientIOEventTLSConfirm, client, NULL);
#ifndef WIN32
/* Block SIGWINCH from interrupting poll in curses programs */
@@ -1617,6 +1619,7 @@ static int virNetClientIOEventLoop(virNetClientPtr client,
#endif /* !WIN32 */
int timeout = -1;
virNetMessagePtr msg = NULL;
+ g_autoptr(GSource) source = NULL;
GIOCondition ev = 0;
struct virNetClientIOEventData data = {
.client = client,
@@ -1651,10 +1654,10 @@ static int virNetClientIOEventLoop(virNetClientPtr client,
if (client->nstreams)
ev |= G_IO_IN;
- virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock),
- ev,
- client->eventCtx,
- virNetClientIOEventFD, &data, NULL);
+ source = virEventGLibAddSocketWatch(virNetSocketGetFD(client->sock),
+ ev,
+ client->eventCtx,
+ virNetClientIOEventFD, &data, NULL);
/* Release lock while poll'ing so other threads
* can stuff themselves on the queue */
diff --git a/src/util/vireventglib.c b/src/util/vireventglib.c
index 803332a6f8..6e334b3398 100644
--- a/src/util/vireventglib.c
+++ b/src/util/vireventglib.c
@@ -45,7 +45,7 @@ struct virEventGLibHandle
int fd;
int events;
int removed;
- guint source;
+ GSource *source;
virEventHandleCallback cb;
void *opaque;
virFreeCallback ff;
@@ -56,7 +56,7 @@ struct virEventGLibTimeout
int timer;
int interval;
int removed;
- guint source;
+ GSource *source;
virEventTimeoutCallback cb;
void *opaque;
virFreeCallback ff;
@@ -210,23 +210,25 @@ virEventGLibHandleUpdate(int watch,
if (events == data->events)
goto cleanup;
- if (data->source != 0) {
- VIR_DEBUG("Removed old handle watch=%d", data->source);
- g_source_remove(data->source);
+ if (data->source != NULL) {
+ VIR_DEBUG("Removed old handle source=%p", data->source);
+ g_source_destroy(data->source);
+ g_source_unref(data->source);
}
data->source = virEventGLibAddSocketWatch(
data->fd, cond, NULL, virEventGLibHandleDispatch, data, NULL);
data->events = events;
- VIR_DEBUG("Added new handle watch=%d", data->source);
+ VIR_DEBUG("Added new handle source=%p", data->source);
} else {
- if (data->source == 0)
+ if (data->source == NULL)
goto cleanup;
- VIR_DEBUG("Removed old handle watch=%d", data->source);
- g_source_remove(data->source);
- data->source = 0;
+ VIR_DEBUG("Removed old handle source=%p", data->source);
+ g_source_destroy(data->source);
+ g_source_unref(data->source);
+ data->source = NULL;
data->events = 0;
}
@@ -272,9 +274,10 @@ virEventGLibHandleRemove(int watch)
VIR_DEBUG("Remove handle data=%p watch=%d fd=%d",
data, watch, data->fd);
- if (data->source != 0) {
- g_source_remove(data->source);
- data->source = 0;
+ if (data->source != NULL) {
+ g_source_destroy(data->source);
+ g_source_unref(data->source);
+ data->source = NULL;
data->events = 0;
}
@@ -309,6 +312,22 @@ virEventGLibTimeoutDispatch(void *opaque)
return TRUE;
}
+
+static GSource *
+virEventGLibTimeoutCreate(int interval,
+ struct virEventGLibTimeout *data)
+{
+ GSource *source = g_timeout_source_new(interval);
+
+ g_source_set_callback(source,
+ virEventGLibTimeoutDispatch,
+ data, NULL);
+ g_source_attach(source, NULL);
+
+ return source;
+}
+
+
static int
virEventGLibTimeoutAdd(int interval,
virEventTimeoutCallback cb,
@@ -327,9 +346,7 @@ virEventGLibTimeoutAdd(int interval,
data->opaque = opaque;
data->ff = ff;
if (interval >= 0)
- data->source = g_timeout_add(interval,
- virEventGLibTimeoutDispatch,
- data);
+ data->source = virEventGLibTimeoutCreate(interval, data);
g_ptr_array_add(timeouts, data);
@@ -390,19 +407,20 @@ virEventGLibTimeoutUpdate(int timer,
VIR_DEBUG("Update timeout data=%p timer=%d interval=%d ms", data, timer,
interval);
if (interval >= 0) {
- if (data->source != 0)
- g_source_remove(data->source);
+ if (data->source != NULL) {
+ g_source_destroy(data->source);
+ g_source_unref(data->source);
+ }
data->interval = interval;
- data->source = g_timeout_add(data->interval,
- virEventGLibTimeoutDispatch,
- data);
+ data->source = virEventGLibTimeoutCreate(interval, data);
} else {
- if (data->source == 0)
+ if (data->source == NULL)
goto cleanup;
- g_source_remove(data->source);
- data->source = 0;
+ g_source_destroy(data->source);
+ g_source_unref(data->source);
+ data->source = NULL;
}
cleanup:
@@ -448,9 +466,10 @@ virEventGLibTimeoutRemove(int timer)
VIR_DEBUG("Remove timeout data=%p timer=%d",
data, timer);
- if (data->source != 0) {
- g_source_remove(data->source);
- data->source = 0;
+ if (data->source != NULL) {
+ g_source_destroy(data->source);
+ g_source_unref(data->source);
+ data->source = NULL;
}
/* since the actual timeout deletion is done asynchronously, a timeoutUpdate call
may
diff --git a/src/util/vireventglibwatch.c b/src/util/vireventglibwatch.c
index 178707f6b7..b7f3a8786a 100644
--- a/src/util/vireventglibwatch.c
+++ b/src/util/vireventglibwatch.c
@@ -233,17 +233,20 @@ GSource *virEventGLibCreateSocketWatch(int fd,
#endif /* WIN32 */
-guint virEventGLibAddSocketWatch(int fd,
- GIOCondition condition,
- GMainContext *context,
- virEventGLibSocketFunc func,
- gpointer opaque,
- GDestroyNotify notify)
+GSource *
+virEventGLibAddSocketWatch(int fd,
+ GIOCondition condition,
+ GMainContext *context,
+ virEventGLibSocketFunc func,
+ gpointer opaque,
+ GDestroyNotify notify)
{
- g_autoptr(GSource) source = NULL;
+ GSource *source = NULL;
source = virEventGLibCreateSocketWatch(fd, condition);
g_source_set_callback(source, (GSourceFunc)func, opaque, notify);
- return g_source_attach(source, context);
+ g_source_attach(source, context);
+
+ return source;
}
diff --git a/src/util/vireventglibwatch.h b/src/util/vireventglibwatch.h
index 2f7e61cfba..f57be1f503 100644
--- a/src/util/vireventglibwatch.h
+++ b/src/util/vireventglibwatch.h
@@ -40,9 +40,10 @@ typedef gboolean (*virEventGLibSocketFunc)(int fd,
GIOCondition condition,
gpointer data);
-guint virEventGLibAddSocketWatch(int fd,
- GIOCondition condition,
- GMainContext *context,
- virEventGLibSocketFunc func,
- gpointer opaque,
- GDestroyNotify notify);
+GSource *virEventGLibAddSocketWatch(int fd,
+ GIOCondition condition,
+ GMainContext *context,
+ virEventGLibSocketFunc func,
+ gpointer opaque,
+ GDestroyNotify notify)
+ G_GNUC_WARN_UNUSED_RESULT;
--
2.24.1
11:10 a.m.
New subject: [libvirt PATCH 2/2] util: avoid crash due to race in glib event loop code
There is a fairly long standing race condition bug in glib which can hit
if you call g_source_destroy or g_source_unref from a non-main thread:
https://gitlab.gnome.org/GNOME/glib/-/merge_requests/1358
Unfortunately it is really common for libvirt to call g_source_destroy
from a non-main thread. This glib bug is the cause of non-determinstic
crashes in eventtest, and probably in libvirtd too.
To work around the problem we need to ensure that we never release
the last reference on a GSource from a non-main thread. The previous
patch replaced our use of g_source_destroy with a pair of
g_source_remove and g_source_unref. We can now delay the g_source_unref
call by using a idle callback to invoke it from the main thread which
avoids the race condition.
Signed-off-by: Daniel P. Berrangé <berrange(a)redhat.com>
---
src/util/vireventglib.c | 30 ++++++++++++++++++++++++------
1 file changed, 24 insertions(+), 6 deletions(-)
diff --git a/src/util/vireventglib.c b/src/util/vireventglib.c
index 6e334b3398..7e61b096ed 100644
--- a/src/util/vireventglib.c
+++ b/src/util/vireventglib.c
@@ -185,6 +185,24 @@ virEventGLibHandleFind(int watch)
return NULL;
}
+/*
+ * If the last refernce to a GSource is released in a non-main
+ * thread we're exposed to a race condition that causes a
+ * crash:
+ * https://gitlab.gnome.org/GNOME/glib/-/merge_requests/1358
+ * Thus we're using an idle func to release our ref
+ */
+static gboolean
+virEventGLibSourceUnrefIdle(gpointer data)
+{
+ GSource *src = data;
+
+ g_source_unref(src);
+
+ return FALSE;
+}
+
+
static void
virEventGLibHandleUpdate(int watch,
int events)
@@ -213,7 +231,7 @@ virEventGLibHandleUpdate(int watch,
if (data->source != NULL) {
VIR_DEBUG("Removed old handle source=%p", data->source);
g_source_destroy(data->source);
- g_source_unref(data->source);
+ g_idle_add(virEventGLibSourceUnrefIdle, data->source);
}
data->source = virEventGLibAddSocketWatch(
@@ -227,7 +245,7 @@ virEventGLibHandleUpdate(int watch,
VIR_DEBUG("Removed old handle source=%p", data->source);
g_source_destroy(data->source);
- g_source_unref(data->source);
+ g_idle_add(virEventGLibSourceUnrefIdle, data->source);
data->source = NULL;
data->events = 0;
}
@@ -276,7 +294,7 @@ virEventGLibHandleRemove(int watch)
if (data->source != NULL) {
g_source_destroy(data->source);
- g_source_unref(data->source);
+ g_idle_add(virEventGLibSourceUnrefIdle, data->source);
data->source = NULL;
data->events = 0;
}
@@ -409,7 +427,7 @@ virEventGLibTimeoutUpdate(int timer,
if (interval >= 0) {
if (data->source != NULL) {
g_source_destroy(data->source);
- g_source_unref(data->source);
+ g_idle_add(virEventGLibSourceUnrefIdle, data->source);
}
data->interval = interval;
@@ -419,7 +437,7 @@ virEventGLibTimeoutUpdate(int timer,
goto cleanup;
g_source_destroy(data->source);
- g_source_unref(data->source);
+ g_idle_add(virEventGLibSourceUnrefIdle, data->source);
data->source = NULL;
}
@@ -468,7 +486,7 @@ virEventGLibTimeoutRemove(int timer)
if (data->source != NULL) {
g_source_destroy(data->source);
- g_source_unref(data->source);
+ g_idle_add(virEventGLibSourceUnrefIdle, data->source);
data->source = NULL;
}
--
2.24.1
2:59 a.m.
On 7/29/20 6:10 PM, Daniel P. Berrangé wrote:
A while ago Cole reported that libvirt is exposing a long
term bug in GLib's event loop that results in non-deterministic
crashes:
https://www.redhat.com/archives/libvir-list/2020-February/msg00422.html
I kept forgetting to investigate it until now.
Daniel P. Berrangé (2):
util: keep track of full GSource object not source ID number
util: avoid crash due to race in glib event loop code
src/rpc/virnetclient.c | 27 ++++++-----
src/util/vireventglib.c | 91 +++++++++++++++++++++++++-----------
src/util/vireventglibwatch.c | 19 ++++----
src/util/vireventglibwatch.h | 13 +++---
4 files changed, 97 insertions(+), 53 deletions(-)
Reviewed-by: Michal Privoznik <mprivozn(a)redhat.com>
Michal
1576
days inactive
1577
days old
3 comments
2 participants
participants (2)
-
Daniel P. Berrangé -
Michal Privoznik