From: "Daniel P. Berrange" <berrange(a)redhat.com&gt; When libvirt calls virInitialize it creates a thread local for the virErrorPtr storage, and registers a callback to cleanup memory when a thread exits. When libvirt is dlclose()d or otherwise made non-resident, the callback function is removed from memory, but the thread local may still exist and if a thread later exists, it will invoke the callback and SEGV. There may also be other thread locals with callbacks pointing to libvirt code, so it is in general never safe to unload libvirt.so from memory once initialized. To allow dlclose() to succeed, but keep libvirt.so resident in memory, link with '-z nodelete'. This issue was first found with the libvirt CIM provider, but can potentially hit many of the dynamic language bindings which all ultimately involve dlopen() in some way, either on libvirt.so itself, or on the glue code for the binding which in turns links to libvirt * configure.ac, src/Makefile.am: Ensure libvirt.so is linked with -z nodelete * tests/.gitignore, tests/Makefile.am, tests/shunloadhelper.c, tests/shunloadtest.c: A test case to unload libvirt while a thread is still running. --- configure.ac | 9 +++ src/Makefile.am | 2 +- tests/.gitignore | 3 + tests/Makefile.am | 14 ++++- tests/shunloadhelper.c | 51 ++++++++++++++++ tests/shunloadtest.c | 151 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 tests/shunloadhelper.c create mode 100644 tests/shunloadtest.c diff --git a/configure.ac b/configure.ac index 3c08a19..148c1da 100644 --- a/configure.ac +++ b/configure.ac @@ -96,9 +96,18 @@ AM_PROG_LIBTOOL AM_PROG_CC_C_O AM_PROG_LD +AC_MSG_CHECKING([for how to mark DSO non-deletable at runtime]) +LIBVIRT_NODELETE= +`$LD --help 2>&1 | grep -- "-z nodelete" >/dev/null` && \ + LIBVIRT_NODELETE="-Wl,-z -Wl,nodelete" +AC_MSG_RESULT([$LIBVIRT_NODELETE]) +AC_SUBST([LIBVIRT_NODELETE]) + +AC_MSG_CHECKING([for how to set DSO symbol versions]) VERSION_SCRIPT_FLAGS=-Wl,--version-script= `$LD --help 2>&1 | grep -- --version-script >/dev/null` || \ VERSION_SCRIPT_FLAGS="-Wl,-M -Wl," +AC_MSG_RESULT([$VERSION_SCRIPT_FLAGS]) LIBVIRT_COMPILE_WARNINGS([maximum]) diff --git a/src/Makefile.am b/src/Makefile.am index 322c900..ae1b9e0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1228,7 +1228,7 @@ libvirt_qemu.def: $(srcdir)/libvirt_qemu.syms libvirt_la_SOURCES = libvirt_la_LDFLAGS = $(VERSION_SCRIPT_FLAGS)$(LIBVIRT_SYMBOL_FILE) \ -version-info $(LIBVIRT_VERSION_INFO) \ - $(AM_LDFLAGS) \ + $(AM_LDFLAGS) $(LIBVIRT_NODELETE) \ $(CYGWIN_EXTRA_LDFLAGS) $(MINGW_EXTRA_LDFLAGS) libvirt_la_BUILT_LIBADD += ../gnulib/lib/libgnu.la libvirt_la_LIBADD += $(LIBXML_LIBS) \ diff --git a/tests/.gitignore b/tests/.gitignore index 7159c37..37ca034 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,6 +1,7 @@ *.exe .deps .libs +*.lo ssh commandhelper commandhelper.log @@ -41,3 +42,5 @@ xencapstest xmconfigtest xml2sexprtest xml2vmxtest +libshunload.la +shunloadtest diff --git a/tests/Makefile.am b/tests/Makefile.am index f4afcb9..cbbbc6f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -89,7 +89,9 @@ check_PROGRAMS = virshtest conftest sockettest \ nodeinfotest qparamtest virbuftest \ commandtest commandhelper seclabeltest \ hashtest virnetmessagetest virnetsockettest ssh \ - utiltest virnettlscontexttest + utiltest virnettlscontexttest shunloadtest + +check_LTLIBRARIES = libshunload.la # This is a fake SSH we use from virnetsockettest ssh_SOURCES = ssh.c @@ -207,6 +209,7 @@ TESTS = virshtest \ virnetmessagetest \ virnetsockettest \ virnettlscontexttest \ + shunloadtest \ utiltest \ $(test_scripts) @@ -503,6 +506,15 @@ eventtest_SOURCES = \ eventtest_LDADD = -lrt $(LDADDS) endif +libshunload_la_SOURCES = shunloadhelper.c +libshunload_la_LIBADD = ../src/libvirt.la +libshunload_la_LDFLAGS = -module -avoid-version -rpath /evil/libtool/hack/to/force/shared/lib/creation + +shunloadtest_SOURCES = \ + shunloadtest.c +shunloadtest_LDADD = -lpthread +shunloadtest_DEPENDENCIES = libshunload.la + if WITH_CIL CILOPTFLAGS = CILOPTINCS = diff --git a/tests/shunloadhelper.c b/tests/shunloadhelper.c new file mode 100644 index 0000000..6cbf5f4 --- /dev/null +++ b/tests/shunloadhelper.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2011 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +/* + * This is a helper for shunloadtest.c. This function is built into + * a shared library and linked with libvirto.so + * + * The function initializes libvirt and primes the thread local with + * an error which needs to be freed at thread exit + */ + +#include "internal.h" + +#include <libvirt/libvirt.h> +#include <libvirt/virterror.h> +#include <stdlib.h> + +static void shunloadError(void *userData ATTRIBUTE_UNUSED, + virErrorPtr error ATTRIBUTE_UNUSED) +{ +} + +void shunloadStart(void); + +void shunloadStart(void) { + virConnectPtr conn; + + virSetErrorFunc(NULL, shunloadError); + virInitialize(); + + conn = virConnectOpen("test:///default"); + virDomainDestroy(NULL); + if (conn) + virConnectClose(conn); +} diff --git a/tests/shunloadtest.c b/tests/shunloadtest.c new file mode 100644 index 0000000..cbcbf45 --- /dev/null +++ b/tests/shunloadtest.c @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2011 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +/* + * When libvirt initializes, it creates a thread local for storing + * the last virErrorPtr instance. It also registers a cleanup + * callback for the thread local that will be invoked whenever + * a thread exits. + * + * If the libvirt.so library was dlopen()'d and is dlclose()'d + * while there is still a thread present, then when that thread + * later exits, the libvirt cleanup callback will be invoked. + * Unfortunately libvirt.so will no longer be in memory so the + * callback SEGVs (if you're lucky), or invokes unlreated + * code at the same address as the old callback (in you're + * unlucky). + * + * To fix the problem libvirt is linked '-z nodelete' which + * prevents the code being removed from memory at dlclose(). + * + * This test case demonstrates this SEGV scenario. If this + * test does not SEGV, then the '-z nodelete' fix is working + */ + +#ifdef linux + +#include <dlfcn.h> +#include <pthread.h> +#include <stdbool.h> +#include <stdio.h> +#include <unistd.h> +#include <signal.h> + +#include "internal.h" +#include "ignore-value.h" +#include "testutils.h" + +pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +bool running = false; +bool quit = false; + +static void *threadMain(void *arg) +{ + void (*startup)(void) = arg; + + startup(); + + pthread_mutex_lock(&lock); + running = true; + pthread_cond_signal(&cond); + + while (!quit) { + pthread_cond_wait(&cond, &lock); + } + pthread_mutex_unlock(&lock); + + return NULL; +} + +static void sigHandler(int sig) +{ + ignore_value(write(STDERR_FILENO, "FAIL\n", 5)); + signal(sig, SIG_DFL); + raise(sig); +} + +/* We're not using the testutils.c main() wrapper because + * we don't want 'shunloadtest' itself to link against + * libvirt.so. We need to test dlopen()'ing of libvirt.so + */ +int main(int argc ATTRIBUTE_UNUSED, char **argv) +{ + void (*startup)(void); + pthread_t t; + void *lib; + char *theprogname; + + theprogname = argv[0]; + if (STRPREFIX(theprogname, "./")) + theprogname += 2; + + fprintf(stderr, "TEST: %s\n", theprogname); + fprintf(stderr, " .%*s 1 ", 39, ""); + signal(SIGSEGV, sigHandler); + + if (!(lib = dlopen("./.libs/libshunload.so", RTLD_NOW))) { + fprintf(stderr, "Cannot load ./.libs/libshunload.so %s\n", dlerror()); + return 1; + } + if (!(startup = dlsym(lib, "shunloadStart"))) { + fprintf(stderr, "Cannot find shunloadStart %s\n", dlerror()); + return 1; + } + + /* + * Create a thread which is going to initialize libvirt + * and raise an error + */ + pthread_create(&t, NULL, threadMain, startup); + + /* Wait for the thread to start and call libvirt */ + pthread_mutex_lock(&lock); + while (!running) { + pthread_cond_wait(&cond, &lock); + } + + /* Close the shared library (and thus make libvirt.so + * non-resident */ + dlclose(lib); + + /* Tell the thread to quit */ + quit = true; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&lock); + + pthread_join(t, NULL); + + /* If we got to hear the thread succesfully exited without + * causing a SEGV ! + */ + + fprintf(stderr, "OK\n"); + + return 0; +} + +#else + +int main(void) +{ + return EXIT_AM_SKIP; +} + +#endif -- 1.7.6