Re: [libvirt] libvirtd (from git) no longer responds to 'kill'

This seems to be some sort of deadlock, easily reproduced by running the libguestfs test suite, or even just 'libguestfs-test-tool'. Here is a stack trace: (gdb) t a a bt Thread 11 (Thread 0x7fe6505d7700 (LWP 20021)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943c80, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bdb in virThreadPoolWorker (opaque=opaque@entry=0x1934930) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #4 0x000000328ca07d15 in start_thread (arg=0x7fe6505d7700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 10 (Thread 0x7fe64fdd6700 (LWP 20022)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943c80, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bdb in virThreadPoolWorker (opaque=opaque@entry=0x1934b50) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #4 0x000000328ca07d15 in start_thread (arg=0x7fe64fdd6700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 9 (Thread 0x7fe64f5d5700 (LWP 20023)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943c80, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bdb in virThreadPoolWorker (opaque=opaque@entry=0x1934c50) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #4 0x000000328ca07d15 in start_thread (arg=0x7fe64f5d5700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 8 (Thread 0x7fe64edd4700 (LWP 20024)): #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 ---Type <return> to continue, or q <return> to quit--- #1 0x000000328ca09ca6 in _L_lock_836 () from /lib64/libpthread.so.0 #2 0x000000328ca09ba8 in __GI___pthread_mutex_lock ( mutex=mutex@entry=0x7fe64006eb30) at pthread_mutex_lock.c:64 #3 0x00007fe650ad142d in virMutexLock (m=m@entry=0x7fe64006eb30) at util/virthreadpthread.c:85 #4 0x00007fe650ac38de in virObjectLock (anyobj=anyobj@entry=0x7fe64006eb20) at util/virobject.c:322 #5 0x00007fe650ce65b1 in virSecurityManagerGetModel ( mgr=mgr@entry=0x7fe64006eb20) at security/security_manager.c:236 #6 0x00007fe650ce994c in virSecuritySELinuxSecurityVerify ( mgr=0x7fe64006eb20, def=<optimized out>) at security/security_selinux.c:1806 #7 0x00007fe650ce7251 in virSecurityManagerVerify (mgr=0x7fe64006eb20, def=def@entry=0x7fe63400ac20) at security/security_manager.c:573 #8 0x00007fe650ce3cd4 in virSecurityStackVerify (mgr=<optimized out>, def=0x7fe63400ac20) at security/security_stack.c:125 #9 0x00007fe650ce7251 in virSecurityManagerVerify (mgr=0x7fe64001cc50, def=def@entry=0x7fe63400ac20) at security/security_manager.c:573 #10 0x00007fe64893e63d in qemuDomainCreate (conn=0x7fe638000bd0, xml=0x7fe6340009a0 "<?xml version=\"1.0\"?>\n<domain type=\"kvm\" xmlns:qemu=\"http://libvirt.org/schemas/domain/qemu/1.0\">\n <name>guestfs-492qa31a2ntfmk0j</name>\n <memory unit=\"MiB\">500</memory>\n <currentMemory unit=\"MiB\">"..., flags=<optimized out>) at qemu/qemu_driver.c:1538 #11 0x00007fe650b4fa39 in virDomainCreateXML (conn=0x7fe638000bd0, xmlDesc=0x7fe6340009a0 "<?xml version=\"1.0\"?>\n<domain type=\"kvm\" xmlns:qemu=\"http://libvirt.org/schemas/domain/qemu/1.0\">\n <name>guestfs-492qa31a2ntfmk0j</name>\n <memory unit=\"MiB\">500</memory>\n <currentMemory unit=\"MiB\">"..., flags=2) at libvirt.c:1988 #12 0x000000000042c915 in remoteDispatchDomainCreateXML ( server=<optimized out>, msg=<optimized out>, ret=0x7fe6340008c0, args=0x7fe634000970, rerr=0x7fe64edd3c50, client=0x196eb60) at remote_dispatch.h:1172 #13 remoteDispatchDomainCreateXMLHelper (server=<optimized out>, client=0x196eb60, msg=<optimized out>, rerr=0x7fe64edd3c50, args=0x7fe634000970, ret=0x7fe6340008c0) at remote_dispatch.h:1152 #14 0x00007fe650bbe602 in virNetServerProgramDispatchCall (msg=0x196cd40, client=0x196eb60, server=0x1943b10, prog=0x1968460) at rpc/virnetserverprogram.c:432 #15 virNetServerProgramDispatch (prog=0x1968460, server=server@entry=0x1943b10, client=0x196eb60, msg=0x196cd40) at rpc/virnetserverprogram.c:305 #16 0x00007fe650bb8838 in virNetServerProcessMsg (msg=<optimized out>, prog=<optimized out>, client=<optimized out>, srv=0x1943b10) at rpc/virnetserver.c:162 #17 virNetServerHandleJob (jobOpaque=<optimized out>, opaque=0x1943b10) at rpc/virnetserver.c:183 ---Type <return> to continue, or q <return> to quit--- #18 0x00007fe650ad1b3e in virThreadPoolWorker (opaque=opaque@entry=0x1934b50) at util/virthreadpool.c:144 #19 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #20 0x000000328ca07d15 in start_thread (arg=0x7fe64edd4700) at pthread_create.c:308 #21 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 7 (Thread 0x7fe64e5d3700 (LWP 20025)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943c80, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bdb in virThreadPoolWorker (opaque=opaque@entry=0x1934c50) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #4 0x000000328ca07d15 in start_thread (arg=0x7fe64e5d3700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 6 (Thread 0x7fe64ddd2700 (LWP 20026)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943d18, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque@entry=0x1934930) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #4 0x000000328ca07d15 in start_thread (arg=0x7fe64ddd2700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 5 (Thread 0x7fe64d5d1700 (LWP 20027)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943d18, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque@entry=0x1934b50) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 ---Type <return> to continue, or q <return> to quit--- #4 0x000000328ca07d15 in start_thread (arg=0x7fe64d5d1700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 4 (Thread 0x7fe64cdd0700 (LWP 20028)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943d18, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque@entry=0x1934930) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #4 0x000000328ca07d15 in start_thread (arg=0x7fe64cdd0700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 3 (Thread 0x7fe64c5cf700 (LWP 20029)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943d18, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque@entry=0x1934c50) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #4 0x000000328ca07d15 in start_thread (arg=0x7fe64c5cf700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 Thread 2 (Thread 0x7fe64bdce700 (LWP 20030)): #0 pthread_cond_wait@(a)GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165 #1 0x00007fe650ad155a in virCondWait (c=c@entry=0x1943d18, m=m@entry=0x1943c58) at util/virthreadpthread.c:117 #2 0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque@entry=0x1934b50) at util/virthreadpool.c:103 #3 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>) at util/virthreadpthread.c:161 #4 0x000000328ca07d15 in start_thread (arg=0x7fe64bdce700) at pthread_create.c:308 #5 0x000000328c6f246d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114 ---Type <return> to continue, or q <return> to quit--- Thread 1 (Thread 0x7fe6509f9840 (LWP 20007)): #0 0x000000328c6e998d in poll () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007fe650aa96e5 in poll (__timeout=-1, __nfds=9, __fds=<optimized out>) at /usr/include/bits/poll2.h:46 #2 virEventPollRunOnce () at util/vireventpoll.c:630 #3 0x00007fe650aa805b in virEventRunDefaultImpl () at util/virevent.c:273 #4 0x00007fe650bb9e55 in virNetServerRun (srv=srv@entry=0x1943b10) at rpc/virnetserver.c:1108 #5 0x000000000040c749 in main (argc=<optimized out>, argv=<optimized out>) at libvirtd.c:1481 Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Fedora Windows cross-compiler. Compile Windows programs, test, and build Windows installers. Over 100 libraries supported. http://fedoraproject.org/wiki/MinGW