[libvirt] libvirtd segfault

### Libvirt version: ### libvirt tagged v1.0.0 from git , with f0e72b2f5c675f927d04545dc5095f9e5998f171 applied ### Problem: ### Libvirtd segfaults ### Steps to reproduce: ### This is difficult to reproduce; but appears to happen only when doing 'virsh destroys', and when they are ran in concurrent intervals, over extended periods of time. Here are the steps I have taken to reproduce this problem (debian_6.0_amd64_kvm.img.gz is just a zipped up squeeze stock ISO install): First, copy this script to ease the steps (assumed to be at /root/libvirt_crash.pl later): #!/usr/bin/env perl use strict; use warnings; use Getopt::Long qw(:config no_ignore_case); use String::Random; use UUID::Random; my ($lvms,$sleep,$restarts_per_lvm); GetOptions( 'lvms=i' => \$lvms, 'restarts_per_lvm=i' => \$restarts_per_lvm, 'sleep=i' => \$sleep, ); die "USAGE:\n$0 [--lvms int | --restarts_per_lvm int | --sleep int (optional) ]\n" unless ( $lvms && $restarts_per_lvm ); sub _create_lvm { my $opts = shift; print "\tlvcreate -n $opts->{lvm} -L 5G /dev/LVM\n"; system("lvcreate -n $opts->{lvm} -L 5G /dev/LVM"); } sub _kpartx_lvm { my $opts = shift; die "_kpartx_lvm needs modifier!\n" unless ( $opts->{modifier} ); print "\tkpartx -p p -$opts->{modifier} -v /dev/LVM/$opts->{lvm}\n"; system("kpartx -p p -$opts->{modifier} -v /dev/LVM/$opts->{lvm}"); } sub _remove_lvm { my $opts = shift; print "\tlvremove -f /dev/LVM/$opts->{lvm}\n"; system("lvremove -f /dev/LVM/$opts->{lvm}"); unlink("/xen/configs/$opts->{lvm}.cfg"); } sub _display_lvm { my $opts = shift; print "\tparted -s /dev/LVM/$opts->{lvm} unit gb p\n"; system("parted -s /dev/LVM/$opts->{lvm} unit gb p"); } sub _dd_template { my $opts = shift; print "\tdd if=/root/ssullivan/debian_6.0_amd64_kvm.img.gz bs=1M | gunzip -c | dd of=/dev/LVM/$opts->{lvm} bs=1M\n"; system("dd if=/root/ssullivan/debian_6.0_amd64_kvm.img.gz bs=1M | gunzip -c | dd of=/dev/LVM/$opts->{lvm} bs=1M"); } sub _e2fsck_lvm { my $opts = shift; print "\te2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p1\n"; system("e2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p1"); print "\te2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p3\n"; system("e2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p3") } sub _place_virtconf { my $opts = shift; my $uuid = UUID::Random::generate; my $virt_conf = <<END; <domain type="kvm"> <name>$opts->{lvm}</name> <uuid>$uuid</uuid> <memory>450560</memory> <currentMemory>450560</currentMemory> <vcpu>2</vcpu> <cpu> <topology sockets="2" cores="4" threads="1"/> </cpu> <os> <type arch="x86_64" machine="pc-1.1">hvm</type> <boot dev="hd"/> </os> <features> <acpi/> <apic/> <pae/> </features> <clock offset="localtime"/> <on_poweroff>destroy</on_poweroff> <on_reboot>restart</on_reboot> <on_crash>restart</on_crash> <devices> <emulator>/opt/libexec/qemu-kvm-wrapper</emulator> <disk type="block" device="disk"> <driver name="qemu" cache="none"/> <source dev="/dev/LVM/$opts->{lvm}"/> <target dev="vda" bus="virtio"/> </disk> <serial type="pty"> <target port="1"/> </serial> <console type="pty"> <target port="1"/> </console> <input type="tablet" bus="usb"/> <input type="mouse" bus="ps2"/> <graphics type="vnc" port="-1" autoport="yes" keymap="en-us"/> </devices> </domain> END open(my $fh, '>', "/xen/configs/$opts->{lvm}.cfg") || die "Error opening [/xen/configs/$opts->{lvm}.cfg] [$!]"; print $fh "$virt_conf\n"; close($fh) || die "Error closing [/xen/configs/$opts->{lvm}.cfg] [$!]"; } sub _shutdown_lvm { my $opts = shift; print "\tvirsh destroy $opts->{lvm}\n"; system("virsh destroy $opts->{lvm}"); _virsh_list(); } sub _virsh_list { print "\tvirsh list\n"; system('virsh list'); } sub _start_lvm { my $opts = shift; print "\tvirsh create /xen/configs/$opts->{lvm}.cfg\n"; system("virsh create /xen/configs/$opts->{lvm}.cfg"); _virsh_list(); } my $rnd = new String::Random; my $count = 0; while ( $count < $lvms ) { my $id = $rnd->randpattern("cccccccc"); print "####\nLVM PASS: $count\nLVM: $id\n####\n"; print "Creating LVM...\n"; _create_lvm({ lvm => $id }); print "DD'ing template...\n"; _dd_template({ lvm => $id }); print "kpartxA LVM...\n"; _kpartx_lvm({ lvm => $id, modifier => 'a' }); print "e2fsck LVM..\n"; _e2fsck_lvm({ lvm => $id }); print "kpartxD LVM...\n"; _kpartx_lvm({ lvm => $id, modifier => 'd' }); print "Placing libvirt conf...\n"; _place_virtconf({ lvm => $id }); print "Partition setup:\n"; _display_lvm({ lvm => $id }); print "Performing $restarts_per_lvm restarts on $id...\n"; my $restart_cnt = 0; while ( $restart_cnt < $restarts_per_lvm ) { print "####\nRESTART PASS: $restart_cnt\nLVM: $id\n####\n"; print "Shutting down LVM...\n"; _shutdown_lvm({ lvm => $id }); sleep $sleep if ( $sleep ); print "Starting LVM...\n"; _start_lvm({ lvm => $id }); $restart_cnt++; } print "Removing LVM...\n"; _shutdown_lvm({ lvm => $id }); _remove_lvm({ lvm => $id }); $count++; } I then started 21 of these scripts, with the below commands (preferably in a screen to reduce spamming): perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 --sleep 10 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 --sleep 10 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 & perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 & I already had libvirtd started under GDB at this point. I waited about 3-6 hours, and I found this in my GDB session: 2012-12-27 17:23:35.922+0000: 7445: error : qemuMonitorIO:614 : internal error End of file from monitor 2012-12-27 17:23:36.558+0000: 7445: error : qemuMonitorIO:614 : internal error End of file from monitor Program received signal SIGSEGV, Segmentation fault. [Switching to Thread 0x7fff8e1fc700 (LWP 31142)] qemuDomainObjBeginJobInternal (driver=0x7fffe401d740, driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY, asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768 768 priv->jobs_queued++; Missing separate debuginfos, use: debuginfo-install audit-libs-2.1.3-3.el6.x86_64 augeas-libs-0.9.0-1.el6.x86_64 avahi-libs-0.6.25-11.el6.x86_64 cyrus-sasl-gssapi-2.1.23-13.el6.x86_64 cyrus-sasl-lib-2.1.23-13.el6.x86_64 cyrus-sasl-md5-2.1.23-13.el6.x86_64 cyrus-sasl-plain-2.1.23-13.el6.x86_64 db4-4.7.25-16.el6.x86_64 dbus-libs-1.2.24-5.el6_1.x86_64 device-mapper-libs-1.02.66-6.el6.x86_64 glibc-2.12-1.47.el6.x86_64 gnutls-2.8.5-4.el6.x86_64 keyutils-libs-1.4-3.el6.x86_64 krb5-libs-1.9-22.el6_2.1.x86_64 libblkid-2.17.2-12.4.el6.x86_64 libcap-ng-0.6.4-3.el6_0.1.x86_64 libcom_err-1.41.12-11.el6.x86_64 libcurl-7.19.7-26.el6_1.2.x86_64 libgcrypt-1.4.5-9.el6.x86_64 libgpg-error-1.7-4.el6.x86_64 libidn-1.18-2.el6.x86_64 libnl-1.1-14.el6.x86_64 libpcap-1.0.0-6.20091201git117cb5.el6.x86_64 libpciaccess-0.12.1-1.el6.x86_64 libselinux-2.0.94-5.2.el6.x86_64 libsepol-2.0.41-4.el6.x86_64 libtasn1-2.3-3.el6.x86_64 libudev-147-2.40.el6.x86_64 libuuid-2.17.2-12.4.el6.x86_64 libxml2-2.7.6-4.el6.x86_64 libxslt-1.1.26-2.el6.x86_64 netcf-libs-0.1.9-2.el6.x86_64 nspr-4.8.8-3.el6.x86_64 nss-3.12.10-17.el6_2.x86_64 nss-softokn-freebl-3.12.9-11.el6.x86_64 nss-util-3.12.10-2.el6.x86_64 numactl-2.0.3-9.el6.x86_64 openldap-2.4.23-20.el6.x86_64 openssl-1.0.0-20.el6.x86_64 yajl-1.0.7-3.el6.x86_64 zlib-1.2.3-27.el6.x86_64 (gdb) Here is the output of 'thread apply all bt' from this session: Thread 30 (Thread 0x7fff8e1fc700 (LWP 31142)): #0 qemuDomainObjBeginJobInternal (driver=0x7fffe401d740, driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY, asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768 #1 0x00007fffeac2b223 in qemuDomainDestroyFlags (dom=<value optimized out>, flags=<value optimized out>) at qemu/qemu_driver.c:2052 #2 0x00000039f10f97df in virDomainDestroy (domain=0x7fff741b8540) at libvirt.c:2201 #3 0x0000000000428e22 in remoteDispatchDomainDestroy (server=<value optimized out>, client=<value optimized out>, msg=<value optimized out>, rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value optimized out>) at remote_dispatch.h:1277 #4 remoteDispatchDomainDestroyHelper (server=<value optimized out>, client=<value optimized out>, msg=<value optimized out>, rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value optimized out>) at remote_dispatch.h:1255 #5 0x00000039f1146152 in virNetServerProgramDispatchCall (prog=0x6884a0, server=0x67fe60, client=0x68eb50, msg=0x6930b0) at rpc/virnetserverprogram.c:431 #6 virNetServerProgramDispatch (prog=0x6884a0, server=0x67fe60, client=0x68eb50, msg=0x6930b0) at rpc/virnetserverprogram.c:304 #7 0x00000039f1143fee in virNetServerProcessMsg (srv=<value optimized out>, client=0x68eb50, prog=<value optimized out>, msg=0x6930b0) at rpc/virnetserver.c:171 #8 0x00000039f1144a8b in virNetServerHandleJob (jobOpaque=<value optimized out>, opaque=<value optimized out>) at rpc/virnetserver.c:192 #9 0x00000039f10643ec in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:144 #10 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #11 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #12 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 29 (Thread 0x7fff8ebfd700 (LWP 20241)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 28 (Thread 0x7fff8f5fe700 (LWP 20235)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 27 (Thread 0x7fff8ffff700 (LWP 29349)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 26 (Thread 0x7fffb8dfa700 (LWP 29348)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 25 (Thread 0x7fffb97fb700 (LWP 28577)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 24 (Thread 0x7fffba1fc700 (LWP 28576)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 23 (Thread 0x7fffbabfd700 (LWP 28180)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 22 (Thread 0x7fffbb5fe700 (LWP 28178)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 21 (Thread 0x7fffe0dfa700 (LWP 12076)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 20 (Thread 0x7fffe17fb700 (LWP 5934)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 19 (Thread 0x7fffe21fc700 (LWP 5904)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 18 (Thread 0x7fffe2bfd700 (LWP 5860)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 17 (Thread 0x7fffbbfff700 (LWP 855)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 16 (Thread 0x7fffe97c4700 (LWP 835)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7fffec73f700 (LWP 7457)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7fffed140700 (LWP 7456)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7fffedb41700 (LWP 7455)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x7fffee542700 (LWP 7454)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x7fffeef43700 (LWP 7453)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7fffef944700 (LWP 7452)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7ffff0345700 (LWP 7451)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7ffff0d46700 (LWP 7450)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7ffff1747700 (LWP 7449)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7ffff2148700 (LWP 7448)): #0 0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117 #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103 #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161 #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003009ae570d in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7ffff7fda800 (LWP 7445)): #0 0x0000003009a46f60 in vfprintf () from /lib64/libc.so.6 #1 0x0000003009afc970 in __vsnprintf_chk () from /lib64/libc.so.6 #2 0x0000003009afc8aa in __snprintf_chk () from /lib64/libc.so.6 #3 0x00000039f108144a in snprintf (when=<value optimized out>, buf=0x7fffffffd470 "2012-12-27 17:23:36.561+0000") at /usr/include/bits/stdio2.h:65 #4 virTimeStringThenRaw (when=<value optimized out>, buf=0x7fffffffd470 "2012-12-27 17:23:36.561+0000") at util/virtime.c:219 #5 0x00000039f108167a in virTimeStringNowRaw (buf=0x7fffffffd470 "2012-12-27 17:23:36.561+0000") at util/virtime.c:195 #6 0x00000039f105834b in virLogVMessage (source=VIR_LOG_FROM_FILE, priority=<value optimized out>, filename=0x39f11ad70b "util/event_poll.c", linenr=378, funcname=0x39f11adcc0 "virEventPollMakePollFDs", fmt=<value optimized out>, vargs=0x7fffffffd4f0) at util/logging.c:822 #7 0x00000039f10586ec in virLogMessage (source=<value optimized out>, priority=<value optimized out>, filename=<value optimized out>, linenr=<value optimized out>, funcname=<value optimized out>, fmt=<value optimized out>) at util/logging.c:753 #8 0x00000039f1052241 in virEventPollMakePollFDs () at util/event_poll.c:374 #9 virEventPollRunOnce () at util/event_poll.c:605 #10 0x00000039f1051666 in virEventRunDefaultImpl () at util/event.c:247 #11 0x00000039f11437dd in virNetServerRun (srv=0x67fe60) at rpc/virnetserver.c:1004 #12 0x000000000040c5a3 in main (argc=<value optimized out>, argv=<value optimized out>) at libvirtd.c:1354 (gdb) Does anyone have any insights as to the cause of this, or any potential solutions? If there's anything I can add that would help let me know.