[libvirt] libvirtd segfault

Thursday, 27 December 2012

### Libvirt version: ###

libvirt tagged v1.0.0 from git , with 
f0e72b2f5c675f927d04545dc5095f9e5998f171 applied

### Problem: ###

Libvirtd segfaults

### Steps to reproduce: ###

This is difficult to reproduce; but appears to happen only when doing 
'virsh destroys', and when they are ran in concurrent intervals, over 
extended periods of time.

Here are the steps I have taken to reproduce this problem 
(debian_6.0_amd64_kvm.img.gz is just a zipped up squeeze stock ISO install):

First, copy this script to ease the steps (assumed to be at 
/root/libvirt_crash.pl later):

#!/usr/bin/env perl

use strict;
use warnings;

use Getopt::Long qw(:config no_ignore_case);
use String::Random;
use UUID::Random;

my ($lvms,$sleep,$restarts_per_lvm);
GetOptions(
         'lvms=i' => \$lvms,
         'restarts_per_lvm=i' => \$restarts_per_lvm,
         'sleep=i' => \$sleep,
);

die "USAGE:\n$0 [--lvms int | --restarts_per_lvm int | --sleep int 
(optional) ]\n" unless ( $lvms && $restarts_per_lvm );

sub _create_lvm {
         my $opts = shift;
         print "\tlvcreate -n $opts->{lvm} -L 5G /dev/LVM\n";
         system("lvcreate -n $opts->{lvm} -L 5G /dev/LVM");
}

sub _kpartx_lvm {
         my $opts = shift;
         die "_kpartx_lvm needs modifier!\n" unless ( $opts->{modifier} );
         print "\tkpartx -p p -$opts->{modifier} -v 
/dev/LVM/$opts->{lvm}\n";
         system("kpartx -p p -$opts->{modifier} -v
/dev/LVM/$opts->{lvm}");
}

sub _remove_lvm {
         my $opts = shift;
         print "\tlvremove -f /dev/LVM/$opts->{lvm}\n";
         system("lvremove -f /dev/LVM/$opts->{lvm}");
         unlink("/xen/configs/$opts->{lvm}.cfg");
}

sub _display_lvm {
         my $opts = shift;
         print "\tparted -s /dev/LVM/$opts->{lvm} unit gb p\n";
         system("parted -s /dev/LVM/$opts->{lvm} unit gb p");
}

sub _dd_template {
         my $opts = shift;
         print "\tdd if=/root/ssullivan/debian_6.0_amd64_kvm.img.gz 
bs=1M | gunzip -c | dd of=/dev/LVM/$opts->{lvm} bs=1M\n";
         system("dd if=/root/ssullivan/debian_6.0_amd64_kvm.img.gz bs=1M 
| gunzip -c | dd of=/dev/LVM/$opts->{lvm} bs=1M");
}

sub _e2fsck_lvm {
         my $opts = shift;
         print "\te2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p1\n";
         system("e2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p1");
         print "\te2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p3\n";
         system("e2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p3")
}

sub _place_virtconf {
         my $opts = shift;

         my $uuid = UUID::Random::generate;
         my $virt_conf = <<END;
<domain type="kvm">
<name>$opts->{lvm}</name>
<uuid>$uuid</uuid>
<memory>450560</memory>
<currentMemory>450560</currentMemory>
<vcpu>2</vcpu>
<cpu>
<topology sockets="2" cores="4" threads="1"/>
</cpu>
<os>
<type arch="x86_64" machine="pc-1.1">hvm</type>
<boot dev="hd"/>
</os>
<features>
<acpi/>
<apic/>
<pae/>
</features>
<clock offset="localtime"/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/opt/libexec/qemu-kvm-wrapper</emulator>
<disk type="block" device="disk">
<driver name="qemu" cache="none"/>
<source dev="/dev/LVM/$opts->{lvm}"/>
<target dev="vda" bus="virtio"/>
</disk>

<serial type="pty">
<target port="1"/>
</serial>
<console type="pty">
<target port="1"/>
</console>

<input type="tablet" bus="usb"/>
<input type="mouse" bus="ps2"/>
<graphics type="vnc" port="-1" autoport="yes"
keymap="en-us"/>
</devices>
</domain>
END

         open(my $fh, '>', "/xen/configs/$opts->{lvm}.cfg") || die

"Error opening [/xen/configs/$opts->{lvm}.cfg] [$!]";
         print $fh "$virt_conf\n";
         close($fh) || die "Error closing 
[/xen/configs/$opts->{lvm}.cfg] [$!]";
}

sub _shutdown_lvm {
         my $opts = shift;
         print "\tvirsh destroy $opts->{lvm}\n";
         system("virsh destroy $opts->{lvm}");
         _virsh_list();
}

sub _virsh_list {
         print "\tvirsh list\n";
         system('virsh list');
}

sub _start_lvm {
         my $opts = shift;
         print "\tvirsh create /xen/configs/$opts->{lvm}.cfg\n";
         system("virsh create /xen/configs/$opts->{lvm}.cfg");
         _virsh_list();
}

my $rnd = new String::Random;
my $count = 0;
while ( $count < $lvms ) {

         my $id = $rnd->randpattern("cccccccc");
         print "####\nLVM PASS: $count\nLVM: $id\n####\n";

         print "Creating LVM...\n";
         _create_lvm({ lvm => $id });

         print "DD'ing template...\n";
         _dd_template({ lvm => $id });

         print "kpartxA LVM...\n";
         _kpartx_lvm({ lvm => $id, modifier => 'a' });

         print "e2fsck LVM..\n";
         _e2fsck_lvm({ lvm => $id });

         print "kpartxD LVM...\n";
         _kpartx_lvm({ lvm => $id, modifier => 'd' });

         print "Placing libvirt conf...\n";
         _place_virtconf({ lvm => $id });

         print "Partition setup:\n";
         _display_lvm({ lvm => $id });

         print "Performing $restarts_per_lvm restarts on $id...\n";
         my $restart_cnt = 0;
         while ( $restart_cnt < $restarts_per_lvm ) {

                 print "####\nRESTART PASS: $restart_cnt\nLVM: $id\n####\n";

                 print "Shutting down LVM...\n";
                 _shutdown_lvm({ lvm => $id });

                 sleep $sleep if ( $sleep );

                 print "Starting LVM...\n";
                 _start_lvm({ lvm => $id });

                 $restart_cnt++;
         }

         print "Removing LVM...\n";
         _shutdown_lvm({ lvm => $id });
         _remove_lvm({ lvm => $id });

         $count++;
}

I then started 21 of these scripts, with the below commands (preferably 
in a screen to reduce spamming):

perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 --sleep 10 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 --sleep 10 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &

I already had libvirtd started under GDB at this point. I waited about 
3-6 hours, and I found this in my GDB session:

2012-12-27 17:23:35.922+0000: 7445: error : qemuMonitorIO:614 : internal 
error End of file from monitor
2012-12-27 17:23:36.558+0000: 7445: error : qemuMonitorIO:614 : internal 
error End of file from monitor

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fff8e1fc700 (LWP 31142)]
qemuDomainObjBeginJobInternal (driver=0x7fffe401d740, 
driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY, 
asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768
768         priv->jobs_queued++;
Missing separate debuginfos, use: debuginfo-install 
audit-libs-2.1.3-3.el6.x86_64 augeas-libs-0.9.0-1.el6.x86_64 
avahi-libs-0.6.25-11.el6.x86_64 cyrus-sasl-gssapi-2.1.23-13.el6.x86_64 
cyrus-sasl-lib-2.1.23-13.el6.x86_64 cyrus-sasl-md5-2.1.23-13.el6.x86_64 
cyrus-sasl-plain-2.1.23-13.el6.x86_64 db4-4.7.25-16.el6.x86_64 
dbus-libs-1.2.24-5.el6_1.x86_64 device-mapper-libs-1.02.66-6.el6.x86_64 
glibc-2.12-1.47.el6.x86_64 gnutls-2.8.5-4.el6.x86_64 
keyutils-libs-1.4-3.el6.x86_64 krb5-libs-1.9-22.el6_2.1.x86_64 
libblkid-2.17.2-12.4.el6.x86_64 libcap-ng-0.6.4-3.el6_0.1.x86_64 
libcom_err-1.41.12-11.el6.x86_64 libcurl-7.19.7-26.el6_1.2.x86_64 
libgcrypt-1.4.5-9.el6.x86_64 libgpg-error-1.7-4.el6.x86_64 
libidn-1.18-2.el6.x86_64 libnl-1.1-14.el6.x86_64 
libpcap-1.0.0-6.20091201git117cb5.el6.x86_64 
libpciaccess-0.12.1-1.el6.x86_64 libselinux-2.0.94-5.2.el6.x86_64 
libsepol-2.0.41-4.el6.x86_64 libtasn1-2.3-3.el6.x86_64 
libudev-147-2.40.el6.x86_64 libuuid-2.17.2-12.4.el6.x86_64 
libxml2-2.7.6-4.el6.x86_64 libxslt-1.1.26-2.el6.x86_64 
netcf-libs-0.1.9-2.el6.x86_64 nspr-4.8.8-3.el6.x86_64 
nss-3.12.10-17.el6_2.x86_64 nss-softokn-freebl-3.12.9-11.el6.x86_64 
nss-util-3.12.10-2.el6.x86_64 numactl-2.0.3-9.el6.x86_64 
openldap-2.4.23-20.el6.x86_64 openssl-1.0.0-20.el6.x86_64 
yajl-1.0.7-3.el6.x86_64 zlib-1.2.3-27.el6.x86_64
(gdb)

Here is the output of 'thread apply all bt' from this session:

Thread 30 (Thread 0x7fff8e1fc700 (LWP 31142)):
#0  qemuDomainObjBeginJobInternal (driver=0x7fffe401d740, 
driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY, 
asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768
#1  0x00007fffeac2b223 in qemuDomainDestroyFlags (dom=<value optimized 
out>, flags=<value optimized out>) at qemu/qemu_driver.c:2052
#2  0x00000039f10f97df in virDomainDestroy (domain=0x7fff741b8540) at 
libvirt.c:2201
#3  0x0000000000428e22 in remoteDispatchDomainDestroy (server=<value 
optimized out>, client=<value optimized out>, msg=<value optimized out>, 
rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value optimized 
out>) at remote_dispatch.h:1277
#4  remoteDispatchDomainDestroyHelper (server=<value optimized out>, 
client=<value optimized out>, msg=<value optimized out>, 
rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value optimized 
out>) at remote_dispatch.h:1255
#5  0x00000039f1146152 in virNetServerProgramDispatchCall 
(prog=0x6884a0, server=0x67fe60, client=0x68eb50, msg=0x6930b0) at 
rpc/virnetserverprogram.c:431
#6  virNetServerProgramDispatch (prog=0x6884a0, server=0x67fe60, 
client=0x68eb50, msg=0x6930b0) at rpc/virnetserverprogram.c:304
#7  0x00000039f1143fee in virNetServerProcessMsg (srv=<value optimized 
out>, client=0x68eb50, prog=<value optimized out>, msg=0x6930b0) at 
rpc/virnetserver.c:171
#8  0x00000039f1144a8b in virNetServerHandleJob (jobOpaque=<value 
optimized out>, opaque=<value optimized out>) at rpc/virnetserver.c:192
#9  0x00000039f10643ec in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:144
#10 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#11 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#12 0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 29 (Thread 0x7fff8ebfd700 (LWP 20241)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 28 (Thread 0x7fff8f5fe700 (LWP 20235)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 27 (Thread 0x7fff8ffff700 (LWP 29349)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 26 (Thread 0x7fffb8dfa700 (LWP 29348)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 25 (Thread 0x7fffb97fb700 (LWP 28577)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 24 (Thread 0x7fffba1fc700 (LWP 28576)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 23 (Thread 0x7fffbabfd700 (LWP 28180)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 22 (Thread 0x7fffbb5fe700 (LWP 28178)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 21 (Thread 0x7fffe0dfa700 (LWP 12076)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 20 (Thread 0x7fffe17fb700 (LWP 5934)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 19 (Thread 0x7fffe21fc700 (LWP 5904)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 18 (Thread 0x7fffe2bfd700 (LWP 5860)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 17 (Thread 0x7fffbbfff700 (LWP 855)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 16 (Thread 0x7fffe97c4700 (LWP 835)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 11 (Thread 0x7fffec73f700 (LWP 7457)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 10 (Thread 0x7fffed140700 (LWP 7456)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 9 (Thread 0x7fffedb41700 (LWP 7455)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x7fffee542700 (LWP 7454)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 7 (Thread 0x7fffeef43700 (LWP 7453)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x7fffef944700 (LWP 7452)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 5 (Thread 0x7ffff0345700 (LWP 7451)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x7ffff0d46700 (LWP 7450)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x7ffff1747700 (LWP 7449)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x7ffff2148700 (LWP 7448)):
#0  0x000000300a20b3dc in pthread_cond_wait@(a)GLIBC_2.3.2 () from 
/lib64/libpthread.so.0
#1  0x00000039f1063eb6 in virCondWait (c=<value optimized out>, m=<value 
optimized out>) at util/threads-pthread.c:117
#2  0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized 
out>) at util/threadpool.c:103
#3  0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>) 
at util/threads-pthread.c:161
#4  0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#5  0x0000003009ae570d in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7ffff7fda800 (LWP 7445)):
#0  0x0000003009a46f60 in vfprintf () from /lib64/libc.so.6
#1  0x0000003009afc970 in __vsnprintf_chk () from /lib64/libc.so.6
#2  0x0000003009afc8aa in __snprintf_chk () from /lib64/libc.so.6
#3  0x00000039f108144a in snprintf (when=<value optimized out>, 
buf=0x7fffffffd470 "2012-12-27 17:23:36.561+0000") at 
/usr/include/bits/stdio2.h:65
#4  virTimeStringThenRaw (when=<value optimized out>, buf=0x7fffffffd470 
"2012-12-27 17:23:36.561+0000") at util/virtime.c:219
#5  0x00000039f108167a in virTimeStringNowRaw (buf=0x7fffffffd470 
"2012-12-27 17:23:36.561+0000") at util/virtime.c:195
#6  0x00000039f105834b in virLogVMessage (source=VIR_LOG_FROM_FILE, 
priority=<value optimized out>, filename=0x39f11ad70b 
"util/event_poll.c", linenr=378, funcname=0x39f11adcc0 
"virEventPollMakePollFDs", fmt=<value optimized out>, 
vargs=0x7fffffffd4f0) at util/logging.c:822
#7  0x00000039f10586ec in virLogMessage (source=<value optimized out>, 
priority=<value optimized out>, filename=<value optimized out>, 
linenr=<value optimized out>, funcname=<value optimized out>, fmt=<value 
optimized out>) at util/logging.c:753
#8  0x00000039f1052241 in virEventPollMakePollFDs () at 
util/event_poll.c:374
#9  virEventPollRunOnce () at util/event_poll.c:605
#10 0x00000039f1051666 in virEventRunDefaultImpl () at util/event.c:247
#11 0x00000039f11437dd in virNetServerRun (srv=0x67fe60) at 
rpc/virnetserver.c:1004
#12 0x000000000040c5a3 in main (argc=<value optimized out>, argv=<value 
optimized out>) at libvirtd.c:1354
(gdb)

Does anyone have any insights as to the cause of this, or any potential 
solutions? If there's anything I can add that would help let me know.

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005