2018-01-17 17:32 GMT+02:00 Michal Privoznik <mprivozn@redhat.com>:
Sounds like qemuConnectGetAllDomainStats() forgot to unset the job on a
domain. Can you please attach gdb to libvirtd and run 't a a bt' and
share the output?

Michal

This is output of gdb  't a a bt' ~5 seconds after that error message appeared in log with remoteDispatchConnectGetAllDomainStats() mentioned.

Thread 18 (Thread 0x7fd395ffb700 (LWP 3803)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 17 (Thread 0x7fd3967fc700 (LWP 2858)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3a6589a20 in ?? () from /usr/lib64/libvirt/connection-driver/libvirt_driver_nodedev.so
#3  0x00007fd3bc5b2de2 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 16 (Thread 0x7fd3977fe700 (LWP 2854)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 15 (Thread 0x7fd397fff700 (LWP 2852)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 14 (Thread 0x7fd3a4bbd700 (LWP 2851)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 13 (Thread 0x7fd3a53be700 (LWP 2850)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 12 (Thread 0x7fd3a5bbf700 (LWP 2849)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 11 (Thread 0x7fd3ae7fc700 (LWP 2848)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b24 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 10 (Thread 0x7fd3aeffd700 (LWP 2847)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b24 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 9 (Thread 0x7fd3af7fe700 (LWP 2846)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b24 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x7fd3affff700 (LWP 2845)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b24 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 7 (Thread 0x7fd3b4fef700 (LWP 2844)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b24 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x7fd3a7fff700 (LWP 2843)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 5 (Thread 0x7fd3b57f0700 (LWP 2842)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x7fd3b5ff1700 (LWP 2841)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x7fd3b67f2700 (LWP 2840)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc5b3b63 in ?? () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#4  0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x7fd3b6ff3700 (LWP 2839)):
#0  0x00007fd3bbc1f746 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fd3bc5b3016 in virCondWait () from /usr/lib64/libvirt.so.0
#2  0x00007fd3a5c7776b in qemuMonitorSend () from /usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#3  0x00007fd3a5c8b355 in ?? () from /usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#4  0x00007fd3a5c8d85f in qemuMonitorJSONGetMemoryStats () from /usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#5  0x00007fd3a5c9efc3 in ?? () from /usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#6  0x00007fd3a5c9f18a in ?? () from /usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#7  0x00007fd3a5c9f77f in ?? () from /usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#8  0x00007fd3bc672abe in virDomainListGetStats () from /usr/lib64/libvirt.so.0
#9  0x000055f1c9fbb3b5 in ?? ()
#10 0x00007fd3bc6cd308 in virNetServerProgramDispatch () from /usr/lib64/libvirt.so.0
#11 0x000055f1c9feaef8 in ?? ()
#12 0x00007fd3bc5b3a2b in ?? () from /usr/lib64/libvirt.so.0
#13 0x00007fd3bc5b2db8 in ?? () from /usr/lib64/libvirt.so.0
#14 0x00007fd3bbc186b3 in start_thread () from /lib64/libpthread.so.0
#15 0x00007fd3bb94c76f in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7fd3bd295800 (LWP 2838)):
#0  0x00007fd3bb9409cc in poll () from /lib64/libc.so.6
#1  0x00007fd3bc55cc84 in virEventPollRunOnce () from /usr/lib64/libvirt.so.0
#2  0x00007fd3bc55b971 in virEventRunDefaultImpl () from /usr/lib64/libvirt.so.0
#3  0x00007fd3bc6c864d in virNetDaemonRun () from /usr/lib64/libvirt.so.0
#4  0x000055f1c9fb1700 in ?? ()
#5  0x00007fd3bb869fe1 in __libc_start_main () from /lib64/libc.so.6
#6  0x000055f1c9fb1ffa in ?? ()