1:34 p.m.
2018-01-17 18:38 GMT+02:00 Serhii Kharchenko <harrykas(a)gmail.com
<mailto:harrykas@gmail.com>>:
2018-01-17 17:32 GMT+02:00 Michal Privoznik <mprivozn(a)redhat.com
<mailto:mprivozn@redhat.com>>:
Sounds like qemuConnectGetAllDomainStats() forgot to unset the
job on a
domain. Can you please attach gdb to libvirtd and run 't a a bt' and
share the output?
Michal
This is output of gdb 't a a bt' ~5 seconds after that error
message appeared in log with
remoteDispatchConnectGetAllDomainStats() mentioned.
And this is output of gdb 't a a bt' executed after the VPS is stuck in
'in shutdown' state but error message is not yet logged. It logged after
I executed gdb command.
Thread 18 (Thread 0x7ff36ffff700 (LWP 29836)):
#0 0x00007ff3c0fe4ca6 in pthread_cond_timedwait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978082 in virCondWaitUntil () from /usr/lib64/libvirt.so.0
#2 0x00007ff378017a28 in ?? () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#3 0x00007ff37801b9ab in qemuDomainObjBeginJob () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#4 0x00007ff378047306 in qemuProcessBeginStopJob () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#5 0x00007ff3780b1689 in ?? () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#6 0x00007ff3c1978a2b in ?? () from /usr/lib64/libvirt.so.0
#7 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#8 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#9 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 17 (Thread 0x7ff374fa1700 (LWP 28463)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff378971a20 in ?? () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_nodedev.so
#3 0x00007ff3c1977de2 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 16 (Thread 0x7ff375fa3700 (LWP 28459)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7ff3767a4700 (LWP 28458)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7ff376fa5700 (LWP 28457)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7ff3777a6700 (LWP 28456)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7ff377fa7700 (LWP 28455)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7ff3b37fe700 (LWP 28454)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b24 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7ff3b3fff700 (LWP 28453)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b24 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7ff3b8bb1700 (LWP 28452)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b24 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7ff3b93b2700 (LWP 28451)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b24 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7ff3b9bb3700 (LWP 28450)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b24 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7ff3ba3b4700 (LWP 28449)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7ff3babb5700 (LWP 28448)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7ff3bb3b6700 (LWP 28447)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7ff3bbbb7700 (LWP 28446)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff37805f76b in qemuMonitorSend () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#3 0x00007ff378073355 in ?? () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#4 0x00007ff378075237 in qemuMonitorJSONQueryCPUs () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#5 0x00007ff378061dd2 in qemuMonitorGetCpuHalted () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#6 0x00007ff378023c9e in qemuDomainRefreshVcpuHalted () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#7 0x00007ff3780a3329 in ?? () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#8 0x00007ff37808777f in ?? () from
/usr/lib64/libvirt/connection-driver/libvirt_driver_qemu.so
#9 0x00007ff3c1a37abe in virDomainListGetStats () from
/usr/lib64/libvirt.so.0
#10 0x0000560fd00243b5 in ?? ()
#11 0x00007ff3c1a92308 in virNetServerProgramDispatch () from
/usr/lib64/libvirt.so.0
#12 0x0000560fd0053ef8 in ?? ()
#13 0x00007ff3c1978a2b in ?? () from /usr/lib64/libvirt.so.0
#14 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#15 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#16 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7ff3bc3b8700 (LWP 28445)):
#0 0x00007ff3c0fe4746 in pthread_cond_wait@(a)GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007ff3c1978016 in virCondWait () from /usr/lib64/libvirt.so.0
#2 0x00007ff3c1978b63 in ?? () from /usr/lib64/libvirt.so.0
#3 0x00007ff3c1977db8 in ?? () from /usr/lib64/libvirt.so.0
#4 0x00007ff3c0fdd6b3 in start_thread () from /lib64/libpthread.so.0
#5 0x00007ff3c0d1176f in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7ff3c265a800 (LWP 28444)):
#0 0x00007ff3c0d059cc in poll () from /lib64/libc.so.6
#1 0x00007ff3c1921c84 in virEventPollRunOnce () from
/usr/lib64/libvirt.so.0
#2 0x00007ff3c1920971 in virEventRunDefaultImpl () from
/usr/lib64/libvirt.so.0
#3 0x00007ff3c1a8d64d in virNetDaemonRun () from /usr/lib64/libvirt.so.0
#4 0x0000560fd001a700 in ?? ()
#5 0x00007ff3c0c2efe1 in __libc_start_main () from /lib64/libc.so.6
#6 0x0000560fd001affa in ?? ()