Hey Jim, Andrew, and Ian, This is libvirt v1.2.14 + three patches: c82a59b libxl: drop virDomainObj lock when destroying a domain a1c9d30 libxl: acquire a job when destroying a domain 5bd5406 libxl: Move job acquisition in libxlDomainStart to callers For fun I've set up an guest with PCI passthrough and tried to save it (HAHAH) with an disastrous result (xc_save_helper was stuck). Probably due to outstanding DMA operations wreaking havoc. I will shortly send an patch to disallow 'save' with hosted devices but this scenario exposed an interesting livelock that can happen without passthrough if things go wrong. I have not dug entirely in the code so my observation and theory are most likely incorrect. CC-ing Ian and Andrew. The full dump is below, the end result is that libvirtd is locked. Doing 'libvirt list' or any other commands ends with such command hanged. libvirtd itself locks to be deadlocked on CTX_LOCK. On one thread it has received an event (libxl_osevent_occurred_fd) and tries to take the CTX_LOCK, while another thread (number 10) is stuck in eventloop_iteration which holds the CTX_LOCK. It looks like thread #10 is blocking in libxl_read_exactly waiting for 'libxl-save-helper'. Said application (see below) has dispatched an message through helper_getreply and is blocking on __read_nocancel. I presume the message it has dispatched is the same message the 'libxl_osevent_occurred_fd' is trying to get. In other words, one thread has taken the lock, and is blocked waiting for 'libxl-save-helper'. While 'libxl-save-helper' is blocked waiting for its message to be replied to - which another libvirtd thread tries to do but it cannot as the lock has been already taken. Since I did not dig completely in the code I am probably very wrong about this. Would appreciate thoughts whether this is possible and where I've gotten it wrong. Sorry about the control characters and such. I was on a serial console and capturing this. thread apply bt full (gdb) bt #0 0x00007fd3c50d5f1d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fd3c50d0906 in pthread_mutex_lock () from /lib64/libpthread.so.0 #2 0x00007fd3c5b849b9 in libxl__ctx_lock (ctx=0x7fd38c027ed0) at libxl_internal.h:3259 #3 0x00007fd3c5b87c8e in libxl_osevent_occurred_fd (ctx=0x7fd38c027ed0, for_libxl=0x7fd3ac004230, fd=23, events_ign=0, revents_ign=1) at libxl_event.c:1235 #4 0x00007fd3c6825dc3 in virEventPollDispatchHandles (fds=<optimized out>, nfds=<optimized out>) at util/vireventpoll.c:508 #5 virEventPollRunOnce () at util/vireventpoll.c:657 #6 0x00007fd3c6824651 in virEventRunDefaultImpl () at util/virevent.c:308 #7 0x00007fd3c72aee2d in virNetServerRun (srv=0x7fd3c8bfab40) at rpc/virnetserver.c:1139 #8 0x00007fd3c727add5 in main (argc=<optimized out>, argv=<optimized out>) at libvirtd.c:1491 (gdb) thread apply all bt full Thread 12 (Thread 0x7fd3bc1b6700 (LWP 1263)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfad98, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8bfa7a0) at util/virthreadpool.c:105 data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfad98 priority = false job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8bfa7a0} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 11 (Thread 0x7fd3bb9b5700 (LWP 1264)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfad98, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8bfa960) at util/virthreadpool.c:105 data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfad98 priority = false job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8bfa960} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 10 (Thread 0x7fd3bb1b4700 (LWP 1265)): ---Type <return> to continue, or q <return> to quit--- #0 0x00007fd3c50d61cd in read () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c5b78df2 in libxl_read_exactly (ctx=0x7fd38c027ed0, fd=31, data=0x7fd3bb1b37ce, sz=2, source=0x7fd3b4001520 "domain 5 save/restore helper stdout pipe", what=0x7fd3c5bb7cca "ipc msg header") at libxl_utils.c:430 got = 8313484836034408292 __func__ = "libxl_read_exactly" #2 0x00007fd3c5b80fce in helper_stdout_readable (egc=0x7fd3bb1b3930, ev=0x7fd3b4002868, fd=31, events=3, revents=1) at libxl_save_callout.c:279 shs = 0x7fd3b40027f0 ao = 0x7fd3b4000e50 gc = 0x7fd3b4000e60 rc = 32723 errnoval = 32723 __func__ = "helper_stdout_readable" msglen = 62 msg = <error reading variable msg (Insufficient memory in host GDB for object of size 8026576060204213623 bytes, maximum allowed 2305843009213693951 bytes.)> #3 0x00007fd3c5b87928 in afterpoll_internal (egc=0x7fd3bb1b3930, poller=0x7fd3ac000e30, nfds=4, fds=0x7fd3ac0016f0, now=...) at libxl_event.c:1178 revents = 1 gc = 0x7fd3bb1b3930 efd = 0x7fd3b4002868 __func__ = "afterpoll_internal" __PRETTY_FUNCTION__ = "afterpoll_internal" #4 0x00007fd3c5b89070 in eventloop_iteration (egc=0x7fd3bb1b3930, poller=0x7fd3ac000e30) at libxl_event.c:1638 gc = 0x7fd3bb1b3930 rc = 0 nfds = 4 now = {tv_sec = 1428428961, tv_usec = 930003} timeout = -1 __func__ = "eventloop_iteration" #5 0x00007fd3c5b89bb6 in libxl__ao_inprogress (ao=0x7fd3b4000e50, file=0x7fd3c5baca3b "libxl.c", line=982, func=0x7fd3c5baf300 <__func__.17587> "libxl_domain_suspend") at libxl_event.c:1889 egc = {gc = {alloc_maxsize = 0, alloc_ptrs = 0x0, owner = 0x7fd38c027ed0}, occurred_for_callback = {tqh_first = 0x0, tqh_last = 0x7fd3bb1b3948}, aos_for_callback = {tqh_first = 0x0, tqh_last = 0x7fd3bb1b3958}, aops_for_callback = { tqh_first = 0x0, tqh_last = 0x7fd3bb1b3968}} gc = 0x7fd3b4000e60 rc = 0 __PRETTY_FUNCTION__ = "libxl__ao_inprogress" __func__ = "libxl__ao_inprogress" #6 0x00007fd3c5b46178 in libxl_domain_suspend (ctx=0x7fd38c027ed0, domid=5, fd=27, flags=0, ao_how=0x0) at libxl.c:982 ao__ctx = 0x7fd38c027ed0 ---Type <return> to continue, or q <return> to quit--- ao__rc = 0 ao = 0x7fd3b4000e50 __func__ = "libxl_domain_suspend" egc = {{gc = {alloc_maxsize = 0, alloc_ptrs = 0x0, owner = 0x7fd38c027ed0}, occurred_for_callback = {tqh_first = 0x0, tqh_last = 0x7fd3bb1b3a08}, aos_for_callback = {tqh_first = 0x0, tqh_last = 0x7fd3bb1b3a18}, aops_for_callback = { tqh_first = 0x0, tqh_last = 0x7fd3bb1b3a28}}} gc = 0x7fd3b4000e60 rc = 0 type = LIBXL_DOMAIN_TYPE_HVM dss = 0x7fd3b40025f0 __PRETTY_FUNCTION__ = "libxl_domain_suspend" #7 0x00007fd399cce9a8 in libxlDoDomainSave (driver=driver@entry=0x7fd38c0a3af0, vm=vm@entry=0x7fd38c00cb80, to=to@entry=0x7fd3b4000980 "/root/save") at libxl/libxl_driver.c:1584 cfg = 0x7fd38c014050 hdr = {magic = "libvirt-xml\n \000 \r", version = 1, xmlLen = 1555, unused = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}} event = 0x0 xml = 0x7fd3b40019d0 "<domain type='xen' id='5'>\n <name>OL6</name>\n <uuid>8c46a459-f9b8-42bb-94aa-c469adda5a97</uuid>\n <memory unit='KiB'>2097152</memory>\n <currentMemory unit='KiB'>2097152</currentMemory>\n <vcpu plac"... xml_len = 1555 fd = 27 ret = -1 __FUNCTION__ = "libxlDoDomainSave" #8 0x00007fd399ccefc2 in libxlDomainSaveFlags (dom=0x7fd3b4001410, to=0x7fd3b4000980 "/root/save", dxml=<optimized out>, flags=<optimized out>) at libxl/libxl_driver.c:1653 driver = 0x7fd38c0a3af0 vm = 0x7fd38c00cb80 ret = -1 remove_dom = false __FUNCTION__ = "libxlDomainSaveFlags" #9 0x00007fd3c68e95a4 in virDomainSave (domain=domain@entry=0x7fd3b4001410, to=0x7fd3b4001390 "/root/save") at libvirt-domain.c:841 ret = <optimized out> absolute_to = 0x7fd3b4000980 "/root/save" conn = 0x7fd3b4001240 __func__ = "virDomainSave" __FUNCTION__ = "virDomainSave" #10 0x00007fd3c729be8f in remoteDispatchDomainSave (server=0x7fd3c8bfab40, msg=0x7fd3c8bfded0, args=0x7fd3b4001450, rerr=0x7fd3bb1b3ca0, client=<optimized out>) at remote_dispatch.h:8071 dom = 0x7fd3b4001410 priv = <optimized out> #11 remoteDispatchDomainSaveHelper (server=0x7fd3c8bfab40, client=<optimized out>, msg=0x7fd3c8bfded0, rerr=0x7fd3bb1b3ca0, args=0x7fd3b4001450, ret=0x7fd3b4000e30) at remote_dispatch.h:8047 ---Type <return> to continue, or q <return> to quit--- rv = -1275063216 __func__ = "remoteDispatchDomainSaveHelper" #12 0x00007fd3c6948789 in virNetServerProgramDispatchCall (msg=0x7fd3c8bfded0, client=0x7fd3c8c02330, server=0x7fd3c8bfab40, prog=0x7fd3c8bf4d10) at rpc/virnetserverprogram.c:437 ret = 0x7fd3b4000e30 "" rv = -1 i = <optimized out> identity = 0x7fd3ac0009a0 arg = 0x7fd3b4001450 "\300\r" dispatcher = 0x7fd3c74d8c90 <remoteProcs+2640> rerr = {code = 0, domain = 0, message = 0x0, level = 0, dom = 0x0, str1 = 0x0, str2 = 0x0, str3 = 0x0, int1 = 0, int2 = 0, net = 0x0} #13 virNetServerProgramDispatch (prog=0x7fd3c8bf4d10, server=server@entry=0x7fd3c8bfab40, client=0x7fd3c8c02330, msg=0x7fd3c8bfded0) at rpc/virnetserverprogram.c:307 rerr = {code = 0, domain = 0, message = 0x0, level = 0, dom = 0x0, str1 = 0x0, str2 = 0x0, str3 = 0x0, int1 = 0, int2 = 0, net = 0x0} __func__ = "virNetServerProgramDispatch" __FUNCTION__ = "virNetServerProgramDispatch" #14 0x00007fd3c72ad9f8 in virNetServerProcessMsg (msg=<optimized out>, prog=<optimized out>, client=<optimized out>, srv=0x7fd3c8bfab40) at rpc/virnetserver.c:172 ret = -1 #15 virNetServerHandleJob (jobOpaque=<optimized out>, opaque=0x7fd3c8bfab40) at rpc/virnetserver.c:193 srv = 0x7fd3c8bfab40 job = 0x7fd3c8c0e9b0 __func__ = "virNetServerHandleJob" #16 0x00007fd3c685c8fe in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8bfa650) at util/virthreadpool.c:145 data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfad98 priority = false job = 0x7fd3c8bf51b0 #17 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8bfa650} #18 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #19 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 9 (Thread 0x7fd3ba9b3700 (LWP 1266)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfad98, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8bfa5e0) at util/virthreadpool.c:105 data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfad98 priority = false job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8bfa5e0} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 8 (Thread 0x7fd3ba1b2700 (LWP 1267)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfad98, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8c048a0) at util/virthreadpool.c:105 data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfad98 priority = false job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8c048a0} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 7 (Thread 0x7fd3b99b1700 (LWP 1268)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfae30, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ---Type <return> to continue, or q <return> to quit--- ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8bfa960) at util/virthreadpool.c:105 data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfae30 priority = true job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8bfa960} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 6 (Thread 0x7fd3b91b0700 (LWP 1269)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfae30, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8bfa5e0) at util/virthreadpool.c:105 data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfae30 priority = true job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8bfa5e0} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 5 (Thread 0x7fd3b89af700 (LWP 1270)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfae30, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8c048a0) at util/virthreadpool.c:105 ---Type <return> to continue, or q <return> to quit--- data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfae30 priority = true job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8c048a0} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 4 (Thread 0x7fd39bfff700 (LWP 1271)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfae30, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8bfa7a0) at util/virthreadpool.c:105 data = 0x0 pool = 0x7fd3c8bfad30 cond = 0x7fd3c8bfae30 priority = true job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8bfa7a0} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 3 (Thread 0x7fd39b7fe700 (LWP 1272)): #0 0x00007fd3c50d3590 in pthread_cond_wait@(a)GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c685c0e6 in virCondWait (c=c@entry=0x7fd3c8bfae30, m=m@entry=0x7fd3c8bfad70) at util/virthread.c:154 ret = <optimized out> #2 0x00007fd3c685c99b in virThreadPoolWorker (opaque=opaque@entry=0x7fd3c8bfa5e0) at util/virthreadpool.c:105 data = 0x0 pool = 0x7fd3c8bfad30 ---Type <return> to continue, or q <return> to quit--- cond = 0x7fd3c8bfae30 priority = true job = 0x0 #3 0x00007fd3c685be88 in virThreadHelper (data=<optimized out>) at util/virthread.c:206 args = 0x0 local = {func = 0x7fd3c685c7b0 <virThreadPoolWorker>, funcName = 0x7fd3c72c0bb1 "virNetServerHandleJob", worker = true, opaque = 0x7fd3c8bfa5e0} #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 2 (Thread 0x7fd3c7208700 (LWP 1862)): #0 0x00007fd3c50d61cd in read () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c2f61c70 in read_all (fd=16, data=0x7fd38c02a450, len=16, nonblocking=0) at xs.c:378 done = 0 #2 0x00007fd3c2f636fc in read_message (h=0x7fd38c028150, nonblocking=0) at xs.c:1150 __cancel_buf = {__cancel_jmp_buf = {{__cancel_jmp_buf = {0, -6517440478529039152, 140547370732032, 0, 140547555624704, 140547555625408, -6517440478602439472, -6517432384286599984}, __mask_was_saved = 0}}, __pad = {0x7fd3c7207df0, 0x0, 0x0, 0x0}} __cancel_routine = 0x7fd3c4d8d440 <free> __cancel_arg = 0x7fd38c02a440 __not_first_call = 0 msg = 0x7fd38c02a440 body = 0x0 saved_errno = 0 ret = -1 #3 0x00007fd3c2f63a4c in read_thread (arg=0x7fd38c028150) at xs.c:1222 h = 0x7fd38c028150 fd = 0 #4 0x00007fd3c50ce52a in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00007fd3c4e0a22d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 1 (Thread 0x7fd3c7209880 (LWP 1262)): #0 0x00007fd3c50d5f1d in __lll_lock_wait () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00007fd3c50d0906 in pthread_mutex_lock () from /lib64/libpthread.so.0 No symbol table info available. #2 0x00007fd3c5b849b9 in libxl__ctx_lock (ctx=0x7fd38c027ed0) at libxl_internal.h:3259 ---Type <return> to continue, or q <return> to quit--- r = 32767 __PRETTY_FUNCTION__ = "libxl__ctx_lock" #3 0x00007fd3c5b87c8e in libxl_osevent_occurred_fd (ctx=0x7fd38c027ed0, for_libxl=0x7fd3ac004230, fd=23, events_ign=0, revents_ign=1) at libxl_event.c:1235 egc = {{gc = {alloc_maxsize = 0, alloc_ptrs = 0x0, owner = 0x7fd38c027ed0}, occurred_for_callback = {tqh_first = 0x0, tqh_last = 0x7fff5b0bf878}, aos_for_callback = {tqh_first = 0x0, tqh_last = 0x7fff5b0bf888}, aops_for_callback = { tqh_first = 0x0, tqh_last = 0x7fff5b0bf898}}} gc = 0x7fff5b0bf860 __PRETTY_FUNCTION__ = "libxl_osevent_occurred_fd" ev = 0x7fd3c6976f38 check = {fd = 11, events = 0, revents = 0} __func__ = "libxl_osevent_occurred_fd" #4 0x00007fd3c6825dc3 in virEventPollDispatchHandles (fds=<optimized out>, nfds=<optimized out>) at util/vireventpoll.c:508 cb = 0x7fd399cc70d0 <libxlFDEventCallback> watch = 66 opaque = <optimized out> hEvents = 1 i = 6 n = 6 #5 virEventPollRunOnce () at util/vireventpoll.c:657 fds = 0x7fd3c8c0ee10 ret = <optimized out> timeout = <optimized out> nfds = <optimized out> __func__ = "virEventPollRunOnce" __FUNCTION__ = "virEventPollRunOnce" #6 0x00007fd3c6824651 in virEventRunDefaultImpl () at util/virevent.c:308 __func__ = "virEventRunDefaultImpl" #7 0x00007fd3c72aee2d in virNetServerRun (srv=0x7fd3c8bfab40) at rpc/virnetserver.c:1139 timerid = -1 timerActive = false i = <optimized out> __FUNCTION__ = "virNetServerRun" __func__ = "virNetServerRun" #8 0x00007fd3c727add5 in main (argc=<optimized out>, argv=<optimized out>) at libvirtd.c:1491 srv = 0x7fd3c8bfab40 remote_config_file = 0x7fd3c8bf32a0 "/etc/libvirt/libvirtd.conf" statuswrite = -1 ret = 1 pid_file_fd = 4 pid_file = 0x7fd3c8bfa880 "/var/run/libvirtd.pid" sock_file = 0x7fd3c8c04aa0 "/var/run/libvirt/libvirt-sock" ---Type <return> to continue, or q <return> to quit--- sock_file_ro = 0x7fd3c8c04a30 "/var/run/libvirt/libvirt-sock-ro" timeout = -1 verbose = 0 godaemon = 100 ipsock = 0 config = 0x7fd3c8bf6840 privileged = <optimized out> implicit_conf = <optimized out> run_dir = 0x7fd3c8bfa810 "/var/run/libvirt" old_umask = <optimized out> opts = {{name = 0x7fd3c72b1afe "verbose", has_arg = 0, flag = 0x7fff5b0bf9e8, val = 118}, { name = 0x7fd3c72b1b06 "daemon", has_arg = 0, flag = 0x7fff5b0bf9ec, val = 100}, {name = 0x7fd3c72b1b0d "listen", has_arg = 0, flag = 0x7fff5b0bf9f0, val = 108}, {name = 0x7fd3c72b1c15 "config", has_arg = 1, flag = 0x0, val = 102}, {name = 0x7fd3c72b1b75 "timeout", has_arg = 1, flag = 0x0, val = 116}, {name = 0x7fd3c72b1b14 "pid-file", has_arg = 1, flag = 0x0, val = 112}, {name = 0x7fd3c72b1b1d "version", has_arg = 0, flag = 0x0, val = 86}, { name = 0x7fd3c72b1b25 "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __func__ = "main" (gdb) (gdb) quit A debugging session is active. Inferior 1 [process 1262] will be detached. Quit anyway? (y or n) y Detaching from program: /usr/sbin/libvirtd, process 1262 kroot@nic54:/root/libvirt/src\[root@nic54 src]# ps -eff|grep virsh root 3333 2603 0 13:46 pts/3 00:00:00 virsh console OL6 root 3412 2856 0 13:49 pts/4 00:00:00 virsh list root 3466 1830 0 13:56 pts/0 00:00:00 grep --color=auto virsh kroot@nic54:/root/libvirt/src\[root@nic54 src]# gdb virsh 3412 GNU gdb (GDB) Fedora 7.8.2-38.fc21 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>;. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>;. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from virsh...done. Attaching to program: /usr/bin/virsh, process 3412 Reading symbols from /lib64/libvirt.so.0...done. Loaded symbols for /lib64/libvirt.so.0 Reading symbols from /lib64/libvirt-lxc.so.0...done. Loaded symbols for /lib64/libvirt-lxc.so.0 Reading symbols from /lib64/libvirt-qemu.so.0...done. Loaded symbols for /lib64/libvirt-qemu.so.0 Reading symbols from /lib64/libyajl.so.2...Reading symbols from /lib64/libyajl.so.2...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libyajl.so.2 Reading symbols from /lib64/libdevmapper.so.1.02...Reading symbols from /lib64/libdevmapper.so.1.02...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libdevmapper.so.1.02 Reading symbols from /lib64/libsystemd.so.0...Reading symbols from /lib64/libsystemd.so.0...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libsystemd.so.0 Reading symbols from /lib64/libselinux.so.1...Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libselinux.so.1 Reading symbols from /lib/libxenlight.so.4.5...done. Loaded symbols for /lib/libxenlight.so.4.5 Reading symbols from /lib/libxenctrl.so.4.5...done. Loaded symbols for /lib/libxenctrl.so.4.5 Reading symbols from /lib/libxlutil.so.4.3...done. Loaded symbols for /lib/libxlutil.so.4.3 Reading symbols from /lib64/libutil.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/libutil.so.1 Reading symbols from /lib64/libxml2.so.2...Reading symbols from /lib64/libxml2.so.2...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libxml2.so.2 Reading symbols from /lib64/libncurses.so.5...Reading symbols from /lib64/libncurses.so.5...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libncurses.so.5 Reading symbols from /lib64/libtinfo.so.5...Reading symbols from /lib64/libtinfo.so.5...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libtinfo.so.5 Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/libdl.so.2 Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done. [New LWP 3413] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Loaded symbols for /lib64/libpthread.so.0 Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libc.so.6 Reading symbols from /lib64/libsepol.so.1...Reading symbols from /lib64/libsepol.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libsepol.so.1 Reading symbols from /lib64/libudev.so.1...Reading symbols from /lib64/libudev.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libudev.so.1 Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/librt.so.1 Reading symbols from /lib64/liblzma.so.5...Reading symbols from /lib64/liblzma.so.5...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/liblzma.so.5 Reading symbols from /lib64/libgcrypt.so.20...Reading symbols from /lib64/libgcrypt.so.20...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libgcrypt.so.20 Reading symbols from /lib64/libgpg-error.so.0...Reading symbols from /lib64/libgpg-error.so.0...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libgpg-error.so.0 Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/libresolv.so.2 Reading symbols from /lib64/libdw.so.1...Reading symbols from /lib64/libdw.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libdw.so.1 Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/libgcc_s.so.1 Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/ld-linux-x86-64.so.2 Reading symbols from /lib64/libpcre.so.1...Reading symbols from /lib64/libpcre.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libpcre.so.1 Reading symbols from /lib/libxenguest.so.4.5...done. Loaded symbols for /lib/libxenguest.so.4.5 Reading symbols from /lib/libxenstore.so.3.0...done. Loaded symbols for /lib/libxenstore.so.3.0 Reading symbols from /lib/libblktapctl.so.1.0...done. Loaded symbols for /lib/libblktapctl.so.1.0 Reading symbols from /lib64/libuuid.so.1...Reading symbols from /lib64/libuuid.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libuuid.so.1 Reading symbols from /lib64/libz.so.1...Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libz.so.1 Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libm.so.6 Reading symbols from /lib64/libelf.so.1...Reading symbols from /lib64/libelf.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libelf.so.1 Reading symbols from /lib64/libbz2.so.1...Reading symbols from /lib64/libbz2.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libbz2.so.1 0x00007f08a9cdcc8d in poll () from /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.6-14.fc21.x86_64 device-mapper-libs-1.02.93-3.fc21.x86_64 elfutils-libelf-0.161-2.fc21.x86_64 elfutils-libs-0.161-2.fc21.x86_64 glibc-2.20-8.fc21.x86_64 libgcc-4.9.2-6.fc21.x86_64 libgcrypt-1.6.1-7.fc21.x86_64 libgpg-error-1.17-2.fc21.x86_64 libselinux-2.3-5.fc21.x86_64 libsepol-2.3-4.fc21.x86_64 libuuid-2.25.2-2.fc21.x86_64 libxml2-2.9.1-6.fc21.x86_64 ncurses-libs-5.9-16.20140323.fc21.x86_64 pcre-8.35-8.fc21.x86_64 systemd-libs-216-21.fc21.x86_64 xz-libs-5.1.2-14alpha.fc21.x86_64 yajl-2.1.0-3.fc21.x86_64 zlib-1.2.8-7.fc21.x86_64 (gdb) bthreads Undefined command: "threads". Try "help". (gdb) thread [Current thread is 1 (Thread 0x7f08ac538880 (LWP 3412))] (gdb) bt full #0 0x00007f08a9cdcc8d in poll () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f08ac078b6f in poll (__timeout=-1, __nfds=2, __fds=0x7fffc5a59e20) at /usr/include/bits/poll2.h:46 No locals. #2 virNetClientIOEventLoop (client=client@entry=0x7f08aced6fa0, thiscall=thiscall@entry=0x7f08aced72f0) at rpc/virnetclient.c:1513 ignore = 0 '\000' oldmask = {__val = {0, 140736509353752, 140736509353552, 8926616876308275712, 140736509353568, 8926616876308275712, 139675237709568, 139675221565240, 139675223111672, 140736509353600, 206158430256, 140736509353832, 140736509353632, 8926616876308275712, 139675237798912, 139675223112640}} timeout = -1 msg = 0x0 blockedsigs = {__val = {134287360, 0 <repeats 15 times>}} fds = {{fd = 5, events = 1, revents = 0}, {fd = 6, events = 1, revents = 0}} ret = <optimized out> __FUNCTION__ = "virNetClientIOEventLoop" __func__ = "virNetClientIOEventLoop" #3 0x00007f08ac079316 in virNetClientIO (thiscall=0x7f08aced72f0, client=0x7f08aced6fa0) at rpc/virnetclient.c:1784 rv = -1 #4 virNetClientSendInternal (client=client@entry=0x7f08aced6fa0, msg=msg@entry=0x7f08aced7700, expectReply=expectReply@entry=true, nonBlock=nonBlock@entry=false) at rpc/virnetclient.c:1956 call = 0x7f08aced72f0 ret = -1 __func__ = "virNetClientSendInternal" __FUNCTION__ = "virNetClientSendInternal" #5 0x00007f08ac07a293 in virNetClientSendWithReply (client=client@entry=0x7f08aced6fa0, msg=msg@entry=0x7f08aced7700) at rpc/virnetclient.c:1984 ret = <optimized out> #6 0x00007f08ac07aa62 in virNetClientProgramCall (prog=prog@entry=0x7f08aced71f0, client=client@entry=0x7f08aced6fa0, serial=serial@entry=0, proc=proc@entry=66, noutfds=noutfds@entry=0, outfds=outfds@entry=0x0, ninfds=0x0, infds=0x0, args_filter=0x7f08a9d1a880 <xdr_void>, args=0x0, ret_filter=0x7f08ac072b50 <xdr_remote_auth_list_ret>, ret=0x7fffc5a5a240) at rpc/virnetclientprogram.c:329 msg = 0x7f08aced7700 i = <optimized out> __FUNCTION__ = "virNetClientProgramCall" #7 0x00007f08ac054b54 in callFull (priv=priv@entry=0x7f08aced7460, flags=flags@entry=0, fdin=fdin@entry=0x0, fdinlen=fdinlen@entry=0, fdout=fdout@entry=0x0, fdoutlen=fdoutlen@entry=0x0, proc_nr=66, args_filter=0x7f08a9d1a880 <xdr_void>, args=0x0, ret_filter=0x7f08ac072b50 <xdr_remote_auth_list_ret>, ret=0x7fffc5a5a240 "", conn=0x7f08acedaa90) at remote/remote_driver.c:6600 rv = <optimized out> prog = 0x7f08aced71f0 counter = 0 ---Type <return> to continue, or q <return> to quit--- client = 0x7f08aced6fa0 #8 0x00007f08ac06d20e in call (conn=0x7f08acedaa90, ret=0x7fffc5a5a240 "", ret_filter=<optimized out>, args=0x0, args_filter=<optimized out>, proc_nr=66, flags=0, priv=0x7f08aced7460) at remote/remote_driver.c:6622 No locals. #9 remoteAuthenticate (conn=0x7f08acedaa90, auth=0x7f08ac36d400 <virConnectAuthDefault>, authtype=0x0, priv=0x7f08aced7460) at remote/remote_driver.c:3885 ret = {types = {types_len = 0, types_val = 0x0}} err = <optimized out> type = 0 #10 doRemoteOpen (conn=conn@entry=0x7f08acedaa90, priv=priv@entry=0x7f08aced7460, auth=auth@entry=0x7f08ac36d400 <virConnectAuthDefault>, flags=flags@entry=0) at remote/remote_driver.c:1002 transport_str = 0x0 transport = <optimized out> daemonPath = 0x0 __FUNCTION__ = "doRemoteOpen" name = 0x7f08aced7500 "" command = 0x0 sockname = 0x7f08aced6270 "/var/run/libvirt/libvirt-sock" netcat = 0x0 port = 0x0 authtype = 0x0 username = 0x0 sanity = true verify = true tty = true pkipath = 0x0 keyfile = 0x0 sshauth = 0x0 knownHostsVerify = 0x0 knownHosts = 0x0 retcode = -2 i = <optimized out> __func__ = "doRemoteOpen" __PRETTY_FUNCTION__ = "doRemoteOpen" #11 0x00007f08ac06e57e in remoteConnectOpen (conn=0x7f08acedaa90, auth=0x7f08ac36d400 <virConnectAuthDefault>, flags=<optimized out>) at remote/remote_driver.c:1195 priv = <optimized out> ret = <optimized out> rflags = 0 autostart = 0x0 __func__ = "remoteConnectOpen" #12 0x00007f08ac01d65d in do_open (name=name@entry=0x0, auth=auth@entry=0x7f08ac36d400 <virConnectAuthDefault>, ---Type <return> to continue, or q <return> to quit--- flags=flags@entry=0) at libvirt.c:1189 i = 0 res = <optimized out> ret = 0x7f08acedaa90 conf = 0x7f08acedaa60 __FUNCTION__ = "do_open" __func__ = "do_open" #13 0x00007f08ac01ebf0 in virConnectOpenAuth (name=0x0, auth=0x7f08ac36d400 <virConnectAuthDefault>, flags=0) at libvirt.c:1341 ret = 0x0 __func__ = "virConnectOpenAuth" #14 0x00007f08ac5ba66c in vshConnect (ctl=0x7fffc5a5a510, uri=<optimized out>, readonly=<optimized out>) at virsh.c:355 c = 0x0 interval = 5 count = 6 keepalive_forced = false #15 0x00007f08ac5bd773 in vshReconnect (ctl=0x7fffc5a5a510) at virsh.c:401 connected = false #16 0x00007f08ac5bdc08 in vshCommandRun (ctl=0x7fffc5a5a510, cmd=0x7f08acedd320) at virsh.c:1951 before = {tv_sec = 139675222862792, tv_usec = 1} after = {tv_sec = 4294967295, tv_usec = 139675228146928} enable_timing = false ret = true #17 0x00007f08ac5b88f7 in main (argc=<optimized out>, argv=0x7fffc5a5aca8) at virsh.c:3738 _ctl = {name = 0x0, conn = 0x0, cmd = 0x7f08acedd320, cmdstr = 0x0, imode = false, quiet = false, debug = 4, timing = false, readonly = false, logfile = 0x0, log_fd = -1, historydir = 0x0, historyfile = 0x0, useGetInfo = false, useSnapshotOld = false, blockJobNoBytes = false, eventLoop = {thread = 139675038205696}, lock = {lock = {__data = { __lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 512, __spins = 0, __elision = 0, __list = { __prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 17 times>, "\002", '\000' <repeats 21 times>, __align = 0}}, eventLoopStarted = true, quit = false, eventPipe = {-1, -1}, eventTimerId = 1, escapeChar = 0x7f08ac5eee65 "^]", keepalive_interval = -1, keepalive_count = -1, termattr = {c_iflag = 1280, c_oflag = 5, c_cflag = 191, c_lflag = 35387, c_line = 0 '\000', c_cc = "\003\034\177\025\004\000\001\000\021\023\032\000\022\017\027\026", '\000' <repeats 15 times>, c_ispeed = 15, c_ospeed = 15}, istty = true} ctl = 0x7fffc5a5a510 defaultConn = <optimized out> ret = true (gdb) quit A debugging session is active. Inferior 1 [process 3412] will be detached. Quit anyway? (y or n) y Detaching from program: /usr/bin/virsh, process 3412 kroot@nic54:/root/libvirt/src\[root@nic54 src]# ps -eff|Grep ps -eff|Grep grep xc root 3257 1 1 13:46 ? 00:00:07 /usr/lib/xen/bin/qemu-system-i386 -xen-domid 5 -chardev socket,id=libxl-cmd,path=/var/run/xen/qmp-libxl-5,server,nowait -no-shutdown -mon chardev=libxl-cmd,mode=control -nodefaults -name OL6 -vnc 0.0.0.0:0 -display none -serial pty -device cirrus-vga,vgamem_mb=8 -boot order=c -usb -usbdevice tablet -smp 4,maxcpus=4 -net none -incoming fd:26 -machine xenfv -m 2040 -drive file=/dev/G/OL6,if=ide,index=0,media=disk,format=raw,cache=writeback root 3473 1830 0 13:57 pts/0 00:00:00 grep --color=auto xc kroot@nic54:/root/libvirt/src\[root@nic54 src]# cd ../ kroot@nic54:/root/libvirt\[root@nic54 libvirt]# lps -eff|grep xl root 3257 1 1 13:46 ? 00:00:07 /usr/lib/xen/bin/qemu-system-i386 -xen-domid 5 -chardev socket,id=libxl-cmd,path=/var/run/xen/qmp-libxl-5,server,nowait -no-shutdown -mon chardev=libxl-cmd,mode=control -nodefaults -name OL6 -vnc 0.0.0.0:0 -display none -serial pty -device cirrus-vga,vgamem_mb=8 -boot order=c -usb -usbdevice tablet -smp 4,maxcpus=4 -net none -incoming fd:26 -machine xenfv -m 2040 -drive file=/dev/G/OL6,if=ide,index=0,media=disk,format=raw,cache=writeback root 3411 1262 0 13:49 ? 00:00:00 /usr/lib/xen/bin/libxl-save-helper --save-domain 27 5 0 0 4 1 28 49 72 root 3475 1830 0 13:57 pts/0 00:00:00 grep --color=auto xl kroot@nic54:/root/libvirt\[root@nic54 libvirt]# cd kroot@nic54:/root\[root@nic54 ~]# cd xen kroot@nic54:/root/xen\[root@nic54 xen]# ls autogen.sh config.guess config.status configure.ac dist INSTALL MAINTAINERS README tools xen CODING_STYLE config.log config.sub COPYING docs install.sh Makefile scripts unmodified_drivers config Config.mk configure CREDITS extras m4 misc stubdom version.sh kroot@nic54:/root/xen\[root@nic54 xen]# cd tools/ kroot@nic54:/root/xen/tools\[root@nic54 tools]# ls blktap2 configure examples libfsimage memshr qemu-xen-dir tests xenpmd config.h configure.ac firmware libvchan misc qemu-xen-dir-remote xcutils xenstat config.h.in console flask libxc ocaml qemu-xen-traditional-dir xenbackendd xenstore config.log cross-install hotplug libxl pygrub qemu-xen-traditional-dir-remote xenmon xentrace config.status debugger include Makefile python Rules.mk xenpaging kroot@nic54:/root/xen/tools\[root@nic54 tools]# find ./ -cd libxl kroot@nic54:/root/xen/tools/libxl\[root@nic54 libxl]# gdb libxl-save-helper 3411 GNU gdb (GDB) Fedora 7.8.2-38.fc21 Copyright (C) 2014 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>;. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>;. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from libxl-save-helper...done. Attaching to program: /root/xen/tools/libxl/libxl-save-helper, process 3411 Reading symbols from /lib/libxenctrl.so.4.5...done. Loaded symbols for /lib/libxenctrl.so.4.5 Reading symbols from /lib/libxenguest.so.4.5...done. Loaded symbols for /lib/libxenguest.so.4.5 Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done. [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Loaded symbols for /lib64/libpthread.so.0 Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libc.so.6 Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/libdl.so.2 Reading symbols from /lib64/libz.so.1...Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /lib64/libz.so.1 Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/ld-linux-x86-64.so.2 0x00007f31876761b0 in __read_nocancel () from /lib64/libpthread.so.0 Missing separate debuginfos, use: debuginfo-install glibc-2.20-8.fc21.x86_64 zlib-1.2.8-7.fc21.x86_64 (gdb) bthread [Current thread is 1 (Thread 0x7f3187ef7700 (LWP 3411))] (gdb) bt full #0 0x00007f31876761b0 in __read_nocancel () from /lib64/libpthread.so.0 No symbol table info available. #1 0x0000000000400ef3 in read_exactly (fd=0, buf=0x7fff06a86078, len=4) at libxl_save_helper.c:105 r = 4198631 __PRETTY_FUNCTION__ = "read_exactly" #2 0x000000000040110b in helper_getreply (user=0x0) at libxl_save_helper.c:153 v = 39437520 r = 131072 #3 0x00000000004020c2 in helper_stub_suspend (user=0x0) at _libxl_save_msgs_helper.c:128 buf = 0x259c4d0 "" len = 2 allocd = 2 __PRETTY_FUNCTION__ = "helper_stub_suspend" r = 111698000 #4 0x00007f3187893c3c in suspend_and_state (suspend=0x401fec <helper_stub_suspend>, data=0x0, xch=0x259c010, io_fd=27, dom=5, info=0x7fff06a8a9c0) at xc_domain_save.c:364 No locals. #5 0x00007f3187895fe8 in xc_domain_save (xch=0x259c010, io_fd=27, dom=5, max_iters=29, max_factor=3, flags=4, callbacks=0x604160 <helper_save_callbacks>, hvm=1) at xc_domain_save.c:987 info = {domid = 5, ssidref = 0, dying = 0, crashed = 0, shutdown = 0, paused = 0, blocked = 1, running = 0, hvm = 1, debugged = 0, pvh = 0, shutdown_reason = 255, nr_pages = 524298, nr_outstanding_pages = 0, nr_shared_pages = 0, nr_paged_pages = 0, shared_info_frame = 7640, cpu_time = 32660372500, max_memkb = 2098176, nr_online_vcpus = 4, max_vcpu_id = 3, handle = "\214F\244Y\371\270B\273\224\252\304i\255\332Z\227", cpupool = 0} domctl = {cmd = 2280706376, interface_version = 32561, domain = 0, u = {createdomain = {ssidref = 2280625784, handle = "1\177\000\000\370Gχ1\177\000\000\004\000\000", flags = 0}, getdomaininfo = {domain = 38520, flags = 32561, tot_pages = 139850708633592, max_pages = 4, outstanding_pages = 139850710750840, shr_pages = 1, paged_pages = 0, shared_info_frame = 1, cpu_time = 139850710831432, nr_online_vcpus = 279491920, max_vcpu_id = 0, ssidref = 4196058, handle = "\000\000\000\000\377\377\377\377\000\000\000\000\000\000\000", cpupool = 0}, getmemlist = {max_pfns = 139850710750840, start_pfn = 139850708633592, buffer = {p = 0x4}, num_pfns = 139850710750840}, getpageframeinfo = {gmfn = 139850710750840, type = 2278508536}, getpageframeinfo2 = { num = 139850710750840, array = {p = 0x7f3187cf47f8 <_dl_lookup_symbol_x+312>}}, getpageframeinfo3 = { num = 139850710750840, array = {p = 0x7f3187cf47f8 <_dl_lookup_symbol_x+312>}}, nodeaffinity = {nodemap = { bitmap = {p = 0x7f3187ef9678 "\210\a@"}, nr_bits = 2278508536}}, vcpuaffinity = {vcpu = 2280625784, flags = 32561, cpumap_hard = {bitmap = { p = 0x7f3187cf47f8 <_dl_lookup_symbol_x+312> "H\203\304\060\203", <incomplete sequence \370>}, nr_bits = 4}, cpumap_soft = {bitmap = {p = 0x7f3187ef9678 "\210\a@"}, nr_bits = 1}}, shadow_op = {op = 2280625784, mode = 32561, mb = 2278508536, dirty_bitmap = {p = 0x4 <error: Cannot access memory at address 0x4>}, pages = 139850710750840, stats = {fault_count = 1, dirty_count = 0}}, max_mem = {max_memkb = 139850710750840}, vcpucontext = { vcpu = 2280625784, ctxt = {p = 0x7f3187cf47f8 <_dl_lookup_symbol_x+312>}}, getvcpuinfo = {vcpu = 2280625784, online = 49 '1', blocked = 127 '\177', running = 0 '\000', cpu_time = 139850708633592, cpu = 4}, max_vcpus = { max = 2280625784}, scheduler_op = {sched_id = 2280625784, cmd = 32561, u = {sedf = {period = 139850708633592, slice = 4, latency = 139850710750840, extratime = 1, weight = 0}, credit = {weight = 18424, cap = 34767}, ---Type <return> to continue, or q <return> to quit--- credit2 = {weight = 18424}, rtds = {period = 2278508536, budget = 32561}}}, setdomainhandle = { handle = "x\226\357\207\061\177\000\000\370Gχ1\177\000"}, setdebugging = {enable = 120 'x'}, irq_permission = { pirq = 120 'x', allow_access = 150 '\226'}, iomem_permission = {first_mfn = 139850710750840, nr_mfns = 139850708633592, allow_access = 4 '\004'}, ioport_permission = {first_port = 2280625784, nr_ports = 32561, allow_access = 248 '\370'}, hypercall_init = {gmfn = 139850710750840}, settimeoffset = { time_offset_seconds = 139850710750840}, disable_migrate = {disable = 2280625784}, tsc_info = {out_info = { p = 0x7f3187ef9678}, info = {tsc_mode = 2278508536, gtsc_khz = 32561, incarnation = 4, pad = 0, elapsed_nsec = 139850710750840}}, hvmcontext = {size = 2280625784, buffer = { p = 0x7f3187cf47f8 <_dl_lookup_symbol_x+312> "H\203\304\060\203", <incomplete sequence \370>}}, hvmcontext_partial = {type = 2280625784, instance = 32561, buffer = { p = 0x7f3187cf47f8 <_dl_lookup_symbol_x+312> "H\203\304\060\203", <incomplete sequence \370>}}, address_size = { size = 2280625784}, sendtrigger = {trigger = 2280625784, vcpu = 32561}, get_device_group = { machine_sbdf = 2280625784, max_sdevs = 32561, num_sdevs = 2278508536, sdev_array = {p = 0x4}}, assign_device = { machine_sbdf = 2280625784}, bind_pt_irq = {machine_irq = 2280625784, irq_type = 32561, hvm_domid = 2278508536, u = { isa = {isa_irq = 4 '\004'}, pci = {bus = 4 '\004', device = 0 '\000', intx = 0 '\000'}, msi = {gvec = 4 '\004', gflags = 0, gtable = 139850710750840}}}, memory_mapping = {first_gfn = 139850710750840, first_mfn = 139850708633592, nr_mfns = 4, add_mapping = 2280625784, padding = 32561}, ioport_mapping = { first_gport = 2280625784, first_mport = 32561, nr_ports = 2278508536, add_mapping = 32561}, pin_mem_cacheattr = { start = 139850710750840, end = 139850708633592, type = 4}, ext_vcpucontext = {vcpu = 2280625784, size = 32561, syscall32_callback_eip = 139850708633592, sysenter_callback_eip = 4, syscall32_callback_cs = 38520, sysenter_callback_cs = 34799, syscall32_disables_events = 49 '1', sysenter_disables_events = 127 '\177', { mcg_cap = 1, vmce = {caps = 1, mci_ctl2_bank0 = 0, mci_ctl2_bank1 = 1}}}, set_target = {target = 38520}, subscribe = {port = 2280625784}, debug_op = {op = 2280625784, vcpu = 32561}, mem_event_op = {op = 2280625784, mode = 32561, port = 2278508536}, mem_sharing_op = {op = 120 'x', u = {enable = 150 '\226'}}, cpuid = {input = { 2280625784, 32561}, eax = 2278508536, ebx = 32561, ecx = 4, edx = 0}, vcpuextstate = {vcpu = 2280625784, xfeature_mask = 139850708633592, size = 4, buffer = {p = 0x7f3187ef9678}}, vcpu_msrs = {vcpu = 2280625784, msr_count = 32561, msrs = {p = 0x7f3187cf47f8 <_dl_lookup_symbol_x+312>}}, access_required = { access_required = 120 'x'}, audit_p2m = {orphans = 139850710750840, m2p_bad = 139850708633592, p2m_bad = 4}, set_virq_handler = {virq = 2280625784}, set_max_evtchn = {max_port = 2280625784}, gdbsx_guest_memio = { pgd3val = 139850710750840, gva = 139850708633592, uva = 4, len = 2280625784, gwr = 49 '1', remain = 1}, set_broken_page_p2m = {pfn = 139850710750840}, cacheflush = {start_pfn = 139850710750840, nr_pfns = 139850708633592}, gdbsx_pauseunp_vcpu = {vcpu = 2280625784}, gdbsx_domstatus = {paused = 120 'x', vcpu_id = 32561, vcpu_ev = 2278508536}, vnuma = {nr_vnodes = 2280625784, nr_vmemranges = 32561, nr_vcpus = 2278508536, pad = 32561, vdistance = {p = 0x4}, vcpu_to_vnode = {p = 0x7f3187ef9678}, vnode_to_pnode = {p = 0x1}, vmemrange = {p = 0x0}}, psr_cmt_op = {cmd = 2280625784, data = 32561}, pad = "x\226\357\207\061\177\000\000\370Gχ1\177\000\000\004\000\000\000\000\000\000\000x\226\357\207\061\177\000\000\001", '\000' <repeats 15 times>, "\001\000\000\000\000\000\000\000H\321\360\207\061\177\000\000P\265\250\020\000\000\000\000\332\006@\000\000\000\000\000\377\377\377\377", '\000' <repeats 12 times>, "\240\324\360\207\061\177\000\000੨\006\377\177\000\000\377\377\377\377\001\000\000\000Щ\250\006\377\177\000"}} rc = 0 frc = 0 i = 32767 ---Type <return> to continue, or q <return> to quit--- j = 0 last_iter = 0 iter = 0 live = 0 debug = 0 superpages = 1 race = 0 sent_last_iter = 111717528 skip_this_iter = 0 sent_this_iter = 0 tmem_saved = 0 shared_info_frame = 7640 ctxt = {x64 = {fpu_ctxt = {x = '\000' <repeats 511 times>}, flags = 0, user_regs = {r15 = 0, r14 = 0, r13 = 0, r12 = 0, { rbp = 0, ebp = 0}, {rbx = 0, ebx = 0}, r11 = 0, r10 = 0, r9 = 0, r8 = 0, {rax = 0, eax = 0}, {rcx = 0, ecx = 0}, { rdx = 0, edx = 0}, {rsi = 0, esi = 0}, {rdi = 0, edi = 0}, error_code = 0, entry_vector = 0, {rip = 0, eip = 0}, cs = 0, _pad0 = {0}, saved_upcall_mask = 0 '\000', _pad1 = "\000\000", {rflags = 0, eflags = 0}, {rsp = 0, esp = 0}, ss = 0, _pad2 = {0, 0, 0}, es = 0, _pad3 = {0, 0, 0}, ds = 0, _pad4 = {0, 0, 0}, fs = 0, _pad5 = {0, 0, 0}, gs = 0, _pad6 = {0, 0, 0}}, trap_ctxt = {{vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0} <repeats 157 times>, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708618830}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 12288}, {vector = 104 'h', flags = 33 '!', cs = 0, address = 8552}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708618830}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 86016}, {vector = 108 'l', flags = 70 'F', cs = 1, address = 83564}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 5}, {vector = 0 '\000', flags = 64 '@', cs = 33, address = 2187264}, {vector = 0 '\000', flags = 82 'R', cs = 33, address = 2183688}, {vector = 0 '\000', flags = 64 '@', cs = 1, address = 3}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708688590}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850710757048}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708638876}, { vector = 108 'l', flags = 46 '.', cs = 27, address = 139850710757048}, {vector = 17 '\021', flags = 0 '\000', cs = 0, address = 139850708660020}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850693546923}, { vector = 0 '\000', flags = 144 '\220', cs = 34799, address = 139850708625717}, {vector = 96 '`', flags = 164 '\244', cs = 1704, address = 6573502260}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850701779419}, {vector = 24 '\030', flags = 160 '\240', cs = 34799, address = 139850708625717}, { vector = 144 '\220', flags = 164 '\244', cs = 1704, address = 4294967296}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850695725576}, {vector = 8 '\b', flags = 82 'R', cs = 34570, address = 140733305103152}, { vector = 208 '\320', flags = 161 '\241', cs = 1704, address = 2183688}, {vector = 4 '\004', flags = 0 '\000', cs = 0, address = 6}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 139850710757048}, {vector = 55 '7', flags = 96 '`', cs = 34696, address = 139850710831392}, {vector = 0 '\000', flags = 253 '\375', cs = 0, address = 2100235}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 33261}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 136 '\210', flags = 95 '_', cs = 1, address = 4096}, { ---Type <return> to continue, or q <return> to quit--- vector = 176 '\260', flags = 0 '\000', cs = 0, address = 1428421374}, {vector = 128 '\200', flags = 139 '\213', cs = 8758, address = 1408431017}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 1415029273}, { vector = 145 '\221', flags = 17 '\021', cs = 54, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 139850708660020}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850703986771}, {vector = 48 '0', flags = 171 '\253', cs = 34800, address = 139850708625717}, {vector = 48 '0', flags = 171 '\253', cs = 34800, address = 6573468531}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 140733305103600}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708630484}, {vector = 88 'X', flags = 166 '\246', cs = 34800, address = 139850710754536}, {vector = 180 '\264', flags = 206 '\316', cs = 34602, address = 1298}, {vector = 48 '0', flags = 221 '\335', cs = 34602, address = 139850708630484}, {vector = 18 '\022', flags = 5 '\005', cs = 0, address = 139850710753304}, {vector = 216 '\330', flags = 124 '|', cs = 34662, address = 196}, { vector = 16 '\020', flags = 127 '\177', cs = 34662, address = 139850708632351}, {vector = 196 '\304', flags = 0 '\000', cs = 0, address = 139850701774504}, {vector = 248 '\370', flags = 162 '\242', cs = 34799, address = 139850710754120}, {vector = 152 '\230', flags = 165 '\245', cs = 1704, address = 140733305103764}, { vector = 159 '\237', flags = 20 '\024', cs = 34732, address = 139850701774504}, {vector = 0 '\000', flags = 206 '\316', cs = 34731, address = 139850701771120}, {vector = 152 '\230', flags = 165 '\245', cs = 1704, address = 258797660}, {vector = 193 '\301', flags = 179 '\263', cs = 61, address = 28}, {vector = 112 'p', flags = 166 '\246', cs = 1704, address = 140733305103968}, {vector = 248 '\370', flags = 162 '\242', cs = 34799, address = 139850710750728}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 8}, {vector = 88 'X', flags = 166 '\246', cs = 34800, address = 0}, {vector = 56 '8', flags = 151 '\227', cs = 34799, address = 139850710821296}, {vector = 0 '\000', flags = 167 '\247', cs = 1704, address = 140733305104168}, { vector = 88 'X', flags = 166 '\246', cs = 34800, address = 0}, {vector = 80 'P', flags = 151 '\227', cs = 34799, address = 139850708633592}, {vector = 3 '\003', flags = 0 '\000', cs = 0, address = 139850710751056}, { vector = 1 '\001', flags = 0 '\000', cs = 0, address = 0}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 139850710820440}, {vector = 56 '8', flags = 151 '\227', cs = 34799, address = 140733305103920}, { vector = 174 '\256', flags = 96 '`', cs = 65415, address = 0}, {vector = 176 '\260', flags = 169 '\251', cs = 34800, address = 140733305103984}, {vector = 224 '\340', flags = 86 'V', cs = 34603, address = 140733305103968}, {vector = 92 '\\', flags = 240 '\360', cs = 3948, address = 139850706326687}, { vector = 255 '\377', flags = 255 '\377', cs = 65535, address = 0}, {vector = 112 'p', flags = 145 '\221', cs = 34662, address = 139850710753304}, {vector = 16 '\020', flags = 29 '\035', cs = 34662, address = 140733305104064}, {vector = 65 'A', flags = 89 'Y', cs = 63831, address = 140733305104063}, { vector = 255 '\377', flags = 255 '\377', cs = 65535, address = 139850708586528}, {vector = 64 '@', flags = 12 '\f', cs = 64, address = 140733305106160}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708586696}, {vector = 64 '@', flags = 12 '\f', cs = 64, address = 140733305106160}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708652908}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 0}, {vector = 67 'C', flags = 0 '\000', cs = 0, address = 139850701771120}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 140733305104400}, {vector = 64 '@', flags = 12 '\f', cs = 64, address = 139850708681520}, {vector = 97 'a', flags = 88 'X', cs = 63831, address = 140733305104287}, { vector = 16 '\020', flags = 192 '\300', cs = 601, address = 139850698074918}, {vector = 152 '\230', flags = 196 '\304', cs = 601, address = 0}, {vector = 16 '\020', flags = 168 '\250', cs = 1704, address = 16}, { vector = 224 '\340', flags = 167 '\247', cs = 1704, address = 140733305104288}, {vector = 16 '\020', ---Type <return> to continue, or q <return> to quit--- flags = 192 '\300', cs = 601, address = 1}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 39436288}}, ldt_base = 139850701741336, ldt_ents = 139850710820440, gdt_frames = {0, 139850706447238, 140733305106160, 39436304, 0, 17179869185, 4, 0, 140733305105696, 139850706396927, 88, 0, 6308096, 6308096, 0, 6308096}, gdt_ents = 0, kernel_ss = 6308096, kernel_sp = 0, ctrlreg = {139637976727552, 139850710755272, 139850710755352, 140733305104648, 140733305104644, 4196203, 139850697911728, 4195048}, debugreg = {139850697911440, 140733305104648, 2090266759, 32660418, 7, 140733305104864, 140733305104848, 139850710755272}, event_callback_eip = 139850710750728, failsafe_callback_eip = 139848430125056, syscall_callback_eip = 8, vm_assist = 2280625808, fs_base = 0, gs_base_kernel = 3, gs_base_user = 139850710832288}, x32 = {fpu_ctxt = {x = '\000' <repeats 511 times>}, flags = 0, user_regs = {ebx = 0, ecx = 0, edx = 0, esi = 0, edi = 0, ebp = 0, eax = 0, error_code = 0, entry_vector = 0, eip = 0, cs = 0, saved_upcall_mask = 0 '\000', _pad0 = 0 '\000', eflags = 0, esp = 0, ss = 0, _pad1 = 0, es = 0, _pad2 = 0, ds = 0, _pad3 = 0, fs = 0, _pad4 = 0, gs = 0, _pad5 = 0}, trap_ctxt = {{vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0} <repeats 256 times>}, ldt_base = 0, ldt_ents = 0, gdt_frames = { 0 <repeats 16 times>}, gdt_ents = 0, kernel_ss = 0, kernel_sp = 0, ctrlreg = {0, 0, 0, 0, 0, 0, 0, 0}, debugreg = { 0, 0, 0, 0, 0, 0, 0, 0}, event_callback_cs = 0, event_callback_eip = 0, failsafe_callback_cs = 0, failsafe_callback_eip = 0, vm_assist = 0}, c = {fpu_ctxt = {x = '\000' <repeats 511 times>}, flags = 0, user_regs = { r15 = 0, r14 = 0, r13 = 0, r12 = 0, {rbp = 0, ebp = 0, _ebp = 0}, {rbx = 0, ebx = 0, _ebx = 0}, r11 = 0, r10 = 0, r9 = 0, r8 = 0, {rax = 0, eax = 0, _eax = 0}, {rcx = 0, ecx = 0, _ecx = 0}, {rdx = 0, edx = 0, _edx = 0}, {rsi = 0, esi = 0, _esi = 0}, {rdi = 0, edi = 0, _edi = 0}, error_code = 0, entry_vector = 0, {rip = 0, eip = 0, _eip = 0}, cs = 0, _pad0 = {0}, saved_upcall_mask = 0 '\000', _pad1 = "\000\000", {rflags = 0, eflags = 0, _eflags = 0}, { rsp = 0, esp = 0, _esp = 0}, ss = 0, _pad2 = {0, 0, 0}, es = 0, _pad3 = {0, 0, 0}, ds = 0, _pad4 = {0, 0, 0}, fs = 0, _pad5 = {0, 0, 0}, gs = 0, _pad6 = {0, 0, 0}}, trap_ctxt = {{vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0} <repeats 157 times>, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708618830}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 12288}, {vector = 104 'h', flags = 33 '!', cs = 0, address = 8552}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708618830}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 86016}, {vector = 108 'l', flags = 70 'F', cs = 1, address = 83564}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 5}, {vector = 0 '\000', flags = 64 '@', cs = 33, address = 2187264}, {vector = 0 '\000', flags = 82 'R', cs = 33, address = 2183688}, {vector = 0 '\000', flags = 64 '@', cs = 1, address = 3}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708688590}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850710757048}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708638876}, { vector = 108 'l', flags = 46 '.', cs = 27, address = 139850710757048}, {vector = 17 '\021', flags = 0 '\000', cs = 0, address = 139850708660020}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850693546923}, { vector = 0 '\000', flags = 144 '\220', cs = 34799, address = 139850708625717}, {vector = 96 '`', flags = 164 '\244', cs = 1704, address = 6573502260}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850701779419}, {vector = 24 '\030', flags = 160 '\240', cs = 34799, address = 139850708625717}, { vector = 144 '\220', flags = 164 '\244', cs = 1704, address = 4294967296}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850695725576}, {vector = 8 '\b', flags = 82 'R', cs = 34570, address = 140733305103152}, { vector = 208 '\320', flags = 161 '\241', cs = 1704, address = 2183688}, {vector = 4 '\004', flags = 0 '\000', cs = 0, address = 6}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 139850710757048}, {vector = 55 '7', ---Type <return> to continue, or q <return> to quit--- flags = 96 '`', cs = 34696, address = 139850710831392}, {vector = 0 '\000', flags = 253 '\375', cs = 0, address = 2100235}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 33261}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 136 '\210', flags = 95 '_', cs = 1, address = 4096}, { vector = 176 '\260', flags = 0 '\000', cs = 0, address = 1428421374}, {vector = 128 '\200', flags = 139 '\213', cs = 8758, address = 1408431017}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 1415029273}, { vector = 145 '\221', flags = 17 '\021', cs = 54, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 139850708660020}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850703986771}, {vector = 48 '0', flags = 171 '\253', cs = 34800, address = 139850708625717}, {vector = 48 '0', flags = 171 '\253', cs = 34800, address = 6573468531}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 140733305103600}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708630484}, {vector = 88 'X', flags = 166 '\246', cs = 34800, address = 139850710754536}, {vector = 180 '\264', flags = 206 '\316', cs = 34602, address = 1298}, {vector = 48 '0', flags = 221 '\335', cs = 34602, address = 139850708630484}, {vector = 18 '\022', flags = 5 '\005', cs = 0, address = 139850710753304}, {vector = 216 '\330', flags = 124 '|', cs = 34662, address = 196}, { vector = 16 '\020', flags = 127 '\177', cs = 34662, address = 139850708632351}, {vector = 196 '\304', flags = 0 '\000', cs = 0, address = 139850701774504}, {vector = 248 '\370', flags = 162 '\242', cs = 34799, address = 139850710754120}, {vector = 152 '\230', flags = 165 '\245', cs = 1704, address = 140733305103764}, { vector = 159 '\237', flags = 20 '\024', cs = 34732, address = 139850701774504}, {vector = 0 '\000', flags = 206 '\316', cs = 34731, address = 139850701771120}, {vector = 152 '\230', flags = 165 '\245', cs = 1704, address = 258797660}, {vector = 193 '\301', flags = 179 '\263', cs = 61, address = 28}, {vector = 112 'p', flags = 166 '\246', cs = 1704, address = 140733305103968}, {vector = 248 '\370', flags = 162 '\242', cs = 34799, address = 139850710750728}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 8}, {vector = 88 'X', flags = 166 '\246', cs = 34800, address = 0}, {vector = 56 '8', flags = 151 '\227', cs = 34799, address = 139850710821296}, {vector = 0 '\000', flags = 167 '\247', cs = 1704, address = 140733305104168}, { vector = 88 'X', flags = 166 '\246', cs = 34800, address = 0}, {vector = 80 'P', flags = 151 '\227', cs = 34799, address = 139850708633592}, {vector = 3 '\003', flags = 0 '\000', cs = 0, address = 139850710751056}, { vector = 1 '\001', flags = 0 '\000', cs = 0, address = 0}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 139850710820440}, {vector = 56 '8', flags = 151 '\227', cs = 34799, address = 140733305103920}, { vector = 174 '\256', flags = 96 '`', cs = 65415, address = 0}, {vector = 176 '\260', flags = 169 '\251', cs = 34800, address = 140733305103984}, {vector = 224 '\340', flags = 86 'V', cs = 34603, address = 140733305103968}, {vector = 92 '\\', flags = 240 '\360', cs = 3948, address = 139850706326687}, { vector = 255 '\377', flags = 255 '\377', cs = 65535, address = 0}, {vector = 112 'p', flags = 145 '\221', cs = 34662, address = 139850710753304}, {vector = 16 '\020', flags = 29 '\035', cs = 34662, address = 140733305104064}, {vector = 65 'A', flags = 89 'Y', cs = 63831, address = 140733305104063}, { vector = 255 '\377', flags = 255 '\377', cs = 65535, address = 139850708586528}, {vector = 64 '@', flags = 12 '\f', cs = 64, address = 140733305106160}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708586696}, {vector = 64 '@', flags = 12 '\f', cs = 64, address = 140733305106160}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 0}, { vector = 0 '\000', flags = 0 '\000', cs = 0, address = 139850708652908}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 0}, {vector = 67 'C', flags = 0 '\000', cs = 0, address = 139850701771120}, {vector = 0 '\000', flags = 0 '\000', cs = 0, address = 140733305104400}, {vector = 64 '@', flags = 12 '\f', cs = 64, address = 139850708681520}, {vector = 97 'a', flags = 88 'X', cs = 63831, address = 140733305104287}, { ---Type <return> to continue, or q <return> to quit--- vector = 16 '\020', flags = 192 '\300', cs = 601, address = 139850698074918}, {vector = 152 '\230', flags = 196 '\304', cs = 601, address = 0}, {vector = 16 '\020', flags = 168 '\250', cs = 1704, address = 16}, { vector = 224 '\340', flags = 167 '\247', cs = 1704, address = 140733305104288}, {vector = 16 '\020', flags = 192 '\300', cs = 601, address = 1}, {vector = 1 '\001', flags = 0 '\000', cs = 0, address = 39436288}}, ldt_base = 139850701741336, ldt_ents = 139850710820440, gdt_frames = {0, 139850706447238, 140733305106160, 39436304, 0, 17179869185, 4, 0, 140733305105696, 139850706396927, 88, 0, 6308096, 6308096, 0, 6308096}, gdt_ents = 0, kernel_ss = 6308096, kernel_sp = 0, ctrlreg = {139637976727552, 139850710755272, 139850710755352, 140733305104648, 140733305104644, 4196203, 139850697911728, 4195048}, debugreg = {139850697911440, 140733305104648, 2090266759, 32660418, 7, 140733305104864, 140733305104848, 139850710755272}, event_callback_eip = 139850710750728, failsafe_callback_eip = 139848430125056, syscall_callback_eip = 8, vm_assist = 2280625808, fs_base = 0, gs_base_kernel = 3, gs_base_user = 139850710832288}} pfn_type = 0x0 pfn_batch = 0x0 pfn_err = 0x0 page = '\000' <repeats 4095 times> live_shinfo = 0x0 region_base = 0x0 buffer = 0x0 xc__hypercall_buffer_buffer = {hbuf = 0x0, param_shadow = 0x0, dir = 0, ubuf = 0xffffffffffffffff, sz = 0} to_skip = 0x0 xc__hypercall_buffer_to_skip = {hbuf = 0x0, param_shadow = 0x0, dir = 0, ubuf = 0xffffffffffffffff, sz = 0} to_send = 0x0 xc__hypercall_buffer_to_send = {hbuf = 0x0, param_shadow = 0x0, dir = 0, ubuf = 0xffffffffffffffff, sz = 0} to_fix = 0x0 time_stats = {wall = {tv_sec = 0, tv_usec = 0}, d0_cpu = 0, d1_cpu = 0} shadow_stats = {fault_count = 0, dirty_count = 0} needed_to_fix = 0 total_sent = 0 vcpumap = {1, 0 <repeats 63 times>} hvm_buf_size = 0 hvm_buf = 0x0 magic_pfns = {0, 0, 0} mfn = 140733305105560 ob_pagebuf = {buf = 0x7f3185e8f010, size = 16777216, pos = 0, write_count = 0} ob_tailbuf = {buf = 0x0, size = 0, pos = 0, write_count = 0} ob = 0x0 _ctx = {hvirt_start = 18446603336221196288, pt_levels = 4, max_mfn = 8650240, live_p2m = 0x0, live_m2p = 0x0, m2p_mfn0 = 0, dinfo = {guest_width = 8, p2m_size = 1044481}} ctx = 0x7fff06a88190 dinfo = 0x7fff06a881c0 compress_ctx = 0x0 compressing = 0 ---Type <return> to continue, or q <return> to quit--- completed = 0 __func__ = "xc_domain_save" __PRETTY_FUNCTION__ = "xc_domain_save" #6 0x00000000004016c0 in main (argc=11, argv=0x7fff06a8af50) at libxl_save_helper.c:231 io_fd = 27 max_factor = 0 flags = 4 hvm = 1 dom = 5 max_iters = 0 cbflags = 72 r = 0 mode = 0x7fff06a8cf3c "--save-domain" __PRETTY_FUNCTION__ = "main" (gdb) (gdb) (gdb) (gdb) bt #0 0x00007f31876761b0 in __read_nocancel () from /lib64/libpthread.so.0 #1 0x0000000000400ef3 in read_exactly (fd=0, buf=0x7fff06a86078, len=4) at libxl_save_helper.c:105 #2 0x000000000040110b in helper_getreply (user=0x0) at libxl_save_helper.c:153 #3 0x00000000004020c2 in helper_stub_suspend (user=0x0) at _libxl_save_msgs_helper.c:128 #4 0x00007f3187893c3c in suspend_and_state (suspend=0x401fec <helper_stub_suspend>, data=0x0, xch=0x259c010, io_fd=27, dom=5, info=0x7fff06a8a9c0) at xc_domain_save.c:364 #5 0x00007f3187895fe8 in xc_domain_save (xch=0x259c010, io_fd=27, dom=5, max_iters=29, max_factor=3, flags=4, callbacks=0x604160 <helper_save_callbacks>, hvm=1) at xc_domain_save.c:987 #6 0x00000000004016c0 in main (argc=11, argv=0x7fff06a8af50) at libxl_save_helper.c:231 (gdb) quit A debugging session is active. Inferior 1 [process 3411] will be detached. kroot@nic54:/usr/lib/xen/bin\[root@nic54 bin]# ./xenctx --help -f -D 5[4@-a -Chf -a -C -D 5[1@f[1@  5 5 vcpu0: rip: ffffffff810013a8 flags: 00000046 z p rsp: ffff88007a8bdd50 rax: 0000000000000000 rcx: 0000000000000000 rdx: 0000000000000000 rbx: ffff88007a54bdb8 rsi: deadbeefdeadf00d rdi: deadbeefdeadf00d rbp: ffff88007a8bdd78 r8: 0000000000000000 r9: 0000000000000000 r10: 0000000000000001 r11: 0000000000000010 r12: 0000000000000000 r13: 0000000000000003 r14: ffff88007a54bd4c r15: ffff88007a54bd01 cs: 0010 ss: 0018 ds: 0000 es: 0000 fs: 0000 @ 0000000000000000 gs: 0000 @ ffff88007be00000/0000000000000000 Code (instr addr ffffffff810013a8) cc cc cc cc cc cc cc cc cc cc cc cc cc b8 1d 00 00 00 0f 01 c1 <c3> cc cc cc cc cc cc cc cc cc cc Stack: ffffffff81332553 ffff880000000002 0000000000013fc0 ffff88007a54bd28 0000000000000296 ffff88007a8bddb8 ffffffff810d8dcb ffff88007a8ba200 ffff88007a54bc98 ffff88007a8ba200 ffff88007be0eea0 ffffffff810d8d20 ffff88007a54bd28 ffff88007a8bdeb8 ffffffff810d8a50 000000007a8bde18 ffff88007a8bdfd8 ffff88007a8ba200 ffff88007be0eea8 0000000000000000 Call Trace: [<ffffffff810013a8>] <-- ffff88007a8bdd50: [<ffffffff81332553>] ffff88007a8bdd80: [<ffffffff810d8dcb>] ffff88007a8bdda8: [<ffffffff810d8d20>] ffff88007a8bddc0: [<ffffffff810d8a50>] ffff88007a8bde08: [<ffffffff819032a0>] ffff88007a8bde20: [<ffffffff8159161d>] ffff88007a8bdea8: [<ffffffff810d8960>] ffff88007a8bdec0: [<ffffffff81082a6e>] ffff88007a8bdf28: [<ffffffff810829a0>] ffff88007a8bdf50: [<ffffffff8159b66c>] ffff88007a8bdf80: [<ffffffff810829a0>] ffff88007a8bdfd0: [<ffffffffffffffff>] vcpu1: rip: ffffffff810d8da3 flags: 00000097 s nz a p c rsp: ffff88007abc3d88 rax: 0000000000000003 rcx: dead000000200200 rdx: 0000000000000000 rbx: ffff88007a54bd28 rsi: ffff88007abc00c0 rdi: ffff88007a54bd28 rbp: ffff88007abc3db8 r8: ffff88007be2eec0 r9: 000000444d5210c0 r10: 0000000000000000 r11: 0000000000000010 r12: 0000000000000296 r13: 0000000000000003 r14: ffff88007a54bd4c r15: ffff88007a54bd00 cs: 0010 ss: 0018 ds: 0000 es: 0000 fs: 0000 @ 0000000000000000 gs: 0000 @ ffff88007be20000/0000000000000000 Code (instr addr ffffffff810d8da3) 10 83 c0 01 89 4b 24 89 43 20 41 83 fd 04 74 32 44 89 e8 f3 90 <44> 8b 6b 20 41 39 c5 74 ec 41 83 Stack: ffff88007abc00c0 ffff88007a54bc98 ffff88007abc00c0 ffff88007be2eea0 ffffffff810d8d20 ffff88007a54bd28 ffff88007abc3eb8 ffffffff810d8a50 000000017abc3e18 ffff88007abc3fd8 ffff88007abc00c0 ffff88007be2eea8 0000000000000000 0000000000000000 ffff88007be33fc0 0000000000000000 ffffffff819032a0 0000000000000000 ffff88007abc3ea8 ffffffff8159161d Call Trace: [<ffffffff810d8da3>] <-- ffff88007abc3da8: [<ffffffff810d8d20>] ffff88007abc3dc0: [<ffffffff810d8a50>] ffff88007abc3e08: [<ffffffff819032a0>] ffff88007abc3e20: [<ffffffff8159161d>] ffff88007abc3ea8: [<ffffffff810d8960>] ffff88007abc3ec0: [<ffffffff81082a6e>] ffff88007abc3f28: [<ffffffff810829a0>] ffff88007abc3f50: [<ffffffff8159b66c>] ffff88007abc3f80: [<ffffffff810829a0>] ffff88007abc3fd0: [<ffffffffffffffff>] vcpu2: rip: ffffffff810d8da1 flags: 00000097 s nz a p c rsp: ffff88007a401d88 rax: 0000000000000003 rcx: 0000000000000004 rdx: 0000000000000000 rbx: ffff88007a54bd28 rsi: ffff88007abfe200 rdi: ffff88007a54bd28 rbp: ffff88007a401db8 r8: ffff88007be4eec0 r9: 000000444d5210c0 r10: 0000000000000000 r11: 0000000000000010 r12: 0000000000000296 r13: 0000000000000003 r14: ffff88007a54bd4c r15: ffff88007a54bd00 cs: 0010 ss: 0018 ds: 0000 es: 0000 fs: 0000 @ 0000000000000000 gs: 0000 @ ffff88007be40000/0000000000000000 Code (instr addr ffffffff810d8da1) 8b 4b 10 83 c0 01 89 4b 24 89 43 20 41 83 fd 04 74 32 44 89 e8 <f3> 90 44 8b 6b 20 41 39 c5 74 ec Stack: ffff88007abfe200 ffff88007a54bc98 ffff88007abfe200 ffff88007be4eea0 ffffffff810d8d20 ffff88007a54bd28 ffff88007a401eb8 ffffffff810d8a50 000000027a401e18 ffff88007a401fd8 ffff88007abfe200 ffff88007be4eea8 0000000000000000 0000000000000000 ffff88007be53fc0 0000000000000000 ffffffff819032a0 0000000000000000 ffff88007a401ea8 ffffffff8159161d Call Trace: [<ffffffff810d8da1>] <-- ffff88007a401da8: [<ffffffff810d8d20>] ffff88007a401dc0: [<ffffffff810d8a50>] ffff88007a401e08: [<ffffffff819032a0>] ffff88007a401e20: [<ffffffff8159161d>] ffff88007a401ea8: [<ffffffff810d8960>] ffff88007a401ec0: [<ffffffff81082a6e>] ffff88007a401f28: [<ffffffff810829a0>] ffff88007a401f50: [<ffffffff8159b66c>] ffff88007a401f80: [<ffffffff810829a0>] ffff88007a401fd0: [<ffffffffffffffff>] vcpu3: rip: ffffffff810d8da3 flags: 00000097 s nz a p c rsp: ffff88007a45dd88 rax: 0000000000000003 rcx: dead000000200200 rdx: 0000000000000000 rbx: ffff88007a54bd28 rsi: ffff88007a45a340 rdi: ffff88007a54bd28 rbp: ffff88007a45ddb8 r8: ffff88007be6eec0 r9: 0000000000000001 r10: 0000000000000001 r11: 0000000000000000 r12: 0000000000000296 r13: 0000000000000003 r14: ffff88007a54bd4c r15: ffff88007a54bd00 cs: 0010 ss: 0018 ds: 0000 es: 0000 fs: 0000 @ 0000000000000000 gs: 0000 @ ffff88007be60000/0000000000000000 Code (instr addr ffffffff810d8da3) 10 83 c0 01 89 4b 24 89 43 20 41 83 fd 04 74 32 44 89 e8 f3 90 <44> 8b 6b 20 41 39 c5 74 ec 41 83 Stack: ffff88007a45a340 ffff88007a54bc98 ffff88007a45a340 ffff88007be6eea0 ffffffff810d8d20 ffff88007a54bd28 ffff88007a45deb8 ffffffff810d8a50 000000037a45de18 ffff88007a45dfd8 ffff88007a45a340 ffff88007be6eea8 0000000000000000 0000000000000000 ffff88007be73fc0 0000000000000000 ffffffff819032a0 0000000000000000 ffff88007a45dea8 ffffffff8159161d Call Trace: [<ffffffff810d8da3>] <-- ffff88007a45dda8: [<ffffffff810d8d20>] ffff88007a45ddc0: [<ffffffff810d8a50>] ffff88007a45de08: [<ffffffff819032a0>] ffff88007a45de20: [<ffffffff8159161d>] ffff88007a45dea8: [<ffffffff810d8960>] ffff88007a45dec0: [<ffffffff81082a6e>] ffff88007a45df28: [<ffffffff810829a0>] ffff88007a45df50: [<ffffffff8159b66c>] ffff88007a45df80: [<ffffffff810829a0>] ffff88007a45dfd0: [<ffffffffffffffff>]