root@proxmox1:~# gdb -ex "set pagination 0" -ex "thread apply all bt" --batch -p 22453 [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". [New Thread 0x7f9d661ff700 (LWP 22511)] [New Thread 0x7f9d66bff700 (LWP 22510)] [New Thread 0x7f9d65dff700 (LWP 22498)] [New Thread 0x7f9de77fe700 (LWP 22496)] [New Thread 0x7f9dec89d700 (LWP 22495)] [New Thread 0x7f9dec99e700 (LWP 22494)] [New Thread 0x7f9deca9f700 (LWP 22491)] [New Thread 0x7f9decba0700 (LWP 22490)] [New Thread 0x7f9decca1700 (LWP 22489)] [New Thread 0x7f9ded4a2700 (LWP 22488)] [New Thread 0x7f9de7fff700 (LWP 22487)] [New Thread 0x7f9ded5a3700 (LWP 22486)] [New Thread 0x7f9ded6a4700 (LWP 22485)] [New Thread 0x7f9dedea5700 (LWP 22484)] [New Thread 0x7f9dee6a6700 (LWP 22483)] [New Thread 0x7f9deeea7700 (LWP 22482)] [New Thread 0x7f9def6a8700 (LWP 22481)] [New Thread 0x7f9df00b5700 (LWP 22480)] [New Thread 0x7f9df101b700 (LWP 22479)] warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fffde040000 0x00007f9dfa292d5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 Thread 20 (Thread 0x7f9df101b700 (LWP 22479)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dff04e25b in ceph::log::Log::entry (this=0x7f9e02ce10f0) at log/Log.cc:323 #2 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000000000 in ?? () Thread 19 (Thread 0x7f9df00b5700 (LWP 22480)): #0 0x00007f9dfa2906bb in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfee53f72 in WaitUntil (when=..., mutex=..., this=0x7f9e02cfba38) at ./common/Cond.h:71 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7f9e02cfba38) at ./common/Cond.h:79 #3 CephContextServiceThread::entry (this=0x7f9e02cfb9c0) at common/ceph_context.cc:58 #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 18 (Thread 0x7f9def6a8700 (LWP 22481)): #0 0x00007f9df9fe220b in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9df9f76458 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f9df9f747b1 in free () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f9dffb7b2f7 in deallocate (__p=, this=) at /usr/include/c++/4.7/ext/new_allocator.h:100 #4 _M_put_node (__p=, this=) at /usr/include/c++/4.7/bits/stl_list.h:339 #5 std::_List_base >::_M_clear (this=this@entry=0x7f9d340276b8) at /usr/include/c++/4.7/bits/list.tcc:80 #6 0x00007f9dffbc51f0 in ~_List_base (this=0x7f9d340276b8, __in_chrg=) at /usr/include/c++/4.7/bits/stl_list.h:379 #7 ~list (this=0x7f9d340276b8, __in_chrg=) at /usr/include/c++/4.7/bits/stl_list.h:436 #8 ~list (this=0x7f9d340276b8, __in_chrg=) at ./include/buffer.h:317 #9 ~OSDOp (this=0x7f9d34027630, __in_chrg=) at ./osd/osd_types.h:3192 #10 _Destroy (__pointer=0x7f9d34027630) at /usr/include/c++/4.7/bits/stl_construct.h:95 #11 std::_Destroy_aux::__destroy (__first=0x7f9d34027630, __last=0x7f9d34027710) at /usr/include/c++/4.7/bits/stl_construct.h:105 #12 0x00007f9dffbc52ad in _Destroy (__last=, __first=) at /usr/include/c++/4.7/bits/stl_construct.h:128 #13 _Destroy (__last=, __first=) at /usr/include/c++/4.7/bits/stl_construct.h:155 #14 std::vector >::~vector (this=0x7f9def6a7650, __in_chrg=) at /usr/include/c++/4.7/bits/stl_vector.h:403 #15 0x00007f9dffbb7297 in Objecter::handle_osd_op_reply (this=, m=0x7f9d340273b0) at osdc/Objecter.cc:1869 #16 0x00007f9dfed9b6a4 in librados::RadosClient::_dispatch (this=this@entry=0x7f9e02ce3930, m=m@entry=0x7f9d340273b0) at librados/RadosClient.cc:390 #17 0x00007f9dfed9b7db in librados::RadosClient::ms_dispatch (this=0x7f9e02ce3930, m=0x7f9d340273b0) at librados/RadosClient.cc:360 #18 0x00007f9dfef38b3b in ms_deliver_dispatch (m=0x7f9d340273b0, this=0x7f9e02cfd610) at msg/Messenger.h:587 #19 DispatchQueue::entry (this=0x7f9e02cfd708) at msg/DispatchQueue.cc:124 #20 0x00007f9dfefa437d in DispatchQueue::DispatchThread::entry (this=) at msg/DispatchQueue.h:104 #21 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #22 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #23 0x0000000000000000 in ?? () Thread 17 (Thread 0x7f9deeea7700 (LWP 22482)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfef9fba8 in Wait (mutex=..., this=0x7f9e02cfdb58) at ./common/Cond.h:55 #2 SimpleMessenger::reaper_entry (this=0x7f9e02cfd610) at msg/SimpleMessenger.cc:210 #3 0x00007f9dfefa4a9d in SimpleMessenger::ReaperThread::entry (this=) at msg/SimpleMessenger.h:225 #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 16 (Thread 0x7f9dee6a6700 (LWP 22483)): #0 0x00007f9dfa2906bb in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfee391b8 in WaitUntil (when=..., mutex=..., this=0x7f9e02ce3d50) at common/Cond.h:71 #2 SafeTimer::timer_thread (this=0x7f9e02ce3d40) at common/Timer.cc:114 #3 0x00007f9dfee3a0ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 15 (Thread 0x7f9dedea5700 (LWP 22484)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfee3ab42 in Wait (mutex=..., this=0x7f9e02ce3e68) at ./common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x7f9e02ce3e08) at common/Finisher.cc:78 #3 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #5 0x0000000000000000 in ?? () Thread 14 (Thread 0x7f9ded6a4700 (LWP 22485)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfef8f81a in Wait (mutex=..., this=0x7f9e02d034a0) at ./common/Cond.h:55 #2 Pipe::writer (this=0x7f9e02d03290) at msg/Pipe.cc:1730 #3 0x00007f9dfef9bc5d in Pipe::Writer::entry (this=) at msg/Pipe.h:61 #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 13 (Thread 0x7f9ded5a3700 (LWP 22486)): #0 0x00007f9df9fcab73 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9dfef8487c in Pipe::tcp_read_wait (this=this@entry=0x7f9e02d03290) at msg/Pipe.cc:2282 #2 0x00007f9dfef84b10 in Pipe::tcp_read (this=this@entry=0x7f9e02d03290, buf=, buf@entry=0x7f9ded5a2b4f "\377\340+Z\355\235\177", len=len@entry=1) at msg/Pipe.cc:2255 #3 0x00007f9dfef96a9f in Pipe::reader (this=0x7f9e02d03290) at msg/Pipe.cc:1421 #4 0x00007f9dfef9bd7d in Pipe::Reader::entry (this=) at msg/Pipe.h:49 #5 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 12 (Thread 0x7f9de7fff700 (LWP 22487)): #0 0x00007f9dfa292d5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfa295c4e in _L_cond_lock_1001 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007f9dfa295a7b in __pthread_mutex_cond_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f9dfa29076f in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9dfee391b8 in WaitUntil (when=..., mutex=..., this=0x7f9e02ce4248) at common/Cond.h:71 #5 SafeTimer::timer_thread (this=0x7f9e02ce4238) at common/Timer.cc:114 #6 0x00007f9dfee3a0ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #7 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #8 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #9 0x0000000000000000 in ?? () Thread 11 (Thread 0x7f9ded4a2700 (LWP 22488)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfee3ab42 in Wait (mutex=..., this=0x7f9e02ce4388) at ./common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x7f9e02ce4328) at common/Finisher.cc:78 #3 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #5 0x0000000000000000 in ?? () Thread 10 (Thread 0x7f9decca1700 (LWP 22489)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfef8f81a in Wait (mutex=..., this=0x7f9e02d0a0b0) at ./common/Cond.h:55 #2 Pipe::writer (this=0x7f9e02d09ea0) at msg/Pipe.cc:1730 #3 0x00007f9dfef9bc5d in Pipe::Writer::entry (this=) at msg/Pipe.h:61 #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 9 (Thread 0x7f9decba0700 (LWP 22490)): #0 0x00007f9df9fcab73 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9dfef8487c in Pipe::tcp_read_wait (this=this@entry=0x7f9e02d09ea0) at msg/Pipe.cc:2282 #2 0x00007f9dfef84b10 in Pipe::tcp_read (this=this@entry=0x7f9e02d09ea0, buf=, buf@entry=0x7f9decb9fb4f "\377", len=len@entry=1) at msg/Pipe.cc:2255 #3 0x00007f9dfef96a9f in Pipe::reader (this=0x7f9e02d09ea0) at msg/Pipe.cc:1421 #4 0x00007f9dfef9bd7d in Pipe::Reader::entry (this=) at msg/Pipe.h:49 #5 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 8 (Thread 0x7f9deca9f700 (LWP 22491)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfef8f81a in Wait (mutex=..., this=0x7f9e02d05d20) at ./common/Cond.h:55 #2 Pipe::writer (this=0x7f9e02d05b10) at msg/Pipe.cc:1730 #3 0x00007f9dfef9bc5d in Pipe::Writer::entry (this=) at msg/Pipe.h:61 #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 7 (Thread 0x7f9dec99e700 (LWP 22494)): #0 0x00007f9df9fcab73 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9dfef8487c in Pipe::tcp_read_wait (this=this@entry=0x7f9e02d05b10) at msg/Pipe.cc:2282 #2 0x00007f9dfef84b10 in Pipe::tcp_read (this=this@entry=0x7f9e02d05b10, buf=, buf@entry=0x7f9dec99db4f "\377", len=len@entry=1) at msg/Pipe.cc:2255 #3 0x00007f9dfef96a9f in Pipe::reader (this=0x7f9e02d05b10) at msg/Pipe.cc:1421 #4 0x00007f9dfef9bd7d in Pipe::Reader::entry (this=) at msg/Pipe.h:49 #5 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 6 (Thread 0x7f9dec89d700 (LWP 22495)): #0 0x00007f9dfa292d5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfa28e3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007f9dfa28e1cb in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f9e00ca29b9 in ?? () #4 0x00007f9e009eb52d in ?? () #5 0x00007f9e009d780c in ?? () #6 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #8 0x0000000000000000 in ?? () Thread 5 (Thread 0x7f9de77fe700 (LWP 22496)): #0 0x00007f9df9fce807 in ioctl () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9e009eb3e9 in ?? () #2 0x00007f9e009eb525 in ?? () #3 0x00007f9e009d780c in ?? () #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 4 (Thread 0x7f9d65dff700 (LWP 22498)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9e00ca2bd9 in ?? () #2 0x00007f9e00c3be8b in ?? () #3 0x00007f9e00c3c260 in ?? () #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 3 (Thread 0x7f9d66bff700 (LWP 22510)): #0 0x00007f9dfa290344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfef8f81a in Wait (mutex=..., this=0x7f9e02df0020) at ./common/Cond.h:55 #2 Pipe::writer (this=0x7f9e02defe10) at msg/Pipe.cc:1730 #3 0x00007f9dfef9bc5d in Pipe::Writer::entry (this=) at msg/Pipe.h:61 #4 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 2 (Thread 0x7f9d661ff700 (LWP 22511)): #0 0x00007f9df9fcab73 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f9dfef8487c in Pipe::tcp_read_wait (this=this@entry=0x7f9e02defe10) at msg/Pipe.cc:2282 #2 0x00007f9dfef84b10 in Pipe::tcp_read (this=this@entry=0x7f9e02defe10, buf=, buf@entry=0x7f9d661feb4f "\377", len=len@entry=1) at msg/Pipe.cc:2255 #3 0x00007f9dfef96a9f in Pipe::reader (this=0x7f9e02defe10) at msg/Pipe.cc:1421 #4 0x00007f9dfef9bd7d in Pipe::Reader::entry (this=) at msg/Pipe.h:49 #5 0x00007f9dfa28bb50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007f9df9fd57bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 1 (Thread 0x7f9e007d5900 (LWP 22453)): #0 0x00007f9dfa292d5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f9dfa28e3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007f9dfa28e1cb in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f9dfee193ab in Mutex::Lock (this=0x7f9e02ce4150, no_lockdep=) at common/Mutex.cc:89 #4 0x00007f9dfeda58c9 in Locker (m=..., this=0x7f9d5c23fb50) at ./common/Mutex.h:120 #5 librados::IoCtxImpl::aio_operate_read (this=0x7f9e02d09570, oid=..., o=0x7f9d3800aa30, c=, flags=0, pbl=0x0) at librados/IoCtxImpl.cc:593 #6 0x00007f9dfed7ac98 in librados::IoCtx::aio_operate (this=0x7f9e02d092f8, oid=..., c=0x7f9d38009520, o=0x7f9d5c23fc10, flags=0, pbl=) at librados/librados.cc:1106 #7 0x00007f9dffb7ce65 in librbd::AioRead::send (this=0x7f9d38008b30) at librbd/AioRequest.cc:102 #8 0x00007f9dffb8f23a in librbd::aio_read (ictx=ictx@entry=0x7f9e02d091b0, image_extents=..., buf=buf@entry=0x7f9d4c011a00 "\346\243S-k\244\060\322|t\267\223\343}\332\001\217\356\070\270\340\205\255\001ѝ\277\237\237Tn\v\272s\017%\264\372\224\aw\356\304\312Ό.\a\316\035\035b7l\377\r\271\243\\]x", , pbl=pbl@entry=0x0, c=c@entry=0x7f9d38009010) at librbd/internal.cc:3153 #9 0x00007f9dffb8f794 in librbd::aio_read (ictx=0x7f9e02d091b0, off=18611585024, len=4096, buf=0x7f9d4c011a00 "\346\243S-k\244\060\322|t\267\223\343}\332\001\217\356\070\270\340\205\255\001ѝ\277\237\237Tn\v\272s\017%\264\372\224\aw\356\304\312Ό.\a\316\035\035b7l\377\r\271\243\\]x", , bl=0x0, c=0x7f9d38009010) at librbd/internal.cc:3091 #10 0x00007f9e00c925bb in ?? () #11 0x00007f9e00c92850 in ?? () #12 0x00007f9e00c40f8e in ?? () #13 0x00007f9e00c46243 in ?? () #14 0x00007f9e00c46757 in ?? () #15 0x00007f9e00c4615b in ?? () #16 0x00007f9e00c46757 in ?? () #17 0x00007f9e00c476e7 in ?? () #18 0x00007f9e00c5924a in ?? () #19 0x00007f9df9f3bd10 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #20 0x00007fffde026e40 in ?? () #21 0x0000000000000000 in ?? ()