gdb -ex "set pagination 0" -ex "thread apply all bt" --batch -p 24779 [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". [New Thread 0x7fce10dff700 (LWP 24839)] [New Thread 0x7fce117ff700 (LWP 24838)] [New Thread 0x7fce109ff700 (LWP 24837)] [New Thread 0x7fce923f8700 (LWP 24835)] [New Thread 0x7fce92bf9700 (LWP 24834)] [New Thread 0x7fce92cfa700 (LWP 24833)] [New Thread 0x7fce92dfb700 (LWP 24830)] [New Thread 0x7fce92efc700 (LWP 24829)] [New Thread 0x7fce92ffd700 (LWP 24828)] [New Thread 0x7fce937fe700 (LWP 24827)] [New Thread 0x7fce93fff700 (LWP 24826)] [New Thread 0x7fce981bf700 (LWP 24825)] [New Thread 0x7fce982c0700 (LWP 24822)] [New Thread 0x7fce98ac1700 (LWP 24821)] [New Thread 0x7fce992c2700 (LWP 24820)] [New Thread 0x7fce99ac3700 (LWP 24819)] [New Thread 0x7fce9a2c4700 (LWP 24818)] [New Thread 0x7fce9acd1700 (LWP 24817)] [New Thread 0x7fce9bc37700 (LWP 24816)] warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fff53eb2000 0x00007fcea4eaed5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 Thread 20 (Thread 0x7fce9bc37700 (LWP 24816)): #0 0x00007fcea4eac344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9c6a25b in ceph::log::Log::entry (this=0x7fcead4a70f0) at log/Log.cc:323 #2 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000000000 in ?? () Thread 19 (Thread 0x7fce9acd1700 (LWP 24817)): #0 0x00007fcea4eac6bb in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9a6ff72 in WaitUntil (when=..., mutex=..., this=0x7fcead4c1a38) at ./common/Cond.h:71 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fcead4c1a38) at ./common/Cond.h:79 #3 CephContextServiceThread::entry (this=0x7fcead4c19c0) at common/ceph_context.cc:58 #4 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 18 (Thread 0x7fce9a2c4700 (LWP 24818)): #0 0x00007fcea4eaed5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea4eaa3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007fcea4eaa1cb in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fcea9a353ab in Mutex::Lock (this=0x7fcead4aa150, no_lockdep=) at common/Mutex.cc:89 #4 0x00007fcea99b776e in Locker (m=..., this=0x7fce9a2c3880) at ./common/Mutex.h:120 #5 librados::RadosClient::ms_dispatch (this=0x7fcead4a9930, m=0x7fcdf8022d80) at librados/RadosClient.cc:352 #6 0x00007fcea9b54b3b in ms_deliver_dispatch (m=0x7fcdf8022d80, this=0x7fcead4c3610) at msg/Messenger.h:587 #7 DispatchQueue::entry (this=0x7fcead4c3708) at msg/DispatchQueue.cc:124 #8 0x00007fcea9bc037d in DispatchQueue::DispatchThread::entry (this=) at msg/DispatchQueue.h:104 #9 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #10 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #11 0x0000000000000000 in ?? () Thread 17 (Thread 0x7fce99ac3700 (LWP 24819)): #0 0x00007fcea4eac344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9bbbba8 in Wait (mutex=..., this=0x7fcead4c3b58) at ./common/Cond.h:55 #2 SimpleMessenger::reaper_entry (this=0x7fcead4c3610) at msg/SimpleMessenger.cc:210 #3 0x00007fcea9bc0a9d in SimpleMessenger::ReaperThread::entry (this=) at msg/SimpleMessenger.h:225 #4 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 16 (Thread 0x7fce992c2700 (LWP 24820)): #0 0x00007fcea4eac6bb in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9a551b8 in WaitUntil (when=..., mutex=..., this=0x7fcead4a9d50) at common/Cond.h:71 #2 SafeTimer::timer_thread (this=0x7fcead4a9d40) at common/Timer.cc:114 #3 0x00007fcea9a560ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 15 (Thread 0x7fce98ac1700 (LWP 24821)): #0 0x00007fcea4eac344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9a56b42 in Wait (mutex=..., this=0x7fcead4a9e68) at ./common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x7fcead4a9e08) at common/Finisher.cc:78 #3 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #5 0x0000000000000000 in ?? () Thread 14 (Thread 0x7fce982c0700 (LWP 24822)): #0 0x00007fcea4eac344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9bab81a in Wait (mutex=..., this=0x7fcead4c94a0) at ./common/Cond.h:55 #2 Pipe::writer (this=0x7fcead4c9290) at msg/Pipe.cc:1730 #3 0x00007fcea9bb7c5d in Pipe::Writer::entry (this=) at msg/Pipe.h:61 #4 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 13 (Thread 0x7fce981bf700 (LWP 24825)): #0 0x00007fcea4be6b73 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007fcea9ba087c in Pipe::tcp_read_wait (this=this@entry=0x7fcead4c9290) at msg/Pipe.cc:2282 #2 0x00007fcea9ba0b10 in Pipe::tcp_read (this=this@entry=0x7fcead4c9290, buf=, buf@entry=0x7fce981beb4f "\377\340\353\033\230\316\177", len=len@entry=1) at msg/Pipe.cc:2255 #3 0x00007fcea9bb2a9f in Pipe::reader (this=0x7fcead4c9290) at msg/Pipe.cc:1421 #4 0x00007fcea9bb7d7d in Pipe::Reader::entry (this=) at msg/Pipe.h:49 #5 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 12 (Thread 0x7fce93fff700 (LWP 24826)): #0 0x00007fcea4eaed5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea4eb1c4e in _L_cond_lock_1001 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007fcea4eb1a7b in __pthread_mutex_cond_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fcea4eac76f in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007fcea9a551b8 in WaitUntil (when=..., mutex=..., this=0x7fcead4aa248) at common/Cond.h:71 #5 SafeTimer::timer_thread (this=0x7fcead4aa238) at common/Timer.cc:114 #6 0x00007fcea9a560ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #7 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #8 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #9 0x0000000000000000 in ?? () Thread 11 (Thread 0x7fce937fe700 (LWP 24827)): #0 0x00007fcea4eac344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9a56b42 in Wait (mutex=..., this=0x7fcead4aa388) at ./common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x7fcead4aa328) at common/Finisher.cc:78 #3 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #5 0x0000000000000000 in ?? () Thread 10 (Thread 0x7fce92ffd700 (LWP 24828)): #0 0x00007fcea4eac344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9bab81a in Wait (mutex=..., this=0x7fce94007950) at ./common/Cond.h:55 #2 Pipe::writer (this=0x7fce94007740) at msg/Pipe.cc:1730 #3 0x00007fcea9bb7c5d in Pipe::Writer::entry (this=) at msg/Pipe.h:61 #4 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 9 (Thread 0x7fce92efc700 (LWP 24829)): #0 0x00007fcea4be6b73 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007fcea9ba087c in Pipe::tcp_read_wait (this=this@entry=0x7fce94007740) at msg/Pipe.cc:2282 #2 0x00007fcea9ba0b10 in Pipe::tcp_read (this=this@entry=0x7fce94007740, buf=, buf@entry=0x7fce92efbb4f "\377", len=len@entry=1) at msg/Pipe.cc:2255 #3 0x00007fcea9bb2a9f in Pipe::reader (this=0x7fce94007740) at msg/Pipe.cc:1421 #4 0x00007fcea9bb7d7d in Pipe::Reader::entry (this=) at msg/Pipe.h:49 #5 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 8 (Thread 0x7fce92dfb700 (LWP 24830)): #0 0x00007fcea4eac344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea9bab81a in Wait (mutex=..., this=0x7fce94008c20) at ./common/Cond.h:55 #2 Pipe::writer (this=0x7fce94008a10) at msg/Pipe.cc:1730 #3 0x00007fcea9bb7c5d in Pipe::Writer::entry (this=) at msg/Pipe.h:61 #4 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 7 (Thread 0x7fce92cfa700 (LWP 24833)): #0 0x00007fcea4be6b73 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007fcea9ba087c in Pipe::tcp_read_wait (this=this@entry=0x7fce94008a10) at msg/Pipe.cc:2282 #2 0x00007fcea9ba0b10 in Pipe::tcp_read (this=this@entry=0x7fce94008a10, buf=, buf@entry=0x7fce92cf9b4f "\377", len=len@entry=1) at msg/Pipe.cc:2255 #3 0x00007fcea9bb2a9f in Pipe::reader (this=0x7fce94008a10) at msg/Pipe.cc:1421 #4 0x00007fcea9bb7d7d in Pipe::Reader::entry (this=) at msg/Pipe.h:49 #5 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 6 (Thread 0x7fce92bf9700 (LWP 24834)): #0 0x00007fcea4eaed5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea4eaa3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007fcea4eaa1cb in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fceab8be9b9 in ?? () #4 0x00007fceab60752d in ?? () #5 0x00007fceab5f380c in ?? () #6 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #8 0x0000000000000000 in ?? () Thread 5 (Thread 0x7fce923f8700 (LWP 24835)): #0 0x00007fcea4bea807 in ioctl () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007fceab6073e9 in ?? () #2 0x00007fceab607525 in ?? () #3 0x00007fceab5f380c in ?? () #4 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 4 (Thread 0x7fce109ff700 (LWP 24837)): #0 0x00007fcea4eac344 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fceab8bebd9 in ?? () #2 0x00007fceab857e8b in ?? () #3 0x00007fceab858260 in ?? () #4 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 3 (Thread 0x7fce117ff700 (LWP 24838)): #0 0x00007fcea4eaed5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea4eaa3df in _L_lock_1145 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007fcea4eaa32a in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fcea2722909 in PR_Lock () from /usr/lib/x86_64-linux-gnu/libnspr4.so #4 0x00007fce9b215545 in ?? () from /usr/lib/x86_64-linux-gnu/nss/libsoftokn3.so #5 0x00007fce9b203169 in ?? () from /usr/lib/x86_64-linux-gnu/nss/libsoftokn3.so #6 0x00007fcea342ea6c in ?? () from /usr/lib/x86_64-linux-gnu/libnss3.so #7 0x00007fcea341cc8b in PK11_DestroyContext () from /usr/lib/x86_64-linux-gnu/libnss3.so #8 0x00007fcea9be5024 in nss_aes_operation (op=, secret=..., in=..., out=..., error=...) at auth/Crypto.cc:204 #9 0x00007fcea9be3881 in CryptoKey::encrypt (this=0x7fcead4dd888, cct=0x7fcead4a49b0, in=..., out=..., error=...) at auth/Crypto.cc:358 #10 0x00007fcea9be303e in encode_encrypt_enc_bl (cct=cct@entry=0x7fcead4a49b0, t=..., key=..., out=..., error=...) at auth/cephx/CephxProtocol.h:465 #11 0x00007fcea9be3172 in encode_encrypt (cct=0x7fcead4a49b0, t=..., key=..., out=..., error=...) at auth/cephx/CephxProtocol.h:490 #12 0x00007fcea9be2659 in CephxSessionHandler::sign_message (this=0x7fcead4dd870, m=0x7fce080281a0) at auth/cephx/CephxSessionHandler.cc:48 #13 0x00007fcea9babffb in Pipe::writer (this=0x7fce940e71f0) at msg/Pipe.cc:1693 #14 0x00007fcea9bb7c5d in Pipe::Writer::entry (this=) at msg/Pipe.h:61 #15 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #16 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #17 0x0000000000000000 in ?? () Thread 2 (Thread 0x7fce10dff700 (LWP 24839)): #0 0x00007fcea4eaed5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea4eaa3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007fcea4eaa1cb in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fcea9a353ab in Mutex::Lock (this=0x7fce940e7328, no_lockdep=) at common/Mutex.cc:89 #4 0x00007fcea9bb36ae in Pipe::reader (this=0x7fce940e71f0) at msg/Pipe.cc:1485 #5 0x00007fcea9bb7d7d in Pipe::Reader::entry (this=) at msg/Pipe.h:49 #6 0x00007fcea4ea7b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007fcea4bf17bd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #8 0x0000000000000000 in ?? () Thread 1 (Thread 0x7fceab3f1900 (LWP 24779)): #0 0x00007fcea4eaed5c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fcea4eaa3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007fcea4eaa1cb in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fcea9a353ab in Mutex::Lock (this=0x7fce940e7328, no_lockdep=) at common/Mutex.cc:89 #4 0x00007fcea9bbcc3a in SimpleMessenger::submit_message (this=this@entry=0x7fcead4c3610, m=m@entry=0x7fce0800af00, con=con@entry=0x7fce9410bae0, dest_addr=..., dest_type=4, lazy=lazy@entry=false) at msg/SimpleMessenger.cc:429 #5 0x00007fcea9bbd6a9 in SimpleMessenger::_send_message (this=0x7fcead4c3610, m=0x7fce0800af00, con=0x7fce9410bae0, lazy=false) at msg/SimpleMessenger.cc:138 #6 0x00007fceaa7d11b5 in send_message (con=..., m=0x7fce0800af00, this=) at ./msg/Messenger.h:411 #7 Objecter::send_op (this=this@entry=0x7fcead4c4060, op=op@entry=0x7fce0017c900) at osdc/Objecter.cc:1678 #8 0x00007fceaa7d1981 in Objecter::_op_submit (this=0x7fcead4c4060, op=0x7fce0017c900) at osdc/Objecter.cc:1340 #9 0x00007fcea99c18d8 in librados::IoCtxImpl::aio_operate_read (this=0x7fce94006a50, oid=..., o=0x7fcdf4031d80, c=, flags=0, pbl=0x0) at librados/IoCtxImpl.cc:594 #10 0x00007fcea9996c98 in librados::IoCtx::aio_operate (this=0x7fce94006628, oid=..., c=0x7fcdf401d7f0, o=0x7fce00177740, flags=0, pbl=) at librados/librados.cc:1106 #11 0x00007fceaa798e65 in librbd::AioRead::send (this=0x7fcdf402f010) at librbd/AioRequest.cc:102 #12 0x00007fceaa7ab23a in librbd::aio_read (ictx=ictx@entry=0x7fce940064e0, image_extents=..., buf=buf@entry=0x7fcdf402e000 "\302%c\271\327L\221\001\270dKd\351L\224\031\227l-zw\327\305\017\222-\372\070\330\375\235\024\262EV\360=\243%\006\266\310\061\233<\274\203\022\026\071\v\257\356)\254\v\"gV9\270-\230\016\344̙\323\v\351d\034\234\071\001\224T\263B", pbl=pbl@entry=0x0, c=c@entry=0x7fcdf402dec0) at librbd/internal.cc:3153 #13 0x00007fceaa7ab794 in librbd::aio_read (ictx=0x7fce940064e0, off=19147112448, len=4096, buf=0x7fcdf402e000 "\302%c\271\327L\221\001\270dKd\351L\224\031\227l-zw\327\305\017\222-\372\070\330\375\235\024\262EV\360=\243%\006\266\310\061\233<\274\203\022\026\071\v\257\356)\254\v\"gV9\270-\230\016\344̙\323\v\351d\034\234\071\001\224T\263B", bl=0x0, c=0x7fcdf402dec0) at librbd/internal.cc:3091 #14 0x00007fceab8ae5bb in ?? () #15 0x00007fceab8ae850 in ?? () #16 0x00007fceab85cf8e in ?? () #17 0x00007fceab862243 in ?? () #18 0x00007fceab862757 in ?? () #19 0x00007fceab86215b in ?? () #20 0x00007fceab862757 in ?? () #21 0x00007fceab8636e7 in ?? () #22 0x00007fceab87524a in ?? () #23 0x00007fcea4b57d10 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #24 0x00007fff53ea7b30 in ?? () #25 0x0000000000000000 in ?? ()