(gdb) thread apply all bt Thread 25 (Thread 0x7fc1bde78700 (LWP 23704)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000005c970b in ceph::log::Log::entry (this=0x2a2e000) at log/Log.cc:323 #2 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000000000 in ?? () Thread 24 (Thread 0x7fc1be679700 (LWP 23705)): #0 0x00007fc1c03dd0c1 in sem_timedwait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x000000000065aa08 in CephContextServiceThread::entry (this=0x2a6afc0) at common/ceph_context.cc:57 #2 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000000000 in ?? () Thread 23 (Thread 0x7fc1bd677700 (LWP 23706)): #0 0x00007fc1c03dd89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fc1c03d9065 in _L_lock_858 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007fc1c03d8eba in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x0000000000620323 in Mutex::Lock (this=0x2a7eb28, no_lockdep=) at common/Mutex.cc:89 #4 0x00000000004b2b47 in Locker (m=..., this=) at ./common/Mutex.h:120 #5 Monitor::do_admin_command (this=0x2a7eb00, command=..., args=..., ss=...) at mon/Monitor.cc:231 #6 0x00000000004d9662 in AdminHook::call (this=0x38a5dd0, command=..., args=..., out=...) at mon/Monitor.cc:223 #7 0x0000000000646e56 in AdminSocket::do_accept (this=0x2a2b000) at common/admin_socket.cc:343 #8 0x00000000006486c0 in AdminSocket::entry (this=0x2a2b000) at common/admin_socket.cc:241 #9 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #10 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #11 0x0000000000000000 in ?? () Thread 22 (Thread 0x7fc1bce76700 (LWP 23707)): #0 0x00007fc1be980033 in select () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x0000000000596884 in SignalHandler::entry (this=0x3308180) at global/signal_handler.cc:210 #2 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000000000 in ?? () Thread 21 (Thread 0x7fc1bc675700 (LWP 23708)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x000000000063a8b1 in Wait (mutex=..., this=0x32a4b60) at ./common/Cond.h:55 #2 SimpleMessenger::reaper_entry (this=0x32a4680) at msg/SimpleMessenger.cc:206 #3 0x000000000063f0cd in SimpleMessenger::ReaperThread::entry (this=) at msg/SimpleMessenger.h:422 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () ---Type to continue, or q to quit--- Thread 20 (Thread 0x7fc1bbe74700 (LWP 23709)): #0 0x00007fc1c03dd89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007fc1c03e02b8 in _L_cond_lock_874 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007fc1c03e0088 in __pthread_mutex_cond_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fc1c03db1b2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00000000006459bb in WaitUntil (when=..., mutex=..., this=0x2a7eb90) at common/Cond.h:71 #5 SafeTimer::timer_thread (this=0x2a7eb80) at common/Timer.cc:114 #6 0x000000000064636d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #7 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #8 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #9 0x0000000000000000 in ?? () Thread 19 (Thread 0x7fc1bb673700 (LWP 23710)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006effcd in leveldb::port::CondVar::Wait() () #2 0x00000000006d93cb in leveldb::DBImpl::MakeRoomForWrite(bool) () #3 0x00000000006d9713 in leveldb::DBImpl::Write(leveldb::WriteOptions const&, leveldb::WriteBatch*) () #4 0x0000000000581487 in LevelDBStore::submit_transaction_sync (this=, t=...) at os/LevelDBStore.h:145 #5 0x0000000000490e32 in MonitorDBStore::apply_transaction (this=0x7fff6e8007e0, t=...) at mon/MonitorDBStore.h:207 #6 0x00000000004a28f7 in Monitor::handle_sync_chunk (this=0x2a7eb00, m=0x3a32340) at mon/Monitor.cc:1612 #7 0x00000000004c26ab in Monitor::handle_sync (this=0x2a7eb00, m=0x3a32340) at mon/Monitor.cc:1783 #8 0x00000000004c35f0 in Monitor::_ms_dispatch (this=0x2a7eb00, m=0x3a32340) at mon/Monitor.cc:3196 #9 0x00000000004da672 in Monitor::ms_dispatch (this=0x2a7eb00, m=0x3a32340) at mon/Monitor.h:1385 #10 0x00000000006b0221 in ms_deliver_dispatch (m=0x3a32340, this=0x32a4680) at msg/Messenger.h:566 #11 DispatchQueue::entry (this=0x32a4768) at msg/DispatchQueue.cc:119 #12 0x000000000063e8dd in DispatchQueue::DispatchThread::entry (this=) at msg/DispatchQueue.h:104 #13 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #14 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #15 0x0000000000000000 in ?? () Thread 18 (Thread 0x7fc1bae72700 (LWP 23711)): #0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006a4625 in Accepter::entry (this=0x32a4740) at msg/Accepter.cc:197 #2 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x0000000000000000 in ?? () Thread 17 (Thread 0x7fc1ba570700 (LWP 23714)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006c788d in Wait (mutex=..., this=0x2a3d6e8) at ./common/Cond.h:55 #2 Pipe::writer (this=0x2a3d500) at msg/Pipe.cc:1575 #3 0x00000000006d1bed in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () ---Type to continue, or q to quit--- Thread 16 (Thread 0x7fc1ba36e700 (LWP 23716)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006c788d in Wait (mutex=..., this=0x38ecbe8) at ./common/Cond.h:55 #2 Pipe::writer (this=0x38eca00) at msg/Pipe.cc:1575 #3 0x00000000006d1bed in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 15 (Thread 0x7fc1ba16c700 (LWP 23720)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006ceeca in Wait (mutex=..., this=0x2a3d6e8) at ./common/Cond.h:55 #2 Pipe::reader (this=0x2a3d500) at msg/Pipe.cc:1315 #3 0x00000000006d1c0d in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 14 (Thread 0x7fc1ba06b700 (LWP 23721)): #0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006bb782 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2109 #2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x38eca00, buf=0x7fc1ba06adbf "\377", len=1) at msg/Pipe.cc:2082 #3 0x00000000006cec14 in Pipe::reader (this=0x38eca00) at msg/Pipe.cc:1324 #4 0x00000000006d1c0d in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 13 (Thread 0x7fc1b9c68700 (LWP 17969)): #0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006bb782 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2109 #2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x37bf280, buf=0x7fc1b9c67dbf "\377", len=1) at msg/Pipe.cc:2082 #3 0x00000000006cec14 in Pipe::reader (this=0x37bf280) at msg/Pipe.cc:1324 #4 0x00000000006d1c0d in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 12 (Thread 0x7fc1b7844700 (LWP 17970)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006c788d in Wait (mutex=..., this=0x37bf468) at ./common/Cond.h:55 #2 Pipe::writer (this=0x37bf280) at msg/Pipe.cc:1575 #3 0x00000000006d1bed in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 ---Type to continue, or q to quit--- #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 11 (Thread 0x7fc1b8854700 (LWP 17983)): #0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006bb782 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2109 #2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x38ec000, buf=0x7fc1b8853dbf "\377", len=1) at msg/Pipe.cc:2082 #3 0x00000000006cec14 in Pipe::reader (this=0x38ec000) at msg/Pipe.cc:1324 #4 0x00000000006d1c0d in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 10 (Thread 0x7fc1b723e700 (LWP 17984)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006c788d in Wait (mutex=..., this=0x38ec1e8) at ./common/Cond.h:55 #2 Pipe::writer (this=0x38ec000) at msg/Pipe.cc:1575 #3 0x00000000006d1bed in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 9 (Thread 0x7fc1b7c48700 (LWP 17985)): #0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006bb782 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2109 #2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x2a3dc80, buf=0x7fc1b7c47dbf "\377", len=1) at msg/Pipe.cc:2082 #3 0x00000000006cec14 in Pipe::reader (this=0x2a3dc80) at msg/Pipe.cc:1324 #4 0x00000000006d1c0d in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 8 (Thread 0x7fc1b8652700 (LWP 17986)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006c788d in Wait (mutex=..., this=0x2a3de68) at ./common/Cond.h:55 #2 Pipe::writer (this=0x2a3dc80) at msg/Pipe.cc:1575 #3 0x00000000006d1bed in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 7 (Thread 0x7fc1b8955700 (LWP 17987)): #0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006bb782 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2109 #2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x3872c80, buf=0x7fc1b8954dbf "\377", len=1) at msg/Pipe.cc:2082 ---Type to continue, or q to quit--- #3 0x00000000006cec14 in Pipe::reader (this=0x3872c80) at msg/Pipe.cc:1324 #4 0x00000000006d1c0d in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 6 (Thread 0x7fc1b834f700 (LWP 17988)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006c788d in Wait (mutex=..., this=0x3872e68) at ./common/Cond.h:55 #2 Pipe::writer (this=0x3872c80) at msg/Pipe.cc:1575 #3 0x00000000006d1bed in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 5 (Thread 0x7fc1b8b57700 (LWP 17991)): #0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006bb782 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2109 #2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x3872a00, buf=0x7fc1b8b56dbf "\377", len=1) at msg/Pipe.cc:2082 #3 0x00000000006cec14 in Pipe::reader (this=0x3872a00) at msg/Pipe.cc:1324 #4 0x00000000006d1c0d in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 4 (Thread 0x7fc1b7f4b700 (LWP 17992)): #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006c788d in Wait (mutex=..., this=0x3872be8) at ./common/Cond.h:55 #2 Pipe::writer (this=0x3872a00) at msg/Pipe.cc:1575 #3 0x00000000006d1bed in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 3 (Thread 0x7fc1b7945700 (LWP 17993)): #0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006bb782 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2109 #2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x3872500, buf=0x7fc1b7944dbf "\377", len=1) at msg/Pipe.cc:2082 #3 0x00000000006cec14 in Pipe::reader (this=0x3872500) at msg/Pipe.cc:1324 #4 0x00000000006d1c0d in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #7 0x0000000000000000 in ?? () Thread 2 (Thread 0x7fc1b7b47700 (LWP 17994)): ---Type to continue, or q to quit--- #0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006c788d in Wait (mutex=..., this=0x38726e8) at ./common/Cond.h:55 #2 Pipe::writer (this=0x3872500) at msg/Pipe.cc:1575 #3 0x00000000006d1bed in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 1 (Thread 0x7fc1c0800780 (LWP 23703)): #0 0x00007fc1c03d8148 in pthread_join () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006407f2 in Thread::join (this=0x32a48f0, prval=) at common/Thread.cc:121 #2 0x0000000000639608 in SimpleMessenger::wait (this=0x32a4680) at msg/SimpleMessenger.cc:502 #3 0x000000000048c5c1 in main (argc=, argv=) at ceph_mon.cc:500 (gdb)