(gdb) thread apply all bt Thread 13 (Thread 0x7f10b1735700 (LWP 2195)): #0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0 #1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0 #2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0 #3 0x00007f10b347891a in tc_new () from /usr/lib/libtcmalloc.so.0 #4 0x00007f10b22baa89 in std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #5 0x00007f10b22bc495 in char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #6 0x00007f10b22bc61d in std::basic_string, std::allocator >::basic_string(char const*, unsigned long, std::allocator const&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #7 0x0000000000758820 in PrebufferedStreambuf::get_str (this=0x1e662e0) at common/PrebufferedStreambuf.cc:61 #8 0x00000000006cb883 in get_str (this=0x1e66270) at log/Entry.h:49 #9 ceph::log::Log::_flush (this=0x1cf4000, t=0x7f10b1734e00, requeue=0x1cf40e8, crash=false) at log/Log.cc:210 #10 0x00000000006cbd6d in ceph::log::Log::flush (this=0x1cf4000) at log/Log.cc:178 #11 0x00000000006cbf83 in ceph::log::Log::entry (this=0x1cf4000) at log/Log.cc:318 #12 0x00007f10b3232e9a in start_thread (arg=0x7f10b1735700) at pthread_create.c:308 #13 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #14 0x0000000000000000 in ?? () Thread 12 (Thread 0x7f10b0f34700 (LWP 2196)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:215 #1 0x000000000078ca96 in WaitUntil (when=..., mutex=..., this=0x1cdb668) at ./common/Cond.h:71 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x1cdb668) at ./common/Cond.h:79 #3 CephContextServiceThread::entry (this=0x1cdb600) at common/ceph_context.cc:58 #4 0x00007f10b3232e9a in start_thread (arg=0x7f10b0f34700) at pthread_create.c:308 #5 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #6 0x0000000000000000 in ?? () Thread 11 (Thread 0x7f10b0733700 (LWP 2197)): #0 0x00007f10b1a35b03 in __GI___poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x0000000000773c1b in AdminSocket::entry (this=0x1cf4140) at common/admin_socket.cc:238 #2 0x00007f10b3232e9a in start_thread (arg=0x7f10b0733700) at pthread_create.c:308 #3 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 ---Type to continue, or q to quit--- #4 0x0000000000000000 in ?? () Thread 10 (Thread 0x7f10aff32700 (LWP 2198)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x000000000075d199 in Wait (mutex=..., this=0x1d11518) at ./common/Cond.h:55 #2 SimpleMessenger::reaper_entry (this=0x1d11000) at msg/SimpleMessenger.cc:207 #3 0x0000000000762ebd in SimpleMessenger::ReaperThread::entry (this=) at msg/SimpleMessenger.h:225 #4 0x00007f10b3232e9a in start_thread (arg=0x7f10aff32700) at pthread_create.c:308 #5 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #6 0x0000000000000000 in ?? () Thread 9 (Thread 0x7f10af731700 (LWP 2199)): #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:132 #1 0x00007f10b323c2b8 in _L_cond_lock_874 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007f10b323c088 in __pthread_mutex_cond_lock (mutex=0x1d00a40) at ../nptl/pthread_mutex_lock.c:61 #3 0x00007f10b32371b2 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:302 #4 0x000000000076bd5b in WaitUntil (when=..., mutex=..., this=0x1d00a98) at common/Cond.h:71 #5 SafeTimer::timer_thread (this=0x1d00a88) at common/Timer.cc:114 #6 0x000000000076c70d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #7 0x00007f10b3232e9a in start_thread (arg=0x7f10af731700) at pthread_create.c:308 #8 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #9 0x0000000000000000 in ?? () Thread 8 (Thread 0x7f10aef30700 (LWP 2200)): #0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0 #1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0 #2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0 #3 0x00007f10b3476ee3 in tc_realloc () from /usr/lib/libtcmalloc.so.0 #4 0x00007f10b19a766f in _IO_vfscanf_internal (s=, format=, argptr=0x7f10aef2cdf8, errp=0x0) at vfscanf.c:1413 #5 0x00007f10b19c0360 in _IO_vsscanf (string=0x7f10aef2d050 "00400000-009a7000 r-xp 00000000 08:00 28008502", ' ' , "/extra_space/dzafman/ceph/src/ceph-mon", format=0x7f10b347fd4f "%lx-%lx %4s %lx %x:%x %ld %n", args=0x7f10aef2cdf8) at iovsscanf.c:45 #6 0x00007f10b19ab317 in __sscanf (s=, format=) at sscanf.c:34 #7 0x00007f10b3473ccb in ProcMapsIterator::NextExt(unsigned long*, unsigned long*, char**, unsigned long*, long*, char**, unsigned long*, unsigned long*, unsigned long*, unsigned long*, unsigned long*) () from /usr/lib/libtcmalloc.so.0 ---Type to continue, or q to quit--- #8 0x00007f10b3473fdf in ProcMapsIterator::Next(unsigned long*, unsigned long*, char**, unsigned long*, long*, char**) () from /usr/lib/libtcmalloc.so.0 #9 0x00007f10b347420a in tcmalloc::FillProcSelfMaps(char*, int, bool*) () from /usr/lib/libtcmalloc.so.0 #10 0x00007f10b346d9ad in HeapProfileTable::FillOrderedProfile(char*, int) const () from /usr/lib/libtcmalloc.so.0 #11 0x00007f10b346f20b in ?? () from /usr/lib/libtcmalloc.so.0 #12 0x00007f10b346f5ed in ?? () from /usr/lib/libtcmalloc.so.0 #13 0x00007f10b346ff98 in HeapProfilerDump () from /usr/lib/libtcmalloc.so.0 #14 0x000000000058d3a4 in Monitor::handle_command (this=0x1d00a00, m=0x28d7200) at mon/Monitor.cc:2403 #15 0x0000000000593fba in Monitor::dispatch (this=0x1d00a00, s=0x234da80, m=0x28d7200, src_is_mon=) at mon/Monitor.cc:2878 #16 0x0000000000594582 in Monitor::_ms_dispatch (this=0x1d00a00, m=0x28d7200) at mon/Monitor.cc:2847 #17 0x00000000005acfc2 in Monitor::ms_dispatch (this=0x1d00a00, m=0x28d7200) at mon/Monitor.h:735 #18 0x00000000008443b9 in ms_deliver_dispatch (m=0x28d7200, this=0x1d11000) at msg/Messenger.h:587 #19 DispatchQueue::entry (this=0x1d110e8) at msg/DispatchQueue.cc:123 #20 0x00000000007626bd in DispatchQueue::DispatchThread::entry (this=) at msg/DispatchQueue.h:104 #21 0x00007f10b3232e9a in start_thread (arg=0x7f10aef30700) at pthread_create.c:308 #22 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #23 0x0000000000000000 in ?? () Thread 7 (Thread 0x7f10ae72f700 (LWP 2201)): #0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0 #1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0 #2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0 #3 0x00007f10b347891a in tc_new () from /usr/lib/libtcmalloc.so.0 #4 0x000000000075d839 in SimpleMessenger::add_accept_pipe (this=0x1d11000, sd=26) at msg/SimpleMessenger.cc:313 #5 0x000000000084012a in Accepter::entry (this=0x1d110c0) at msg/Accepter.cc:220 #6 0x00007f10b3232e9a in start_thread (arg=0x7f10ae72f700) at pthread_create.c:308 #7 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #8 0x0000000000000000 in ?? () Thread 6 (Thread 0x7f10ade2d700 (LWP 2236)): #0 0x00007f10b1a35b03 in __GI___poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x0000000000878e04 in SignalHandler::entry (this=0x1d0d000) at global/signal_handler.cc:213 #2 0x00007f10b3232e9a in start_thread (arg=0x7f10ade2d700) at pthread_create.c:308 #3 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #4 0x0000000000000000 in ?? () ---Type to continue, or q to quit--- Thread 5 (Thread 0x7f10ad62c700 (LWP 2655)): #0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0 #1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0 #2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0 #3 0x00007f10b347891a in tc_new () from /usr/lib/libtcmalloc.so.0 #4 0x0000000000780322 in ceph::buffer::create (len=1936) at common/buffer.cc:505 #5 0x000000000085a0e1 in Pipe::read_message (this=0x1e56a00, pm=0x7f10ad62bdb0) at msg/Pipe.cc:1794 #6 0x000000000086c7d2 in Pipe::reader (this=0x1e56a00) at msg/Pipe.cc:1442 #7 0x000000000086efcd in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #8 0x00007f10b3232e9a in start_thread (arg=0x7f10ad62c700) at pthread_create.c:308 #9 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #10 0x0000000000000000 in ?? () Thread 4 (Thread 0x7f10ad52b700 (LWP 2656)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008647bd in Wait (mutex=..., this=0x1e56be8) at ./common/Cond.h:55 #2 Pipe::writer (this=0x1e56a00) at msg/Pipe.cc:1671 #3 0x000000000086efad in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f10b3232e9a in start_thread (arg=0x7f10ad52b700) at pthread_create.c:308 #5 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #6 0x0000000000000000 in ?? () Thread 3 (Thread 0x7f10ad228700 (LWP 18841)): #0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0 #1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0 #2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0 #3 0x00007f10b347891a in tc_new () from /usr/lib/libtcmalloc.so.0 #4 0x0000000000780322 in ceph::buffer::create (len=23) at common/buffer.cc:505 #5 0x000000000085a0e1 in Pipe::read_message (this=0x259a280, pm=0x7f10ad227db0) at msg/Pipe.cc:1794 #6 0x000000000086c7d2 in Pipe::reader (this=0x259a280) at msg/Pipe.cc:1442 #7 0x000000000086efcd in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #8 0x00007f10b3232e9a in start_thread (arg=0x7f10ad228700) at pthread_create.c:308 #9 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 ---Type to continue, or q to quit--- #10 0x0000000000000000 in ?? () Thread 2 (Thread 0x7f10ad127700 (LWP 18843)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008647bd in Wait (mutex=..., this=0x259a468) at ./common/Cond.h:55 #2 Pipe::writer (this=0x259a280) at msg/Pipe.cc:1671 #3 0x000000000086efad in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f10b3232e9a in start_thread (arg=0x7f10ad127700) at pthread_create.c:308 #5 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #6 0x0000000000000000 in ?? () Thread 1 (Thread 0x7f10b3ac3780 (LWP 2194)): #0 0x00007f10b3234148 in pthread_join (threadid=139709631366912, thread_return=0x0) at pthread_join.c:89 #1 0x0000000000766232 in Thread::join (this=0x1d11270, prval=) at common/Thread.cc:121 #2 0x000000000075c0a5 in SimpleMessenger::wait (this=0x1d11000) at msg/SimpleMessenger.cc:513 #3 0x000000000054afdd in main (argc=, argv=0x1cd9960) at ceph_mon.cc:674