|
(gdb) thread apply all bt
|
|
|
|
Thread 13 (Thread 0x7f10b1735700 (LWP 2195)):
|
|
#0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0
|
|
#1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0
|
|
#2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0
|
|
#3 0x00007f10b347891a in tc_new () from /usr/lib/libtcmalloc.so.0
|
|
#4 0x00007f10b22baa89 in std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator<char> const&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
|
|
#5 0x00007f10b22bc495 in char* std::string::_S_construct<char const*>(char const*, char const*, std::allocator<char> const&, std::forward_iterator_tag) ()
|
|
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
|
|
#6 0x00007f10b22bc61d in std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, unsigned long, std::allocator<char> const&) ()
|
|
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
|
|
#7 0x0000000000758820 in PrebufferedStreambuf::get_str (this=0x1e662e0) at common/PrebufferedStreambuf.cc:61
|
|
#8 0x00000000006cb883 in get_str (this=0x1e66270) at log/Entry.h:49
|
|
#9 ceph::log::Log::_flush (this=0x1cf4000, t=0x7f10b1734e00, requeue=0x1cf40e8, crash=false) at log/Log.cc:210
|
|
#10 0x00000000006cbd6d in ceph::log::Log::flush (this=0x1cf4000) at log/Log.cc:178
|
|
#11 0x00000000006cbf83 in ceph::log::Log::entry (this=0x1cf4000) at log/Log.cc:318
|
|
#12 0x00007f10b3232e9a in start_thread (arg=0x7f10b1735700) at pthread_create.c:308
|
|
#13 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#14 0x0000000000000000 in ?? ()
|
|
|
|
Thread 12 (Thread 0x7f10b0f34700 (LWP 2196)):
|
|
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:215
|
|
#1 0x000000000078ca96 in WaitUntil (when=..., mutex=..., this=0x1cdb668) at ./common/Cond.h:71
|
|
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x1cdb668) at ./common/Cond.h:79
|
|
#3 CephContextServiceThread::entry (this=0x1cdb600) at common/ceph_context.cc:58
|
|
#4 0x00007f10b3232e9a in start_thread (arg=0x7f10b0f34700) at pthread_create.c:308
|
|
#5 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#6 0x0000000000000000 in ?? ()
|
|
|
|
Thread 11 (Thread 0x7f10b0733700 (LWP 2197)):
|
|
#0 0x00007f10b1a35b03 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
|
|
#1 0x0000000000773c1b in AdminSocket::entry (this=0x1cf4140) at common/admin_socket.cc:238
|
|
#2 0x00007f10b3232e9a in start_thread (arg=0x7f10b0733700) at pthread_create.c:308
|
|
#3 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
---Type <return> to continue, or q <return> to quit---
|
|
#4 0x0000000000000000 in ?? ()
|
|
|
|
Thread 10 (Thread 0x7f10aff32700 (LWP 2198)):
|
|
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
|
|
#1 0x000000000075d199 in Wait (mutex=..., this=0x1d11518) at ./common/Cond.h:55
|
|
#2 SimpleMessenger::reaper_entry (this=0x1d11000) at msg/SimpleMessenger.cc:207
|
|
#3 0x0000000000762ebd in SimpleMessenger::ReaperThread::entry (this=<optimized out>) at msg/SimpleMessenger.h:225
|
|
#4 0x00007f10b3232e9a in start_thread (arg=0x7f10aff32700) at pthread_create.c:308
|
|
#5 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#6 0x0000000000000000 in ?? ()
|
|
|
|
Thread 9 (Thread 0x7f10af731700 (LWP 2199)):
|
|
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:132
|
|
#1 0x00007f10b323c2b8 in _L_cond_lock_874 () from /lib/x86_64-linux-gnu/libpthread.so.0
|
|
#2 0x00007f10b323c088 in __pthread_mutex_cond_lock (mutex=0x1d00a40) at ../nptl/pthread_mutex_lock.c:61
|
|
#3 0x00007f10b32371b2 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:302
|
|
#4 0x000000000076bd5b in WaitUntil (when=..., mutex=..., this=0x1d00a98) at common/Cond.h:71
|
|
#5 SafeTimer::timer_thread (this=0x1d00a88) at common/Timer.cc:114
|
|
#6 0x000000000076c70d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
|
|
#7 0x00007f10b3232e9a in start_thread (arg=0x7f10af731700) at pthread_create.c:308
|
|
#8 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#9 0x0000000000000000 in ?? ()
|
|
|
|
Thread 8 (Thread 0x7f10aef30700 (LWP 2200)):
|
|
#0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0
|
|
#1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0
|
|
#2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0
|
|
#3 0x00007f10b3476ee3 in tc_realloc () from /usr/lib/libtcmalloc.so.0
|
|
#4 0x00007f10b19a766f in _IO_vfscanf_internal (s=<optimized out>, format=<optimized out>, argptr=0x7f10aef2cdf8, errp=0x0) at vfscanf.c:1413
|
|
#5 0x00007f10b19c0360 in _IO_vsscanf (string=0x7f10aef2d050 "00400000-009a7000 r-xp 00000000 08:00 28008502", ' ' <repeats 27 times>, "/extra_space/dzafman/ceph/src/ceph-mon",
|
|
format=0x7f10b347fd4f "%lx-%lx %4s %lx %x:%x %ld %n", args=0x7f10aef2cdf8) at iovsscanf.c:45
|
|
#6 0x00007f10b19ab317 in __sscanf (s=<optimized out>, format=<optimized out>) at sscanf.c:34
|
|
#7 0x00007f10b3473ccb in ProcMapsIterator::NextExt(unsigned long*, unsigned long*, char**, unsigned long*, long*, char**, unsigned long*, unsigned long*, unsigned long*, unsigned long*, unsigned long*) () from /usr/lib/libtcmalloc.so.0
|
|
---Type <return> to continue, or q <return> to quit---
|
|
#8 0x00007f10b3473fdf in ProcMapsIterator::Next(unsigned long*, unsigned long*, char**, unsigned long*, long*, char**) () from /usr/lib/libtcmalloc.so.0
|
|
#9 0x00007f10b347420a in tcmalloc::FillProcSelfMaps(char*, int, bool*) () from /usr/lib/libtcmalloc.so.0
|
|
#10 0x00007f10b346d9ad in HeapProfileTable::FillOrderedProfile(char*, int) const () from /usr/lib/libtcmalloc.so.0
|
|
#11 0x00007f10b346f20b in ?? () from /usr/lib/libtcmalloc.so.0
|
|
#12 0x00007f10b346f5ed in ?? () from /usr/lib/libtcmalloc.so.0
|
|
|
|
#13 0x00007f10b346ff98 in HeapProfilerDump () from /usr/lib/libtcmalloc.so.0
|
|
#14 0x000000000058d3a4 in Monitor::handle_command (this=0x1d00a00, m=0x28d7200) at mon/Monitor.cc:2403
|
|
#15 0x0000000000593fba in Monitor::dispatch (this=0x1d00a00, s=0x234da80, m=0x28d7200, src_is_mon=<optimized out>) at mon/Monitor.cc:2878
|
|
#16 0x0000000000594582 in Monitor::_ms_dispatch (this=0x1d00a00, m=0x28d7200) at mon/Monitor.cc:2847
|
|
#17 0x00000000005acfc2 in Monitor::ms_dispatch (this=0x1d00a00, m=0x28d7200) at mon/Monitor.h:735
|
|
#18 0x00000000008443b9 in ms_deliver_dispatch (m=0x28d7200, this=0x1d11000) at msg/Messenger.h:587
|
|
#19 DispatchQueue::entry (this=0x1d110e8) at msg/DispatchQueue.cc:123
|
|
#20 0x00000000007626bd in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
|
|
#21 0x00007f10b3232e9a in start_thread (arg=0x7f10aef30700) at pthread_create.c:308
|
|
#22 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#23 0x0000000000000000 in ?? ()
|
|
|
|
Thread 7 (Thread 0x7f10ae72f700 (LWP 2201)):
|
|
#0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0
|
|
#1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0
|
|
#2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0
|
|
#3 0x00007f10b347891a in tc_new () from /usr/lib/libtcmalloc.so.0
|
|
#4 0x000000000075d839 in SimpleMessenger::add_accept_pipe (this=0x1d11000, sd=26) at msg/SimpleMessenger.cc:313
|
|
#5 0x000000000084012a in Accepter::entry (this=0x1d110c0) at msg/Accepter.cc:220
|
|
#6 0x00007f10b3232e9a in start_thread (arg=0x7f10ae72f700) at pthread_create.c:308
|
|
#7 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#8 0x0000000000000000 in ?? ()
|
|
|
|
Thread 6 (Thread 0x7f10ade2d700 (LWP 2236)):
|
|
#0 0x00007f10b1a35b03 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
|
|
#1 0x0000000000878e04 in SignalHandler::entry (this=0x1d0d000) at global/signal_handler.cc:213
|
|
#2 0x00007f10b3232e9a in start_thread (arg=0x7f10ade2d700) at pthread_create.c:308
|
|
#3 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#4 0x0000000000000000 in ?? ()
|
|
---Type <return> to continue, or q <return> to quit---
|
|
|
|
Thread 5 (Thread 0x7f10ad62c700 (LWP 2655)):
|
|
#0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0
|
|
#1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0
|
|
#2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0
|
|
#3 0x00007f10b347891a in tc_new () from /usr/lib/libtcmalloc.so.0
|
|
#4 0x0000000000780322 in ceph::buffer::create (len=1936) at common/buffer.cc:505
|
|
#5 0x000000000085a0e1 in Pipe::read_message (this=0x1e56a00, pm=0x7f10ad62bdb0) at msg/Pipe.cc:1794
|
|
#6 0x000000000086c7d2 in Pipe::reader (this=0x1e56a00) at msg/Pipe.cc:1442
|
|
#7 0x000000000086efcd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
|
|
#8 0x00007f10b3232e9a in start_thread (arg=0x7f10ad62c700) at pthread_create.c:308
|
|
#9 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#10 0x0000000000000000 in ?? ()
|
|
|
|
Thread 4 (Thread 0x7f10ad52b700 (LWP 2656)):
|
|
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
|
|
#1 0x00000000008647bd in Wait (mutex=..., this=0x1e56be8) at ./common/Cond.h:55
|
|
#2 Pipe::writer (this=0x1e56a00) at msg/Pipe.cc:1671
|
|
#3 0x000000000086efad in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
|
|
#4 0x00007f10b3232e9a in start_thread (arg=0x7f10ad52b700) at pthread_create.c:308
|
|
#5 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#6 0x0000000000000000 in ?? ()
|
|
|
|
Thread 3 (Thread 0x7f10ad228700 (LWP 18841)):
|
|
#0 0x00007f10b34729e3 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/libtcmalloc.so.0
|
|
#1 0x00007f10b34728d2 in SpinLock::SlowLock() () from /usr/lib/libtcmalloc.so.0
|
|
#2 0x00007f10b346f9a1 in ?? () from /usr/lib/libtcmalloc.so.0
|
|
#3 0x00007f10b347891a in tc_new () from /usr/lib/libtcmalloc.so.0
|
|
#4 0x0000000000780322 in ceph::buffer::create (len=23) at common/buffer.cc:505
|
|
#5 0x000000000085a0e1 in Pipe::read_message (this=0x259a280, pm=0x7f10ad227db0) at msg/Pipe.cc:1794
|
|
#6 0x000000000086c7d2 in Pipe::reader (this=0x259a280) at msg/Pipe.cc:1442
|
|
#7 0x000000000086efcd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
|
|
#8 0x00007f10b3232e9a in start_thread (arg=0x7f10ad228700) at pthread_create.c:308
|
|
#9 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
---Type <return> to continue, or q <return> to quit---
|
|
#10 0x0000000000000000 in ?? ()
|
|
|
|
Thread 2 (Thread 0x7f10ad127700 (LWP 18843)):
|
|
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
|
|
#1 0x00000000008647bd in Wait (mutex=..., this=0x259a468) at ./common/Cond.h:55
|
|
#2 Pipe::writer (this=0x259a280) at msg/Pipe.cc:1671
|
|
#3 0x000000000086efad in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
|
|
#4 0x00007f10b3232e9a in start_thread (arg=0x7f10ad127700) at pthread_create.c:308
|
|
#5 0x00007f10b1a414bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
|
|
#6 0x0000000000000000 in ?? ()
|
|
|
|
Thread 1 (Thread 0x7f10b3ac3780 (LWP 2194)):
|
|
#0 0x00007f10b3234148 in pthread_join (threadid=139709631366912, thread_return=0x0) at pthread_join.c:89
|
|
#1 0x0000000000766232 in Thread::join (this=0x1d11270, prval=<optimized out>) at common/Thread.cc:121
|
|
#2 0x000000000075c0a5 in SimpleMessenger::wait (this=0x1d11000) at msg/SimpleMessenger.cc:513
|
|
#3 0x000000000054afdd in main (argc=<optimized out>, argv=0x1cd9960) at ceph_mon.cc:674
|