Project

General

Profile

Bug #4851 » bt-mon3.txt

Backtrace mon3 02-05-2013 - Wido den Hollander, 05/02/2013 10:56 AM

 
(gdb) thread apply all bt

Thread 25 (Thread 0x7fc1bde78700 (LWP 23704)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000005c970b in ceph::log::Log::entry (this=0x2a2e000) at log/Log.cc:323
#2 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#3 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 24 (Thread 0x7fc1be679700 (LWP 23705)):
#0 0x00007fc1c03dd0c1 in sem_timedwait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x000000000065aa08 in CephContextServiceThread::entry (this=0x2a6afc0) at common/ceph_context.cc:57
#2 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#3 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 23 (Thread 0x7fc1bd677700 (LWP 23706)):
#0 0x00007fc1c03dd89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fc1c03d9065 in _L_lock_858 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fc1c03d8eba in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#3 0x0000000000620323 in Mutex::Lock (this=0x2a7eb28, no_lockdep=<optimized out>) at common/Mutex.cc:89
#4 0x00000000004b2b47 in Locker (m=..., this=<synthetic pointer>) at ./common/Mutex.h:120
#5 Monitor::do_admin_command (this=0x2a7eb00, command=..., args=..., ss=...) at mon/Monitor.cc:231
#6 0x00000000004d9662 in AdminHook::call (this=0x38a5dd0, command=..., args=..., out=...) at mon/Monitor.cc:223
#7 0x0000000000646e56 in AdminSocket::do_accept (this=0x2a2b000) at common/admin_socket.cc:343
#8 0x00000000006486c0 in AdminSocket::entry (this=0x2a2b000) at common/admin_socket.cc:241
#9 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#11 0x0000000000000000 in ?? ()

Thread 22 (Thread 0x7fc1bce76700 (LWP 23707)):
#0 0x00007fc1be980033 in select () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x0000000000596884 in SignalHandler::entry (this=0x3308180) at global/signal_handler.cc:210
#2 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#3 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 21 (Thread 0x7fc1bc675700 (LWP 23708)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x000000000063a8b1 in Wait (mutex=..., this=0x32a4b60) at ./common/Cond.h:55
#2 SimpleMessenger::reaper_entry (this=0x32a4680) at msg/SimpleMessenger.cc:206
#3 0x000000000063f0cd in SimpleMessenger::ReaperThread::entry (this=<optimized out>) at msg/SimpleMessenger.h:422
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

---Type <return> to continue, or q <return> to quit---
Thread 20 (Thread 0x7fc1bbe74700 (LWP 23709)):
#0 0x00007fc1c03dd89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fc1c03e02b8 in _L_cond_lock_874 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2 0x00007fc1c03e0088 in __pthread_mutex_cond_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#3 0x00007fc1c03db1b2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#4 0x00000000006459bb in WaitUntil (when=..., mutex=..., this=0x2a7eb90) at common/Cond.h:71
#5 SafeTimer::timer_thread (this=0x2a7eb80) at common/Timer.cc:114
#6 0x000000000064636d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#7 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#9 0x0000000000000000 in ?? ()

Thread 19 (Thread 0x7fc1bb673700 (LWP 23710)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006effcd in leveldb::port::CondVar::Wait() ()
#2 0x00000000006d93cb in leveldb::DBImpl::MakeRoomForWrite(bool) ()
#3 0x00000000006d9713 in leveldb::DBImpl::Write(leveldb::WriteOptions const&, leveldb::WriteBatch*) ()
#4 0x0000000000581487 in LevelDBStore::submit_transaction_sync (this=<optimized out>, t=...) at os/LevelDBStore.h:145
#5 0x0000000000490e32 in MonitorDBStore::apply_transaction (this=0x7fff6e8007e0, t=...) at mon/MonitorDBStore.h:207
#6 0x00000000004a28f7 in Monitor::handle_sync_chunk (this=0x2a7eb00, m=0x3a32340) at mon/Monitor.cc:1612
#7 0x00000000004c26ab in Monitor::handle_sync (this=0x2a7eb00, m=0x3a32340) at mon/Monitor.cc:1783
#8 0x00000000004c35f0 in Monitor::_ms_dispatch (this=0x2a7eb00, m=0x3a32340) at mon/Monitor.cc:3196
#9 0x00000000004da672 in Monitor::ms_dispatch (this=0x2a7eb00, m=0x3a32340) at mon/Monitor.h:1385
#10 0x00000000006b0221 in ms_deliver_dispatch (m=0x3a32340, this=0x32a4680) at msg/Messenger.h:566
#11 DispatchQueue::entry (this=0x32a4768) at msg/DispatchQueue.cc:119
#12 0x000000000063e8dd in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
#13 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#14 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#15 0x0000000000000000 in ?? ()

Thread 18 (Thread 0x7fc1bae72700 (LWP 23711)):
#0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00000000006a4625 in Accepter::entry (this=0x32a4740) at msg/Accepter.cc:197
#2 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#3 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 17 (Thread 0x7fc1ba570700 (LWP 23714)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006c788d in Wait (mutex=..., this=0x2a3d6e8) at ./common/Cond.h:55
#2 Pipe::writer (this=0x2a3d500) at msg/Pipe.cc:1575
#3 0x00000000006d1bed in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()
---Type <return> to continue, or q <return> to quit---

Thread 16 (Thread 0x7fc1ba36e700 (LWP 23716)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006c788d in Wait (mutex=..., this=0x38ecbe8) at ./common/Cond.h:55
#2 Pipe::writer (this=0x38eca00) at msg/Pipe.cc:1575
#3 0x00000000006d1bed in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 15 (Thread 0x7fc1ba16c700 (LWP 23720)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006ceeca in Wait (mutex=..., this=0x2a3d6e8) at ./common/Cond.h:55
#2 Pipe::reader (this=0x2a3d500) at msg/Pipe.cc:1315
#3 0x00000000006d1c0d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 14 (Thread 0x7fc1ba06b700 (LWP 23721)):
#0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00000000006bb782 in Pipe::tcp_read_wait (this=<optimized out>) at msg/Pipe.cc:2109
#2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x38eca00, buf=0x7fc1ba06adbf "\377", len=1) at msg/Pipe.cc:2082
#3 0x00000000006cec14 in Pipe::reader (this=0x38eca00) at msg/Pipe.cc:1324
#4 0x00000000006d1c0d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 13 (Thread 0x7fc1b9c68700 (LWP 17969)):
#0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00000000006bb782 in Pipe::tcp_read_wait (this=<optimized out>) at msg/Pipe.cc:2109
#2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x37bf280, buf=0x7fc1b9c67dbf "\377", len=1) at msg/Pipe.cc:2082
#3 0x00000000006cec14 in Pipe::reader (this=0x37bf280) at msg/Pipe.cc:1324
#4 0x00000000006d1c0d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 12 (Thread 0x7fc1b7844700 (LWP 17970)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006c788d in Wait (mutex=..., this=0x37bf468) at ./common/Cond.h:55
#2 Pipe::writer (this=0x37bf280) at msg/Pipe.cc:1575
#3 0x00000000006d1bed in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
---Type <return> to continue, or q <return> to quit---
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 11 (Thread 0x7fc1b8854700 (LWP 17983)):
#0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00000000006bb782 in Pipe::tcp_read_wait (this=<optimized out>) at msg/Pipe.cc:2109
#2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x38ec000, buf=0x7fc1b8853dbf "\377", len=1) at msg/Pipe.cc:2082
#3 0x00000000006cec14 in Pipe::reader (this=0x38ec000) at msg/Pipe.cc:1324
#4 0x00000000006d1c0d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 10 (Thread 0x7fc1b723e700 (LWP 17984)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006c788d in Wait (mutex=..., this=0x38ec1e8) at ./common/Cond.h:55
#2 Pipe::writer (this=0x38ec000) at msg/Pipe.cc:1575
#3 0x00000000006d1bed in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 9 (Thread 0x7fc1b7c48700 (LWP 17985)):
#0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00000000006bb782 in Pipe::tcp_read_wait (this=<optimized out>) at msg/Pipe.cc:2109
#2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x2a3dc80, buf=0x7fc1b7c47dbf "\377", len=1) at msg/Pipe.cc:2082
#3 0x00000000006cec14 in Pipe::reader (this=0x2a3dc80) at msg/Pipe.cc:1324
#4 0x00000000006d1c0d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 8 (Thread 0x7fc1b8652700 (LWP 17986)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006c788d in Wait (mutex=..., this=0x2a3de68) at ./common/Cond.h:55
#2 Pipe::writer (this=0x2a3dc80) at msg/Pipe.cc:1575
#3 0x00000000006d1bed in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 7 (Thread 0x7fc1b8955700 (LWP 17987)):
#0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00000000006bb782 in Pipe::tcp_read_wait (this=<optimized out>) at msg/Pipe.cc:2109
#2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x3872c80, buf=0x7fc1b8954dbf "\377", len=1) at msg/Pipe.cc:2082
---Type <return> to continue, or q <return> to quit---
#3 0x00000000006cec14 in Pipe::reader (this=0x3872c80) at msg/Pipe.cc:1324
#4 0x00000000006d1c0d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 6 (Thread 0x7fc1b834f700 (LWP 17988)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006c788d in Wait (mutex=..., this=0x3872e68) at ./common/Cond.h:55
#2 Pipe::writer (this=0x3872c80) at msg/Pipe.cc:1575
#3 0x00000000006d1bed in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 5 (Thread 0x7fc1b8b57700 (LWP 17991)):
#0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00000000006bb782 in Pipe::tcp_read_wait (this=<optimized out>) at msg/Pipe.cc:2109
#2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x3872a00, buf=0x7fc1b8b56dbf "\377", len=1) at msg/Pipe.cc:2082
#3 0x00000000006cec14 in Pipe::reader (this=0x3872a00) at msg/Pipe.cc:1324
#4 0x00000000006d1c0d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 4 (Thread 0x7fc1b7f4b700 (LWP 17992)):
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006c788d in Wait (mutex=..., this=0x3872be8) at ./common/Cond.h:55
#2 Pipe::writer (this=0x3872a00) at msg/Pipe.cc:1575
#3 0x00000000006d1bed in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 3 (Thread 0x7fc1b7945700 (LWP 17993)):
#0 0x00007fc1be97b313 in poll () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00000000006bb782 in Pipe::tcp_read_wait (this=<optimized out>) at msg/Pipe.cc:2109
#2 0x00000000006bbaa0 in Pipe::tcp_read (this=0x3872500, buf=0x7fc1b7944dbf "\377", len=1) at msg/Pipe.cc:2082
#3 0x00000000006cec14 in Pipe::reader (this=0x3872500) at msg/Pipe.cc:1324
#4 0x00000000006d1c0d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#5 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 2 (Thread 0x7fc1b7b47700 (LWP 17994)):
---Type <return> to continue, or q <return> to quit---
#0 0x00007fc1c03dad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006c788d in Wait (mutex=..., this=0x38726e8) at ./common/Cond.h:55
#2 Pipe::writer (this=0x3872500) at msg/Pipe.cc:1575
#3 0x00000000006d1bed in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4 0x00007fc1c03d6e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#5 0x00007fc1be986ccd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 1 (Thread 0x7fc1c0800780 (LWP 23703)):
#0 0x00007fc1c03d8148 in pthread_join () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00000000006407f2 in Thread::join (this=0x32a48f0, prval=<optimized out>) at common/Thread.cc:121
#2 0x0000000000639608 in SimpleMessenger::wait (this=0x32a4680) at msg/SimpleMessenger.cc:502
#3 0x000000000048c5c1 in main (argc=<optimized out>, argv=<optimized out>) at ceph_mon.cc:500
(gdb)
(6-6/7)