root@node2:~# ps -ef | grep ceph root 12633 1 0 00:18 ? 00:00:00 /usr/bin/ceph-mon -i a --pid-file /var/run/ceph/mon.a.pid -c /etc/ceph/ceph.conf root 14642 12858 0 00:22 pts/1 00:00:00 grep --color=auto ceph root@node2:~# gdb /usr/bin/ceph-mon 12633 GNU gdb (GDB) 7.5.91.20130417-cvs-ubuntu Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /usr/bin/ceph-mon...(no debugging symbols found)...done. Attaching to program: /usr/bin/ceph-mon, process 12633 Reading symbols from /lib/x86_64-linux-gnu/libpthread.so.0...(no debugging symbols found)...done. [New LWP 12650] [New LWP 12651] [New LWP 12647] [New LWP 12646] [New LWP 12641] [New LWP 12640] [New LWP 12639] [New LWP 12638] [New LWP 12637] [New LWP 12636] [New LWP 12635] [New LWP 12634] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Loaded symbols for /lib/x86_64-linux-gnu/libpthread.so.0 Reading symbols from /usr/lib/libcrypto++.so.9...(no debugging symbols found)...done. Loaded symbols for /usr/lib/libcrypto++.so.9 Reading symbols from /lib/x86_64-linux-gnu/libuuid.so.1...(no debugging symbols found)...done. Loaded symbols for /lib/x86_64-linux-gnu/libuuid.so.1 Reading symbols from /lib/x86_64-linux-gnu/librt.so.1...(no debugging symbols found)...done. Loaded symbols for /lib/x86_64-linux-gnu/librt.so.1 Reading symbols from /usr/lib/libtcmalloc.so.4...(no debugging symbols found)...done. Loaded symbols for /usr/lib/libtcmalloc.so.4 Reading symbols from /usr/lib/x86_64-linux-gnu/libleveldb.so.1...(no debugging symbols found)...done. Loaded symbols for /usr/lib/x86_64-linux-gnu/libleveldb.so.1 Reading symbols from /usr/lib/x86_64-linux-gnu/libstdc++.so.6...(no debugging symbols found)...done. Loaded symbols for /usr/lib/x86_64-linux-gnu/libstdc++.so.6 Reading symbols from /lib/x86_64-linux-gnu/libm.so.6...(no debugging symbols found)...done. Loaded symbols for /lib/x86_64-linux-gnu/libm.so.6 Reading symbols from /lib/x86_64-linux-gnu/libgcc_s.so.1...(no debugging symbols found)...done. Loaded symbols for /lib/x86_64-linux-gnu/libgcc_s.so.1 Reading symbols from /lib/x86_64-linux-gnu/libc.so.6...(no debugging symbols found)...done. Loaded symbols for /lib/x86_64-linux-gnu/libc.so.6 Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/ld-linux-x86-64.so.2 Reading symbols from /usr/lib/x86_64-linux-gnu/libunwind.so.8...(no debugging symbols found)...done. Loaded symbols for /usr/lib/x86_64-linux-gnu/libunwind.so.8 Reading symbols from /usr/lib/libsnappy.so.1...(no debugging symbols found)...done. Loaded symbols for /usr/lib/libsnappy.so.1 0x00007f3ac4553218 in pthread_join () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) thread apply all bt Thread 13 (Thread 0x7f3abdb70700 (LWP 12634)): #0 0x00007f3ac4555ca4 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000005cc62b in ceph::log::Log::entry() () #2 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 12 (Thread 0x7f3ac2581700 (LWP 12635)): #0 0x00007f3ac4558071 in sem_timedwait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x000000000065b318 in CephContextServiceThread::entry() () #2 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 11 (Thread 0x7f3a7d0f2700 (LWP 12636)): #0 0x00007f3ac2a8f3cd in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x000000000064a0bd in AdminSocket::entry() () #2 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 10 (Thread 0x7f3a76da5700 (LWP 12637)): #0 0x00007f3ac2a94233 in select () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x000000000059b8ba in SignalHandler::entry() () #2 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 9 (Thread 0x7f3a6bc10700 (LWP 12638)): #0 0x00007f3ac4555ca4 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x000000000063cad0 in SimpleMessenger::reaper_entry() () #2 0x00000000006409dd in SimpleMessenger::ReaperThread::entry() () #3 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 8 (Thread 0x7f3a656f2700 (LWP 12639)): #0 0x00007f3ac455882c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f3ac455b1c8 in _L_cond_lock_987 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007f3ac455af8b in __pthread_mutex_cond_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f3ac4556130 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x0000000000646feb in SafeTimer::timer_thread() () #5 0x0000000000647f1d in SafeTimerThread::entry() () #6 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #7 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 7 (Thread 0x7f3a62b35700 (LWP 12640)): #0 0x00007f3ac4555ca4 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f3ac35c373d in leveldb::port::CondVar::Wait() () from /usr/lib/x86_64-linux-gnu/libleveldb.so.1 #2 0x00007f3ac359d77c in leveldb::DBImpl::MakeRoomForWrite(bool) () from /usr/lib/x86_64-linux-gnu/libleveldb.so.1 #3 0x00007f3ac35a0b65 in leveldb::DBImpl::Write(leveldb::WriteOptions const&, leveldb::WriteBatch*) () from /usr/lib/x86_64-linux-gnu/libleveldb.so.1 #4 0x0000000000588a29 in LevelDBStore::submit_transaction_sync(std::tr1::shared_ptr) () #5 0x00000000004a679a in MonitorDBStore::apply_transaction(MonitorDBStore::Transaction&) () #6 0x00000000004b8909 in Monitor::handle_sync_chunk(MMonSync*) () #7 0x00000000004caebb in Monitor::handle_sync(MMonSync*) () #8 0x00000000004d4b6b in Monitor::_ms_dispatch(Message*) () #9 0x00000000004eec82 in Monitor::ms_dispatch(Message*) () ---Type to continue, or q to quit--- #10 0x00000000006ab9c3 in DispatchQueue::entry() () #11 0x000000000063ff8d in DispatchQueue::DispatchThread::entry() () #12 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #13 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 6 (Thread 0x7f3a62334700 (LWP 12641)): #0 0x00007f3ac2a8f3cd in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006a0601 in Accepter::entry() () #2 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 5 (Thread 0x7f3ab97f7700 (LWP 12646)): #0 0x00007f3ac4555ca4 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006bf15e in Pipe::writer() () #2 0x00000000006c8f6d in Pipe::Writer::entry() () #3 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 4 (Thread 0x7f3ab7d62700 (LWP 12647)): #0 0x00007f3ac4555ca4 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006bf15e in Pipe::writer() () #2 0x00000000006c8f6d in Pipe::Writer::entry() () #3 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #4 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 3 (Thread 0x7f3abdcb7700 (LWP 12651)): #0 0x00007f3ac2a8f3cd in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006b433c in Pipe::tcp_read_wait() () #2 0x00000000006b4600 in Pipe::tcp_read(char*, int) () #3 0x00000000006c5c04 in Pipe::reader() () #4 0x00000000006c908d in Pipe::Reader::entry() () #5 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 2 (Thread 0x7f3ab4c27700 (LWP 12650)): #0 0x00007f3ac2a8f3cd in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00000000006b433c in Pipe::tcp_read_wait() () #2 0x00000000006b4600 in Pipe::tcp_read(char*, int) () #3 0x00000000006c5c04 in Pipe::reader() () #4 0x00000000006c908d in Pipe::Reader::entry() () #5 0x00007f3ac4551f8e in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #6 0x00007f3ac2a9be1d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 1 (Thread 0x7f3ac49787c0 (LWP 12633)): #0 0x00007f3ac4553218 in pthread_join () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00000000006423d2 in Thread::join(void**) () #2 0x000000000063bc98 in SimpleMessenger::wait() () #3 0x00000000004a218a in main () (gdb) (gdb) quit A debugging session is active. Inferior 1 [process 12633] will be detached. Quit anyway? (y or n) y Detaching from program: /usr/bin/ceph-mon, process 12633 root@node2:~#