Ceph

ubuntu@teuthology:/var/lib/teuthworker/archive/sage-2014-08-09_14:13:44-rados-next-testing-basic-multi/410713

3 (!) osds died with this deadlock.

(gdb) t 67
[Switching to thread 67 (Thread 0x7f0add460700 (LWP 17247))]
#0  0x00007f0afadbb89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
(gdb) bt
#0  0x00007f0afadbb89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007f0afadb7065 in _L_lock_858 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007f0afadb6eba in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#3  0x0000000000a3db83 in Mutex::Lock (this=0x1934bd0, no_lockdep=<optimized out>) at common/Mutex.cc:89
#4  0x0000000000a634f6 in Locker (m=..., this=<synthetic pointer>) at ./common/Mutex.h:120
#5  SimpleMessenger::get_connection (this=0x1934800, dest=...) at msg/SimpleMessenger.cc:382
#6  0x0000000000646842 in OSDService::send_message_osd_cluster (this=0x1db4da0, peer=1, m=0x2c1a600,
    from_epoch=104) at osd/OSD.cc:687
#7  0x000000000083a66b in ReplicatedPG::send_message (this=0x1fb5000, to_osd=1, m=0x2c1a600)
    at osd/ReplicatedPG.h:292
#8  0x000000000093d416 in ECBackend::dispatch_recovery_messages (this=0x1fe56c0, m=..., priority=127)
    at osd/ECBackend.cc:426
#9  0x000000000093e054 in ECBackend::handle_message (this=0x1fe56c0, _op=...) at osd/ECBackend.cc:681
#10 0x00000000007bdb1a in ReplicatedPG::do_request (this=0x1fb5000, op=..., handle=...)
    at osd/ReplicatedPG.cc:1109
#11 0x0000000000647182 in OSD::dequeue_op (this=0x1db3700, pg=..., op=..., handle=...) at osd/OSD.cc:8278
#12 0x0000000000647c51 in OSD::ShardedOpWQ::_process (this=0x1db46a8, thread_index=<optimized out>,
    hb=<optimized out>) at osd/OSD.cc:8180
#13 0x0000000000a7797c in ShardedThreadPool::shardedthreadpool_worker (this=0x1db3cf8, thread_index=8)
    at common/WorkQueue.cc:320
#14 0x0000000000a79260 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>)
    at common/WorkQueue.h:504
#15 0x00007f0afadb4e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#16 0x00007f0af97653fd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#17 0x0000000000000000 in ?? ()

Thread 74 (Thread 0x7f0ad482f700 (LWP 18889)):
#0  0x00007f0afadb8d84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000b411e1 in Wait (mutex=..., this=0x5de9fd8) at ./common/Cond.h:55
#2  Pipe::stop_and_wait (this=0x5de9dc0) at msg/Pipe.cc:1412
#3  0x0000000000a5f4fd in SimpleMessenger::mark_down (this=0x1934800, con=<optimized out>)
    at msg/SimpleMessenger.cc:673
#4  0x0000000000618824 in OSD::require_same_peer_instance (this=0x1db3700, op=..., map=...) at osd/OSD.cc:6710
#5  0x0000000000698b45 in OSD::handle_replica_op<MOSDECSubOpWriteReply, 109> (this=0x1db3700, op=..., osdmap=...)
    at osd/OSD.cc:8073
#6  0x000000000065246e in OSD::dispatch_op_fast (this=0x1db3700, op=..., osdmap=...) at osd/OSD.cc:5692
#7  0x0000000000652534 in OSD::dispatch_session_waiting (this=0x1db3700, session=0x25c9240, osdmap=...)
    at osd/OSD.cc:5409
#8  0x00000000006544cc in OSD::ms_fast_dispatch (this=0x1db3700, m=<optimized out>) at osd/OSD.cc:5432
#9  0x0000000000b2eba9 in ms_fast_dispatch (m=0x1be0400, this=0x1934800) at msg/Messenger.h:607
#10 DispatchQueue::fast_dispatch (this=0x1934908, m=0x1be0400) at msg/DispatchQueue.cc:71
#11 0x0000000000b5781b in Pipe::reader (this=0x5de9dc0) at msg/Pipe.cc:1566
#12 0x0000000000b594bd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:49
#13 0x00007f0afadb4e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#14 0x00007f0af97653fd in clone () from /lib/x86_64-linux-gnu/libc.so.6
#15 0x0000000000000000 in ?? ()