Bug #3836: osd: common/Mutex.cc: 94: FAILED assert(r == 0) in PG::start_flush() - Ceph - Ceph

Actions

Copy link

Bug #3836

closed

osd: common/Mutex.cc: 94: FAILED assert(r == 0) in PG::start_flush()

Added by Sage Weil over 11 years ago. Updated about 11 years ago.

Status:

Resolved

Priority:

High

Assignee:

Samuel Just

Category:

OSD

Target version:

% Done:

Source:

Q/A

Tags:

Backport:

bobtail

Regression:

Severity:

3 - minor

Reviewed:

Affected Versions:

ceph-qa-suite:

Pull request ID:

Crash signature (v1):

Crash signature (v2):

Description

     0> 2013-01-16 23:27:20.943045 7fed0fabf700 -1 common/Mutex.cc: In function 'void Mutex::Lock(bool)' thread 7fed0fabf700 time 2013-01-16 23:27:20.895172
common/Mutex.cc: 94: FAILED assert(r == 0)

 ceph version 0.56-287-gb0162fa (b0162fab3d927544885f2b9609b9ab3dc4aaff74)
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x95) [0x11af291]
 2: (Mutex::Lock(bool)+0x14e) [0x1167ba4]
 3: (Mutex::Locker::Locker(Mutex&)+0x2f) [0xe2de9f]
 4: (SharedPtrRegistry<pg_t, DeletingState>::lookup(pg_t const&)+0xed) [0xfdacd3]
 5: (PG::start_flush(ObjectStore::Transaction*, std::list<Context*, std::allocator<Context*> >*, std::list<Context*, std::allocator<Context*> >*)+0x125) [0xfb9817]
 6: (PG::RecoveryState::Started::Started(boost::statechart::state<PG::RecoveryState::Started, PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Start, (boost::statechart::history_mode)0>::my_context)+0xe6) [0xfbe44a]
 7: (boost::statechart::state<PG::RecoveryState::Started, PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Start, (boost::statechart::history_mode)0>::shallow_construct(boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std:
:allocator<void>, boost::statechart::null_exception_translator>* const&, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>&)+0x49) [0xffa8fb]
 8: (boost::statechart::state<PG::RecoveryState::Started, PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Start, (boost::statechart::history_mode)0>::deep_construct(boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::al
locator<void>, boost::statechart::null_exception_translator>* const&, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>&)+0x28) [0xffac22]
 9: (boost::statechart::detail::inner_constructor<boost::mpl::l_item<mpl_::long_<1l>, PG::RecoveryState::Started, boost::mpl::l_end>, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null
_exception_translator> >::construct(boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>* const&, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachin
e, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>&)+0x23) [0xff2c3d]
 10: (boost::statechart::detail::safe_reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, m
pl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::transit_impl<PG::RecoveryState::Started, PG::RecoveryState::RecoveryMachine, boost::statechart::detail::no_transition_function>(boost::statechart::de
tail::no_transition_function const&)+0x80) [0xfe6160]
 11: (boost::statechart::detail::safe_reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, m
pl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::transit<PG::RecoveryState::Started>()+0x27) [0xfdba69]
 12: (PG::RecoveryState::Reset::react(PG::ActMap const&)+0x178) [0xfbf346]
 13: (boost::statechart::detail::reaction_result boost::statechart::custom_reaction<PG::ActMap>::react<PG::RecoveryState::Reset, boost::statechart::event_base, void const*>(PG::RecoveryState::Reset&, boost::statechart::event_base const&, void const* const&)+0x4d) [0x101975a]
 14: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::
na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list4<boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::Null
Evt>, boost::statechart::custom_reaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::detail::no_context<boost::statechart::
event_base>::no_function(boost::statechart::event_base const&))> >, boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::n
a, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl
_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x33) [0x
1018b17]
 15: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::
na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react<boost::mpl::list4<boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::NullEvt>, boost::statechart::custom_r
eaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::detail::no_context<boost::statechart::event_base>::no_function(boost::s
tatechart::event_base const&))> > >(boost::statechart::event_base const&, void const*)+0x2b) [0x1017e59]
 16: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::
na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list5<boost::statechart::custom_reaction<PG::AdvMap>, boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::NullEvt>, boost::statechart::custom_reaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::detail::no_context<boost::statechart::event_base>::no_function(boost::statechart::event_base const&))> >, boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x53) [0x1016cb2]
 17: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react<boost::mpl::list5<boost::statechart::custom_reaction<PG::AdvMap>, boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::NullEvt>, boost::statechart::custom_reaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::detail::no_context<boost::statechart::event_base>::no_function(boost::statechart::event_base const&))> > >(boost::statechart::event_base const&, void const*)+0x2b) [0x10157c9]
 18: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list<boost::statechart::custom_reaction<PG::QueryState>, boost::statechart::custom_reaction<PG::AdvMap>, boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::NullEvt>, boost::statechart::custom_reaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::detail::no_context<boost::statechart::event_base>::no_function(boost::statechart::event_base const&))>, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x53) [0x1012f82]
 19: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react<boost::mpl::list<boost::statechart::custom_reaction<PG::QueryState>, boost::statechart::custom_reaction<PG::AdvMap>, boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::NullEvt>, boost::statechart::custom_reaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::detail::no_context<boost::statechart::event_base>::no_function(boost::statechart::event_base const&))>, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na> >(boost::statechart::event_base const&, void const*)+0x2b) [0x1010027]
 20: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x2b) [0x100ae29]
 21: (boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>::operator()()+0x41) [0xe74d97]
 22: (boost::statechart::detail::safe_reaction_result boost::statechart::null_exception_translator::operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::exception_event_handler>(boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<void>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::exception_event_handler)+0x2b) [0xfed299]
 23: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x139) [0xfe0b81]
 24: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x24) [0xfd75dc]
 25: (PG::RecoveryState::handle_event(boost::statechart::event_base const&, PG::RecoveryCtx*)+0x3a) [0xfd2e0e]
 26: (PG::handle_create(PG::RecoveryCtx*)+0x14c) [0xfbdd5e]
 27: (OSD::get_or_create_pg(pg_info_t const&, std::map<unsigned int, pg_interval_t, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, pg_interval_t> > >&, unsigned int, int, int&, bool)+0x88c) [0xea86da]
 28: (OSD::handle_pg_log(std::tr1::shared_ptr<OpRequest>)+0x567) [0xecc3b1]
 29: (OSD::dispatch_op(std::tr1::shared_ptr<OpRequest>)+0x151) [0xeb94a1]
 30: (OSD::_dispatch(Message*)+0x81b) [0xeba0f3]
 31: (OSD::ms_dispatch(Message*)+0x1b1) [0xeb82af]

ubuntu@teuthology:/a/sage-2013-01-16_21:39:19-regression-master-testing-basic/350

cat orig.config.yaml 
kernel:
  kdb: true
  sha1: 08e8a5a3a8be539d3c0d502e161e53d3539e0bf5
nuke-on-error: true
overrides:
  ceph:
    conf:
      global:
        ms inject socket failures: 5000
    fs: ext4
    log-whitelist:
    - slow request
    sha1: b0162fab3d927544885f2b9609b9ab3dc4aaff74
  s3tests:
    branch: master
  workunit:
    sha1: b0162fab3d927544885f2b9609b9ab3dc4aaff74
roles:
- - mon.a
  - osd.0
  - osd.1
  - osd.2
- - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
tasks:
- chef: null
- clock: null
- ceph:
    log-whitelist:
    - wrongly marked me down
    - objects unfound and apparently lost
- thrashosds:
    timeout: 1200
- rados:
    clients:
    - client.0
    objects: 50
    op_weights:
      delete: 50
      read: 100
      snap_create: 50
      snap_remove: 50
      snap_rollback: 50
      write: 100
    ops: 4000

Actions

Copy link

Updated by Ian Colle over 11 years ago

Assignee set to Sage Weil

Actions

Copy link

Updated by Sage Weil over 11 years ago

(gdb) thr app all bt

Thread 70 (Thread 0x7fed0c2b8700 (LWP 31469)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3ad94c0, mutex=..., when=...) at ./common/Cond.h:71
#2  0x0000000000edb1ca in Cond::WaitInterval (this=0x3ad94c0, cct=0x3a9d000, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x00000000011a48e3 in ThreadPool::worker (this=0x3ad9448, wt=0x3ac73c0) at common/WorkQueue.cc:131
#4  0x00000000011a61a7 in ThreadPool::WorkThread::entry (this=0x3ac73c0) at common/WorkQueue.h:288
#5  0x000000000119da7d in Thread::_entry_func (arg=0x3ac73c0) at common/Thread.cc:41
#6  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 69 (Thread 0x7fed0719f700 (LWP 818)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000010 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 68 (Thread 0x7fed14ac9700 (LWP 31385)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3aba460, mutex=...) at ./common/Cond.h:55
#2  0x00000000011903fa in SimpleMessenger::reaper_entry (this=0x3aba000) at msg/SimpleMessenger.cc:206
#3  0x0000000001196530 in SimpleMessenger::ReaperThread::entry (this=0x3aba220) at msg/SimpleMessenger.h:411
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3aba220) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 67 (Thread 0x7fed06c9a700 (LWP 32621)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0xa790e60, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0xa790c80) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0xa790ca8) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0xa790ca8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 66 (Thread 0x7fed1aad5700 (LWP 31324)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00000000010eb9a4 in ceph::log::Log::entry (this=0x3aa5000) at log/Log.cc:323
#2  0x000000000119da7d in Thread::_entry_func (arg=0x3aa5000) at common/Thread.cc:41
#3  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#4  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 65 (Thread 0x7fed11ac3700 (LWP 31458)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3abe508, mutex=...) at ./common/Cond.h:55
#2  0x00000000010e2277 in Finisher::finisher_thread_entry (this=0x3abe4a8) at common/Finisher.cc:80
#3  0x0000000000ede824 in Finisher::FinisherThread::entry (this=0x3abe5c0) at ./common/Finisher.h:46
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3abe5c0) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 64 (Thread 0x7fed142c8700 (LWP 31453)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3abe078, mutex=...) at ./common/Cond.h:55
#2  0x00000000010e2277 in Finisher::finisher_thread_entry (this=0x3abe018) at common/Finisher.cc:80
#3  0x0000000000ede824 in Finisher::FinisherThread::entry (this=0x3abe130) at ./common/Finisher.h:46
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3abe130) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 63 (Thread 0x7fed132c6700 (LWP 31455)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3abea80, mutex=..., when=...) at ./common/Cond.h:71
#2  0x0000000000edb1ca in Cond::WaitInterval (this=0x3abea80, cct=0x3a9d000, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x00000000011a48e3 in ThreadPool::worker (this=0x3abea08, wt=0x3a93fe0) at common/WorkQueue.cc:131
#4  0x00000000011a61a7 in ThreadPool::WorkThread::entry (this=0x3a93fe0) at common/WorkQueue.h:288
#5  0x000000000119da7d in Thread::_entry_func (arg=0x3a93fe0) at common/Thread.cc:41
#6  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 62 (Thread 0x7fed06f9d700 (LWP 32735)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007fed1b8a8520 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#2  0x0000000000000001 in ?? ()
#3  0x000000000000e318 in ?? ()
#4  0x00007fed06f9ca20 in ?? ()
#5  0x00007fed1b6590cb in std::basic_streambuf<char, std::char_traits<char> >::xsputn(char const*, long) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#6  0x0000000003be1120 in ?? ()
#7  0x0000000000000000 in ?? ()

---Type <return> to continue, or q <return> to quit---
Thread 61 (Thread 0x7fed080ae700 (LWP 31601)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000010 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 60 (Thread 0x7fed16acd700 (LWP 31451)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3ae14b8, mutex=...) at ./common/Cond.h:55
#2  0x00000000010301f8 in FileJournal::write_thread_entry (this=0x3ae1380) at os/FileJournal.cc:1108
#3  0x0000000000ee23ec in FileJournal::Writer::entry (this=0x3ae1978) at ./os/FileJournal.h:293
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3ae1978) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 59 (Thread 0x7fed13ac7700 (LWP 31454)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3abea80, mutex=..., when=...) at ./common/Cond.h:71
#2  0x0000000000edb1ca in Cond::WaitInterval (this=0x3abea80, cct=0x3a9d000, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x00000000011a48e3 in ThreadPool::worker (this=0x3abea08, wt=0x3a93fc0) at common/WorkQueue.cc:131
#4  0x00000000011a61a7 in ThreadPool::WorkThread::entry (this=0x3a93fc0) at common/WorkQueue.h:288
#5  0x000000000119da7d in Thread::_entry_func (arg=0x3a93fc0) at common/Thread.cc:41
#6  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 58 (Thread 0x7fed102c0700 (LWP 31461)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000000 in ?? ()

Thread 57 (Thread 0x7fed18ad1700 (LWP 31390)):
#0  0x00007fed1cc5d89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007fed1cc602b8 in _L_cond_lock_874 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007fed1cc60088 in __pthread_mutex_cond_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#3  0x00007fed1cc5b1b2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#4  0x0000000000edb163 in Cond::WaitUntil (this=0x3ad9078, mutex=..., when=...) at ./common/Cond.h:71
#5  0x00000000011a07ea in SafeTimer::timer_thread (this=0x3ad9068) at common/Timer.cc:115
#6  0x00000000011a1938 in SafeTimerThread::entry (this=0x3a93d60) at common/Timer.cc:38
#7  0x000000000119da7d in Thread::_entry_func (arg=0x3a93d60) at common/Thread.cc:41
#8  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#9  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#10 0x0000000000000000 in ?? ()

Thread 56 (Thread 0x7fed1a2d4700 (LWP 31328)):
#0  0x00007fed1cc5d0c1 in sem_timedwait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00000000011bb633 in CephContextServiceThread::entry (this=0x3a9b1c0) at common/ceph_context.cc:57
#2  0x000000000119da7d in Thread::_entry_func (arg=0x3a9b1c0) at common/Thread.cc:41
#3  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#4  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 55 (Thread 0x7fed112c2700 (LWP 31459)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3abe6e8, mutex=..., when=...) at ./common/Cond.h:71
#2  0x00000000011a07ea in SafeTimer::timer_thread (this=0x3abe6d8) at common/Timer.cc:115
#3  0x00000000011a1938 in SafeTimerThread::entry (this=0x3ac7540) at common/Timer.cc:38
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3ac7540) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 54 (Thread 0x7fed1d27c780 (LWP 31319)):
#0  0x00007fed1cc58148 in pthread_join () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x000000000119dd3f in Thread::join (this=0x3aba200, prval=0x0) at common/Thread.cc:121
#2  0x0000000001275a73 in DispatchQueue::wait (this=0x3aba0e8) at msg/DispatchQueue.cc:145
#3  0x00000000011932b5 in SimpleMessenger::wait (this=0x3aba000) at msg/SimpleMessenger.cc:496
#4  0x0000000000dcfd2c in main (argc=6, argv=0x7fff7a22cbe8) at ceph_osd.cc:446

Thread 53 (Thread 0x7fed06e9c700 (LWP 788)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000010 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 52 (Thread 0x7fed152ca700 (LWP 31386)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3aba960, mutex=...) at ./common/Cond.h:55
#2  0x00000000011903fa in SimpleMessenger::reaper_entry (this=0x3aba500) at msg/SimpleMessenger.cc:206
#3  0x0000000001196530 in SimpleMessenger::ReaperThread::entry (this=0x3aba720) at msg/SimpleMessenger.h:411
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3aba720) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 51 (Thread 0x7fed081af700 (LWP 783)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x6f051e0, mutex=...) at ./common/Cond.h:55
---Type <return> to continue, or q <return> to quit---
#2  0x0000000001290786 in Pipe::writer (this=0x6f05000) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x6f05028) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x6f05028) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 50 (Thread 0x7fed0709e700 (LWP 836)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000013 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 49 (Thread 0x7fed12ac5700 (LWP 31456)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3abebc0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001059f2d in FileStore::flusher_entry (this=0x3abe000) at os/FileStore.cc:3272
#3  0x000000000106c9b6 in FileStore::FlusherThread::entry (this=0x3abec18) at os/FileStore.h:262
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3abec18) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 48 (Thread 0x7fed172ce700 (LWP 31450)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3abe638, mutex=..., when=...) at ./common/Cond.h:71
#2  0x0000000000edb1ca in Cond::WaitInterval (this=0x3abe638, cct=0x3a9d000, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x000000000105a416 in FileStore::sync_entry (this=0x3abe000) at os/FileStore.cc:3312
#4  0x000000000106c01e in FileStore::SyncThread::entry (this=0x3abe7b8) at os/FileStore.h:115
#5  0x000000000119da7d in Thread::_entry_func (arg=0x3abe7b8) at common/Thread.cc:41
#6  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 47 (Thread 0x7fed078a6700 (LWP 835)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x6f38be0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x6f38a00) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x6f38a28) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x6f38a28) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 46 (Thread 0x7fed19ad3700 (LWP 31330)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000000 in ?? ()

Thread 45 (Thread 0x7fed06492700 (LWP 778)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000010 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 44 (Thread 0x7fed122c4700 (LWP 31457)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3abe938, mutex=...) at ./common/Cond.h:55
#2  0x00000000010e2277 in Finisher::finisher_thread_entry (this=0x3abe8d8) at common/Finisher.cc:80
#3  0x0000000000ede824 in Finisher::FinisherThread::entry (this=0x3abe9f0) at ./common/Finisher.h:46
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3abe9f0) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 43 (Thread 0x7fed182d0700 (LWP 31388)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3abae60, mutex=...) at ./common/Cond.h:55
#2  0x00000000011903fa in SimpleMessenger::reaper_entry (this=0x3abaa00) at msg/SimpleMessenger.cc:206
#3  0x0000000001196530 in SimpleMessenger::ReaperThread::entry (this=0x3abac20) at msg/SimpleMessenger.h:411
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3abac20) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 42 (Thread 0x7fed07caa700 (LWP 791)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x545a460, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x545a280) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x545a2a8) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x545a2a8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 41 (Thread 0x7fed07aa8700 (LWP 782)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000dd1882 in std::_List_base<ceph::buffer::ptr, std::allocator<ceph::buffer::ptr> >::_List_impl::~_List_impl (this=0x6f05000, __in_chrg=<optimized out>) at /usr/include/c++/4.6/bits/stl_list.h:309
Backtrace stopped: frame did not save the PC
---Type <return> to continue, or q <return> to quit---

Thread 40 (Thread 0x7fed0e2bc700 (LWP 31465)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3abbb50, mutex=...) at ./common/Cond.h:55
#2  0x00000000012757c9 in DispatchQueue::entry (this=0x3abbae8) at msg/DispatchQueue.cc:117
#3  0x0000000001195cf6 in DispatchQueue::DispatchThread::entry (this=0x3abbc00) at msg/DispatchQueue.h:85
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3abbc00) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 39 (Thread 0x7fed06391700 (LWP 820)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x8b0cbe0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x8b0ca00) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x8b0ca28) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x8b0ca28) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 38 (Thread 0x7fed07dab700 (LWP 31615)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3be11e0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x3be1000) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x3be1028) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3be1028) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 37 (Thread 0x7fed073a1700 (LWP 769)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x5f3ebe0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x5f3ea00) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x5f3ea28) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x5f3ea28) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 36 (Thread 0x7fed10ac1700 (LWP 31460)):
#0  0x00007fed1cc5d89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007fed1cc59065 in _L_lock_858 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007fed1cc58eba in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#3  0x0000000001167b0c in Mutex::Lock (this=0x3ad9010, no_lockdep=false) at common/Mutex.cc:90
#4  0x0000000000eb8130 in OSD::ms_dispatch (this=0x3ad9000, m=0x6d24900) at osd/OSD.cc:3211
#5  0x0000000001275e69 in Messenger::ms_deliver_dispatch (this=0x3aba000, m=0x6d24900) at msg/Messenger.h:549
#6  0x00000000012755d1 in DispatchQueue::entry (this=0x3aba0e8) at msg/DispatchQueue.cc:107
#7  0x0000000001195cf6 in DispatchQueue::DispatchThread::entry (this=0x3aba200) at msg/DispatchQueue.h:85
#8  0x000000000119da7d in Thread::_entry_func (arg=0x3aba200) at common/Thread.cc:41
#9  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#11 0x0000000000000000 in ?? ()

Thread 35 (Thread 0x7fed06795700 (LWP 777)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000007 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 34 (Thread 0x7fed15acb700 (LWP 31387)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3abbe60, mutex=...) at ./common/Cond.h:55
#2  0x00000000011903fa in SimpleMessenger::reaper_entry (this=0x3abba00) at msg/SimpleMessenger.cc:206
#3  0x0000000001196530 in SimpleMessenger::ReaperThread::entry (this=0x3abbc20) at msg/SimpleMessenger.h:411
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3abbc20) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 33 (Thread 0x7fed162cc700 (LWP 31452)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3ae16f0, mutex=...) at ./common/Cond.h:55
#2  0x000000000103239e in FileJournal::write_finish_thread_entry (this=0x3ae1380) at os/FileJournal.cc:1316
#3  0x0000000000ee2446 in FileJournal::WriteFinisher::entry (this=0x3ae1990) at ./os/FileJournal.h:303
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3ae1990) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 32 (Thread 0x7fed06593700 (LWP 762)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x50866e0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x5086500) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x5086528) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x5086528) at common/Thread.cc:41
---Type <return> to continue, or q <return> to quit---
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 31 (Thread 0x7fed06694700 (LWP 714)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3ac6e60, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x3ac6c80) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x3ac6ca8) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3ac6ca8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 30 (Thread 0x7fed074a2700 (LWP 799)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000010 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 29 (Thread 0x7fed0608e700 (LWP 774)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000007 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 28 (Thread 0x7fed072a0700 (LWP 768)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x5f3e960, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x5f3e780) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x5f3e7a8) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x5f3e7a8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 27 (Thread 0x7fed0618f700 (LWP 801)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x545a1e0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x545a000) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x545a028) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x545a028) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 26 (Thread 0x7fed077a5700 (LWP 761)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000010 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 25 (Thread 0x7fed076a4700 (LWP 713)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000d00000010 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 24 (Thread 0x7fed079a7700 (LWP 715)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000dd1882 in std::_List_base<ceph::buffer::ptr, std::allocator<ceph::buffer::ptr> >::_List_impl::~_List_impl (this=0x545ac80, __in_chrg=<optimized out>) at /usr/include/c++/4.6/bits/stl_list.h:309
Backtrace stopped: frame did not save the PC

Thread 23 (Thread 0x7fed06a98700 (LWP 31602)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3ac6be0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x3ac6a00) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x3ac6a28) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3ac6a28) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 22 (Thread 0x7fed17acf700 (LWP 31391)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3adaba8, mutex=...) at ./common/Cond.h:55
#2  0x00000000011a0798 in SafeTimer::timer_thread (this=0x3adab98) at common/Timer.cc:113
#3  0x00000000011a1938 in SafeTimerThread::entry (this=0x3a93d40) at common/Timer.cc:38
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3a93d40) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 21 (Thread 0x7fed1d262700 (LWP 324)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000010 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 20 (Thread 0x7fed06997700 (LWP 779)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
---Type <return> to continue, or q <return> to quit---
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x545a960, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x545a780) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x545a7a8) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x545a7a8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 19 (Thread 0x7fed075a3700 (LWP 716)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x545ae60, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x545ac80) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x545aca8) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x545aca8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 18 (Thread 0x7fed092b2700 (LWP 31475)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3adad20, mutex=...) at ./common/Cond.h:55
#2  0x00000000010e2277 in Finisher::finisher_thread_entry (this=0x3adacc0) at common/Finisher.cc:80
#3  0x0000000000ede824 in Finisher::FinisherThread::entry (this=0x3adadd8) at ./common/Finisher.h:46
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3adadd8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 17 (Thread 0x7fed06290700 (LWP 32038)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000003 in ?? ()
#2  0x0000000000000000 in ?? ()

Thread 16 (Thread 0x7fed08ab1700 (LWP 31476)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3adaa80, mutex=...) at ./common/Cond.h:55
#2  0x00000000011a0798 in SafeTimer::timer_thread (this=0x3adaa70) at common/Timer.cc:113
#3  0x00000000011a1938 in SafeTimerThread::entry (this=0x3ac72e0) at common/Timer.cc:38
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3ac72e0) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 15 (Thread 0x7fed192d2700 (LWP 31389)):
#0  0x00007fed1adda823 in ptrace () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000000 in ?? ()

Thread 14 (Thread 0x7fed06896700 (LWP 325)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3bf01e0, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x3bf0000) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x3bf0028) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3bf0028) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 13 (Thread 0x7fed05e8d700 (LWP 32620)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000dd1882 in std::_List_base<ceph::buffer::ptr, std::allocator<ceph::buffer::ptr> >::_List_impl::~_List_impl (this=0xa790c80, __in_chrg=<optimized out>) at /usr/include/c++/4.6/bits/stl_list.h:309
Backtrace stopped: frame did not save the PC

Thread 12 (Thread 0x7fed082b0700 (LWP 32034)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3b6de60, mutex=...) at ./common/Cond.h:55
#2  0x0000000001290786 in Pipe::writer (this=0x3b6dc80) at msg/Pipe.cc:1527
#3  0x00000000012953cc in Pipe::Writer::entry (this=0x3b6dca8) at msg/Pipe.h:59
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3b6dca8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 11 (Thread 0x7fed09ab3700 (LWP 31474)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3ad9af8, mutex=..., when=...) at ./common/Cond.h:71
#2  0x0000000000edb1ca in Cond::WaitInterval (this=0x3ad9af8, cct=0x3a9d000, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x0000000000eac5ff in OSD::heartbeat_entry (this=0x3ad9000) at osd/OSD.cc:2083
#4  0x0000000000edfdc4 in OSD::T_Heartbeat::entry (this=0x3ad9b88) at osd/OSD.h:558
#5  0x000000000119da7d in Thread::_entry_func (arg=0x3ad9b88) at common/Thread.cc:41
#6  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 10 (Thread 0x7fed0a2b4700 (LWP 31473)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3ad9958, mutex=..., when=...) at ./common/Cond.h:71
---Type <return> to continue, or q <return> to quit---
#2  0x0000000000edb1ca in Cond::WaitInterval (this=0x3ad9958, cct=0x3a9d000, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x00000000011a48e3 in ThreadPool::worker (this=0x3ad98e0, wt=0x3ac7880) at common/WorkQueue.cc:131
#4  0x00000000011a61a7 in ThreadPool::WorkThread::entry (this=0x3ac7880) at common/WorkQueue.h:288
#5  0x000000000119da7d in Thread::_entry_func (arg=0x3ac7880) at common/Thread.cc:41
#6  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 9 (Thread 0x7fed0eabd700 (LWP 31464)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x3aba650, mutex=...) at ./common/Cond.h:55
#2  0x00000000012757c9 in DispatchQueue::entry (this=0x3aba5e8) at msg/DispatchQueue.cc:117
#3  0x0000000001195cf6 in DispatchQueue::DispatchThread::entry (this=0x3aba700) at msg/DispatchQueue.h:85
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3aba700) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 8 (Thread 0x7fed0b2b6700 (LWP 31471)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3ad9648, mutex=..., when=...) at ./common/Cond.h:71
#2  0x0000000000edb1ca in Cond::WaitInterval (this=0x3ad9648, cct=0x3a9d000, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x00000000011a48e3 in ThreadPool::worker (this=0x3ad95d0, wt=0x3ac7380) at common/WorkQueue.cc:131
#4  0x00000000011a61a7 in ThreadPool::WorkThread::entry (this=0x3ac7380) at common/WorkQueue.h:288
#5  0x000000000119da7d in Thread::_entry_func (arg=0x3ac7380) at common/Thread.cc:41
#6  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 7 (Thread 0x7fed0bab7700 (LWP 31470)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x3ad94c0, mutex=..., when=...) at ./common/Cond.h:71
#2  0x0000000000edb1ca in Cond::WaitInterval (this=0x3ad94c0, cct=0x3a9d000, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x00000000011a48e3 in ThreadPool::worker (this=0x3ad9448, wt=0x3ac7360) at common/WorkQueue.cc:131
#4  0x00000000011a61a7 in ThreadPool::WorkThread::entry (this=0x3ac7360) at common/WorkQueue.h:288
#5  0x000000000119da7d in Thread::_entry_func (arg=0x3ac7360) at common/Thread.cc:41
#6  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 6 (Thread 0x7fed0cab9700 (LWP 31468)):
#0  0x00007fed1cc5ad84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000e2e0f8 in Cond::Wait (this=0x7fff7a22c730, mutex=...) at ./common/Cond.h:55
#2  0x00000000010e2277 in Finisher::finisher_thread_entry (this=0x7fff7a22c6d0) at common/Finisher.cc:80
#3  0x0000000000ede824 in Finisher::FinisherThread::entry (this=0x7fff7a22c7e8) at ./common/Finisher.h:46
#4  0x000000000119da7d in Thread::_entry_func (arg=0x7fff7a22c7e8) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 5 (Thread 0x7fed0aab5700 (LWP 31472)):
#0  0x00007fed1cc5d89c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007fed1cc59065 in _L_lock_858 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007fed1cc58eba in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#3  0x0000000001167b0c in Mutex::Lock (this=0x3ada688, no_lockdep=false) at common/Mutex.cc:90
#4  0x0000000000e2de9f in Mutex::Locker::Locker (this=0x7fed0aab4810, m=...) at ./common/Mutex.h:120
#5  0x0000000000f323cc in SharedPtrRegistry<pg_t, DeletingState>::OnRemoval::operator() (this=0x8b5c738, to_remove=0x7c4cbd0) at ./common/sharedptr_registry.hpp:43
#6  0x0000000000f3e8d5 in std::tr1::_Sp_counted_base_impl<DeletingState*, SharedPtrRegistry<pg_t, DeletingState>::OnRemoval, (__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x8b5c720) at /usr/include/c++/4.6/tr1/shared_ptr.h:264
#7  0x0000000000e4b48e in std::tr1::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (this=0x8b5c720) at /usr/include/c++/4.6/tr1/shared_ptr.h:147
#8  0x0000000000e3c0ff in std::tr1::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count (this=0x55d8020, __in_chrg=<optimized out>) at /usr/include/c++/4.6/tr1/shared_ptr.h:348
#9  0x0000000000ee2020 in std::tr1::__shared_ptr<DeletingState, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x55d8018, __in_chrg=<optimized out>) at /usr/include/c++/4.6/tr1/shared_ptr.h:548
#10 0x0000000000ee203a in std::tr1::shared_ptr<DeletingState>::~shared_ptr (this=0x55d8018, __in_chrg=<optimized out>) at /usr/include/c++/4.6/tr1/shared_ptr.h:992
#11 0x0000000000ee2054 in boost::tuples::cons<std::tr1::shared_ptr<DeletingState>, boost::tuples::null_type>::~cons (this=0x55d8018, __in_chrg=<optimized out>) at /usr/include/boost/tuple/detail/tuple_basic.hpp:348
#12 0x0000000000ee2073 in boost::tuples::cons<std::tr1::shared_ptr<ObjectStore::Sequencer>, boost::tuples::cons<std::tr1::shared_ptr<DeletingState>, boost::tuples::null_type> >::~cons (this=0x55d8008, __in_chrg=<optimized out>) at /usr/include/boost/tuple/detail/tuple_basic.hpp:257
#13 0x0000000000ee20bd in boost::tuples::cons<coll_t, boost::tuples::cons<std::tr1::shared_ptr<ObjectStore::Sequencer>, boost::tuples::cons<std::tr1::shared_ptr<DeletingState>, boost::tuples::null_type> > >::~cons (this=0x55d8000, __in_chrg=<optimized out>) at /usr/include/boost/tuple/detail/tuple_basic.hpp:257
#14 0x0000000000ee2102 in boost::tuples::tuple<coll_t, std::tr1::shared_ptr<ObjectStore::Sequencer>, std::tr1::shared_ptr<DeletingState>, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type>::~tuple (this=0x55d8000, __in_chrg=<optimized out>) at /usr/include/boost/tuple/detail/tuple_basic.hpp:470
#15 0x0000000000eaf153 in OSD::RemoveWQ::_process (this=0x3ada570, item=0x55d8000) at osd/OSD.cc:2323
#16 0x0000000000f3eea6 in ThreadPool::WorkQueue<boost::tuples::tuple<coll_t, std::tr1::shared_ptr<ObjectStore::Sequencer>, std::tr1::shared_ptr<DeletingState>, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type> >::_void_process (this=0x3ada570, p=0x55d8000) at ./common/WorkQueue.h:226
#17 0x00000000011a455c in ThreadPool::worker (this=0x3ad9758, wt=0x3ac7340) at common/WorkQueue.cc:113
#18 0x00000000011a61a7 in ThreadPool::WorkThread::entry (this=0x3ac7340) at common/WorkQueue.h:288
#19 0x000000000119da7d in Thread::_entry_func (arg=0x3ac7340) at common/Thread.cc:41
#20 0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#21 0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#22 0x0000000000000000 in ?? ()

Thread 4 (Thread 0x7fed0f2be700 (LWP 31463)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000000 in ?? ()

Thread 3 (Thread 0x7fed0d2ba700 (LWP 31467)):
#0  0x00007fed1cc5b0fe in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x0000000000edb163 in Cond::WaitUntil (this=0x7fff7a22c618, mutex=..., when=...) at ./common/Cond.h:71
#2  0x00000000011a07ea in SafeTimer::timer_thread (this=0x7fff7a22c608) at common/Timer.cc:115
#3  0x00000000011a1938 in SafeTimerThread::entry (this=0x3ac73e0) at common/Timer.cc:38
---Type <return> to continue, or q <return> to quit---
#4  0x000000000119da7d in Thread::_entry_func (arg=0x3ac73e0) at common/Thread.cc:41
#5  0x00007fed1cc56e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x00007fed1ade14bd in klogctl () from /lib/x86_64-linux-gnu/libc.so.6
#7  0x0000000000000000 in ?? ()

Thread 2 (Thread 0x7fed0dabb700 (LWP 31466)):
#0  0x00007fed1add5b03 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x0000000000000000 in ?? ()

Thread 1 (Thread 0x7fed0fabf700 (LWP 31462)):
#0  0x00007fed1cc5eb7b in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00000000010d1e42 in reraise_fatal (signum=6) at global/signal_handler.cc:58
#2  0x00000000010d2115 in handle_fatal_signal (signum=6) at global/signal_handler.cc:104
#3  <signal handler called>
#4  0x00007fed1ad25445 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#5  0x00007fed1ad28bab in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#6  0x00007fed1d2804e0 in ?? ()
#7  0x0000000000000000 in ?? ()

Actions

Copy link

Updated by Tamilarasi muthamizhan about 11 years ago

ubuntu@teuthology:/a/teuthology-2013-01-22_07:00:04-regression-bobtail-master-basic/3235

ubuntu@teuthology:/a/teuthology-2013-01-22_07:00:04-regression-bobtail-master-basic/3235$ cat config.yaml 
kernel: &id001
  kdb: true
  sha1: 0fa6ebc600bc8e830551aee47a0e929e818a1868
nuke-on-error: true
overrides:
  ceph:
    conf:
      client:
        rbd cache: true
        rbd cache max dirty: 0
      global:
        ms inject socket failures: 5000
    fs: xfs
    log-whitelist:
    - slow request
    sha1: c5fe0965572c074a2a33660719ce3222d18c1464
  s3tests:
    branch: bobtail
  workunit:
    sha1: c5fe0965572c074a2a33660719ce3222d18c1464
roles:
- - mon.a
  - osd.0
  - osd.1
  - osd.2
- - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
targets:
  ubuntu@plana69.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKGuvzfY8V0QDpwGmSa3Fz3ms2SM0syFb8GAnmivOpz29fV0m/kJMrbnobPWjZlZdcPE0Dn6bkcJ0htns9fn6n79nF0G7SGEQo8RFwbWUF2DHkcy8QxwF0HfQPn0OhrZdp3P2pN5lX2m5FwTLdZLprm7eQcW6V2gqD6ouZZKIUXfu2jy9B02DYWLPAG18n9Wq181IOQVoOQ0BP6gddwohnlGyipDgY8VBTiTV0uRV5PKVu3J2Ak5LCmwHRv9T0TyR3R6sRNNGvA85uaOgewk4/fPGsvPXRblILlKSi44ZMp0gFSA3flG9Coal3s0KTE8t8q35+MTGDfGd5o4Y4Hivv
  ubuntu@plana81.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzwyoEUguCYhu0TNdXrk7aWVUvePF6F6coBynLZ73Y/7eqTKAzxnNCiwUQx3kGjK33kliZDk7g/x4FdsiwzknDuGGCXP1pZyIGVtU5wNJ6ZM29XyH2SZvyU0MNfmoMzygxHR53TGcsK3hzwSbaW1woEpJmoqgIFQJ6BJr3nc2foKl79wBQdCNumqJ7sbh26xYVI29vYsJHUTYAdmyE4QrLaLOkZKU5Q/OvUnbKQbURcs7ArxFooObu9h0ENRPK4MKuxBFfgpAYTr/rMeWfVQxvSWsuOMpOdzLaNLt5UBYVVU+wxIFdDwcHb+2Er0rEV49W9xUD6JGXnaFjrxDDocfj
  ubuntu@plana82.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoUwiVhSsjFPU46JeiJdYRXuw5Y26MGiIj90va3uLJGz6UPar/mpUkJgcx55uQhpp/8hdCMj5CRBTbLgnKURRX4iGVi9WltSXSGATWQC8ZXcSIOrr919a7pzvnWw9KviHFmKz+kqWby0NhrdHsk6x/bk9S7+fTP23+84aXsSGMS7uavPtjUuidOpUNMyw/QrazNKSE7CDLk21lBPs4dnFV+Ei17odL/opthGCNycuzL73geV9yQ5MpPtzB/wYL72Nbkx069ONmqFVItumN+v6km6XAvzb3QEhO62hqRW9avrW99vVfAhKgdeaEMeLYyR6ZrRCExAZ8Xn4rte0tCJHX
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph:
    log-whitelist:
    - wrongly marked me down
    - objects unfound and apparently lost
- thrashosds:
    timeout: 1200
- rbd_fsx:
    clients:
    - client.0
    ops: 2000
ubuntu@teuthology:/a/teuthology-2013-01-22_07:00:04-regression-bobtail-master-basic/3235$ cat summary.yaml 
ceph-sha1: c5fe0965572c074a2a33660719ce3222d18c1464
client.0-kernel-sha1: 0fa6ebc600bc8e830551aee47a0e929e818a1868
description: collection:rbd-thrash clusters:6-osd-3-machine.yaml fs:xfs.yaml msgr-failures:few.yaml
  thrashers:default.yaml workloads:rbd_fsx_cache_writethrough.yaml
duration: 3761.7431240081787
failure_reason: 'Command failed with status 1: ''/tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage
  /tmp/cephtest/archive/coverage /tmp/cephtest/daemon-helper kill /tmp/cephtest/binary/usr/local/bin/ceph-osd
  -f -i 4 -c /tmp/cephtest/ceph.conf'''
flavor: basic
mds.a-kernel-sha1: 0fa6ebc600bc8e830551aee47a0e929e818a1868
mon.a-kernel-sha1: 0fa6ebc600bc8e830551aee47a0e929e818a1868
owner: scheduled_teuthology@teuthology
success: false

Actions

Copy link

Updated by Sage Weil about 11 years ago

Priority changed from Urgent to High

Actions

Copy link

Updated by Sage Weil about 11 years ago

Assignee deleted (~~Sage Weil~~)

Actions

Copy link

Updated by Samuel Just about 11 years ago

Status changed from New to Fix Under Review
Assignee set to Samuel Just
Backport set to bobtail

D'oh. sharedptr_registry.hpp has an extaneous Mutex::Locker l(lock) declaration in the retry loop. It only actually is instantiated if we found a WeakPtr to the key which happened to be no longer valid. We should backport to bobtail. wip_3836

Actions

Copy link