2013-04-06 20:48:53.152576 7f78d5dbd700 -1 *** Caught signal (Aborted) **
 in thread 7f78d5dbd700

 ceph version 0.56.2 (586538e22afba85c59beda49789ec42024e7a061)
 1: /usr/bin/ceph-osd() [0x7a1b99]
 2: (()+0xeff0) [0x7f78e82ebff0]
 3: (gsignal()+0x35) [0x7f78e6d751b5]
 4: (abort()+0x180) [0x7f78e6d77fc0]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x115) [0x7f78e7609dc5]
 6: (()+0xcb166) [0x7f78e7608166]
 7: (()+0xcb193) [0x7f78e7608193]
 8: (()+0xcb28e) [0x7f78e760828e]
 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x7c9) [0x7f93c9]
 10: (PG::RecoveryState::Crashed::Crashed(boost::statechart::state<PG::RecoveryState::Crashed, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::my_context)+0xc4) [0x67b034]
 11: (boost::statechart::detail::inner_constructor<boost::mpl::l_item<mpl_::long_<1l>, PG::RecoveryState::Crashed, boost::mpl::l_end>, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator> >::construct(boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>* const&, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>&)+0x26) [0x6df836]
 12: (boost::statechart::simple_state<PG::RecoveryState::Started, PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Start, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xf2) [0x6ec002]
 13: (boost::statechart::simple_state<PG::RecoveryState::Stray, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x199) [0x6ec319]
 14: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x6e665b]
 15: (PG::handle_peering_event(std::tr1::shared_ptr<PG::CephPeeringEvt>, PG::RecoveryCtx*)+0x316) [0x6a12d6]
 16: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x321) [0x62c321]
 17: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x10) [0x659ab0]
 18: (ThreadPool::worker(ThreadPool::WorkThread*)+0x879) [0x831a49]
 19: (ThreadPool::WorkThread::entry()+0x10) [0x834250]
 20: (()+0x68ca) [0x7f78e82e38ca]
 21: (clone()+0x6d) [0x7f78e6e12b6d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- begin dump of recent events ---
   -56> 2013-04-06 20:48:53.138423 7f78dadc7700  1 -- 10.251.128.6:6801/6972 <== osd.3 10.252.0.7:6825/1740 4 ==== pg_notify(2.14c(20) epoch 17599) v4 ==== 1536+0+0 (2378376889 0 0) 0x658c1c0 con 0x6725000
   -55> 2013-04-06 20:48:53.138431 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 189, time: 2013-04-06 20:48:53.138363, event: header_read, request: pg_notify(2.14c(20) epoch 17599) v4
   -54> 2013-04-06 20:48:53.138437 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 189, time: 2013-04-06 20:48:53.138364, event: throttled, request: pg_notify(2.14c(20) epoch 17599) v4
   -53> 2013-04-06 20:48:53.138445 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 189, time: 2013-04-06 20:48:53.138413, event: all_read, request: pg_notify(2.14c(20) epoch 17599) v4
   -52> 2013-04-06 20:48:53.138448 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 189, time: 2013-04-06 20:48:53.138430, event: dispatched, request: pg_notify(2.14c(20) epoch 17599) v4
   -51> 2013-04-06 20:48:53.138452 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 189, time: 2013-04-06 20:48:53.138452, event: waiting_for_osdmap, request: pg_notify(2.14c(20) epoch 17599) v4
   -50> 2013-04-06 20:48:53.138457 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 189, time: 2013-04-06 20:48:53.138457, event: started, request: pg_notify(2.14c(20) epoch 17599) v4
   -49> 2013-04-06 20:48:53.138468 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 189, time: 2013-04-06 20:48:53.138467, event: done, request: pg_notify(2.14c(20) epoch 17599) v4
   -48> 2013-04-06 20:48:53.138482 7f78dadc7700  1 -- 10.251.128.6:6801/6972 <== osd.3 10.252.0.7:6825/1740 5 ==== pg_notify(1.14d(20) epoch 17599) v4 ==== 1536+0+0 (3505868558 0 0) 0x6569700 con 0x6725000
   -47> 2013-04-06 20:48:53.138489 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 190, time: 2013-04-06 20:48:53.138424, event: header_read, request: pg_notify(1.14d(20) epoch 17599) v4
   -46> 2013-04-06 20:48:53.138493 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 190, time: 2013-04-06 20:48:53.138424, event: throttled, request: pg_notify(1.14d(20) epoch 17599) v4
   -45> 2013-04-06 20:48:53.138496 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 190, time: 2013-04-06 20:48:53.138474, event: all_read, request: pg_notify(1.14d(20) epoch 17599) v4
   -44> 2013-04-06 20:48:53.138500 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 190, time: 2013-04-06 20:48:53.138488, event: dispatched, request: pg_notify(1.14d(20) epoch 17599) v4
   -43> 2013-04-06 20:48:53.138503 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 190, time: 2013-04-06 20:48:53.138503, event: waiting_for_osdmap, request: pg_notify(1.14d(20) epoch 17599) v4
   -42> 2013-04-06 20:48:53.138507 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 190, time: 2013-04-06 20:48:53.138507, event: started, request: pg_notify(1.14d(20) epoch 17599) v4
   -41> 2013-04-06 20:48:53.138516 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 190, time: 2013-04-06 20:48:53.138516, event: done, request: pg_notify(1.14d(20) epoch 17599) v4
   -40> 2013-04-06 20:48:53.138525 7f78dadc7700  1 -- 10.251.128.6:6801/6972 <== osd.25 10.252.0.7:6817/2691 20 ==== osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7 ==== 1364+0+32768 (1836186821 0 2998824952) 0x6513000 con 0x642e9a0
   -39> 2013-04-06 20:48:53.138533 7f78dadc7700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.137006, event: header_read, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -38> 2013-04-06 20:48:53.138539 7f78dadc7700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.137007, event: throttled, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -37> 2013-04-06 20:48:53.138544 7f78dadc7700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.138511, event: all_read, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -36> 2013-04-06 20:48:53.138549 7f78dadc7700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.138532, event: dispatched, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -35> 2013-04-06 20:48:53.138553 7f78dadc7700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.138553, event: waiting_for_osdmap, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -34> 2013-04-06 20:48:53.138571 7f78d75c0700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.138571, event: reached_pg, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -33> 2013-04-06 20:48:53.138582 7f78d75c0700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.138582, event: started, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -32> 2013-04-06 20:48:53.138685 7f78dcdcb700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.138685, event: committed, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -31> 2013-04-06 20:48:53.138697 7f78dcdcb700  5 --OSD::tracker-- reqid: osd.25.0:7900215, seq: 191, time: 2013-04-06 20:48:53.138697, event: done, request: osd_sub_op(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] v 17598'359102 snapset=0=[]:[] snapc=0=[]) v7
   -30> 2013-04-06 20:48:53.138925 7f78dadc7700  1 -- 10.251.128.6:6801/6972 <== osd.0 10.252.0.7:6802/26726 23 ==== osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7 ==== 1364+0+32768 (2943928352 0 382434317) 0x6513000 con 0x642e2c0
   -29> 2013-04-06 20:48:53.138937 7f78dadc7700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.137743, event: header_read, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -28> 2013-04-06 20:48:53.138943 7f78dadc7700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.137744, event: throttled, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -27> 2013-04-06 20:48:53.138948 7f78dadc7700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.138917, event: all_read, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -26> 2013-04-06 20:48:53.138953 7f78dadc7700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.138935, event: dispatched, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -25> 2013-04-06 20:48:53.138957 7f78dadc7700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.138957, event: waiting_for_osdmap, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -24> 2013-04-06 20:48:53.138976 7f78d6dbf700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.138976, event: reached_pg, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -23> 2013-04-06 20:48:53.138987 7f78d6dbf700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.138987, event: started, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -22> 2013-04-06 20:48:53.139105 7f78dcdcb700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.139105, event: committed, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -21> 2013-04-06 20:48:53.139116 7f78dcdcb700  5 --OSD::tracker-- reqid: osd.0.0:170, seq: 192, time: 2013-04-06 20:48:53.139116, event: done, request: osd_sub_op(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] v 17598'2407017 snapset=0=[]:[] snapc=0=[]) v7
   -20> 2013-04-06 20:48:53.142639 7f78dd5cc700  1 -- 10.251.128.6:6801/6972 --> 10.252.0.7:6817/2691 -- osd_sub_op_reply(osd.25.0:7900215 2.120 b03a5b20/rb.0.3bfe.2ae8944a.000000053687/head//2 [push] ack, result = 0) v1 -- ?+0 0x6709780 con 0x642e9a0
   -19> 2013-04-06 20:48:53.142775 7f78dd5cc700  1 -- 10.251.128.6:6801/6972 --> 10.252.0.7:6802/26726 -- osd_sub_op_reply(osd.0.0:169 2.4e 8d89504e/rb.0.3303.238e1f29.0000000bb3c0/head//2 [push] ack, result = 0) v1 -- ?+0 0x6572500 con 0x642e2c0
   -18> 2013-04-06 20:48:53.144256 7f78dadc7700  1 -- 10.251.128.6:6801/6972 <== osd.15 10.251.0.5:6833/20089 11 ==== pg_log(2.39 epoch 17599 query_epoch 17599) v3 ==== 1140+0+0 (3348972873 0 0) 0x2d53a00 con 0x6594580
   -17> 2013-04-06 20:48:53.144268 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 193, time: 2013-04-06 20:48:53.144210, event: header_read, request: pg_log(2.39 epoch 17599 query_epoch 17599) v3
   -16> 2013-04-06 20:48:53.144274 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 193, time: 2013-04-06 20:48:53.144211, event: throttled, request: pg_log(2.39 epoch 17599 query_epoch 17599) v3
   -15> 2013-04-06 20:48:53.144278 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 193, time: 2013-04-06 20:48:53.144246, event: all_read, request: pg_log(2.39 epoch 17599 query_epoch 17599) v3
   -14> 2013-04-06 20:48:53.144281 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 193, time: 2013-04-06 20:48:53.144266, event: dispatched, request: pg_log(2.39 epoch 17599 query_epoch 17599) v3
   -13> 2013-04-06 20:48:53.144285 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 193, time: 2013-04-06 20:48:53.144285, event: waiting_for_osdmap, request: pg_log(2.39 epoch 17599 query_epoch 17599) v3
   -12> 2013-04-06 20:48:53.144292 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 193, time: 2013-04-06 20:48:53.144292, event: started, request: pg_log(2.39 epoch 17599 query_epoch 17599) v3
   -11> 2013-04-06 20:48:53.144300 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 193, time: 2013-04-06 20:48:53.144300, event: done, request: pg_log(2.39 epoch 17599 query_epoch 17599) v3
   -10> 2013-04-06 20:48:53.145620 7f78dd5cc700  1 -- 10.251.128.6:6801/6972 --> 10.252.0.7:6802/26726 -- osd_sub_op_reply(osd.0.0:170 2.4e 4ba2aa4e/rb.0.3bfe.2ae8944a.000000053684/head//2 [push] ack, result = 0) v1 -- ?+0 0x6709c80 con 0x642e2c0
    -9> 2013-04-06 20:48:53.145877 7f78dadc7700  1 -- 10.251.128.6:6801/6972 <== osd.15 10.251.0.5:6833/20089 12 ==== MRecoveryReserve REQUEST  pgid: 2.39, query_epoch: 17599 v1 ==== 25+0+0 (270078198 0 0) 0x6569540 con 0x6594580
    -8> 2013-04-06 20:48:53.145886 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 194, time: 2013-04-06 20:48:53.145843, event: header_read, request: MRecoveryReserve REQUEST  pgid: 2.39, query_epoch: 17599 v1
    -7> 2013-04-06 20:48:53.145892 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 194, time: 2013-04-06 20:48:53.145843, event: throttled, request: MRecoveryReserve REQUEST  pgid: 2.39, query_epoch: 17599 v1
    -6> 2013-04-06 20:48:53.145895 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 194, time: 2013-04-06 20:48:53.145868, event: all_read, request: MRecoveryReserve REQUEST  pgid: 2.39, query_epoch: 17599 v1
    -5> 2013-04-06 20:48:53.145898 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 194, time: 2013-04-06 20:48:53.145885, event: dispatched, request: MRecoveryReserve REQUEST  pgid: 2.39, query_epoch: 17599 v1
    -4> 2013-04-06 20:48:53.145903 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 194, time: 2013-04-06 20:48:53.145903, event: waiting_for_osdmap, request: MRecoveryReserve REQUEST  pgid: 2.39, query_epoch: 17599 v1
    -3> 2013-04-06 20:48:53.146925 7f78dadc7700  5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 194, time: 2013-04-06 20:48:53.146925, event: done, request: MRecoveryReserve REQUEST  pgid: 2.39, query_epoch: 17599 v1
    -2> 2013-04-06 20:48:53.146977 7f78d6dbf700  1 -- 10.251.128.6:6801/6972 --> osd.15 10.251.0.5:6833/20089 -- MRecoveryReserve GRANT  pgid: 2.39, query_epoch: 17599 v1 -- ?+0 0x658ce00
    -1> 2013-04-06 20:48:53.147198 7f78dcdcb700  1 -- 10.251.128.6:6801/6972 --> osd.15 10.251.0.5:6833/20089 -- pg_info(1 pgs e17599:2.39) v3 -- ?+0 0x64fea80
     0> 2013-04-06 20:48:53.152576 7f78d5dbd700 -1 *** Caught signal (Aborted) **
 in thread 7f78d5dbd700

 ceph version 0.56.2 (586538e22afba85c59beda49789ec42024e7a061)
 1: /usr/bin/ceph-osd() [0x7a1b99]
 2: (()+0xeff0) [0x7f78e82ebff0]
 3: (gsignal()+0x35) [0x7f78e6d751b5]
 4: (abort()+0x180) [0x7f78e6d77fc0]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x115) [0x7f78e7609dc5]
 6: (()+0xcb166) [0x7f78e7608166]
 7: (()+0xcb193) [0x7f78e7608193]
 8: (()+0xcb28e) [0x7f78e760828e]
 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x7c9) [0x7f93c9]
 10: (PG::RecoveryState::Crashed::Crashed(boost::statechart::state<PG::RecoveryState::Crashed, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::my_context)+0xc4) [0x67b034]
 11: (boost::statechart::detail::inner_constructor<boost::mpl::l_item<mpl_::long_<1l>, PG::RecoveryState::Crashed, boost::mpl::l_end>, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator> >::construct(boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>* const&, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>&)+0x26) [0x6df836]
 12: (boost::statechart::simple_state<PG::RecoveryState::Started, PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Start, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xf2) [0x6ec002]
 13: (boost::statechart::simple_state<PG::RecoveryState::Stray, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x199) [0x6ec319]
 14: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x6e665b]
 15: (PG::handle_peering_event(std::tr1::shared_ptr<PG::CephPeeringEvt>, PG::RecoveryCtx*)+0x316) [0x6a12d6]
 16: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x321) [0x62c321]
 17: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x10) [0x659ab0]
 18: (ThreadPool::worker(ThreadPool::WorkThread*)+0x879) [0x831a49]
 19: (ThreadPool::WorkThread::entry()+0x10) [0x834250]
 20: (()+0x68ca) [0x7f78e82e38ca]
 21: (clone()+0x6d) [0x7f78e6e12b6d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
   0/ 5 none
   0/ 1 lockdep
   0/ 1 context
   1/ 1 crush
   1/ 5 mds
   1/ 5 mds_balancer
   1/ 5 mds_locker
   1/ 5 mds_log
   1/ 5 mds_log_expire
   1/ 5 mds_migrator
   0/ 1 buffer
   0/ 1 timer
   0/ 1 filer
   0/ 1 striper
   0/ 1 objecter
   0/ 5 rados
   0/ 5 rbd
   0/ 5 journaler
   0/ 5 objectcacher
   0/ 5 client
   0/ 5 osd
   0/ 5 optracker
   0/ 5 objclass
   1/ 3 filestore
   1/ 3 journal
   0/ 5 ms
   1/ 5 mon
   0/10 monc
   0/ 5 paxos
   0/ 5 tp
   1/ 5 auth
   1/ 5 crypto
   1/ 1 finisher
   1/ 5 heartbeatmap
   1/ 5 perfcounter
   1/ 5 rgw
   1/ 5 hadoop
   1/ 5 javaclient
   1/ 5 asok
   1/ 1 throttle
  -2/-2 (syslog threshold)
  -1/-1 (stderr threshold)
  max_recent    100000
  max_new         1000
  log_file /var/log/ceph/ceph-osd.2.log
--- end dump of recent events ---