Ceph

    -1> 2014-03-27 17:44:05.912691 7f7746636700 -1 osd.1 pg_epoch: 85 pg[4.1( v 83'14281 (44'9114,83'14281] local-les=85 n=988 ec=8 les/c 85/83 82/84/84) [1,4] r=-1 lpr=84 pi=68-83/5 crt=83'14281 mlcod 0'0 active] check_local 1/hit_set_4.1_archive_2014-03-27 17:24:38.134425_2014-03-27 17:25:24.545798/head/.ceph-int
ernal/4 exists, but should have been deleted
     0> 2014-03-27 17:44:05.933057 7f7746636700 -1 osd/ReplicatedPG.cc: In function 'virtual void ReplicatedPG::check_local()' thread 7f7746636700 time 2014-03-27 17:44:05.912706
osd/ReplicatedPG.cc: 10190: FAILED assert(0 == "erroneously present object")

 ceph version 0.78-367-gd9a2dea (d9a2dea755a62e4f9fe0795410f37b68a15ae054)
 1: (ReplicatedPG::check_local()+0x4d7) [0x7c01a7]
 2: (PG::RecoveryState::Active::react(PG::AllReplicasActivated const&)+0x42) [0x73edf2]
 3: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list2<boost::statechart::custom_reaction<PG::Backfilled>
, boost::statechart::custom_reaction<PG::AllReplicasActivated> >, boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::Rec
overyState::Activating, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0xf0) [0x7a3240]
 4: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list4<boost::statechart::custom_reaction<PG::MNotifyRec>
, boost::statechart::custom_reaction<PG::MLogRec>, boost::statechart::custom_reaction<PG::Backfilled>, boost::statechart::custom_reaction<PG::AllReplicasActivated> >, boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mod
e)0> >(boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x85) [0x7a3315]
 5: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list6<boost::statechart::custom_reaction<PG::AdvMap>, bo
ost::statechart::custom_reaction<PG::MInfoRec>, boost::statechart::custom_reaction<PG::MNotifyRec>, boost::statechart::custom_reaction<PG::MLogRec>, boost::statechart::custom_reaction<PG::Backfilled>, boost::statechart::custom_reaction<PG::AllReplicasActivated> >, boost::statechart::simple_state<PG::RecoveryState::
Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x85
) [0x7a3455]
 6: (boost::statechart::simple_state<PG::RecoveryState::Active, PG::RecoveryState::Primary, PG::RecoveryState::Activating, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x8d) [0x7a358d]
 7: (boost::statechart::simple_state<PG::RecoveryState::Activating, PG::RecoveryState::Active, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (bo
ost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x11d) [0x7a25cd]
 8: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x78543b]
 9: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x785791]
 10: (PG::handle_peering_event(std::tr1::shared_ptr<PG::CephPeeringEvt>, PG::RecoveryCtx*)+0x303) [0x73d863]
 11: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x2c6) [0x656656]
 12: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x12) [0x6a4c52]
 13: (ThreadPool::worker(ThreadPool::WorkThread*)+0x4e6) [0xa52376]
 14: (ThreadPool::WorkThread::entry()+0x10) [0xa54180]
 15: (()+0x7e9a) [0x7f775aedbe9a]
 16: (clone()+0x6d) [0x7f775949cccd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---

ubuntu@teuthology:/var/lib/teuthworker/archive/teuthology-2014-03-27_02:30:16-rados-firefly-distro-basic-plana/148961