Ceph

    -1> 2014-03-07 06:49:17.235206 7fd356528700 10 _merge_object_divergent_entries: merging hoid 2cc18063/plana2120563-294/head//4 entries: 39'796 (39'790) delete   2cc18063/plana2120563-294/head//4 by osd.0.0:910 2014-03-07 06:49:06.878493,39'797 (39'790) promote  2cc18063/plana2120563-294/head//4 by osd.0.0:911 2
014-03-07 06:49:06.958373
     0> 2014-03-07 06:49:17.240435 7fd356528700 -1 osd/PGLog.cc: In function 'static void PGLog::_merge_object_divergent_entries(const PGLog::IndexedLog&, const hobject_t&, const std::list<pg_log_entry_t>&, const pg_info_t&, eversion_t, pg_missing_t&, boost::optional<std::pair<eversion_t, hobject_t> >*, PGLog::LogE
ntryHandler*)' thread 7fd356528700 time 2014-03-07 06:49:17.235227
osd/PGLog.cc: 291: FAILED assert(i->prior_version == last)

 ceph version 0.77-757-g8221a8e (8221a8ecba14f80eb7e35e3b1e6fe8487502b2d9)
 1: (PGLog::_merge_object_divergent_entries(PGLog::IndexedLog const&, hobject_t const&, std::list<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, pg_info_t const&, eversion_t, pg_missing_t&, boost::optional<std::pair<eversion_t, hobject_t> >*, PGLog::LogEntryHandler*)+0x323) [0x8467f3]
 2: (PGLog::_merge_divergent_entries(PGLog::IndexedLog const&, std::list<pg_log_entry_t, std::allocator<pg_log_entry_t> >&, pg_info_t const&, eversion_t, pg_missing_t&, std::map<eversion_t, hobject_t, std::less<eversion_t>, std::allocator<std::pair<eversion_t const, hobject_t> > >*, PGLog::LogEntryHandler*)+0x271) 
[0x854221]
 3: (PGLog::proc_replica_log(ObjectStore::Transaction&, pg_info_t&, pg_log_t const&, pg_missing_t&, pg_shard_t) const+0x62b) [0x84d14b]
 4: (PG::proc_replica_log(ObjectStore::Transaction&, pg_info_t&, pg_log_t&, pg_missing_t&, pg_shard_t)+0x15a) [0x7f143a]
 5: (PG::RecoveryState::GetMissing::react(PG::MLogRec const&)+0xaf) [0x7f17cf]
 6: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::GetMissing, PG::RecoveryState::Peering, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, 
mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<PG::MLogRec>, boost::statechart::transition<PG::NeedUpThru, PG::RecoveryState::WaitUpThru, boost::statechart::detail::no_context<PG::NeedUpT
hru>, &(boost::statechart::detail::no_context<PG::NeedUpThru>::no_function(PG::NeedUpThru const&))>, boost::statechart::transition<PG::CheckRepops, PG::RecoveryState::WaitFlushedPeering, boost::statechart::detail::no_context<PG::CheckRepops>, &(boost::statechart::detail::no_context<PG::CheckRepops>::no_function(PG:
:CheckRepops const&))> >, boost::statechart::simple_state<PG::RecoveryState::GetMissing, PG::RecoveryState::Peering, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mp
l_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::GetMissing, PG::RecoveryState::Peering, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, m
pl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x110) [0x842f00]
 7: (boost::statechart::simple_state<PG::RecoveryState::GetMissing, PG::RecoveryState::Peering, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (b
oost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x56) [0x842fc6]
 8: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x8236db]
 9: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x823a31]
 10: (PG::handle_peering_event(std::tr1::shared_ptr<PG::CephPeeringEvt>, PG::RecoveryCtx*)+0x303) [0x7db193]
 11: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x2c6) [0x658266]
 12: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x12) [0x6a6132]
 13: (ThreadPool::worker(ThreadPool::WorkThread*)+0x4e6) [0xa53296]
 14: (ThreadPool::WorkThread::entry()+0x10) [0xa550a0]
 15: (()+0x7e9a) [0x7fd36abcae9a]
 16: (clone()+0x6d) [0x7fd36938f3fd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

ubuntu@teuthology:/var/lib/teuthworker/archive/teuthology-2014-03-07_02:30:02-rados-firefly-testing-basic-plana/121107