Bug #7655
osd/ReplicatedPG.cc: 9293: FAILED assert(latest->is_update())
0%
Description
ceph version 0.77-757-g8221a8e (8221a8ecba14f80eb7e35e3b1e6fe8487502b2d9)
1: (ReplicatedPG::recover_primary(int, ThreadPool::TPHandle&)+0x1e87) [0x892c57]
2: (ReplicatedPG::start_recovery_ops(int, PG::RecoveryCtx*, ThreadPool::TPHandle&, int*)+0x2e8) [0x8c7e78]
3: (OSD::do_recovery(PG*, ThreadPool::TPHandle&)+0x1c8) [0x64f598]
4: (OSD::RecoveryWQ::_process(PG*, ThreadPool::TPHandle&)+0x11) [0x69ea31]
5: (ThreadPool::worker(ThreadPool::WorkThread*)+0x4e6) [0xa53296]
6: (ThreadPool::WorkThread::entry()+0x10) [0xa550a0]
7: (()+0x7e9a) [0x7f17ceda9e9a]
8: (clone()+0x6d) [0x7f17cd56e3fd]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
Related issues
Associated revisions
ReplicatedPG::finish_ctx: clear object_info if !obs.exists
Otherwise, we see a different object_info_t depending on whether the
transaction deleting the object clears before another op recreating it appears.
In particular, we use oi.version to set the prior_version on the log entries in
finish_ctx. If the oi is allowed to stick around the recreation log event will
have a prior version of the deletion event when it should have a prior version
of eversion_t().
Fixes: #7655
Signed-off-by: Samuel Just <sam.just@inktank.com>
Reviewed-by: Sage Weil <sage@inktank.com>
History
#1 Updated by Samuel Just about 10 years ago
Also caused:
95> 2014-03-07 15:04:11.803637 7f504ecb4700 10 _merge_object_divergent_entries: merging hoid c520ed7f/plana4423920-36/head//4 entries: 97'1631 (0'0) promote c520ed7f/plana4423920-36/head//4 by osd.0.0:1024 2014-03-07 15:00:28.356604,97'1636 (97'1631) delete c520ed7f/plana4423920-36/head//4 by osd.0.0:1029 2014
03-07 15:00:29.070537,97'1639 (97'1631) promote c520ed7f/plana4423920-36/head//4 by osd.0.0:1032 2014-03-07 15:00:29.190700,97'1641 (97'1639) delete c520ed7f/plana4423920-36/head//4 by osd.0.0:1034 2014-03-07 15:00:29.400128
...
osd/PGLog.cc: 291: FAILED assert(i->prior_version == last)
ceph version 0.77-741-g135c27e (135c27ec74be352416d06a9d0ad78e63cf477433)
1: (PGLog::_merge_object_divergent_entries(PGLog::IndexedLog const&, hobject_t const&, std::list<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, pg_info_t const&, eversion_t, pg_missing_t&, boost::optional<std::pair<eversion_t, hobject_t> >, PGLog::LogEntryHandler)+0x323) [0x846e83]
2: (PGLog::_merge_divergent_entries(PGLog::IndexedLog const&, std::list<pg_log_entry_t, std::allocator<pg_log_entry_t> >&, pg_info_t const&, eversion_t, pg_missing_t&, std::map<eversion_t, hobject_t, std::less<eversion_t>, std::allocator<std::pair<eversion_t const, hobject_t> > >, PGLog::LogEntryHandler)+0x271) [0x8548b1]
3: (PGLog::proc_replica_log(ObjectStore::Transaction&, pg_info_t&, pg_log_t const&, pg_missing_t&, pg_shard_t) const+0x62b) [0x84d7db]
4: (PG::proc_replica_log(ObjectStore::Transaction&, pg_info_t&, pg_log_t&, pg_missing_t&, pg_shard_t)+0x15a) [0x7f1aca]
5: (PG::RecoveryState::GetMissing::react(PG::MLogRec const&)+0xaf) [0x7f1e5f]
6: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::GetMissing, PG::RecoveryState::Peering, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<PG::MLogRec>, boost::statechart::transition<PG::NeedUpThru, PG::RecoveryState::WaitUpThru, boost::statechart::detail::no_context<PG::NeedUpThru>, &(boost::statechart::detail::no_context<PG::NeedUpThru>::no_function(PG::NeedUpThru const&))>, boost::statechart::transition<PG::CheckRepops, PG::RecoveryState::WaitFlushedPeering, boost::statechart::detail::no_context<PG::CheckRepops>, &(boost::statechart::detail::no_context<PG::CheckRepops>::no_function(PG::CheckRepops const&))> >, boost::statechart::simple_state<PG::RecoveryState::GetMissing, PG::RecoveryState::Peering, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::GetMissing, PG::RecoveryState::Peering, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x110) [0x843590]
7: (boost::statechart::simple_state<PG::RecoveryState::GetMissing, PG::RecoveryState::Peering, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x56) [0x843656]
8: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x823d6b]
9: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x8240c1]
10: (PG::handle_peering_event(std::tr1::shared_ptr<PG::CephPeeringEvt>, PG::RecoveryCtx*)+0x303) [0x7db823]
11: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x2c6) [0x6586b6]
12: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x12) [0x6a6942]
13: (ThreadPool::worker(ThreadPool::WorkThread*)+0x4e6) [0xa532f6]
14: (ThreadPool::WorkThread::entry()+0x10) [0xa55100]
15: (()+0x7e9a) [0x7f5063b57e9a]
16: (clone()+0x6d) [0x7f506231c3fd]
#2 Updated by Sage Weil about 10 years ago
- Status changed from 7 to Resolved