Project

General

Profile

Actions

Bug #62550

closed

osd crashes when doing peering

Added by Xuehan Xu 9 months ago. Updated 19 days ago.

Status:
Resolved
Priority:
Normal
Assignee:
Category:
-
Target version:
-
% Done:

0%

Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):

Description

#0  __pthread_kill_implementation (no_tid=0, signo=11, threadid=140478027361984) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (signo=11, threadid=140478027361984) at ./nptl/pthread_kill.c:78
#2  __GI___pthread_kill (threadid=140478027361984, signo=signo@entry=11) at ./nptl/pthread_kill.c:89
#3  0x00007fc3977bc476 in __GI_raise (sig=11) at ../sysdeps/posix/raise.c:26
#4  0x000055cb6f38b616 in reraise_fatal (signum=11) at /home/xuxuehan/src/ceph/src/crimson/common/fatal_signal.cc:41
#5  FatalSignal::signal_entry (signum=11, info=<optimized out>) at /home/xuxuehan/src/ceph/src/crimson/common/fatal_signal.cc:62
#6  <signal handler called>
#7  boost::intrusive::bstree_algorithms_base<boost::intrusive::rbtree_node_traits<void*, false> >::next_node (node=<optimized out>) at boost/include/boost/intrusive/detail/bstree_algorithms_base.hpp:54
#8  boost::intrusive::tree_iterator<boost::intrusive::mhtraits<ceph::common::intrusive_lru_base<ceph::common::intrusive_lru_config<hobject_t, crimson::osd::ObjectContext, crimson::osd::obc_to_hoid<crimson::osd::ObjectContext> > >, boost::intrusive::set_member_hook<>, &ceph::common::intrusive_lru_base<ceph::common::intrusive_lru_config<hobject_t, crimson::osd::ObjectContext, crimson::osd::obc_to_hoid<crimson::osd::ObjectContext> > >::set_hook>, false>::operator++() (this=<optimized out>)
    at boost/include/boost/intrusive/detail/tree_iterator.hpp:98
#9  ceph::common::intrusive_lru<ceph::common::intrusive_lru_config<hobject_t, crimson::osd::ObjectContext, crimson::osd::obc_to_hoid<crimson::osd::ObjectContext> > >::for_each<crimson::osd::PG::context_registry_on_change()::$_35>(crimson::osd::PG::context_registry_on_change()::$_35&&) (this=0x7fc38d39dd40, f=...) at /home/xuxuehan/src/ceph/src/common/intrusive_lru.h:217
#10 crimson::osd::ObjectContextRegistry::for_each<crimson::osd::PG::context_registry_on_change()::$_35>(crimson::osd::PG::context_registry_on_change()::$_35&&) (this=0x7fc33b591fc0, f=...) at /home/xuxuehan/src/ceph/src/crimson/osd/object_context.h:282
#11 crimson::osd::PG::context_registry_on_change (this=this@entry=0x7fc38d39c000) at /home/xuxuehan/src/ceph/src/crimson/osd/pg.cc:1478
#12 0x000055cb6ed562dc in crimson::osd::PG::on_change (this=0x7fc38d39c000, t=...) at /home/xuxuehan/src/ceph/src/crimson/osd/pg.cc:1448
#13 0x000055cb6efdb71e in PeeringState::start_peering_interval (this=this@entry=0x7fc38d39c500, lastmap=..., newup=std::vector of length 3, capacity 3 = {...}, new_up_primary=new_up_primary@entry=1, newacting=std::vector of length 3, capacity 3 = {...}, 
    new_acting_primary=new_acting_primary@entry=1, t=...) at /home/xuxuehan/src/ceph/src/osd/PeeringState.cc:700
#14 0x000055cb6f01259d in PeeringState::Reset::react (this=this@entry=0x7fc38e3d6090, advmap=...) at /home/xuxuehan/src/ceph/src/osd/PeeringState.cc:4740
#15 0x000055cb6f06e27c in boost::statechart::custom_reaction<PeeringState::AdvMap>::react<PeeringState::Reset, boost::statechart::event_base, void const*> (stt=..., evt=..., eventType=<optimized out>) at boost/include/boost/statechart/custom_reaction.hpp:42
#16 boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list5<boost::statechart::custom_reaction<PeeringState::AdvMap>, boost::statechart::custom_reaction<PeeringState::ActMap>, boost::statechart::custom_reaction<NullEvt>, boost::statechart::custom_reaction<PeeringState::IntervalFlush>, boost::statechart::transition<boost::statechart::event_base, PeeringState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function> >, boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (stt=..., evt=..., eventType=0x55cb7154ac58 <boost::statechart::detail::id_holder<PeeringState::AdvMap>::idProvider_>) at boost/include/boost/statechart/simple_state.hpp:814
#17 0x000055cb6f043e2c in boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react<boost::mpl::list6<boost::statechart::custom_reaction<PeeringState::QueryUnfound>, boost::statechart::custom_reaction<PeeringState::AdvMap>, boost::statechart::custom_reaction<PeeringState::ActMap>, boost::statechart::custom_reaction<NullEvt>, boost::statechart::custom_reaction<PeeringState::IntervalFlush>, boost::statechart::transition<boost::statechart::event_base, PeeringState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function> > > (this=0x7fc38e3d6090, evt=..., 
    eventType=0xffffffffffffaaf8) at boost/include/boost/statechart/simple_state.hpp:850
#18 boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list<boost::statechart::custom_reaction<PeeringState::QueryState>, boost::statechart::custom_reaction<PeeringState::QueryUnfound>, boost::statechart::custom_reaction<PeeringState::AdvMap>, boost::statechart::custom_reaction<PeeringState::ActMap>, boost::statechart::custom_reaction<NullEvt>, boost::statechart::custom_reaction<PeeringState::IntervalFlush>, boost::statechart::transition<boost::statechart::event_base, PeeringState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function>, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> > (stt=..., eventType=0xffffffffffffaaf8, evt=...) at boost/include/boost/statechart/simple_state.hpp:820
#19 boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react<boost::mpl::list<boost::statechart::custom_reaction<PeeringState::QueryState>, boost::statechart::custom_reaction<PeeringState::QueryUnfound>, boost::statechart::custom_reaction<PeeringState::AdvMap>, boost::statechart::custom_reaction<PeeringState::ActMap>, boost::statechart::custom_reaction<NullEvt>, boost::statechart::custom_reaction<PeeringState::IntervalFlush>, boost::statechart::transition<boost::statechart::event_base, PeeringState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function>, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na> > (this=0x7fc38e3d6090, eventType=<optimized out>, 
    evt=...) at boost/include/boost/statechart/simple_state.hpp:850
#20 boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl (this=0x7fc38e3d6090, evt=..., eventType=<optimized out>)
    at boost/include/boost/statechart/simple_state.hpp:489
#21 0x000055cb6ed8e16e in boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<boost::statechart::none>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>::operator() (this=<optimized out>)
    at boost/include/boost/statechart/state_machine.hpp:87
#22 boost::statechart::null_exception_translator::operator()<boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<boost::statechart::none>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine<PeeringState::PeeringMachine, PeeringState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::exception_event_handler> (this=0x7fc38d39dc68, action=...)
    at boost/include/boost/statechart/null_exception_translator.hpp:33
#23 boost::statechart::state_machine<PeeringState::PeeringMachine, PeeringState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::send_event (this=this@entry=0x7fc38d39dbf8, evt=...)
    at boost/include/boost/statechart/state_machine.hpp:885
#24 0x000055cb6ed8e2d1 in boost::statechart::state_machine<PeeringState::PeeringMachine, PeeringState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_queued_events (this=this@entry=0x7fc38d39dbf8)
    at boost/include/boost/statechart/state_machine.hpp:910
#25 0x000055cb6ed8e0c5 in boost::statechart::state_machine<PeeringState::PeeringMachine, PeeringState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event (this=this@entry=0x7fc38d39dbf8, evt=...)
    at boost/include/boost/statechart/state_machine.hpp:280
#26 0x000055cb6efd7934 in PeeringState::handle_event (this=0x7fc38d39c500, evt=..., rctx=<optimized out>) at /home/xuxuehan/src/ceph/src/osd/PeeringState.h:1753
#27 PeeringState::advance_map (this=0x7fc38d39c500, osdmap=..., lastmap=..., newup=std::vector of length 3, capacity 3 = {...}, up_primary=1, newacting=std::vector of length 3, capacity 3 = {...}, acting_primary=1, rctx=...)
    at /home/xuxuehan/src/ceph/src/osd/PeeringState.cc:449
#28 0x000055cb6ed6415b in crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11::operator()() const (this=0x7fc38494bfb0) at /home/xuxuehan/src/ceph/src/crimson/osd/pg.cc:695
#29 std::__invoke_impl<void, crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11>(std::__invoke_other, crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11&&) (__f=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61
#30 std::__invoke<crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11>(crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11&&) (__fn=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96
#31 std::__apply_impl<crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11, std::tuple<>>(crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11&&, std::tuple<>&&, std::integer_sequence<unsigned long>) (__f=..., __t=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/tuple:1854
#32 std::apply<crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11, std::tuple<> >(crimson::osd::PG::handle_advance_map(crimson::local_shared_foreign_ptr<boost::local_shared_ptr<OSDMap const> >, PeeringCtx&)::$_11&&, std::tuple<>&&) (__f=..., __t=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/tuple:1865
Actions #1

Updated by Matan Breizman 2 months ago

  • Status changed from New to Need More Info

Was this resolved? If not, is it reproducible?

Actions #2

Updated by Xuehan Xu 2 months ago

Matan Breizman wrote:

Was this resolved? If not, is it reproducible?

This should have been fixed by https://github.com/ceph/ceph/pull/55223

Actions #3

Updated by Matan Breizman 19 days ago

  • Status changed from Need More Info to Resolved
  • Assignee set to Samuel Just
  • Pull request ID set to 55223
Actions

Also available in: Atom PDF