Bug #64332
Updated by Matan Breizman 3 months ago
See: https://github.com/scylladb/seastar/pull/1329 Seastar added a debug macro that is used to count the number of gate holders in debug mode and assert it's zero when the gate is moved or destroyed. This exposed a bug in the current way gates are handled in Crimson and causes a crash. The macro is disabled in out seastar submodule to allow upgrading. The macro should be re-enabled once the crash is fixed. Can be reproduced locally using `ceph_test_cls_rbd`. <pre><code class="text"> (gdb) bt #0 0x00000000098ac0c0 in boost::intrusive::list_node_traits<void*>::set_next (next=0x60e00010acf8, n=0x0) at ceph/build/boost/include/boost/intrusive/detail/list_node.hpp:66 #1 boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*> >::unlink ( this_node=0x7fc7710e02a8) at ceph/build/boost/include/boost/intrusive/circular_list_algorithms.hpp:154 #2 boost::intrusive::generic_hook<(boost::intrusive::algo_types)0, boost::intrusive::list_node_traits<void*>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)2, (boost::intrusive::base_hook_type)0>::unlink ( this=0x7fc7710e02a8) at ceph/build/boost/include/boost/intrusive/detail/generic_hook.hpp:214 #3 boost::intrusive::detail::destructor_impl<boost::intrusive::generic_hook<(boost::intrusive::algo_types)0, boost::intrusive::list_node_traits<void*>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)2, (boost::intrusive::base_hook_type)0> > (hook=...) at ceph/build/boost/include/boost/intrusive/detail/generic_hook.hpp:53 #4 boost::intrusive::generic_hook<(boost::intrusive::algo_types)0, boost::intrusive::list_node_traits<void*>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)2, (boost::intrusive::base_hook_type)0>::~generic_hook (this=0x7fc7710e02a8, __in_chrg=<optimized out>) at ceph/build/boost/include/boost/intrusive/detail/generic_hook.hpp:193 #5 boost::intrusive::list_member_hook<boost::intrusive::link_mode<(boost::intrusive::link_mode_type)2> >::~list_member_hook (this=0x7fc7710e02a8, __in_chrg=<optimized out>) at ceph/build/boost/include/boost/intrusive/list_hook.hpp:207 #6 seastar::gate::holder::~holder (this=0x7fc7710e02a0, __in_chrg=<optimized out>) at ceph/src/seastar/include/seastar/core/gate.hh:236 #7 0x000000000a9a7775 in seastar::with_gate<crimson::os::AlienStore::do_with_op_gate<std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<const std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list> > >, crimson::os::AlienStore::omap_get_values(crimson::os::FuturizedStore::Shard::CollectionRef, const ghobject_t&, const std::optional<std::__cxx11::basic_string<char> >&)::<lambda(auto:128&)> >(std::map<std::__cxx11::basic_string<char>, ceph::buffer::v15_2_0::list, std::less<void> >&&, crimson::os::AlienStore::omap_get_values(crimson::os::FuturizedStore::Shard::CollectionRef, const ghobject_t&, const std::optional<std::__cxx11::basic_string<char> >&)::<lambda(auto:128&)>&&) const::<lambda()> > (func=..., g=...) at ceph/src/seastar/include/seastar/core/gate.hh:327 #8 crimson::os::AlienStore::do_with_op_gate<std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<const std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list> > >, crimson::os::AlienStore::omap_get_values(crimson::os::FuturizedStore::Shard::CollectionRef, const ghobject_t&, const std::optional<std::__cxx11::basic_string<char> >&)::<lambda(auto:128&)> >(void) const (this=this@entry=0x628000010100) at ceph/src/crimson/os/alienstore/alien_store.h:117 #9 0x000000000a9bf439 in crimson::os::AlienStore::omap_get_values (this=0x628000010100, ch=..., oid=..., start=std::optional<std::string> = {...}) at ceph/src/crimson/os/alienstore/alien_store.cc:442 #10 0x000000000666a017 in OSDriver::get_next (this=0x625000ed0158, key="SNA_2_", '0' <repeats 15 times>, "2_", '0' <repeats 15 times>, "2.2B", next=<optimized out>) at ceph/src/osd/SnapMapper.cc:119 #11 0x00000000066b1a79 in MapCacher::MapCacher<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list>::get_next (this=this@entry=0x625000ed0210, key="SNA_2_", '0' <repeats 15 times>, "2_", '0' <repeats 15 times>, "2.2B", next=next@entry=0x7fc7710e1ae0) at ceph/src/common/map_cacher.hpp:99 #12 0x000000000661abed in SnapMapper::get_objects_by_prefixes (this=this@entry=0x625000ed0200, snap=..., max=max@entry=2, out=out@entry=0x7fc7710e2be0) at ceph/src/osd/SnapMapper.cc:591 #13 0x00000000066227bc in SnapMapper::get_next_objects_to_trim (this=this@entry=0x625000ed0200, snap=..., max=max@entry=2, out=out@entry=0x7fc7710e2be0) at ceph/src/osd/SnapMapper.cc:652 </code></pre> [WIP]