Ceph » crimson

1. create a new rbd image
2. test with fio
[global]
ioengine=rbd
clientname=admin
pool=test_pool
rbdname=test_img
direct=1
sync=1
verify=crc32
do_verify=1
randseed=1259
[write-4k]
rw=write
bs=4K
numjobs=1
iodepth=768
random_distribution=zoned:90/10:10/90
3. restart crimson-osd then assertion fails
DEBUG 2022-03-10 09:46:57,201 [shard 0] seastore_omap - OMapInnerNode: list
crimson-osd: ../src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc:203: virtual crimson::os::seastore::omap_manager::OMapNode::list_ret crimson::os::seastore::omap_manager::OMapInnerNode::list(crimson::os::seastore::omap_manager::omap_context_t, const std::optional<std::__cxx11::basic_string<char> >&, crimson::os::seastore::omap_manager::OMapNode::omap_list_config_t): Assertion `child_iter != iter_cend()' failed.
Aborting on shard 0.
Backtrace:
Reactor stalled for 6459 ms on shard 0. Backtrace: 0x45bad 0xd6a792f 0xd45bc35 0xd470c53 0xd471104 0xd4713c7 0xd4716a7 0x12b1f 0xc8e3b 0x6d85655 0x6d89167 0x6d8cdf5 0x6d8da6c 0x6d8e136 0x6d81aff 0x6d82006 0x6d82804 0x12b1f 0x3737e 0x21db4 0x21c88 0x2fa75 0xb333db1 0xb1fb928 0xb2174e4 0xb2bbf61 0xb2bcbff 0xb2bd4b2 0xb2be3ad 0xb3ae8f1 0xd42f35f 0xd48977a 0xd5e9a42 0xd5eb90a 0xd04d218 0xd051613 0x1916f61 0x23492 0x15b752d
Reactor stalled for 26546 ms on shard 0. Backtrace: 0x45bad 0xd6a792f 0xd45bc35 0xd470c53 0xd471104 0xd4713c7 0xd4716a7 0x12b1f 0xc8e3b 0x6d85655 0x6d89167 0x6d8cdf5 0x6d8da6c 0x6d8e136 0x6d81aff 0x6d82006 0x6d82804 0x12b1f 0x3737e 0x21db4 0x21c88 0x2fa75 0xb333db1 0xb1fb928 0xb2174e4 0xb2bbf61 0xb2bcbff 0xb2bd4b2 0xb2be3ad 0xb3ae8f1 0xd42f35f 0xd48977a 0xd5e9a42 0xd5eb90a 0xd04d218 0xd051613 0x1916f61 0x23492 0x15b752d
0# gsignal in /lib64/libc.so.6
1# abort in /lib64/libc.so.6
2# 0x00002AFC0AA61C89 in /lib64/libc.so.6
3# 0x00002AFC0AA6FA76 in /lib64/libc.so.6
4# crimson::os::seastore::omap_manager::OMapInnerNode::list(crimson::os::seastore::omap_manager::omap_context_t, std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&, crimson::os::seastore::OMapManager::omap_list_config_t) at /da1/zhangsong02/ceph-master/build/../src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc:247
5# auto crimson::os::seastore::omap_manager::OMapInnerNode::list(crimson::os::seastore::omap_manager::omap_context_t, std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&, crimson::os::seastore::OMapManager::omap_list_config_t)::{lambda(auto:1&, auto:2&, auto:3&)#1}::operator()<crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > > >(crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>&, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t&, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > >&) const::{lambda()#1}::operator()() const::{lambda(auto:1&&)#1}::operator()<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >({lambda()#1}) const at /da1/zhangsong02/ceph-master/build/../src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc:242
6# auto seastar::futurize_invoke<crimson::os::seastore::omap_manager::OMapInnerNode::list(crimson::os::seastore::omap_manager::omap_context_t, std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&, crimson::os::seastore::OMapManager::omap_list_config_t)::{lambda(auto:1&, auto:2&, auto:3&)#1}::operator()<crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > > >(crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>&, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t&, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > >&) const::{lambda()#1}::operator()() const::{lambda(auto:1&&)#1}, boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >({lambda()#1}, boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode>&&) at /da1/zhangsong02/ceph-master/build/../src/crimson/common/interruptible_future.h:1524
7# auto crimson::interruptible::internal::call_with_interruption_impl<crimson::os::seastore::TransactionConflictCondition, crimson::os::seastore::omap_manager::OMapInnerNode::list(crimson::os::seastore::omap_manager::omap_context_t, std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&, crimson::os::seastore::OMapManager::omap_list_config_t)::{lambda(auto:1&, auto:2&, auto:3&)#1}::operator()<crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > > >(crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>&, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t&, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > >&) const::{lambda()#1}::operator()() const::{lambda(auto:1&&)#1}, boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >(seastar::lw_shared_ptr<crimson::os::seastore::TransactionConflictCondition>, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t&&, boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode>&&) at /da1/zhangsong02/ceph-master/build/../src/crimson/common/interruptible_future.h:193
8# auto crimson::interruptible::call_with_interruption<crimson::os::seastore::TransactionConflictCondition, crimson::os::seastore::omap_manager::OMapInnerNode::list(crimson::os::seastore::omap_manager::omap_context_t, std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&, crimson::os::seastore::OMapManager::omap_list_config_t)::{lambda(auto:1&, auto:2&, auto:3&)#1}::operator()<crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > > >(crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>&, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t&, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > >&) const::{lambda()#1}::operator()() const::{lambda(auto:1&&)#1}, boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode>, crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::stop_iteration_tag> > > >, 0>(crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::lw_shared_ptr<crimson::os::seastore::TransactionConflictCondition>, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t&&, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > >&&) at /da1/zhangsong02/ceph-master/build/../src/crimson/common/interruptible_future.h:253
9# operator() at /da1/zhangsong02/ceph-master/build/../src/crimson/common/interruptible_future.h:872
10# seastar::noncopyable_function<crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<seastar::stop_iteration_tag> > > > (seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&)>::indirect_vtable_for<seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >::then_wrapped_maybe_erase<false, crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<seastar::stop_iteration_tag> > > >, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> > >::safe_then<crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> > > >::safe_then_interruptible<true, crimson::os::seastore::omap_manager::OMapInnerNode::list(crimson::os::seastore::omap_manager::omap_context_t, std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&, crimson::os::seastore::OMapManager::omap_list_config_t)::{lambda(auto:1&, auto:2&, auto:3&)#1}::operator()<crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > > >(crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>&, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t&, std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > >&) const::{lambda()#1}::operator()() const::{lambda(auto:1&&)#1}, boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode>, 0>(crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t&&)::{lambda(boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode>&&)#1}, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::pass_further>({lambda()#1}, crimson::os::seastore::omap_manager::StringKVInnerNodeLayout::iter_t<true>&&)::{lambda({lambda()#1})#1}>(std::tuple<bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0> > > >&&)::{lambda(seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&)#1}>::call(seastar::noncopyable_function<crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<seastar::stop_iteration_tag> > > > (seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&)> const*, seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&) at /da1/zhangsong02/ceph-master/build/../src/seastar/include/seastar/util/noncopyable_function.hh:152
11# seastar::continuation<seastar::internal::promise_base_with_type<seastar::bool_class<seastar::stop_iteration_tag> >, seastar::noncopyable_function<crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<seastar::stop_iteration_tag> > > > (seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&)>, seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >::then_wrapped_nrvo<crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<seastar::stop_iteration_tag> > > >, seastar::noncopyable_function<crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<seastar::stop_iteration_tag> > > > (seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&)> >(seastar::noncopyable_function<crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<seastar::stop_iteration_tag> > > > (seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&)>&&)::{lambda(seastar::internal::promise_base_with_type<seastar::bool_class<seastar::stop_iteration_tag> >&&, seastar::noncopyable_function<crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<seastar::bool_class<seastar::stop_iteration_tag> > > > (seastar::future<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&)>&, seastar::future_state<boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >&&)#1}, boost::intrusive_ptr<crimson::os::seastore::omap_manager::OMapNode> >::run_and_dispose() at /da1/zhangsong02/ceph-master/build/../src/seastar/include/seastar/util/noncopyable_function.hh:209
12# seastar::reactor::run_tasks(seastar::reactor::task_queue&) at /da1/zhangsong02/ceph-master/build/../src/seastar/src/core/reactor.cc:2345
13# seastar::reactor::run_some_tasks() at /da1/zhangsong02/ceph-master/build/../src/seastar/src/core/reactor.cc:2755
14# seastar::reactor::do_run() at /da1/zhangsong02/ceph-master/build/../src/seastar/src/core/reactor.cc:2923
15# seastar::reactor::run() at /da1/zhangsong02/ceph-master/build/../src/seastar/src/core/reactor.cc:2806
16# seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at /da1/zhangsong02/ceph-master/build/../src/seastar/src/core/app-template.cc:265
17# seastar::app_template::run(int, char**, std::function<seastar::future<int> ()>&&) at /da1/zhangsong02/ceph-master/build/../src/seastar/src/core/app-template.cc:156
18# main at /da1/zhangsong02/ceph-master/build/../src/crimson/osd/main.cc:238 (discriminator 1)
19# __libc_start_main in /lib64/libc.so.6
20# _start in /da1/zhangsong02/ceph-master/build/bin/crimson-osd