Ceph » crimson

Cluster based on dc9b89d619920da9b69b72e80ffdf057f865be50 deployed with:

MDS=0 MGR=1 OSD=1 MON=1 ../src/vstart.sh -n --crimson --seastore --nolockdep --nodaemon --redirect-output --without-dashboard  --no-restart -o "debug_objclass=20" -o "debug_osd=20" -o "debug_none=20" 

explodes during:

CRIMSON_COMPAT=true RBD_FEATURES=1 bin/ceph_test_librbd
...
[       OK ] TestLibRBD.TestIOPPWithIOHint (17868 ms)
[ RUN      ] TestLibRBD.TestIOToSnapshot
using new format!
num snaps is: 0
expected: 0
num snaps is: 1
expected: 1
snap: orig
found orig with size 2097152
read: 80
write test data!

On all the OSDs:

DEBUG 2022-09-16 13:59:04,001 [shard 0] osd - write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 119'33, trimmed: , trimmed_dups: , clear_divergent_priors: 0
DEBUG 2022-09-16 13:59:04,001 [shard 0] osd - final snapset 18={}:{18={18}} in 2:c5e87adb:::rbd_data.101d3949a6b0.0000000000000000:head
DEBUG 2022-09-16 13:59:04,001 [shard 0] osd - ReplicatedBackend::_submit_transaction: do_transaction...
ERROR 2022-09-16 13:59:04,001 [shard 0] none - ../src/crimson/os/seastore/seastore.cc:1345 : In function 'crimson::os::seastore::SeaStore::_do_transaction_step(crimson::os::seastore::SeaStore::internal_context_t&, crimson::os::FuturizedStore::CollectionRef&, std::vector<boost::intrusive_ptr<crimson::os::seastore::Onode> >&, std::vector<boost::intrusive_ptr<crimson::os::seastore::Onode> >&, ceph::os::Transaction::iterator&)::<lambda()>', abort(%s)
unexpected enoent error

Backtrace:

 0# gsignal in /lib64/libc.so.6
 1# abort in /lib64/libc.so.6
 2# ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) at /home/rzarzynski/ceph1/build/../src/seastar/include/seastar/util/log.hh:106
 3# _ZN7seastar15futurize_invokeIZN7crimsonL8composerINS1_9erroratorIJNS1_19unthrowable_wrapperIRKSt10error_codeL_ZNS1_2ecILi5EEEEEEEE12pass_furtherEJZNS4_IS7_L_ZNS8_ILi2EEEEE6handleIZNS1_2os8seastore8SeaStore20_do_transaction_stepERNSG_18internal_context_tERN5boost13intrusive_ptrINSE_19FuturizedCollectionEEERSt6vectorINSK_INSF_5OnodeEEESaISQ_EEST_RN4ceph2os11Transaction8iteratorEEUlvE0_EEDaOT_EUlRKSC_E_NS1_8ct_error10assert_allEEEEDaS11_DpOT0_EUlDpOT_E_JS12_EEEDaS11_S19_ at /opt/rh/gcc-toolset-11/root/usr/include/c++/11/bits/basic_string.h:672
 4# _ZN7crimson13interruptible8internal27call_with_interruption_implINS_2os8seastore28TransactionConflictConditionEZNS_L8composerINS_9erroratorIJNS_19unthrowable_wrapperIRKSt10error_codeL_ZNS_2ecILi5EEEEEEEE12pass_furtherEJZNS8_ISB_L_ZNSC_ILi2EEEEE6handleIZNS4_8SeaStore20_do_transaction_stepERNSI_18internal_context_tERN5boost13intrusive_ptrINS3_19FuturizedCollectionEEERSt6vectorINSM_INS4_5OnodeEEESaISS_EESV_RN4ceph2os11Transaction8iteratorEEUlvE0_EEDaOT_EUlRKSG_E_NS_8ct_error10assert_allEEEEDaS13_DpOT0_EUlDpOT_E_JS14_EEEDaN7seastar13lw_shared_ptrIS12_EEOT0_DpOT1_ at /home/rzarzynski/ceph1/build/../src/crimson/common/interruptible_future.h:193
 5# _ZN7crimson13interruptible22call_with_interruptionINS_2os8seastore28TransactionConflictConditionEZNS_L8composerINS_9erroratorIJNS_19unthrowable_wrapperIRKSt10error_codeL_ZNS_2ecILi5EEEEEEEE12pass_furtherEJZNS7_ISA_L_ZNSB_ILi2EEEEE6handleIZNS3_8SeaStore20_do_transaction_stepERNSH_18internal_context_tERN5boost13intrusive_ptrINS2_19FuturizedCollectionEEERSt6vectorINSL_INS3_5OnodeEEESaISR_EESU_RN4ceph2os11Transaction8iteratorEEUlvE0_EEDaOT_EUlRKSF_E_NS_8ct_error10assert_allEEEEDaS12_DpOT0_EUlDpOT_E_S13_N7seastar6futureIvEELi0EEEDaNS1F_13lw_shared_ptrIS11_EEOT0_OT1_ at /home/rzarzynski/ceph1/build/../src/crimson/common/interruptible_future.h:253
 6# operator()<const crimson::unthrowable_wrapper<const std::error_code&, ((const std::error_code&)(& crimson::ec<2>))>&> at /home/rzarzynski/ceph1/build/../src/crimson/common/interruptible_future.h:951
 7# _ZN7crimson9erroratorIJNS_19unthrowable_wrapperIRKSt10error_codeL_ZNS_2ecILi5EEEEEENS1_IS4_L_ZNS5_ILi2EEEEEENS1_IS4_L_ZNS5_ILi75EEEEEEEE7_futureINS_23errorated_future_markerIvEEE24_safe_then_handle_errorsINS0_IJS6_EE8futurizeIN7seastar6futureIvEEEESJ_ZNS_13interruptible27interruptible_future_detailINS_2os8seastore28TransactionConflictConditionESD_E26handle_error_interruptibleILb1EZNS_L8composerINSF_12pass_furtherEJZNS7_6handleIZNSO_8SeaStore20_do_transaction_stepERNSV_18internal_context_tERN5boost13intrusive_ptrINSN_19FuturizedCollectionEEERSt6vectorINSZ_INSO_5OnodeEEESaIS15_EES18_RN4ceph2os11Transaction8iteratorEEUlvE0_EEDaOT_EUlRKS7_E_NS_8ct_error10assert_allEEEEDaS1G_DpOT0_EUlDpOT_E_EEDaOT0_EUlS1G_E_EEDaS1U_OT1_ at /home/rzarzynski/ceph1/build/../src/crimson/common/errorator.h:458
 8# operator()<seastar::future<> > at /home/rzarzynski/ceph1/build/../src/crimson/common/errorator.h:718
 9# seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::noncopyable_function<crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<5> > >::_future<crimson::errorated_future_marker<void> > (seastar::future<void>&&)>, seastar::future<void>::then_wrapped_nrvo<crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<5> > >::_future<crimson::errorated_future_marker<void> >, seastar::noncopyable_function<crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<5> > >::_future<crimson::errorated_future_marker<void> > (seastar::future<void>&&)> >(seastar::noncopyable_function<crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<5> > >::_future<crimson::errorated_future_marker<void> > (seastar::future<void>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<5> > >::_future<crimson::errorated_future_marker<void> > (seastar::future<void>&&)>&, seastar::future_state<seastar::internal::monostate>&&)#1}, void>::run_and_dispose() at /home/rzarzynski/ceph1/build/../src/seastar/include/seastar/util/noncopyable_function.hh:209
10# seastar::reactor::run_tasks(seastar::reactor::task_queue&) at /home/rzarzynski/ceph1/build/../src/seastar/src/core/reactor.cc:2353
11# seastar::reactor::run_some_tasks() at /home/rzarzynski/ceph1/build/../src/seastar/src/core/reactor.cc:2766
12# seastar::reactor::do_run() at /home/rzarzynski/ceph1/build/../src/seastar/src/core/reactor.cc:2934
13# seastar::reactor::run() at /home/rzarzynski/ceph1/build/../src/seastar/src/core/reactor.cc:2817
14# seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at /home/rzarzynski/ceph1/build/../src/seastar/src/core/app-template.cc:265
15# seastar::app_template::run(int, char**, std::function<seastar::future<int> ()>&&) at /home/rzarzynski/ceph1/build/../src/seastar/src/core/app-template.cc:156
16# main at /home/rzarzynski/ceph1/build/../src/crimson/osd/main.cc:231
17# __libc_start_main in /lib64/libc.so.6
18# _start in /home/rzarzynski/ceph1/build/bin/crimson-osd