Bug #45857
Updated by Deepika Upadhyay almost 4 years ago
setup: setting debug level 20 for bluestore, filestore and osd and using seastar with seastar_default_allocator + Release mode <pre> turns out filestore is getting mounted but after this seeing failure in open_co WARN 2020-06-03 12:18:18,342 [shard 0] osd - OSD warning: got an error loading one or more classes: (5) Input/output error INFO 2020-06-03 12:18:18,342 [shard 0] osd - start DEBUG 2020-06-03 12:18:18,342 [shard 0] filestore - mount . </pre> . .(a lot of threads created) [Thread 0x7fffc67fc700 (LWP 3872158) exited] [New Thread 0x7fffc5ffb700 (LWP 3872159)] [Thread 0x7fffc5ffb700 (LWP 3872159) exited] [New Thread 0x7fffc5ffb700 (LWP 3872160)] [New Thread 0x7fffc67fc700 (LWP 3872161)] [New Thread 0x7fffc6ffd700 (LWP 3872162)] [New Thread 0x7fffc77fe700 (LWP 3872163)] [New Thread 0x7fffd77fe700 (LWP 3872164)] [New Thread 0x7fffd6ffd700 (LWP 3872165)] DEBUG 2020-06-03 12:18:20,360 [shard 0] filestore - open_collection </pre> receives segfault after open_collection breakpoint: <pre> => 0xf402b7 <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+23>: mov r13,QWORD PTR [rsi+0x50] 0xf402bb <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+27>: mov rbx,rsp 0xf402be <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+30>: lea rax,[rbx+0x10] 0xf402c2 <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+34>: mov QWORD PTR [rsp],rax 0xf402c6 <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+38>: test r13,r13 0xf402c9 <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+41>: je 0xf402d8 <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+56> 0xf402cb <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+43>: mov rdi,r13 0xf402ce <_ZNSt10_HashtableI6coll_tSt4pairIKS0_N5boost13intrusive_ptrIN7crimson2os19FuturizedCollectionEEEESaIS9_ENSt8__detail10_Select1stESt8equal_toIS0_ESt4hashIS0_ENSB_18_Mod_range_hashingENSB_20_Default_ranged_hashENSB_20_Prime_rehash_policyENSB_17_Hashtable_traitsILb1ELb0ELb1EEEE4findERS2_+46>: call 0x80e290 <strlen@plt> ----------------------------------------------------------------------------------------------------------------------------- 0x0000000000f402b7 in std::_Hashtable<coll_t, std::pair<coll_t const, boost::intrusive_ptr<crimson::os::FuturizedCollection> >, std::allocator<std::pair<coll_t const, boost::intrusive_ptr<crimson::os::FuturizedCollection> > >, std::__detail::_Select1st, std::equal_to<coll_t>, std::hash<coll_t>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::find(coll_t const&) () </pre> seastar backtrace: <pre> [Backtrace #0] void seastar::backtrace<seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}>(seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}&&) at :? seastar::print_with_backtrace(seastar::backtrace_buffer&) at reactor.cc:? seastar::print_with_backtrace(char const*) at reactor.cc:? seastar::install_oneshot_signal_handler<11, &seastar::sigsegv_action>()::{lambda(int, siginfo_t*, void*)#1}::_FUN(int, siginfo_t*, void*) at reactor.cc:? .annobin_sigaction.c at sigaction.c:? std::_Hashtable<coll_t, std::pair<coll_t const, boost::intrusive_ptr<crimson::os::FuturizedCollection> >, std::allocator<std::pair<coll_t const, boost::intrusive_ptr<crimson::os::FuturizedCollection> > >, std::__detail::_Select1st, std::equal_to<coll_t>, std::hash<coll_t>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::find(coll_t const&) at ??:? crimson::os::AlienStore::create_new_collection(coll_t const&)::{lambda(boost::intrusive_ptr<ObjectStore::CollectionImpl>)#2}::operator()(boost::intrusive_ptr<ObjectStore::CollectionImpl>) const [clone .isra.1258] at alien_store.cc:? seastar::continuation<seastar::internal::promise_base_with_type<boost::intrusive_ptr<crimson::os::FuturizedCollection> >, seastar::future<boost::intrusive_ptr<ObjectStore::CollectionImpl> >::then_impl_nrvo<crimson::os::AlienStore::open_collection(coll_t const&)::{lambda(boost::intrusive_ptr<ObjectStore::CollectionImpl>)#2}, seastar::future<boost::intrusive_ptr<crimson::os::FuturizedCollection> > >(crimson::os::AlienStore::open_collection(coll_t const&)::{lambda(boost::intrusive_ptr<ObjectStore::CollectionImpl>)#2}&&)::{lambda()#1}::operator()() const::{lambda(seastar::internal::promise_base_with_type<boost::intrusive_ptr<crimson::os::FuturizedCollection> >&, seastar::future_state<boost::intrusive_ptr<ObjectStore::CollectionImpl> >&&)#1}, boost::intrusive_ptr<ObjectStore::CollectionImpl> >::run_and_dispose() at alien_store.cc:? seastar::reactor::run_tasks(seastar::reactor::task_queue&) at :? seastar::reactor::run_some_tasks() [clone .part.1352] at reactor.cc:? seastar::reactor::run() at :? seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at :? main at ??:? __libc_start_main at ??:? _start at ??:? </pre>