Ceph » rbd

An IO can race attempting to acquire the exclusive lock and therefore can restart while the 'owner_lock' is still being held.

 recursive lock of librbd::ImageCtx::owner_lock (0x55fa78bb6730) (14)
 ceph version 16.0.0-6741-g21b24e02476 (21b24e02476c0530231c9902bf6ca41087697f77) pacific (dev)
 1: (ceph::mutex_debug_detail::mutex_debugging_base::_will_lock(bool)+0x4a) [0x7fdf6029a316]
 2: (ceph::shared_mutex_debug::lock_shared()+0x30) [0x7fdf60361f70]
 3: (std::shared_lock<ceph::shared_mutex_debug>::shared_lock(ceph::shared_mutex_debug&)+0x37) [0x55fa769d0bf9]
 4: (librbd::exclusive_lock::ImageDispatch<librbd::ImageCtx>::needs_exclusive_lock(bool, unsigned long, librbd::io::DispatchResult*, Context*)+0xcf) [0x55fa77800e01]
 5: (librbd::exclusive_lock::ImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)+0x1b2) [0x55fa777fff62]
 6: (librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor::operator()(librbd::io::ImageDispatchSpec::Read&) const+0x14a) [0x55fa77831232]
 7: (boost::disable_if_c<(false)&&boost::is_same<librbd::io::ImageDispatchSpec::Read&, librbd::io::ImageDispatchSpec::Read&>::value, bool>::type boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>::internal_visit<librbd::io::ImageDispatchSpec::Read&>(librbd::io::ImageDispatchSpec::Read&, int)+0x29) [0x55fa77834615]
 8: (boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>::result_type boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>, void*, librbd::io::ImageDispatchSpec::Read>(int, boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>&, void*, librbd::io::ImageDispatchSpec::Read*, mpl_::bool_<true>)+0x3a) [0x55fa7783426c]
 9: (boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>::result_type boost::detail::variant::visitation_impl_invoke<boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>, void*, librbd::io::ImageDispatchSpec::Read, boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>::has_fallback_type_>(int, boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>&, void*, librbd::io::ImageDispatchSpec::Read*, boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>::has_fallback_type_, int)+0x31) [0x55fa77833e1c]
 10: (librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const::result_type boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>::apply_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const>(librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const&) &+0x100) [0x55fa778329fe]
 11: (librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor::result_type boost::apply_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor, boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>&>(librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const&, boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>&)+0x2b) [0x55fa778324cb]
 12: (librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)+0xa2) [0x55fa7783103a]
 13: (librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)+0x2ba) [0x55fa77835594]
 14: (librbd::io::ImageDispatchSpec::send()+0x35) [0x55fa7782fc4d]
 15: (librbd::io::ImageDispatchSpec::C_Dispatcher::complete(int)+0xce) [0x55fa7782fb38]
 16: (librbd::exclusive_lock::ImageDispatch<librbd::ImageCtx>::handle_acquire_lock(int)+0x548) [0x55fa77801848]
 17: (librbd::util::detail::C_CallbackAdapter<librbd::exclusive_lock::ImageDispatch<librbd::ImageCtx>, &librbd::exclusive_lock::ImageDispatch<librbd::ImageCtx>::handle_acquire_lock>::finish(int)+0x24) [0x55fa77801cc0]
 18: (Context::complete(int)+0x27) [0x55fa769978c5]
 19: (librbd::ManagedLock<librbd::ImageCtx>::acquire_lock(Context*)+0x1d8) [0x55fa76c4e992]
 20: (librbd::exclusive_lock::ImageDispatch<librbd::ImageCtx>::needs_exclusive_lock(bool, unsigned long, librbd::io::DispatchResult*, Context*)+0x51e) [0x55fa77801250]
 21: (librbd::exclusive_lock::ImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)+0x1b2) [0x55fa777fff62]
 22: (librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor::operator()(librbd::io::ImageDispatchSpec::Read&) const+0x14a) [0x55fa77831232]
 23: (boost::disable_if_c<(false)&&boost::is_same<librbd::io::ImageDispatchSpec::Read&, librbd::io::ImageDispatchSpec::Read&>::value, bool>::type boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>::internal_visit<librbd::io::ImageDispatchSpec::Read&>(librbd::io::ImageDispatchSpec::Read&, int)+0x29) [0x55fa77834615]
 24: (boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>::result_type boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>, void*, librbd::io::ImageDispatchSpec::Read>(int, boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>&, void*, librbd::io::ImageDispatchSpec::Read*, mpl_::bool_<true>)+0x3a) [0x55fa7783426c]
 25: (boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>::result_type boost::detail::variant::visitation_impl_invoke<boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>, void*, librbd::io::ImageDispatchSpec::Read, boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>::has_fallback_type_>(int, boost::detail::variant::invoke_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const, false>&, void*, librbd::io::ImageDispatchSpec::Read*, boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>::has_fallback_type_, int)+0x31) [0x55fa77833e1c]
 26: (librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const::result_type boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>::apply_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const>(librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const&) &+0x100) [0x55fa778329fe]
 27: (librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor::result_type boost::apply_visitor<librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor, boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>&>(librbd::io::ImageDispatcher<librbd::ImageCtx>::SendVisitor const&, boost::variant<librbd::io::ImageDispatchSpec::Read, librbd::io::ImageDispatchSpec::Discard, librbd::io::ImageDispatchSpec::Write, librbd::io::ImageDispatchSpec::WriteSame, librbd::io::ImageDispatchSpec::CompareAndWrite, librbd::io::ImageDispatchSpec::Flush, librbd::io::ImageDispatchSpec::ListSnaps>&)+0x2b) [0x55fa778324cb]
 28: (librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)+0xa2) [0x55fa7783103a]
 29: (librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)+0x2ba) [0x55fa77835594]
 30: (librbd::io::ImageDispatchSpec::send()+0x35) [0x55fa7782fc4d]
 31: (librbd::io::ImageDispatchSpec::C_Dispatcher::complete(int)+0xce) [0x55fa7782fb38]
 32: ./bin/unittest_librbd(+0x27f24da) [0x55fa776cc4da]
 33: ./bin/unittest_librbd(+0x27f3ad4) [0x55fa776cdad4]
 34: ./bin/unittest_librbd(+0x27f3a4f) [0x55fa776cda4f]
 35: ./bin/unittest_librbd(+0x27f39f7) [0x55fa776cd9f7]
 36: ./bin/unittest_librbd(+0x27f3906) [0x55fa776cd906]
 37: ./bin/unittest_librbd(+0x27f386c) [0x55fa776cd86c]
 38: ./bin/unittest_librbd(+0x27f3786) [0x55fa776cd786]
 39: ./bin/unittest_librbd(+0x27f3564) [0x55fa776cd564]
 40: (boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long)+0x36) [0x55fa7718fd4c]
 41: (boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&)+0x1c2) [0x55fa77193262]
 42: (boost::asio::detail::scheduler::run(boost::system::error_code&)+0xf4) [0x55fa77192e3c]
 43: (boost::asio::io_context::run()+0x48) [0x55fa77a130c2]
 44: (ceph::async::io_context_pool::start(short)::{lambda()#1}::operator()() const+0x1f) [0x55fa77a1367d]
 45: (void std::__invoke_impl<void, ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::__invoke_other, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)+0x20) [0x55fa77a188fe]
 46: (std::__invoke_result<ceph::async::io_context_pool::start(short)::{lambda()#1}>::type std::__invoke<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::__invoke_result&&, (ceph::async::io_context_pool::start(short)::{lambda()#1}&&)...)+0x20) [0x55fa77a17d41]
 47: (std::invoke_result<ceph::async::io_context_pool::start(short)::{lambda()#1}>::type std::invoke<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::invoke_result&&, (ceph::async::io_context_pool::start(short)::{lambda()#1}&&)...)+0x20) [0x55fa77a16e2a]
 48: _ZZ17make_named_threadIZN4ceph5async15io_context_pool5startEsEUlvE_JEESt6threadSt17basic_string_viewIcSt11char_traitsIcEEOT_DpOT0_ENKUlSA_SD_E_clIS3_JEEEDaSA_SD_()
 49: (std::basic_string_view<char, std::char_traits<char> > std::__invoke_impl<void, make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}>(std::__invoke_other, make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}&&, {lambda()#1}&&)+0x37) [0x55fa77a1af80]
 50: (std::__invoke_result<std::basic_string_view<char, std::char_traits<char> >, (make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}&&)...>::type std::__invoke<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}>(make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}&&)+0x37) [0x55fa77a1acec]
 51: (void std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>)+0x43) [0x55fa77a1aabb]
 52: (std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}> >::operator()()+0x18) [0x55fa77a1a7d8]
 53: (std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}> > >::_M_run()+0x1c) [0x55fa77a1a18a]
 54: /usr/lib64/libstdc++.so.6(+0xd84d4) [0x7fdf5e6104d4]
 55: /lib64/libpthread.so.0(+0x8eaa) [0x7fdf693f5eaa]
 56: clone()