Ceph » rbd

Adding the backtrace for better searchability:

/home/jenkins-build/build/workspace/ceph-pull-requests/src/common/mutex_debug.h: In function 'void ceph::mutex_debug_detail::mutex_debug_impl<<anonymous> >::unlock_impl() [with bool Recursive = false]' thread 7f49c2883700 time 2021-03-01T15:03:25.977622+0000
/home/jenkins-build/build/workspace/ceph-pull-requests/src/common/mutex_debug.h: 134: FAILED ceph_assert(r == 0)
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x19d) [0x7f49e26209dd]
 2: /home/jenkins-build/build/workspace/ceph-pull-requests/build/lib/libceph-common.so.2(+0x1675c5f) [0x7f49e2620c5f]
 3: (ceph::mutex_debug_detail::mutex_debug_impl<false>::unlock_impl()+0x31) [0x5565c55f19f5]
 4: (ceph::mutex_debug_detail::mutex_debug_impl<false>::unlock(bool)+0x4b) [0x5565c55d6d43]
 5: (std::lock_guard<ceph::mutex_debug_detail::mutex_debug_impl<false> >::~lock_guard()+0x20) [0x5565c55d2ecc]
 6: (librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}::operator()() const+0x5c) [0x5565c5d0e9ae]
 7: (void boost::asio::asio_handler_invoke<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}>(librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}&, ...)+0x68) [0x5565c5d1bd10]
 8: (void boost_asio_handler_invoke_helpers::invoke<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}, {lambda(int)#5}>(librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}&, {lambda(int)#5}&)+0x33) [0x5565c5d1ba8a]
 9: (void boost::asio::system_executor::dispatch<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}, std::allocator<void> >(librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}&&, std::allocator<void> const&) const+0x49) [0x5565c5d1b7ab]
 10: (boost::asio::detail::work_dispatcher<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}>::operator()()+0x52) [0x5565c5d1b434]
 11: (void boost::asio::asio_handler_invoke<boost::asio::detail::work_dispatcher<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}> >(boost::asio::detail::work_dispatcher<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}>&, ...)+0x68) [0x5565c5d1af70]
 12: (void boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::work_dispatcher<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}>, {lambda()#2}>(boost::asio::detail::work_dispatcher<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}>&, {lambda()#2}&)+0x33) [0x5565c5d1a7af]
 13: (boost::asio::detail::executor_op<boost::asio::detail::work_dispatcher<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::work_dispatcher<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::schedule_dispatch_delayed_requests()::{lambda(int)#5}::operator()(int) const::{lambda()#2}>*, boost::system::error_code const&, unsigned long)+0x9e) [0x5565c5d19c68]
 14: (boost::asio::detail::scheduler_operation::complete(void*, boost::system::error_code const&, unsigned long)+0x36) [0x5565c5e24e56]
 15: (boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock&, boost::asio::detail::scheduler_thread_info&, boost::system::error_code const&)+0x1c2) [0x5565c5e281e6]
 16: (boost::asio::detail::scheduler::run(boost::system::error_code&)+0xf4) [0x5565c5e27dba]
 17: (boost::asio::io_context::run()+0x48) [0x5565c707083a]
 18: (ceph::async::io_context_pool::start(short)::{lambda()#1}::operator()() const+0x1f) [0x5565c7070d6f]
 19: (void std::__invoke_impl<void, ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::__invoke_other, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)+0x20) [0x5565c70753b7]
 20: (std::__invoke_result<ceph::async::io_context_pool::start(short)::{lambda()#1}>::type std::__invoke<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::__invoke_result&&, (ceph::async::io_context_pool::start(short)::{lambda()#1}&&)...)+0x20) [0x5565c7074a4d]
 21: (std::invoke_result<ceph::async::io_context_pool::start(short)::{lambda()#1}>::type std::invoke<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::invoke_result&&, (ceph::async::io_context_pool::start(short)::{lambda()#1}&&)...)+0x20) [0x5565c7073dfe]
 22: _ZZ17make_named_threadIZN4ceph5async15io_context_pool5startEsEUlvE_JEESt6threadSt17basic_string_viewIcSt11char_traitsIcEEOT_DpOT0_ENKUlSA_SD_E_clIS3_JEEEDaSA_SD_()
 23: (std::basic_string_view<char, std::char_traits<char> > std::__invoke_impl<void, make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}>(std::__invoke_other, make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}&&, {lambda()#1}&&)+0x37) [0x5565c70775f8]
 24: (std::__invoke_result<std::basic_string_view<char, std::char_traits<char> >, (make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}&&)...>::type std::__invoke<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}>(make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}&&)+0x37) [0x5565c7077363]
 25: (void std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>)+0x43) [0x5565c707712f]
 26: (std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}> >::operator()()+0x18) [0x5565c7076e4e]
 27: (std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1, auto:2&&)#1}, {lambda()#1}> > >::_M_run()+0x1c) [0x5565c70767da]
 28: /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xd0d80) [0x7f49e03b3d80]
 29: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76db) [0x7f49ebe736db]
 30: clone()

This ticket was about an actual ceph_assert in the mutex code. AFAIK we haven't seen it since. The comments highlight a gtest assertion, which is different and is being tracked in https://tracker.ceph.com/issues/46875.