Ceph » crimson

Noticed while testing: https://github.com/ceph/ceph/pull/46062

https://pulpito.ceph.com/sjust-2023-05-26_22:44:48-crimson-rados-wip-sjust-crimson-testing-2023-05-25-2-distro-default-smithi/7287711/

osd.1 crash:

DEBUG 2023-05-26 23:36:36,101 [shard 0] osd - ShardServices::dispatch_context_transaction: empty transaction
Reactor stalled for 95 ms on shard 1. Backtrace: 0x45d4d 0x47328a37 0x47046135 0x470607b5 0x47060c5c 0x47060db2 0x470611fd 0x12cef 0x2a6de432fcde 0x2a6de4330d18 0x2a6de43336cf 0x2a6de4334360 0x168b86 0x46c06173 0x2a6de43345d4 0x2a6de4323b19 0x2a6de4323be5 0x2a6de43150a5 0x2a6de4316f2a 0xd61c0 0x32402 0xbd8a7 0xbd134 0xbdf7a 0x3c4cdbd7 0x3c4d6633 0x470249ea 0x47081727 0x47278455 0x47299364 0x47299939 0x46e0e063 0x81c9 0x39e72
kernel callstack:
    #0 0x55698378bbd7 in auto seastar::futurize_invoke<crimson::osd::Watch::notify_ack(unsigned long, ceph::buffer::v15_2_0::list const&)::{lambda(auto:1)#1}&, seastar::shared_ptr<crimson::osd::Notify> const&>(crimson::osd::Watch::notify_ack(unsigned long, ceph::buffer::v15_2_0::list const&)::{lambda(auto:1)#1}&, seastar::shared_ptr<crimson::osd::Notify> const&) (/usr/bin/ceph-osd+0x3c4cdbd7)
    #1 0x556983794633 in seastar::internal::do_for_each_state<std::_Rb_tree_const_iterator<seastar::shared_ptr<crimson::osd::Notify> >, crimson::osd::Watch::notify_ack(unsigned long, ceph::buffer::v15_2_0::list const&)::{lambda(auto:1)#1}>::run_and_dispose() (/usr/bin/ceph-osd+0x3c4d6633)
    #2 0x55698e2e29ea in seastar::reactor::run_tasks(seastar::reactor::task_queue&) (/usr/bin/ceph-osd+0x470249ea)
    #3 0x55698e33f727 in seastar::reactor::run_some_tasks() (/usr/bin/ceph-osd+0x47081727)
    #4 0x55698e536455 in seastar::reactor::do_run() (/usr/bin/ceph-osd+0x47278455)
    #5 0x55698e557364 in seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::{lambda()#3}::operator()() const (/usr/bin/ceph-osd+0x47299364)
    #6 0x55698e557939 in std::_Function_handler<void (), seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::{lambda()#3}>::_M_invoke(std::_Any_data const&) (/usr/bin/ceph-osd+0x47299939)
    #7 0x55698e0cc063 in seastar::posix_thread::start_routine(void*) (/usr/bin/ceph-osd+0x46e0e063)
    #8 0x7fd7281131c9 in start_thread (/lib64/libpthread.so.0+0x81c9)
    #9 0x7fd72672ee72 in __clone (/lib64/libc.so.6+0x39e72)

0x604000bc2630 is located 32 bytes inside of 48-byte region [0x604000bc2610,0x604000bc2640)
freed by thread T1 (reactor-1) here:
    #0 0x7fd72b5af36f in operator delete(void*, unsigned long) (/lib64/libasan.so.6+0xb736f)
    #1 0x5569837a910a in std::_Rb_tree<seastar::shared_ptr<crimson::osd::Notify>, seastar::shared_ptr<crimson::osd::Notify>, std::_Identity<seastar::shared_ptr<crimson::osd::Notify> >, std::less<void>, std::allocator<seastar::shared_ptr<crimson::osd::Notify> > >::_M_erase(std::_Rb_tree_node<seastar::shared_ptr<crimson::osd::Notify> >*) (/usr/bin/ceph-osd+0x3c4eb10a)

previously allocated by thread T1 (reactor-1) here:
    #0 0x7fd72b5ae307 in operator new(unsigned long) (/lib64/libasan.so.6+0xb6307)
    #1 0x5569837b7cae in std::pair<std::_Rb_tree_iterator<seastar::shared_ptr<crimson::osd::Notify> >, bool> std::_Rb_tree<seastar::shared_ptr<crimson::osd::Notify>, seastar::shared_ptr<crimson::osd::Notify>, std::_Identity<seastar::shared_ptr<crimson::osd::Notify> >, std::less<void>, std::allocator<seastar::shared_ptr<crimson::osd::Notify> > >::_M_emplace_unique<seastar::shared_ptr<crimson::osd::Notify> >(seastar::shared_ptr<crimson::osd::Notify>&&) (/usr/bin/ceph-osd+0x3c4f9cae)

Thread T1 (reactor-1) created by T0 here:
    #0 0x7fd72b5507c5 in pthread_create (/lib64/libasan.so.6+0x587c5)
    #1 0x55698e0d0346 in seastar::posix_thread::posix_thread(seastar::posix_thread::attr, std::function<void ()>) (/usr/bin/ceph-osd+0x46e12346)
    #2 0x5569917b195f  (/usr/bin/ceph-osd+0x4a4f395f)

SUMMARY: AddressSanitizer: heap-use-after-free (/usr/bin/ceph-osd+0x3c4cdbd7) in auto seastar::futurize_invoke<crimson::osd::Watch::notify_ack(unsigned long, ceph::buffer::v15_2_0::list const&)::{lambda(auto:1)#1}&, seastar::shared_ptr<crimson::osd::Notify> const&>(crimson::osd::Watch::notify_ack(unsigned long, ceph::buffer::v15_2_0::list const&)::{lambda(auto:1)#1}&, seastar::shared_ptr<crimson::osd::Notify> const&)

Similar issue: https://github.com/ceph/ceph/pull/40654