Ceph » CephFS

Standby-replay daemon crashes during replay when accessing inode map.
Ref: BZ2218759

[root@49cd6ae8516b working]# gdb -q /usr/bin/ceph-mds core.ceph-mds.167.9df534a325934dd2b50fa68d8b8aee29.2006407.1694647922000000
Core was generated by `ceph-mds --fsid=724e0358-2cfc-4a0f-9a99-419999493584 --keyring=/etc/ceph/keyrin'.
Program terminated with signal SIGSEGV, Segmentation fault.

(gdb) bt
#0  __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=11, no_tid=no_tid@entry=0) at pthread_kill.c:44
#1  0x00007f2ad51495b3 in __pthread_kill_internal (signo=11, threadid=<optimized out>) at pthread_kill.c:78
#2  0x00007f2ad50fcd46 in __GI_raise (sig=11) at ../sysdeps/posix/raise.c:26
#3  0x000055fb3bc25f6a in reraise_fatal (signum=11) at /usr/src/debug/ceph-17.2.6-138.el9cp.x86_64/src/global/signal_handler.cc:88
#4  handle_oneshot_fatal_signal (signum=11) at /usr/src/debug/ceph-17.2.6-138.el9cp.x86_64/src/global/signal_handler.cc:363
#5  <signal handler called>
#6  0x00007f2ad546f0d3 in std::_Rb_tree_decrement(std::_Rb_tree_node_base*) () from /lib64/libstdc++.so.6
#7  0x000055fb3b8f99d5 in std::_Rb_tree_iterator<std::pair<int const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >::operator-- (this=<synthetic pointer>)
    at /usr/include/c++/11/bits/stl_tree.h:302
#8  std::_Rb_tree<int, std::pair<int const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<int const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<int>, std::allocator<std::pair<int const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_get_insert_unique_pos (this=<optimized out>, __k=@0x55fb7b594140: 1) at /usr/include/c++/11/bits/stl_tree.h:2080
#9  0x000055fb3bcc1522 in std::_Rb_tree<int, std::pair<int const, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > >, std::_Select1st<std::pair<int const, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > > >, std::less<int>, std::allocator<std::pair<int const, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > > > >::_M_get_insert_hint_unique_pos (
    __k=@0x55fb7b594140: 1, __position={first = 774547952, second = std::unordered_set with 94540820119553 elements}, this=0x55fc2e2aaa68) at /usr/include/c++/11/bits/stl_tree.h:2209
#10 std::_Rb_tree<int, std::pair<int const, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > >, std::_Select1st<std::pair<int const, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > > >, std::less<int>, std::allocator<std::pair<int const, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > > > >::_M_emplace_hint_unique<std::piecewise_construct_t const&, std::tuple<int const&>, std::tuple<> >(std::_Rb_tree_const_iterator<std::pair<int const, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > > >, std::piecewise_construct_t const&, std::tuple<int const&>&&, std::tuple<>&&) [clone .constprop.0] [clone .isra.0] (this=0x55fc2e2aaa68, __pos=
  {first = 774547952, second = std::unordered_set with 94540820119553 elements}) at /usr/include/c++/11/bits/stl_tree.h:2435
#11 0x000055fb3bb7214c in std::map<int, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> >, std::less<int>, std::allocator<std::pair<int const, std::unordered_set<unsigned long, std::hash<unsigned long>, std::equal_to<unsigned long>, std::allocator<unsigned long> > > > >::operator[] (__k=@0x55fb3f330490: 1, 
    this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:350
#12 MDSTableClient::got_journaled_ack (this=0x55fb3f330480, tid=<optimized out>) at /usr/src/debug/ceph-17.2.6-138.el9cp.x86_64/src/mds/MDSTableClient.cc:222
#13 0x000055fb3bbbd003 in MDLog::_replay_thread (this=0x55fb3f378300) at /usr/src/debug/ceph-17.2.6-138.el9cp.x86_64/src/mds/MDLog.cc:1436
#14 0x000055fb3b932861 in MDLog::ReplayThread::entry (this=<optimized out>) at /usr/src/debug/ceph-17.2.6-138.el9cp.x86_64/src/mds/MDLog.h:192
#15 0x00007f2ad5147802 in start_thread (arg=<optimized out>) at pthread_create.c:443
#16 0x00007f2ad50e7450 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

(gdb) print mds->whoami 
$2 = 0

(gdb) p mds->state
$3 = MDSMap::STATE_STANDBY_REPLAY

(gdb) f 12
#12 MDSTableClient::got_journaled_ack (this=0x55fb3f330480, tid=<optimized out>) at /usr/src/debug/ceph-17.2.6-138.el9cp.x86_64/src/mds/MDSTableClient.cc:222
222        pending_commit[tid]->pending_commit_tids[table].erase(tid);
(gdb) p table
$4 = 1
(gdb) p tid
$5 = <optimized out>