Ceph

I found this crash on one of the clusters to which I have access.

It was not me who told the MDS to dump the inode, so I don't know all the details. However, on that day, the cluster had a problematic rsync workload initiated by an old kernel client (the default kernel in Rocky Linux 8.9, was 4.18.0-513.11.1 at that time) and had multiple warnings during the day:

• SLOW_OPS
• MDS_SLOW_METADATA_IO
• MDS_TRIM
• MDS_SLOW_REQUEST
• OSD_DOWN (was restarted in an attempt to troubleshoot slow ops)
• PG_DEGRADED (consequence of OSD_DOWN)
• MDS_CLIENT_OLDEST_TID (then the MDS evicted the bad client and the cluster went to normal for a while)

What we have seen is that multiple clients piling up for xlock on the same inode. We decided to dump that inode, and apparently, that's what led to the crash.

{
    "assert_condition": "_raw",
    "assert_file": "/home/abuild/rpmbuild/BUILD/ceph-17.2.7/src/common/buffer.cc",
    "assert_func": "const char* ceph::buffer::v15_2_0::ptr::c_str() const",
    "assert_line": 510,
    "assert_msg": "/home/abuild/rpmbuild/BUILD/ceph-17.2.7/src/common/buffer.cc: In function 'const char* ceph::buffer::v15_2_0::ptr::c_str() const' thread 7f152d9f9700 time 2024-02-27T18:29:15.732353+0000\n/home/abuild/rpmbuild/BUILD/ceph-17.2.7/src/common/buffer.cc: 510: FAILED ceph_assert(_raw)\n",
    "assert_thread_name": "admin_socket",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x16910) [0x7f1532f5c910]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17e) [0x7f15331fb5b9]",
        "/usr/lib64/ceph/libceph-common.so.2(+0x2726ff) [0x7f15331fb6ff]",
        "/usr/lib64/ceph/libceph-common.so.2(+0x613aad) [0x7f153359caad]",
        "(InodeStoreBase::dump(ceph::Formatter*) const+0xe0) [0x564100fbec30]",
        "(CInode::dump(ceph::Formatter*, int) const+0x99c) [0x564100fbfc5c]",
        "(MDCache::dump_inode(ceph::Formatter*, unsigned long)+0x6a) [0x564100e2867a]",
        "(MDSRank::command_dump_inode(ceph::Formatter*, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&, std::ostream&)+0xd2) [0x564100d1cc62]",
        "(MDSRankDispatcher::handle_asok_command(std::basic_string_view<char, std::char_traits<char> >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&, ceph::Formatter*, ceph::buffer::v15_2_0::list const&, std::function<void (int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list&)>)+0x1bc8) [0x564100d3a038]",
        "(MDSDaemon::asok_command(std::basic_string_view<char, std::char_traits<char> >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&, ceph::Formatter*, ceph::buffer::v15_2_0::list const&, std::function<void (int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list&)>)+0xafa) [0x564100d0b8ca]",
        "(MDSSocketHook::call_async(std::basic_string_view<char, std::char_traits<char> >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&, ceph::Formatter*, ceph::buffer::v15_2_0::list const&, std::function<void (int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list&)>)+0xa4) [0x564100d13e14]",
        "(AdminSocket::execute_command(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, ceph::buffer::v15_2_0::list const&, std::function<void (int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list&)>)+0x537) [0x7f15332f2047]",
        "(AdminSocket::do_tell_queue()+0x1cb) [0x7f15332f2e3b]",
        "(AdminSocket::entry()+0x325) [0x7f15332f44a5]",
        "/usr/lib64/libstdc++.so.6(+0xe4703) [0x7f1532a9f703]",
        "/lib64/libpthread.so.0(+0xa6ea) [0x7f1532f506ea]",
        "clone()" 
    ],
    "ceph_version": "17.2.7",
    "crash_id": "2024-02-27T18:29:15.734966Z_05b6e380-3d54-4995-9bb8-aa9a27c8b63c",
    "entity_name": "mds.croit-mds01",
    "os_id": "opensuse-leap",
    "os_name": "openSUSE Leap",
    "os_version": "15.5",
    "os_version_id": "15.5",
    "process_name": "ceph-mds",
    "stack_sig": "60da4e4af826129fe6b8f0b0e460172dcf20ea4b26c77375baeb4f3a82676121",
    "timestamp": "2024-02-27T18:29:15.734966Z",
    "utsname_hostname": "croit-mds01",
    "utsname_machine": "x86_64",
    "utsname_release": "5.14.21-150500.55.31-default",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP PREEMPT_DYNAMIC Wed Oct 4 16:52:05 UTC 2023 (5dc23e0)" 
}

I have no other information, and the MDS logs have been rotated automatically since then.