Ceph » RADOS

New crash events were reported via Telemetry with newer versions (['17.2.5']) than encountered in Tracker (17.2.1).

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=7a53cbc0bcdeffa2f26d71d0108e5f5ff3996898e40f82dacac9edba5a79a496

Assert condition: have_pending
Assert function: void PaxosService::propose_pending()

Sanitized backtrace:

    PaxosService::propose_pending()
    KVMonitor::do_osd_destroy(int, uuid_d&)
    OSDMonitor::prepare_command_osd_destroy(int, std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >&)
    OSDMonitor::prepare_command_osd_purge(int, std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >&)
    OSDMonitor::prepare_command_impl(boost::intrusive_ptr<MonOpRequest>, std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&)
    OSDMonitor::prepare_command(boost::intrusive_ptr<MonOpRequest>)
    OSDMonitor::prepare_update(boost::intrusive_ptr<MonOpRequest>)
    PaxosService::dispatch(boost::intrusive_ptr<MonOpRequest>)
    Monitor::handle_command(boost::intrusive_ptr<MonOpRequest>)
    Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)
    Monitor::_ms_dispatch(Message*)
    Monitor::handle_forward(boost::intrusive_ptr<MonOpRequest>)
    Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)
    Monitor::_ms_dispatch(Message*)
    Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)
    DispatchQueue::entry()
    DispatchQueue::DispatchThread::entry()

{
    "assert_condition": "have_pending",
    "assert_file": "mon/PaxosService.cc",
    "assert_func": "void PaxosService::propose_pending()",
    "assert_line": 193,
    "assert_msg": "mon/PaxosService.cc: In function 'void PaxosService::propose_pending()' thread 7fb17abb3700 time 2022-11-27T05:27:46.048417+0000\nmon/PaxosService.cc: 193: FAILED ceph_assert(have_pending)",
    "assert_thread_name": "ms_dispatch",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12cf0) [0x7fb184dc5cf0]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x7fb186e1a499]",
        "/usr/lib64/ceph/libceph-common.so.2(+0x269605) [0x7fb186e1a605]",
        "(PaxosService::propose_pending()+0x472) [0x559456768802]",
        "(KVMonitor::do_osd_destroy(int, uuid_d&)+0x4d5) [0x559456704395]",
        "(OSDMonitor::prepare_command_osd_destroy(int, std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >&)+0x27e) [0x55945678f4fe]",
        "(OSDMonitor::prepare_command_osd_purge(int, std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >&)+0xd7) [0x55945678fa37]",
        "(OSDMonitor::prepare_command_impl(boost::intrusive_ptr<MonOpRequest>, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&)+0x165bf) [0x5594567f212f]",
        "(OSDMonitor::prepare_command(boost::intrusive_ptr<MonOpRequest>)+0x38d) [0x5594567f8ccd]",
        "(OSDMonitor::prepare_update(boost::intrusive_ptr<MonOpRequest>)+0x17b) [0x5594567f931b]",
        "(PaxosService::dispatch(boost::intrusive_ptr<MonOpRequest>)+0x2ce) [0x559456768c0e]",
        "(Monitor::handle_command(boost::intrusive_ptr<MonOpRequest>)+0x1d39) [0x55945660d549]",
        "(Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x9e2) [0x5594566116d2]",
        "(Monitor::_ms_dispatch(Message*)+0x406) [0x5594566125e6]",
        "(Monitor::handle_forward(boost::intrusive_ptr<MonOpRequest>)+0xbf9) [0x559456613c39]",
        "(Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x10c7) [0x559456611db7]",
        "(Monitor::_ms_dispatch(Message*)+0x406) [0x5594566125e6]",
        "(Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x5d) [0x559456642dad]",
        "(Messenger::ms_deliver_dispatch(boost::intrusive_ptr<Message> const&)+0x478) [0x7fb187094c88]",
        "(DispatchQueue::entry()+0x50f) [0x7fb1870920cf]",
        "(DispatchQueue::DispatchThread::entry()+0x11) [0x7fb1871598f1]",
        "/lib64/libpthread.so.0(+0x81ca) [0x7fb184dbb1ca]",
        "clone()" 
    ],
    "ceph_version": "17.2.5",
    "crash_id": "2022-11-27T05:27:46.057187Z_eaac788c-5d4f-4289-9717-67a10cd31f24",
    "entity_name": "mon.7d82cd92777bc108bda33d3d314596e37a3efb71",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mon",
    "stack_sig": "eb8737cc2f0a9176afa6b1225824d8b9b0e09d6b3a386f8979f97f6f8020f347",
    "timestamp": "2022-11-27T05:27:46.057187Z",
    "utsname_machine": "x86_64",
    "utsname_release": "6.0.9-arch1-1",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP PREEMPT_DYNAMIC Wed, 16 Nov 2022 17:01:17 +0000" 
}