Ceph » RADOS

New crash events were reported via Telemetry with newer versions (['16.2.6', '16.2.7', '16.2.9']) than encountered in Tracker (16.2.5).

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=4979bd08b53f6f3ce69bf22220df0458ab2bf89168ea6e9ecb294050d51752a6

Assert condition: abort
Assert function: int MonitorDBStore::apply_transaction(MonitorDBStore::TransactionRef)

Sanitized backtrace:

    MonitorDBStore::apply_transaction(std::shared_ptr<MonitorDBStore::Transaction>)
    Elector::persist_epoch(unsigned int)
    ElectionLogic::bump_epoch(unsigned int)
    ElectionLogic::start()
    Monitor::start_election()
    Monitor::handle_probe_reply(boost::intrusive_ptr<MonOpRequest>)
    Monitor::handle_probe(boost::intrusive_ptr<MonOpRequest>)
    Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)
    Monitor::_ms_dispatch(Message*)
    Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)
    DispatchQueue::entry()
    DispatchQueue::DispatchThread::entry()

{
    "assert_condition": "abort",
    "assert_file": "mon/MonitorDBStore.h",
    "assert_func": "int MonitorDBStore::apply_transaction(MonitorDBStore::TransactionRef)",
    "assert_line": 351,
    "assert_msg": "mon/MonitorDBStore.h: In function 'int MonitorDBStore::apply_transaction(MonitorDBStore::TransactionRef)' thread ffff8d0a4200 time 2022-07-20T08:55:11.540228+0000\nmon/MonitorDBStore.h: 351: ceph_abort_msg(\"failed to write to db\")",
    "assert_thread_name": "ms_dispatch",
    "backtrace": [
        "__kernel_rt_sigreturn()",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b0) [0xffff965b6770]",
        "(MonitorDBStore::apply_transaction(std::shared_ptr<MonitorDBStore::Transaction>)+0x930) [0xaaaabe9d5200]",
        "(Elector::persist_epoch(unsigned int)+0x198) [0xaaaabea91dc8]",
        "(ElectionLogic::bump_epoch(unsigned int)+0x74) [0xaaaabea96b8c]",
        "(ElectionLogic::start()+0x300) [0xaaaabea98730]",
        "(Monitor::start_election()+0x18c) [0xaaaabe9f5a0c]",
        "(Monitor::handle_probe_reply(boost::intrusive_ptr<MonOpRequest>)+0x17c8) [0xaaaabea07af0]",
        "(Monitor::handle_probe(boost::intrusive_ptr<MonOpRequest>)+0x44c) [0xaaaabea087dc]",
        "(Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x11e0) [0xaaaabea16080]",
        "(Monitor::_ms_dispatch(Message*)+0x590) [0xaaaabea16af0]",
        "(Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x64) [0xaaaabea4373c]",
        "(DispatchQueue::entry()+0xff8) [0xffff96796ba8]",
        "(DispatchQueue::DispatchThread::entry()+0x10) [0xffff96839038]",
        "/lib64/libpthread.so.0(+0x7800) [0xffff95bb5800]",
        "/lib64/libc.so.6(+0xd17dc) [0xffff957707dc]" 
    ],
    "ceph_version": "16.2.5",
    "crash_id": "2022-07-20T08:55:11.554574Z_13555780-e7f8-490f-8864-833207e53bd3",
    "entity_name": "mon.d6512682d0d3bfaa32551c819dcbe378249d9bef",
    "os_id": "centos",
    "os_name": "CentOS Linux",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mon",
    "stack_sig": "2ad26627fdbbf79d6bd1ceed17693e8f0a1d5d298129571f4369f4e0ade385bd",
    "timestamp": "2022-07-20T08:55:11.554574Z",
    "utsname_machine": "aarch64",
    "utsname_release": "5.4.100-MARS400",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Thu Jan 6 18:14:12 CST 2022" 
}