Ceph » RADOS

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=8625a9fe17dddf626242a326cfccda209139d6f0639e0fcece04586ff352d218

Assert condition: num_down_in_osds <= num_in_osds
Assert function: void OSDMap::check_health(ceph::common::CephContext*, health_check_map_t*) const

Sanitized backtrace:

    OSDMap::check_health(ceph::common::CephContext*, health_check_map_t*) const
    OSDMonitor::encode_pending(std::shared_ptr<MonitorDBStore::Transaction>)
    PaxosService::propose_pending()
    MgrMonitor::drop_active()
    MgrMonitor::prepare_beacon(boost::intrusive_ptr<MonOpRequest>)
    MgrMonitor::prepare_update(boost::intrusive_ptr<MonOpRequest>)
    PaxosService::dispatch(boost::intrusive_ptr<MonOpRequest>)
    Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)
    Monitor::_ms_dispatch(Message*)
    Monitor::handle_forward(boost::intrusive_ptr<MonOpRequest>)
    Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)
    Monitor::_ms_dispatch(Message*)
    Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)
    DispatchQueue::entry()
    DispatchQueue::DispatchThread::entry()

{
    "assert_condition": "num_down_in_osds <= num_in_osds",
    "assert_file": "osd/OSDMap.cc",
    "assert_func": "void OSDMap::check_health(ceph::common::CephContext*, health_check_map_t*) const",
    "assert_line": 5704,
    "assert_msg": "osd/OSDMap.cc: In function 'void OSDMap::check_health(ceph::common::CephContext*, health_check_map_t*) const' thread 7ff4eb1e3700 time 2022-02-09T13:15:48.885151+0000\nosd/OSDMap.cc: 5704: FAILED ceph_assert(num_down_in_osds <= num_in_osds)",
    "assert_thread_name": "ms_dispatch",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12c20) [0x7ff4f6860c20]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x7ff4f8b24ba3]",
        "/usr/lib64/ceph/libceph-common.so.2(+0x276d6c) [0x7ff4f8b24d6c]",
        "(OSDMap::check_health(ceph::common::CephContext*, health_check_map_t*) const+0x3e5e) [0x7ff4f8f6d29e]",
        "(OSDMonitor::encode_pending(std::shared_ptr<MonitorDBStore::Transaction>)+0x3c38) [0x5572e8cad218]",
        "(PaxosService::propose_pending()+0x21a) [0x5572e8c1e9ea]",
        "(MgrMonitor::drop_active()+0x30c) [0x5572e8d1919c]",
        "(MgrMonitor::prepare_beacon(boost::intrusive_ptr<MonOpRequest>)+0x1d4f) [0x5572e8d1b26f]",
        "(MgrMonitor::prepare_update(boost::intrusive_ptr<MonOpRequest>)+0x5b) [0x5572e8d1f79b]",
        "(PaxosService::dispatch(boost::intrusive_ptr<MonOpRequest>)+0xa6d) [0x5572e8c1f89d]",
        "(Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x6ee) [0x5572e8b099be]",
        "(Monitor::_ms_dispatch(Message*)+0x670) [0x5572e8b0b080]",
        "(Monitor::handle_forward(boost::intrusive_ptr<MonOpRequest>)+0x941) [0x5572e8b0c8b1]",
        "(Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x12bc) [0x5572e8b0a58c]",
        "(Monitor::_ms_dispatch(Message*)+0x670) [0x5572e8b0b080]",
        "(Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x5c) [0x5572e8b39f4c]",
        "(DispatchQueue::entry()+0x126a) [0x7ff4f8d68aba]",
        "(DispatchQueue::DispatchThread::entry()+0x11) [0x7ff4f8e1a5d1]",
        "/lib64/libpthread.so.0(+0x817f) [0x7ff4f685617f]",
        "clone()" 
    ],
    "ceph_version": "16.2.7",
    "crash_id": "2022-02-09T13:15:48.891544Z_346b2551-0314-4cc6-a42f-f23fd9703da5",
    "entity_name": "mon.2cc27d319f666a8deac1826e7fa159f65e898539",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mon",
    "stack_sig": "eec8e259e475bc0896305a855a9cb7988860365956923402901f8111f59deff1",
    "timestamp": "2022-02-09T13:15:48.891544Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.4.0-99-generic",
    "utsname_sysname": "Linux",
    "utsname_version": "#112-Ubuntu SMP Thu Feb 3 13:50:55 UTC 2022" 
}