Ceph » mgr

this is related to https://github.com/ceph/ceph/pull/36964/commits/b9edfbd55f897ebaaad56dc67c15887f57ab61ad

Neha, do you know any new enough case on the teuthology so I could look at the mgr and mon logs?

Mykola Golub wrote:

Neha, do you know any new enough case on the teuthology so I could look at the mgr and mon logs?

https://tracker.ceph.com/issues/49255 is the most recent failure from teuthology I have seen, but it does not have logs because the job died. I have not been able to find logs for any of these yet.

We are experiencing the same crash reports on Octopus 15.2.8. I've not found a backport issue for this issue to Octopus. Can and should this also be backported to Octopus?

Our crash report:

{
    "archived": "2021-04-22 07:53:13.871413",
    "assert_condition": "pending_service_map.epoch > service_map.epoch",
    "assert_file": "/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.8/rpm/el8/BUILD/ceph-15.2.8/src/mgr/Daem
onServer.cc",
    "assert_func": "DaemonServer::got_service_map()::<lambda(const ServiceMap&)>",
    "assert_line": 2796,
    "assert_msg": "/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.8/rpm/el8/BUILD/ceph-15.2.8/src/mgr/Daemo
nServer.cc: In function 'DaemonServer::got_service_map()::<lambda(const ServiceMap&)>' thread 7fe68dfc8700 time 2021-04-22T08:12:49.479400+0200\n/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABL
E_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.8/rpm/el8/BUILD/ceph-15.2.8/src/mgr/DaemonServer.cc: 2796: FAILED ceph_assert(pending_service_map.epoch > service_map.epoch)\n" 
,
    "assert_thread_name": "ms_dispatch",
    "backtrace": [
        "(()+0x12b20) [0x7fe69d76bb20]",
        "(gsignal()+0x10f) [0x7fe69c1bc7ff]",
        "(abort()+0x127) [0x7fe69c1a6c35]",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x7fe69f2ca735]",
        "(()+0x27a8fe) [0x7fe69f2ca8fe]",
        "(DaemonServer::got_service_map()+0x9b5) [0x560a272d3d95]",
        "(Mgr::handle_service_map(boost::intrusive_ptr<MServiceMap>)+0x14b) [0x560a2730cb6b]",
        "(Mgr::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x2c7) [0x560a2730ebf7]",
        "(MgrStandby::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0xb0) [0x560a27317ba0]",
        "(DispatchQueue::entry()+0x126a) [0x7fe69f4e909a]",
        "(DispatchQueue::DispatchThread::entry()+0x11) [0x7fe69f58b8a1]",
        "(()+0x814a) [0x7fe69d76114a]",
        "(clone()+0x43) [0x7fe69c281f23]" 
    ],
    "ceph_version": "15.2.8",
    "crash_id": "2021-04-22T06:12:49.482382Z_c23cd42f-1c72-4598-a909-6519f6fcb842",
    "entity_name": "mgr.alpha",
    "os_id": "centos",
    "os_name": "CentOS Linux",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mgr",
    "stack_sig": "86ab3a48e25206c884ac52034d1562c68826296d4fcbe7ceff0b9d8b9b4a56a1",
    "timestamp": "2021-04-22T06:12:49.482382Z",
    "utsname_hostname": "alpha",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-240.10.1.el8_3.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Mon Jan 18 17:05:51 UTC 2021" 
}

{
"os_version_id": "7",
"assert_condition": "pending_service_map.epoch > service_map.epoch",
"utsname_release": "3.10.0-1062.18.1.el7.x86_64",
"os_name": "CentOS Linux",
"entity_name": "mgr.LNVSFSMDS1502",
"assert_file": "/builddir/build/BUILD/ceph-14.2.16/src/mgr/DaemonServer.cc",
"timestamp": "2021-06-05 06:07:23.855365Z",
"process_name": "ceph-mgr",
"utsname_machine": "x86_64",
"assert_line": 2795,
"utsname_sysname": "Linux",
"os_version": "7 (Core)",
"os_id": "centos",
"assert_thread_name": "ms_dispatch",
"utsname_version": "#1 SMP Tue Mar 17 23:49:17 UTC 2020",
"backtrace": [
"(()+0xc4c7aa) [0x55df1e00e7aa]",
"(()+0xf5f0) [0x7f6435c3e5f0]",
"(gsignal()+0x37) [0x7f643481c337]",
"(abort()+0x148) [0x7f643481da28]",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x379) [0x7f6439394709]",
"(()+0x12a17bc) [0x7f64393947bc]",
"(()+0x9a179c) [0x55df1dd6379c]",
"(()+0x9a470a) [0x55df1dd6670a]",
"(DaemonServer::got_service_map()+0x6a) [0x55df1dd63888]",
"(Mgr::handle_service_map(MServiceMap*)+0x140) [0x55df1ddb83fe]",
"(Mgr::ms_dispatch(Message*)+0x3e8) [0x55df1ddb8aa8]",
"(MgrStandby::ms_dispatch(Message*)+0x20a) [0x55df1ddcfdf2]",
"(Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x5c) [0x55df1dd6d090]",
"(Messenger::ms_deliver_dispatch(boost::intrusive_ptr<Message> const&)+0xe9) [0x7f643955ff99]",
"(DispatchQueue::entry()+0x5ec) [0x7f643955eb3a]",
"(DispatchQueue::DispatchThread::entry()+0x1c) [0x7f64396c9366]",
"(Thread::entry_wrapper()+0x78) [0x7f6439337228]",
"(Thread::_entry_func(void*)+0x18) [0x7f64393371a6]",
"(()+0x7e65) [0x7f6435c36e65]",
"(clone()+0x6d) [0x7f64348e488d]"
],
"utsname_hostname": "MDS002",
"assert_msg": "/builddir/build/BUILD/ceph-14.2.16/src/mgr/DaemonServer.cc: In function 'DaemonServer::got_service_map()::<lambda(const ServiceMap&)>' thread 7f642d0e7700 time 2021-06-05 15:07:23.828349\n/builddir/build/BUILD/ceph-14.2.16/src/mgr/DaemonServer.cc: 2795: FAILED ceph_assert(pending_service_map.epoch > service_map.epoch)\n",
"crash_id": "2021-06-05_06:07:23.855365Z_badf1c8e-7978-4fb1-92a2-82b2137dbde0",
"assert_func": "DaemonServer::got_service_map()::<lambda(const ServiceMap&)>",
"ceph_version": "14.2.16"
}

Can I edit or remove the above message written by me?