Ceph » CephFS

We had a complete cephfs failure tonight caused by crashes of all active and standby MDS.

{
    "crash_id": "2020-01-24_03:52:22.437157Z_75a1514b-5f47-4e42-9821-c1bb8f8fb8fd",
    "timestamp": "2020-01-24 03:52:22.437157Z",
    "process_name": "ceph-mds",
    "entity_name": "mds.m2-1045557",
    "ceph_version": "14.2.4",
    "utsname_hostname": "m2-1045557",
    "utsname_sysname": "Linux",
    "utsname_release": "5.0.0-20-generic",
    "utsname_version": "#21~18.04.1-Ubuntu SMP Thu Jun 27 04:04:37 UTC 2019",
    "utsname_machine": "x86_64",
    "os_name": "Ubuntu",
    "os_id": "ubuntu",
    "os_version_id": "18.04",
    "os_version": "18.04.3 LTS (Bionic Beaver)",
    "assert_condition": "omap_num_objs <= MAX_OBJECTS",
    "assert_func": "void OpenFileTable::commit(MDSContext*, uint64_t, int)",
    "assert_file": "/build/ceph-14.2.4/src/mds/OpenFileTable.cc",
    "assert_line": 473,
    "assert_thread_name": "safe_timer",
    "assert_msg": "/build/ceph-14.2.4/src/mds/OpenFileTable.cc: In function 'void OpenFileTable::commit(MDSContext*, uint64_t, int)' thread 7ff467d8e700 time 2020-01-24 04:52:22.435687\n/build/ceph-14.2.4/src/mds/OpenFileTable.cc: 473: FAILED ceph_assert(omap_num_objs <= MAX_OBJECTS)\n",
    "backtrace": [
        "(()+0x12890) [0x7ff472f29890]",
        "(gsignal()+0xc7) [0x7ff472021e97]",
        "(abort()+0x141) [0x7ff472023801]",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7ff473620bb7]",
        "(()+0x27ad3e) [0x7ff473620d3e]",
        "(OpenFileTable::commit(MDSContext*, unsigned long, int)+0x1ab9) [0x558baea8c119]",
        "(MDLog::trim(int)+0x95d) [0x558baea795ad]",
        "(MDSRankDispatcher::tick()+0x27e) [0x558bae7ed83e]",
        "(FunctionContext::finish(int)+0x2c) [0x558bae7d8c0c]",
        "(Context::complete(int)+0x9) [0x558bae7d6a49]",
        "(SafeTimer::timer_thread()+0xf7) [0x7ff4736f4fe7]",
        "(SafeTimerThread::entry()+0xd) [0x7ff4736f5fed]",
        "(()+0x76db) [0x7ff472f1e6db]",
        "(clone()+0x3f) [0x7ff47210488f]" 
    ]
}