Ceph » mgr » Dashboard

Description of problem¶

Unfortunately I'm dealing with a damaged/unavailable filesystem on my cluster. When expanding the view for this filesystem on the dashboard it makes the mgr crash.

Environment¶

• ceph version string: 16.2.5
• Platform (OS/distro/release): Ubuntu 20.04
• Cluster details (nodes, monitors, OSDs): 3 nodes, 3 mons, 3 mgrs, 4 CephFS, 13 OSDs
• Browser used (e.g.: Version 86.0.4240.198 (Official Build) (64-bit)): Edge Version 94.0.992.31

How reproducible¶

Stop all MDS for a filesystem and then request to expand/view the information for that filesystem on your dashboard.

Actual results¶

Oct 04 12:51:09 node3 systemd¹: ceph-8c6c13ea-e866-11ea-ba42-c5e739a1a644@mgr.node3.hzzoiz.service: Main process exited, code=exited, status=134/n/a

Two crash logs:

root @ node3 # ceph crash info 2021-10-04T10:36:22.973087Z_5b239e07-28ec-48c9-abb3-5a6429680bcc
{
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12b20) [0x7ff5db721b20]",
        "gsignal()",
        "abort()",
        "/lib64/libstdc++.so.6(+0x9009b) [0x7ff5dab2309b]",
        "/lib64/libstdc++.so.6(+0x9653c) [0x7ff5dab2953c]",
        "/lib64/libstdc++.so.6(+0x96597) [0x7ff5dab29597]",
        "/lib64/libstdc++.so.6(+0x967f8) [0x7ff5dab297f8]",
        "/lib64/libstdc++.so.6(+0x9204b) [0x7ff5dab2504b]",
        "(Client::resolve_mds(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<mds_gid_t, std::allocator<mds_gid_t> >*)+0x1413) [0x560ca6761823]",
        "(Client::mds_command(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, ceph::buffer::v15_2_0::list const&, ceph::buffer::v15_2_0::list*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, Context*)+0x116) [0x560ca6761c56]",
        "/usr/bin/ceph-mgr(+0x1bb15a) [0x560ca65bb15a]",
        "/lib64/libpython3.6m.so.1.0(+0x19d287) [0x7ff5e5b35287]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0xf9b54) [0x7ff5e5a91b54]",
        "/lib64/libpython3.6m.so.1.0(+0x17a540) [0x7ff5e5b12540]",
        "/lib64/libpython3.6m.so.1.0(+0x19d527) [0x7ff5e5b35527]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0xf9b54) [0x7ff5e5a91b54]",
        "/lib64/libpython3.6m.so.1.0(+0x17a540) [0x7ff5e5b12540]",
        "/lib64/libpython3.6m.so.1.0(+0x19d527) [0x7ff5e5b35527]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0xf9b54) [0x7ff5e5a91b54]",
        "/lib64/libpython3.6m.so.1.0(+0x19c2ff) [0x7ff5e5b342ff]",
        "PyObject_Call()",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0x17a358) [0x7ff5e5b12358]",
        "/lib64/libpython3.6m.so.1.0(+0x19d527) [0x7ff5e5b35527]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0x17a358) [0x7ff5e5b12358]",
        "/lib64/libpython3.6m.so.1.0(+0x19d527) [0x7ff5e5b35527]",
        "_PyEval_EvalFrameDefault()",
        "_PyFunction_FastCallDict()",
        "_PyObject_FastCallDict()",
        "/lib64/libpython3.6m.so.1.0(+0x10e190) [0x7ff5e5aa6190]",
        "PyObject_Call()",
        "/lib64/libpython3.6m.so.1.0(+0x20e0c2) [0x7ff5e5ba60c2]",
        "/lib64/libpython3.6m.so.1.0(+0x1b4514) [0x7ff5e5b4c514]",
        "/lib64/libpthread.so.0(+0x814a) [0x7ff5db71714a]",
        "clone()" 
    ],
    "ceph_version": "16.2.5",
    "crash_id": "2021-10-04T10:36:22.973087Z_5b239e07-28ec-48c9-abb3-5a6429680bcc",
    "entity_name": "mgr.node3.hzzoiz",
    "os_id": "centos",
    "os_name": "CentOS Linux",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mgr",
    "stack_sig": "9de9126cdc703987461dc75a45ed35d3c116ce87f7cfa4e523a8462075f180de",
    "timestamp": "2021-10-04T10:36:22.973087Z",
    "utsname_hostname": "node3",
    "utsname_machine": "x86_64",
    "utsname_release": "5.11.0-25-generic",
    "utsname_sysname": "Linux",
    "utsname_version": "#27~20.04.1-Ubuntu SMP Tue Jul 13 17:41:23 UTC 2021" 
}

root @ node3 # ceph crash info 2021-10-04T10:26:25.281417Z_fb264d07-5b46-441a-872b-8f37b8270cab
{
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12b20) [0x7f244b1e3b20]",
        "gsignal()",
        "abort()",
        "/lib64/libstdc++.so.6(+0x9009b) [0x7f244a5e509b]",
        "/lib64/libstdc++.so.6(+0x9653c) [0x7f244a5eb53c]",
        "/lib64/libstdc++.so.6(+0x96597) [0x7f244a5eb597]",
        "/lib64/libstdc++.so.6(+0x967f8) [0x7f244a5eb7f8]",
        "/lib64/libstdc++.so.6(+0x9204b) [0x7f244a5e704b]",
        "(Client::resolve_mds(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<mds_gid_t, std::allocator<mds_gid_t> >*)+0x1413) [0x55a99e961823]",
        "(Client::mds_command(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, ceph::buffer::v15_2_0::list const&, ceph::buffer::v15_2_0::list*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, Context*)+0x116) [0x55a99e961c56]",
        "/usr/bin/ceph-mgr(+0x1bb15a) [0x55a99e7bb15a]",
        "/lib64/libpython3.6m.so.1.0(+0x19d287) [0x7f24555f7287]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0xf9b54) [0x7f2455553b54]",
        "/lib64/libpython3.6m.so.1.0(+0x17a540) [0x7f24555d4540]",
        "/lib64/libpython3.6m.so.1.0(+0x19d527) [0x7f24555f7527]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0xf9b54) [0x7f2455553b54]",
        "/lib64/libpython3.6m.so.1.0(+0x17a540) [0x7f24555d4540]",
        "/lib64/libpython3.6m.so.1.0(+0x19d527) [0x7f24555f7527]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0xf9b54) [0x7f2455553b54]",
        "/lib64/libpython3.6m.so.1.0(+0x19c2ff) [0x7f24555f62ff]",
        "PyObject_Call()",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0x17a358) [0x7f24555d4358]",
        "/lib64/libpython3.6m.so.1.0(+0x19d527) [0x7f24555f7527]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0x17a358) [0x7f24555d4358]",
        "/lib64/libpython3.6m.so.1.0(+0x19d527) [0x7f24555f7527]",
        "_PyEval_EvalFrameDefault()",
        "_PyFunction_FastCallDict()",
        "_PyObject_FastCallDict()",
        "/lib64/libpython3.6m.so.1.0(+0x10e190) [0x7f2455568190]",
        "PyObject_Call()",
        "/lib64/libpython3.6m.so.1.0(+0x20e0c2) [0x7f24556680c2]",
        "/lib64/libpython3.6m.so.1.0(+0x1b4514) [0x7f245560e514]",
        "/lib64/libpthread.so.0(+0x814a) [0x7f244b1d914a]",
        "clone()" 
    ],
    "ceph_version": "16.2.5",
    "crash_id": "2021-10-04T10:26:25.281417Z_fb264d07-5b46-441a-872b-8f37b8270cab",
    "entity_name": "mgr.node2.gjvfgq",
    "os_id": "centos",
    "os_name": "CentOS Linux",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mgr",
    "stack_sig": "9de9126cdc703987461dc75a45ed35d3c116ce87f7cfa4e523a8462075f180de",
    "timestamp": "2021-10-04T10:26:25.281417Z",
    "utsname_hostname": "node2",
    "utsname_machine": "x86_64",
    "utsname_release": "5.11.0-37-generic",
    "utsname_sysname": "Linux",
    "utsname_version": "#41~20.04.2-Ubuntu SMP Fri Sep 24 09:06:38 UTC 2021" 
}

Expected results¶

A user-friendly error stating that no MDS are available to handle your request.

Additional info¶