Ceph : Issues
https://tracker.ceph.com/
https://tracker.ceph.com/favicon.ico
2023-05-15T02:48:57Z
Ceph
Redmine
CephFS - Bug #60669 (New): crash: void Server::_unlink_local(MDRequestRef&, CDentry*, CDentry*): ...
https://tracker.ceph.com/issues/60669
2023-05-15T02:48:57Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2cf0c9e2e9b09fc177e74e6f80fbe3421f8ed33add829d5053d5d2117c16c955">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2cf0c9e2e9b09fc177e74e6f80fbe3421f8ed33add829d5053d5d2117c16c955</a></p>
<p>Assert condition: in->first <= straydn->first<br />Assert function: void Server::_unlink_local(MDRequestRef&, CDentry*, CDentry*)</p>
<p>Sanitized backtrace:<br /><pre> Server::_unlink_local(boost::intrusive_ptr<MDRequestImpl>&, CDentry*, CDentry*)
MDSContext::complete(int)
MDSRank::_advance_queues()
MDSRank::ProgressThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-03-22 14:06:38.572934",
"assert_condition": "in->first <= straydn->first",
"assert_file": "mds/Server.cc",
"assert_func": "void Server::_unlink_local(MDRequestRef&, CDentry*, CDentry*)",
"assert_line": 7863,
"assert_msg": "mds/Server.cc: In function 'void Server::_unlink_local(MDRequestRef&, CDentry*, CDentry*)' thread 7f9b10f9e700 time 2023-03-22T14:57:38.762830+0200\nmds/Server.cc: 7863: FAILED ceph_assert(in->first <= straydn->first)",
"assert_thread_name": "mds_rank_progr",
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x13140) [0x7f9b1a9db140]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17e) [0x7f9b1acb7296]",
"/usr/lib/ceph/libceph-common.so.2(+0x25a3da) [0x7f9b1acb73da]",
"(Server::_unlink_local(boost::intrusive_ptr<MDRequestImpl>&, CDentry*, CDentry*)+0x11af) [0x5636fdb3717f]",
"(MDSContext::complete(int)+0x5b) [0x5636fde013cb]",
"(MDSRank::_advance_queues()+0x80) [0x5636fdae4eb0]",
"(MDSRank::ProgressThread::entry()+0x105) [0x5636fdae54d5]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x7ea7) [0x7f9b1a9cfea7]",
"clone()"
],
"ceph_version": "17.2.5",
"crash_id": "2023-03-22T12:57:38.764970Z_0531660d-df0a-4973-96c9-9ba5fa13cbb1",
"entity_name": "mds.4e34e175e72968c4923197eaae6101b61886ab4a",
"os_id": "11",
"os_name": "Debian GNU/Linux 11 (bullseye)",
"os_version": "11 (bullseye)",
"os_version_id": "11",
"process_name": "ceph-mds",
"stack_sig": "d204545d2a103e73a5d2ccbf6cec7b1fb3efdce76038d68266f074961ce06634",
"timestamp": "2023-03-22T12:57:38.764970Z",
"utsname_machine": "x86_64",
"utsname_release": "5.10.0-21-amd64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Debian 5.10.162-1 (2023-01-21)"
}</pre></p>
CephFS - Bug #60629 (New): crash: void interval_set<T, C>::erase(T, T, std::function<bool(T, T)>)...
https://tracker.ceph.com/issues/60629
2023-05-15T02:47:53Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=978159d0d2675e074cadedff0e9abe7fe7254aa08b895f82334c012c99d4b838">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=978159d0d2675e074cadedff0e9abe7fe7254aa08b895f82334c012c99d4b838</a></p>
<p>Assert condition: p != m.end()<br />Assert function: void interval_set<T, C>::erase(T, T, std::function<bool(T, T)>) [with T = inodeno_t; C = std::map]</p>
<p>Sanitized backtrace:<br /><pre> interval_set<inodeno_t, std::map>::erase(inodeno_t, inodeno_t, std::function<bool (inodeno_t, inodeno_t)>)
Server::_session_logged(Session*, unsigned long, bool, unsigned long, interval_set<inodeno_t, std::map> const&, unsigned long, interval_set<inodeno_t, std::map> const&, LogSegment*)
C_MDS_session_finish::finish(int)
MDSContext::complete(int)
MDSIOContextBase::complete(int)
MDSLogContextBase::complete(int)
Finisher::finisher_thread_entry()
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-04-13 09:43:58.669151",
"assert_condition": "p != m.end()",
"assert_file": "include/interval_set.h",
"assert_func": "void interval_set<T, C>::erase(T, T, std::function<bool(T, T)>) [with T = inodeno_t; C = std::map]",
"assert_line": 567,
"assert_msg": "include/interval_set.h: In function 'void interval_set<T, C>::erase(T, T, std::function<bool(T, T)>) [with T = inodeno_t; C = std::map]' thread 7f642c5d1700 time 2023-04-12T23:14:02.750565+0100\ninclude/interval_set.h: 567: FAILED ceph_assert(p != m.end())",
"assert_thread_name": "MR_Finisher",
"backtrace": [
"/lib64/libpthread.so.0(+0x12cf0) [0x7f643a003cf0]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x7f643b00b4e3]",
"/usr/lib64/ceph/libceph-common.so.2(+0x26a64f) [0x7f643b00b64f]",
"(interval_set<inodeno_t, std::map>::erase(inodeno_t, inodeno_t, std::function<bool (inodeno_t, inodeno_t)>)+0x2a9) [0x55abfb247b29]",
"(Server::_session_logged(Session*, unsigned long, bool, unsigned long, interval_set<inodeno_t, std::map> const&, unsigned long, interval_set<inodeno_t, std::map> const&, LogSegment*)+0xf9) [0x55abfb1ece69]",
"(C_MDS_session_finish::finish(int)+0x41) [0x55abfb25f711]",
"(MDSContext::complete(int)+0x5f) [0x55abfb4b8fbf]",
"(MDSIOContextBase::complete(int)+0x534) [0x55abfb4b9754]",
"(MDSLogContextBase::complete(int)+0x5b) [0x55abfb4b9b1b]",
"(Finisher::finisher_thread_entry()+0x18d) [0x7f643b0aa56d]",
"/lib64/libpthread.so.0(+0x81cf) [0x7f6439ff91cf]",
"clone()"
],
"ceph_version": "17.2.6",
"crash_id": "2023-04-12T22:14:02.752804Z_8b75c750-8b43-4c91-b120-446f93f96562",
"entity_name": "mds.3177ab8603c724fe708f15f8148ba595387a8e81",
"os_id": "almalinux",
"os_name": "AlmaLinux",
"os_version": "8.7 (Stone Smilodon)",
"os_version_id": "8.7",
"process_name": "ceph-mds",
"stack_sig": "c4ba364febcb504210a580b8ea77a2c6038c6e20d116ccf592f71543842437fd",
"timestamp": "2023-04-12T22:14:02.752804Z",
"utsname_machine": "x86_64",
"utsname_release": "4.18.0-425.10.1.el8_7.x86_64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Thu Jan 12 10:05:00 EST 2023"
}</pre></p>
mgr - Bug #59852 (New): crash: File "mgr/k8sevents/__init__.py", in <module>: from .module import...
https://tracker.ceph.com/issues/59852
2023-05-15T02:28:40Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=21e6e9fd23b735b90c847971ce484c88795a56a8ea53d3a4b580355eaf945620">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=21e6e9fd23b735b90c847971ce484c88795a56a8ea53d3a4b580355eaf945620</a></p>
<p>Sanitized backtrace:<br /><pre> File "mgr/k8sevents/__init__.py", in <module>: from .module import Module
File "mgr/k8sevents/module.py", in <module>: from kubernetes.client.models.v1_event import V1Event
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
" File \"/usr/share/ceph/mgr/k8sevents/__init__.py\", line 1, in <module>\n from .module import Module",
" File \"/usr/share/ceph/mgr/k8sevents/module.py\", line 70, in <module>\n from kubernetes.client.models.v1_event import V1Event",
"<redacted>"
],
"ceph_version": "17.2.5",
"crash_id": "2023-04-21T13:05:21.451860Z_c8dd8f1a-5098-40eb-8a50-4e0e2d5fbf91",
"entity_name": "mgr.64846ab6a25c4f3895052f8f1d52c1e1d8c246a3",
"mgr_module": "k8sevents",
"mgr_module_caller": "PyModule::load_subclass_of",
"mgr_python_exception": "ModuleNotFoundError",
"os_id": "fedora",
"os_name": "Fedora Linux",
"os_version": "37 (Thirty Seven)",
"os_version_id": "37",
"process_name": "ceph-mgr",
"stack_sig": "3e6f3bd4c0c94e39be24c1a477ad1480cd0475cf18d128fe0503516405c48f88",
"timestamp": "2023-04-21T13:05:21.451860Z",
"utsname_machine": "x86_64",
"utsname_release": "6.2.8-200.fc37.x86_64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP PREEMPT_DYNAMIC Wed Mar 22 19:11:02 UTC 2023"
}</pre></p>
RADOS - Bug #59831 (New): crash: void ECBackend::continue_recovery_op(ECBackend::RecoveryOp&, Rec...
https://tracker.ceph.com/issues/59831
2023-05-15T02:28:09Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=9142b126ec0e6395147219ea2491c7ff242089468db66b588f9b7aef16759c8f">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=9142b126ec0e6395147219ea2491c7ff242089468db66b588f9b7aef16759c8f</a></p>
<p>Assert condition: pop.data.length() == sinfo.aligned_logical_offset_to_chunk_offset( after_progress.data_recovered_to - op.recovery_progress.data_recovered_to)<br />Assert function: void ECBackend::continue_recovery_op(ECBackend::RecoveryOp&, RecoveryMessages*)</p>
<p>Sanitized backtrace:<br /><pre> ECBackend::continue_recovery_op(ECBackend::RecoveryOp&, RecoveryMessages*)
ECBackend::handle_recovery_read_complete(hobject_t const&, boost::tuples::tuple<unsigned long, unsigned long, std::map<pg_shard_t, ceph::buffer::list, std::less<pg_shard_t>, std::allocator<std::pair<pg_shard_t const, ceph::buffer::list> > >, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type>&, std::optional<std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::list, std::less<void>, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::list> > > >, RecoveryMessages*)
OnRecoveryReadComplete::finish(std::pair<RecoveryMessages*, ECBackend::read_result_t&>&)
ECBackend::complete_read_op(ECBackend::ReadOp&, RecoveryMessages*)
ECBackend::handle_sub_read_reply(pg_shard_t, ECSubReadReply&, RecoveryMessages*, ZTracer::Trace const&)
ECBackend::_handle_message(boost::intrusive_ptr<OpRequest>)
PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)
PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)
OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)
ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
ShardedThreadPool::shardedthreadpool_worker(unsigned int)
ShardedThreadPool::WorkThreadSharded::entry()
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-04-11 12:16:54.864155",
"assert_condition": "pop.data.length() == sinfo.aligned_logical_offset_to_chunk_offset( after_progress.data_recovered_to - op.recovery_progress.data_recovered_to)",
"assert_file": "osd/ECBackend.cc",
"assert_func": "void ECBackend::continue_recovery_op(ECBackend::RecoveryOp&, RecoveryMessages*)",
"assert_line": 676,
"assert_msg": "osd/ECBackend.cc: In function 'void ECBackend::continue_recovery_op(ECBackend::RecoveryOp&, RecoveryMessages*)' thread 7f8883209700 time 2023-04-11T10:10:46.038828+0100\nosd/ECBackend.cc: 676: FAILED ceph_assert(pop.data.length() == sinfo.aligned_logical_offset_to_chunk_offset( after_progress.data_recovered_to - op.recovery_progress.data_recovered_to))",
"assert_thread_name": "tp_osd_tp",
"backtrace": [
"/lib64/libpthread.so.0(+0x12cf0) [0x7f88a424acf0]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x55b042d5468b]",
"/usr/bin/ceph-osd(+0x5977f7) [0x55b042d547f7]",
"(ECBackend::continue_recovery_op(ECBackend::RecoveryOp&, RecoveryMessages*)+0x1e30) [0x55b0432ad8b0]",
"(ECBackend::handle_recovery_read_complete(hobject_t const&, boost::tuples::tuple<unsigned long, unsigned long, std::map<pg_shard_t, ceph::buffer::v15_2_0::list, std::less<pg_shard_t>, std::allocator<std::pair<pg_shard_t const, ceph::buffer::v15_2_0::list> > >, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type>&, std::optional<std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > > >, RecoveryMessages*)+0x4c2) [0x55b0432ae052]",
"(OnRecoveryReadComplete::finish(std::pair<RecoveryMessages*, ECBackend::read_result_t&>&)+0x71) [0x55b0432d3e21]",
"(ECBackend::complete_read_op(ECBackend::ReadOp&, RecoveryMessages*)+0x8d) [0x55b04329e88d]",
"(ECBackend::handle_sub_read_reply(pg_shard_t, ECSubReadReply&, RecoveryMessages*, ZTracer::Trace const&)+0xd1c) [0x55b0432b9f4c]",
"(ECBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x2d4) [0x55b0432baa24]",
"(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x56) [0x55b04306d2c6]",
"(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x512) [0x55b042fff9f2]",
"(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x1c0) [0x55b042e55900]",
"(ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x6d) [0x55b0431542ad]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x115f) [0x55b042e68dbf]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x435) [0x55b0435c68c5]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55b0435c8fe4]",
"/lib64/libpthread.so.0(+0x81cf) [0x7f88a42401cf]",
"clone()"
],
"ceph_version": "17.2.5",
"crash_id": "2023-04-11T09:10:46.064419Z_07dabafc-ac37-4c3d-a8bc-2d94e758acd9",
"entity_name": "osd.5124c4347d97042388257a62052a332dbd538ede",
"os_id": "almalinux",
"os_name": "AlmaLinux",
"os_version": "8.7 (Stone Smilodon)",
"os_version_id": "8.7",
"process_name": "ceph-osd",
"stack_sig": "7b7aaff61eb3d064d2d3e9d204575affb0cc72f9424173ad80ce9a743f2eb0b7",
"timestamp": "2023-04-11T09:10:46.064419Z",
"utsname_machine": "x86_64",
"utsname_release": "4.18.0-425.10.1.el8_7.x86_64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Thu Jan 12 10:05:00 EST 2023"
}</pre></p>
bluestore - Bug #56788 (New): crash: void KernelDevice::_aio_thread(): abort
https://tracker.ceph.com/issues/56788
2022-07-28T02:17:12Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=c1fd5f4de1d415baece7e11982350afcdc4bd5d9fcebaf92de8aa1cd60c38f34">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=c1fd5f4de1d415baece7e11982350afcdc4bd5d9fcebaf92de8aa1cd60c38f34</a></p>
<p>Assert condition: abort<br />Assert function: void KernelDevice::_aio_thread()</p>
<p>Sanitized backtrace:<br /><pre> pthread_kill()
raise()
KernelDevice::_aio_thread()
KernelDevice::AioCompletionThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2022-07-09 21:01:05.232857",
"assert_condition": "abort",
"assert_file": "blk/kernel/KernelDevice.cc",
"assert_func": "void KernelDevice::_aio_thread()",
"assert_line": 617,
"assert_msg": "blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7f58457fa640 time 2022-07-09T16:53:00.980762-0400\nblk/kernel/KernelDevice.cc: 617: ceph_abort_msg(\"Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!\")",
"assert_thread_name": "bstore_aio",
"backtrace": [
"/lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f5861dce520]",
"pthread_kill()",
"raise()",
"abort()",
"(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x190) [0x5640326a3e9e]",
"(KernelDevice::_aio_thread()+0xe59) [0x564033285c39]",
"(KernelDevice::AioCompletionThread::entry()+0x11) [0x56403328f551]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7f5861e20b43]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7f5861eb2a00]"
],
"ceph_version": "17.2.0",
"crash_id": "2022-07-09T20:53:00.988485Z_2a09a4a9-254f-4ce6-bc45-2f703d2c4f1c",
"entity_name": "osd.bafedfc7637c5f9baa507101a851d388a1d22b0e",
"io_error": true,
"io_error_code": -5,
"io_error_devname": "dm-2",
"io_error_length": 4096,
"io_error_offset": 22167552,
"io_error_optype": 8,
"io_error_path": "/var/lib/ceph/osd/ceph-6/block",
"os_id": "22.04",
"os_name": "Ubuntu 22.04 LTS",
"os_version": "22.04 LTS (Jammy Jellyfish)",
"os_version_id": "22.04",
"process_name": "ceph-osd",
"stack_sig": "eceb646d37a5aa7dec4dd9a0298f64e5e5519f44e9589e68836110dcd62e82a7",
"timestamp": "2022-07-09T20:53:00.988485Z",
"utsname_machine": "x86_64",
"utsname_release": "5.15.0-39-generic",
"utsname_sysname": "Linux",
"utsname_version": "#42-Ubuntu SMP Thu Jun 9 23:42:32 UTC 2022"
}</pre></p>
RADOS - Bug #56772 (New): crash: uint64_t SnapSet::get_clone_bytes(snapid_t) const: assert(clone_...
https://tracker.ceph.com/issues/56772
2022-07-28T02:16:22Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=62b8a9e7f0bb7fc1fc81b2dcd9ceba2ba36ab9e25e03f08224b6946f9a1fc9d4">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=62b8a9e7f0bb7fc1fc81b2dcd9ceba2ba36ab9e25e03f08224b6946f9a1fc9d4</a></p>
<p>Assert condition: clone_overlap.count(clone)<br />Assert function: uint64_t SnapSet::get_clone_bytes(snapid_t) const</p>
<p>Sanitized backtrace:<br /><pre> SnapSet::get_clone_bytes(snapid_t) const
PrimaryLogPG::add_object_context_to_pg_stat(std::shared_ptr<ObjectContext>, pg_stat_t*)
PrimaryLogPG::recover_backfill(unsigned long, ThreadPool::TPHandle&, bool*)
PrimaryLogPG::start_recovery_ops(unsigned long, ThreadPool::TPHandle&, unsigned long*)
OSD::do_recovery(PG*, unsigned int, unsigned long, ThreadPool::TPHandle&)
ceph::osd::scheduler::PGRecovery::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
ShardedThreadPool::shardedthreadpool_worker(unsigned int)
ShardedThreadPool::WorkThreadSharded::entry()
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "clone_overlap.count(clone)",
"assert_file": "osd/osd_types.cc",
"assert_func": "uint64_t SnapSet::get_clone_bytes(snapid_t) const",
"assert_line": 5783,
"assert_msg": "osd/osd_types.cc: In function 'uint64_t SnapSet::get_clone_bytes(snapid_t) const' thread 7fe44f446700 time 2022-07-26T23:38:26.703737+1200\nosd/osd_types.cc: 5783: FAILED ceph_assert(clone_overlap.count(clone))",
"assert_thread_name": "tp_osd_tp",
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7fe46c5c0140]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x16e) [0x564bdcba0e78]",
"/usr/bin/ceph-osd(+0xac0fb9) [0x564bdcba0fb9]",
"(SnapSet::get_clone_bytes(snapid_t) const+0xe3) [0x564bdce988b3]",
"(PrimaryLogPG::add_object_context_to_pg_stat(std::shared_ptr<ObjectContext>, pg_stat_t*)+0x23e) [0x564bdcd8f16e]",
"(PrimaryLogPG::recover_backfill(unsigned long, ThreadPool::TPHandle&, bool*)+0x17d5) [0x564bdcdf56d5]",
"(PrimaryLogPG::start_recovery_ops(unsigned long, ThreadPool::TPHandle&, unsigned long*)+0xf35) [0x564bdcdfb3f5]",
"(OSD::do_recovery(PG*, unsigned int, unsigned long, ThreadPool::TPHandle&)+0x295) [0x564bdcc6be05]",
"(ceph::osd::scheduler::PGRecovery::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x19) [0x564bdcedc019]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xa27) [0x564bdcc89367]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41a) [0x564bdd3323da]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x564bdd3349b0]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7fe46c5b4ea7]",
"clone()"
],
"ceph_version": "16.2.9",
"crash_id": "2022-07-26T11:38:26.726934Z_7e80ef3c-e1c0-4957-9ca3-e4fc548200da",
"entity_name": "osd.5dd74b9e9d0b76a491b6a1cc86dffd15caf7242f",
"os_id": "11",
"os_name": "Debian GNU/Linux 11 (bullseye)",
"os_version": "11 (bullseye)",
"os_version_id": "11",
"process_name": "ceph-osd",
"stack_sig": "024700b37b3bc297e5ee455cf52e654a74ae210b2b22373907a02cc03ead01dc",
"timestamp": "2022-07-26T11:38:26.726934Z",
"utsname_machine": "x86_64",
"utsname_release": "5.15.39-1-pve",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP PVE 5.15.39-1 (Wed, 22 Jun 2022 17:22:00 +0200)"
}</pre></p>
RADOS - Bug #56770 (New): crash: void OSDShard::register_and_wake_split_child(PG*): assert(p != p...
https://tracker.ceph.com/issues/56770
2022-07-28T02:16:14Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=d9289f1067de7f0cc0e374ff0f760daa4519e04cfa1b7d05fab55e79a19192b3">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=d9289f1067de7f0cc0e374ff0f760daa4519e04cfa1b7d05fab55e79a19192b3</a></p>
<p>Assert condition: p != pg_slots.end()<br />Assert function: void OSDShard::register_and_wake_split_child(PG*)</p>
<p>Sanitized backtrace:<br /><pre> OSDShard::register_and_wake_split_child(PG*)
OSD::_finish_splits(std::set<boost::intrusive_ptr<PG>, std::less<boost::intrusive_ptr<PG> >, std::allocator<boost::intrusive_ptr<PG> > >&)
Context::complete(int)
OSD::ShardedOpWQ::handle_oncommits(std::list<Context*, std::allocator<Context*> >&)
OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
ShardedThreadPool::shardedthreadpool_worker(unsigned int)
ShardedThreadPool::WorkThreadSharded::entry()
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "p != pg_slots.end()",
"assert_file": "osd/OSD.cc",
"assert_func": "void OSDShard::register_and_wake_split_child(PG*)",
"assert_line": 10685,
"assert_msg": "osd/OSD.cc: In function 'void OSDShard::register_and_wake_split_child(PG*)' thread 7f848c1d5700 time 2022-07-25T21:07:35.537660+0100\nosd/OSD.cc: 10685: FAILED ceph_assert(p != pg_slots.end())",
"assert_thread_name": "tp_osd_tp",
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7f84ac363140]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17e) [0x5615e8ee0abc]",
"/usr/bin/ceph-osd(+0xc21c00) [0x5615e8ee0c00]",
"(OSDShard::register_and_wake_split_child(PG*)+0x9c3) [0x5615e8fa67b3]",
"(OSD::_finish_splits(std::set<boost::intrusive_ptr<PG>, std::less<boost::intrusive_ptr<PG> >, std::allocator<boost::intrusive_ptr<PG> > >&)+0x14e) [0x5615e8faac1e]",
"(Context::complete(int)+0x9) [0x5615e8fddf89]",
"(OSD::ShardedOpWQ::handle_oncommits(std::__cxx11::list<Context*, std::allocator<Context*> >&)+0x24) [0x5615e8fef424]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x73f) [0x5615e8fcae5f]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41a) [0x5615e96aa1aa]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5615e96ac780]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7f84ac357ea7]",
"clone()"
],
"ceph_version": "17.2.1",
"crash_id": "2022-07-25T20:07:35.666784Z_6faaa4f1-d8d8-4e75-8264-c32662b68cec",
"entity_name": "osd.80d14b2c0bf175fbe1c342ee221d7eca80db0e5f",
"os_id": "11",
"os_name": "Debian GNU/Linux 11 (bullseye)",
"os_version": "11 (bullseye)",
"os_version_id": "11",
"process_name": "ceph-osd",
"stack_sig": "3f66289ef3dde58c31697747e3918bff9113cac5235150605391ecc28e8a95e3",
"timestamp": "2022-07-25T20:07:35.666784Z",
"utsname_machine": "x86_64",
"utsname_release": "5.15.39-1-pve",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP PVE 5.15.39-1 (Wed, 22 Jun 2022 17:22:00 +0200)"
}</pre></p>
CephFS - Bug #54824 (New): crash: pthread_kill()
https://tracker.ceph.com/issues/54824
2022-03-19T01:25:16Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=6cbe446e1c71d34d8441b9f270e9359495d36b7a8395f1ae9c5563c66cd8b731">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=6cbe446e1c71d34d8441b9f270e9359495d36b7a8395f1ae9c5563c66cd8b731</a></p>
<p>Sanitized backtrace:<br /><pre> pthread_kill()
raise()
std::__throw_out_of_range(char const*)
MDCache::handle_cache_rejoin_ack(boost::intrusive_ptr<MMDSCacheRejoin const> const&)
MDCache::dispatch(boost::intrusive_ptr<Message const> const&)
MDSRank::_dispatch(boost::intrusive_ptr<Message const> const&, bool)
MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr<Message const> const&)
MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)
DispatchQueue::entry()
DispatchQueue::DispatchThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2022-01-28 06:13:47.719023",
"backtrace": [
"/lib/x86_64-linux-gnu/libc.so.6(+0x46520) [0x7fcfd42f9520]",
"pthread_kill()",
"raise()",
"abort()",
"/usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xa19f1) [0x7fcfd467a9f1]",
"/usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xad30c) [0x7fcfd468630c]",
"/usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xad377) [0x7fcfd4686377]",
"/usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xad619) [0x7fcfd4686619]",
"(std::__throw_out_of_range(char const*)+0x41) [0x7fcfd467d3bd]",
"(MDCache::handle_cache_rejoin_ack(boost::intrusive_ptr<MMDSCacheRejoin const> const&)+0x2fde) [0x56095c5e395e]",
"(MDCache::dispatch(boost::intrusive_ptr<Message const> const&)+0xdc) [0x56095c5f644c]",
"(MDSRank::_dispatch(boost::intrusive_ptr<Message const> const&, bool)+0x5b7) [0x56095c491477]",
"(MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr<Message const> const&)+0x53) [0x56095c491a83]",
"(MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x1bf) [0x56095c47d43f]",
"(Messenger::ms_deliver_dispatch(boost::intrusive_ptr<Message> const&)+0x450) [0x7fcfd4cb4420]",
"(DispatchQueue::entry()+0x647) [0x7fcfd4cb1807]",
"(DispatchQueue::DispatchThread::entry()+0x11) [0x7fcfd4d72561]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x98927) [0x7fcfd434b927]",
"clone()"
],
"ceph_version": "16.2.7",
"crash_id": "2022-01-23T06:12:53.319887Z_409478b1-9efc-411a-a421-21890bbee331",
"entity_name": "mds.bba3223b6961d3c90101d3ef927ed6db80baa3ce",
"os_id": "22.04",
"os_name": "Ubuntu Jammy Jellyfish (development branch)",
"os_version": "22.04 (Jammy Jellyfish)",
"os_version_id": "22.04",
"process_name": "ceph-mds",
"stack_sig": "6412208d2c7b5c388ebedf6d9a7df5e20d45c265643a835043b1e1758c662e8f",
"timestamp": "2022-01-23T06:12:53.319887Z",
"utsname_machine": "x86_64",
"utsname_release": "5.8.0-59-generic",
"utsname_sysname": "Linux",
"utsname_version": "#66-Ubuntu SMP Thu Jun 17 00:46:01 UTC 2021"
}</pre></p>
CephFS - Bug #54798 (New): crash: double const ceph::common::ConfigProxy::get_val<double>(std::ba...
https://tracker.ceph.com/issues/54798
2022-03-19T01:24:20Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=a6be1eafcec3445c3e9779d3e775e339dd9c48c7512cf89ac5737c2abff13a12">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=a6be1eafcec3445c3e9779d3e775e339dd9c48c7512cf89ac5737c2abff13a12</a></p>
<p>Sanitized backtrace:<br /><pre> double const ceph::common::ConfigProxy::get_val<double>(std::basic_string_view<char, std::char_traits<char> >) const
MDSRankDispatcher::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&, MDSMap const&)
MDSDaemon::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&)
MDSDaemon::handle_core_message(boost::intrusive_ptr<Message const> const&)
MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)
DispatchQueue::entry()
DispatchQueue::DispatchThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7fd485a76140]",
"gsignal()",
"abort()",
"/lib/x86_64-linux-gnu/libstdc++.so.6(+0x9a7ec) [0x7fd4859277ec]",
"/lib/x86_64-linux-gnu/libstdc++.so.6(+0xa5966) [0x7fd485932966]",
"/lib/x86_64-linux-gnu/libstdc++.so.6(+0xa59d1) [0x7fd4859329d1]",
"/lib/x86_64-linux-gnu/libstdc++.so.6(+0xa5c65) [0x7fd485932c65]",
"/usr/bin/ceph-mds(+0x13c2b6) [0x558c1c8c82b6]",
"(double const ceph::common::ConfigProxy::get_val<double>(std::basic_string_view<char, std::char_traits<char> >) const+0x99) [0x558c1c92ae39]",
"(MDSRankDispatcher::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&, MDSMap const&)+0x1ae) [0x558c1c95119e]",
"(MDSDaemon::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&)+0xd5a) [0x558c1c92178a]",
"(MDSDaemon::handle_core_message(boost::intrusive_ptr<Message const> const&)+0x37f) [0x558c1c92496f]",
"(MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x187) [0x558c1c9251e7]",
"(Messenger::ms_deliver_dispatch(boost::intrusive_ptr<Message> const&)+0x468) [0x7fd485f77448]",
"(DispatchQueue::entry()+0x5ff) [0x7fd485f74abf]",
"(DispatchQueue::DispatchThread::entry()+0xd) [0x7fd486036c3d]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7fd485a6aea7]",
"clone()"
],
"ceph_version": "16.2.7",
"crash_id": "2022-02-20T04:03:27.693374Z_d519e925-1413-42c0-b0ca-49c24cacdd6e",
"entity_name": "mds.20e498aa737f24767ed073d524fb7f19ff86197f",
"os_id": "11",
"os_name": "Debian GNU/Linux 11 (bullseye)",
"os_version": "11 (bullseye)",
"os_version_id": "11",
"process_name": "ceph-mds",
"stack_sig": "7e8e26a42e3616f3f9e5ca48bef80648ffe3f56da687a80cb469f84f71882368",
"timestamp": "2022-02-20T04:03:27.693374Z",
"utsname_machine": "x86_64",
"utsname_release": "5.15.0-0.bpo.2-amd64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Debian 5.15.5-2~bpo11+1 (2022-01-02)"
}</pre></p>
CephFS - Bug #54747 (New): crash: void EMetaBlob::replay(MDSRank*, LogSegment*, MDPeerUpdate*): a...
https://tracker.ceph.com/issues/54747
2022-03-19T01:22:37Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=13291700f923cdc78c6a9b9d8afc158829e3c403b8105fdfba5126230aa8821e">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=13291700f923cdc78c6a9b9d8afc158829e3c403b8105fdfba5126230aa8821e</a></p>
<p>Assert condition: inotablev == mds->inotable->get_version()<br />Assert function: void EMetaBlob::replay(MDSRank*, LogSegment*, MDPeerUpdate*)</p>
<p>Sanitized backtrace:<br /><pre> EMetaBlob::replay(MDSRank*, LogSegment*, MDPeerUpdate*)
EUpdate::replay(MDSRank*)
MDLog::_replay_thread()
MDLog::ReplayThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "inotablev == mds->inotable->get_version()",
"assert_file": "mds/journal.cc",
"assert_func": "void EMetaBlob::replay(MDSRank*, LogSegment*, MDPeerUpdate*)",
"assert_line": 1575,
"assert_msg": "mds/journal.cc: In function 'void EMetaBlob::replay(MDSRank*, LogSegment*, MDPeerUpdate*)' thread 7feec61c9700 time 2022-02-02T15:53:31.579890+0000\nmds/journal.cc: 1575: FAILED ceph_assert(inotablev == mds->inotable->get_version())",
"assert_thread_name": "md_log_replay",
"backtrace": [
"/lib64/libpthread.so.0(+0x12b20) [0x7feed53edb20]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x7feed6641b75]",
"/usr/lib64/ceph/libceph-common.so.2(+0x276d3e) [0x7feed6641d3e]",
"(EMetaBlob::replay(MDSRank*, LogSegment*, MDPeerUpdate*)+0x513e) [0x5575f175112e]",
"(EUpdate::replay(MDSRank*)+0x40) [0x5575f17534f0]",
"(MDLog::_replay_thread()+0xbd9) [0x5575f16da5d9]",
"(MDLog::ReplayThread::entry()+0x11) [0x5575f13dd051]",
"/lib64/libpthread.so.0(+0x814a) [0x7feed53e314a]",
"clone()"
],
"ceph_version": "16.2.5",
"crash_id": "2022-02-02T15:53:31.581539Z_07a5961e-c48b-454b-bd4c-69deb87c2090",
"entity_name": "mds.5d849d82ee6469a510861f454ee3fcc3eef772f1",
"os_id": "centos",
"os_name": "CentOS Linux",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-mds",
"stack_sig": "a8efede3427c2df6d81678e3b210fcc6178d28f0958b807fa777e97fa60b868b",
"timestamp": "2022-02-02T15:53:31.581539Z",
"utsname_machine": "x86_64",
"utsname_release": "5.11.0-44-generic",
"utsname_sysname": "Linux",
"utsname_version": "#48~20.04.2-Ubuntu SMP Tue Dec 14 15:36:44 UTC 2021"
}</pre></p>
RADOS - Bug #54744 (New): crash: void MonMap::add(const mon_info_t&): assert(addr_mons.count(a) =...
https://tracker.ceph.com/issues/54744
2022-03-19T01:22:30Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=0301792b024ebcd170453531530f95fe622fdcc6bb593bef38045e3654d96bcd">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=0301792b024ebcd170453531530f95fe622fdcc6bb593bef38045e3654d96bcd</a></p>
<p>Assert condition: addr_mons.count(a) == 0<br />Assert function: void MonMap::add(const mon_info_t&)</p>
<p>Sanitized backtrace:<br /><pre> MonMap::init_with_addrs(std::vector<entity_addrvec_t, std::allocator<entity_addrvec_t> > const&, bool, std::basic_string_view<char, std::char_traits<char> >)
MonMap::init_with_ips(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, std::basic_string_view<char, std::char_traits<char> >)
MonMap::build_initial(ceph::common::CephContext*, bool, std::ostream&)
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "addr_mons.count(a) == 0",
"assert_file": "mon/MonMap.h",
"assert_func": "void MonMap::add(const mon_info_t&)",
"assert_line": 221,
"assert_msg": "mon/MonMap.h: In function 'void MonMap::add(const mon_info_t&)' thread 7fdbfa1e1580 time 2021-12-14T20:00:23.402334+0000\nmon/MonMap.h: 221: FAILED ceph_assert(addr_mons.count(a) == 0)",
"assert_thread_name": "ceph-mon",
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x153c0) [0x7fdbfabd93c0]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1ad) [0x7fdbfb0f4db0]",
"/usr/lib/ceph/libceph-common.so.2(+0x265f5d) [0x7fdbfb0f4f5d]",
"(MonMap::init_with_addrs(std::vector<entity_addrvec_t, std::allocator<entity_addrvec_t> > const&, bool, std::basic_string_view<char, std::char_traits<char> >)+0x3a8) [0x7fdbfb4e80a8]",
"(MonMap::init_with_ips(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, std::basic_string_view<char, std::char_traits<char> >)+0x93) [0x7fdbfb4e84a3]",
"(MonMap::build_initial(ceph::common::CephContext*, bool, std::ostream&)+0x837) [0x7fdbfb4ea597]",
"main()",
"__libc_start_main()",
"_start()"
],
"ceph_version": "16.2.7",
"crash_id": "2021-12-14T20:00:23.405800Z_673ad289-fd40-4f50-b3d4-e43fa654ecb3",
"entity_name": "mon.8d7b2c1b42f80e192ec02d5fda7e1d93895fe9e0",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.3 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "ceph-mon",
"stack_sig": "0ab303078f78301b00a08a4683ab26737444aad0204ae8af0f8a8fb705db5424",
"timestamp": "2021-12-14T20:00:23.405800Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-91-generic",
"utsname_sysname": "Linux",
"utsname_version": "#102-Ubuntu SMP Fri Nov 5 16:31:28 UTC 2021"
}</pre></p>
CephFS - Bug #54741 (New): crash: MDSTableClient::got_journaled_ack(unsigned long)
https://tracker.ceph.com/issues/54741
2022-03-19T01:22:25Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=366ab44cb3c1d002359d6d1dbe2b4287e261f13517807620ef5a83625e3c0a2b">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=366ab44cb3c1d002359d6d1dbe2b4287e261f13517807620ef5a83625e3c0a2b</a></p>
<p>Sanitized backtrace:<br /><pre> MDSTableClient::got_journaled_ack(unsigned long)
ETableClient::replay(MDSRank*)
MDLog::_replay_thread()
MDLog::ReplayThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
"/lib64/libpthread.so.0(+0x12c20) [0x7f7500e67c20]",
"(std::_Hashtable<unsigned long, unsigned long, std::allocator<unsigned long>, std::__detail::_Identity, std::equal_to<unsigned long>, std::hash<unsigned long>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, true, true> >::_M_erase(std::integral_constant<bool, true>, unsigned long const&)+0x37) [0x55651af0bb97]",
"(MDSTableClient::got_journaled_ack(unsigned long)+0x179) [0x55651af09159]",
"(ETableClient::replay(MDSRank*)+0x1b3) [0x55651afd8d03]",
"(MDLog::_replay_thread()+0xcd1) [0x55651af688e1]",
"(MDLog::ReplayThread::entry()+0x11) [0x55651ac6a311]",
"/lib64/libpthread.so.0(+0x817a) [0x7f7500e5d17a]",
"clone()"
],
"ceph_version": "16.2.7",
"crash_id": "2022-03-01T10:31:29.990280Z_af75c1cc-080f-4364-9410-565aa6812923",
"entity_name": "mds.79d0a07c1449cab02f6cf00fb829f33d1b9f49e6",
"os_id": "centos",
"os_name": "CentOS Linux",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-mds",
"stack_sig": "c009c98d5387ea202da11cd36585a1c06b8d48ee740496c707f1171a5332598c",
"timestamp": "2022-03-01T10:31:29.990280Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-74-generic",
"utsname_sysname": "Linux",
"utsname_version": "#83-Ubuntu SMP Sat May 8 02:35:39 UTC 2021"
}</pre></p>
CephFS - Bug #54665 (New): crash: void ObjectCacher::bh_write_commit(int64_t, sobject_t, std::vec...
https://tracker.ceph.com/issues/54665
2022-03-19T01:18:44Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=40f5abbee44ea6074376240bf7bcd505f895b21c4b0c01a141427fd0637279d8">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=40f5abbee44ea6074376240bf7bcd505f895b21c4b0c01a141427fd0637279d8</a></p>
<p>Assert condition: ob->last_commit_tid < tid<br />Assert function: void ObjectCacher::bh_write_commit(int64_t, sobject_t, std::vector<std::pair<long int, long unsigned int> >&, ceph_tid_t, int)</p>
<p>Sanitized backtrace:<br /><pre> ObjectCacher::bh_write_commit(long, sobject_t, std::vector<std::pair<long, unsigned long>, std::allocator<std::pair<long, unsigned long> > >&, unsigned long, int)
ObjectCacher::C_WriteCommit::finish(int)
Context::complete(int)
C_Lock::finish(int)
Context::complete(int)
Finisher::finisher_thread_entry()
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "ob->last_commit_tid < tid",
"assert_file": "osdc/ObjectCacher.cc",
"assert_func": "void ObjectCacher::bh_write_commit(int64_t, sobject_t, std::vector<std::pair<long int, long unsigned int> >&, ceph_tid_t, int)",
"assert_line": 1254,
"assert_msg": "osdc/ObjectCacher.cc: In function 'void ObjectCacher::bh_write_commit(int64_t, sobject_t, std::vector<std::pair<long int, long unsigned int> >&, ceph_tid_t, int)' thread ffff65ff9880 time 2022-02-21T23:24:53.616748+0100\nosdc/ObjectCacher.cc: 1254: FAILED ceph_assert(ob->last_commit_tid < tid)",
"assert_thread_name": "fn_anonymous",
"backtrace": [
"__kernel_rt_sigreturn()",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x194) [0xffff79a3a958]",
"(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0xffff79a3aac8]",
"(ObjectCacher::bh_write_commit(long, sobject_t, std::vector<std::pair<long, unsigned long>, std::allocator<std::pair<long, unsigned long> > >&, unsigned long, int)+0x1468) [0xaaaaac872b50]",
"(ObjectCacher::C_WriteCommit::finish(int)+0x78) [0xaaaaac877648]",
"(Context::complete(int)+0x1c) [0xaaaaac812624]",
"(C_Lock::finish(int)+0x50) [0xaaaaac815368]",
"(Context::complete(int)+0x1c) [0xaaaaac812624]",
"(Finisher::finisher_thread_entry()+0x164) [0xffff79a9bcfc]",
"/lib/aarch64-linux-gnu/libpthread.so.0(+0x84fc) [0xffff797b44fc]",
"/lib/aarch64-linux-gnu/libc.so.6(+0xd467c) [0xffff794f067c]"
],
"ceph_version": "16.2.7",
"crash_id": "2022-02-21T22:24:53.639829Z_51e0095e-ae2d-46e9-a46a-6cc11047277a",
"entity_name": "client.779720a56c617f4713d46a0389a5f0b5c78d2903",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.4 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "ceph-fuse",
"stack_sig": "b680a903695b7b6fc40b5c3fa66258c3b26f8ad09e710ae266eeb5de9f08e41b",
"timestamp": "2022-02-21T22:24:53.639829Z",
"utsname_machine": "aarch64",
"utsname_release": "5.4.0-1015-raspi",
"utsname_sysname": "Linux",
"utsname_version": "#15-Ubuntu SMP Fri Jul 10 05:34:24 UTC 2020"
}</pre></p>
CephFS - Bug #54644 (New): crash: void SessionMap::replay_open_sessions(version_t, std::map<clien...
https://tracker.ceph.com/issues/54644
2022-03-19T01:17:34Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=b4f84116e68bc35269f3752a633576016917c2224bc891c792dd0600d7e76e2f">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=b4f84116e68bc35269f3752a633576016917c2224bc891c792dd0600d7e76e2f</a></p>
<p>Assert condition: g_conf()->mds_wipe_sessions<br />Assert function: void SessionMap::replay_open_sessions(version_t, std::map<client_t, entity_inst_t>&, std::map<client_t, client_metadata_t>&)</p>
<p>Sanitized backtrace:<br /><pre> SessionMap::replay_open_sessions(unsigned long, std::map<client_t, entity_inst_t, std::less<client_t>, std::allocator<std::pair<client_t const, entity_inst_t> > >&, std::map<client_t, client_metadata_t, std::less<client_t>, std::allocator<std::pair<client_t const, client_metadata_t> > >&)
EImportStart::replay(MDSRank*)
MDLog::_replay_thread()
MDLog::ReplayThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "g_conf()->mds_wipe_sessions",
"assert_file": "mds/SessionMap.cc",
"assert_func": "void SessionMap::replay_open_sessions(version_t, std::map<client_t, entity_inst_t>&, std::map<client_t, client_metadata_t>&)",
"assert_line": 785,
"assert_msg": "mds/SessionMap.cc: In function 'void SessionMap::replay_open_sessions(version_t, std::map<client_t, entity_inst_t>&, std::map<client_t, client_metadata_t>&)' thread 7f0021f33700 time 2022-01-15T00:00:34.437781+0000\nmds/SessionMap.cc: 785: FAILED ceph_assert(g_conf()->mds_wipe_sessions)",
"assert_thread_name": "md_log_replay",
"backtrace": [
"/lib64/libpthread.so.0(+0x12c20) [0x7f0030f46c20]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x7f0031f58ba3]",
"/usr/lib64/ceph/libceph-common.so.2(+0x276d6c) [0x7f0031f58d6c]",
"(SessionMap::replay_open_sessions(unsigned long, std::map<client_t, entity_inst_t, std::less<client_t>, std::allocator<std::pair<client_t const, entity_inst_t> > >&, std::map<client_t, client_metadata_t, std::less<client_t>, std::allocator<std::pair<client_t const, client_metadata_t> > >&)+0x281) [0x5589bb929ac1]",
"(EImportStart::replay(MDSRank*)+0x77e) [0x5589bb9c3e5e]",
"(MDLog::_replay_thread()+0xcd1) [0x5589bb9468e1]",
"(MDLog::ReplayThread::entry()+0x11) [0x5589bb648311]",
"/lib64/libpthread.so.0(+0x817a) [0x7f0030f3c17a]",
"clone()"
],
"ceph_version": "16.2.7",
"crash_id": "2022-01-15T00:00:34.438991Z_fb3a7d15-b402-4ab7-8a7a-9f6cbf6f1672",
"entity_name": "mds.85473b6cddc6781fc8e34f5c98e9b821712ac936",
"os_id": "centos",
"os_name": "CentOS Linux",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-mds",
"stack_sig": "6a0a526b671741cb20ec843c6a1c8311d7ee6fc775fb2463634ea053833fb027",
"timestamp": "2022-01-15T00:00:34.438991Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-74-generic",
"utsname_sysname": "Linux",
"utsname_version": "#83-Ubuntu SMP Sat May 8 02:35:39 UTC 2021"
}</pre></p>
CephFS - Bug #54636 (New): crash: void Locker::file_recover(ScatterLock*): assert(lock->get_state...
https://tracker.ceph.com/issues/54636
2022-03-19T01:17:13Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=b5ee69eea19cfb224a80033a9ad6e8e86b81d1ed57fb2228a4859aa37db00740">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=b5ee69eea19cfb224a80033a9ad6e8e86b81d1ed57fb2228a4859aa37db00740</a></p>
<p>Assert condition: lock->get_state() == LOCK_PRE_SCAN<br />Assert function: void Locker::file_recover(ScatterLock*)</p>
<p>Sanitized backtrace:<br /><pre> Locker::file_recover(ScatterLock*)
MDCache::start_files_to_recover()
MDSRank::recovery_done(int)
MDSRankDispatcher::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&, MDSMap const&)
MDSDaemon::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&)
MDSDaemon::handle_core_message(boost::intrusive_ptr<Message const> const&)
MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)
DispatchQueue::entry()
DispatchQueue::DispatchThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "lock->get_state() == LOCK_PRE_SCAN",
"assert_file": "mds/Locker.cc",
"assert_func": "void Locker::file_recover(ScatterLock*)",
"assert_line": 5682,
"assert_msg": "mds/Locker.cc: In function 'void Locker::file_recover(ScatterLock*)' thread 7f393e96b700 time 2022-03-11T02:00:47.944098+0000\nmds/Locker.cc: 5682: FAILED ceph_assert(lock->get_state() == LOCK_PRE_SCAN)",
"assert_thread_name": "ms_dispatch",
"backtrace": [
"/lib64/libpthread.so.0(+0x12c20) [0x7f394616fc20]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x7f3947181ba3]",
"/usr/lib64/ceph/libceph-common.so.2(+0x276d6c) [0x7f3947181d6c]",
"(Locker::file_recover(ScatterLock*)+0x1bf) [0x55b27fda4ecf]",
"(MDCache::start_files_to_recover()+0x10b) [0x55b27fca2c3b]",
"(MDSRank::recovery_done(int)+0x6f) [0x55b27fbca61f]",
"(MDSRankDispatcher::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&, MDSMap const&)+0x207d) [0x55b27fbdbb2d]",
"(MDSDaemon::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&)+0xeee) [0x55b27fbaf27e]",
"(MDSDaemon::handle_core_message(boost::intrusive_ptr<Message const> const&)+0xcd) [0x55b27fbb2a3d]",
"(MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0xc3) [0x55b27fbb3593]",
"(DispatchQueue::entry()+0x126a) [0x7f39473c5aba]",
"(DispatchQueue::DispatchThread::entry()+0x11) [0x7f39474775d1]",
"/lib64/libpthread.so.0(+0x817a) [0x7f394616517a]",
"clone()"
],
"ceph_version": "16.2.7",
"crash_id": "2022-03-11T02:00:47.948422Z_d57cce74-ff0c-45bb-976e-39aa15263ef3",
"entity_name": "mds.f238d984f4c53535cf55055a45e907f2301806b1",
"os_id": "centos",
"os_name": "CentOS Linux",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-mds",
"stack_sig": "1c91b83ae8f5f093009caf48fc6dd9de533c52749ec2aa4228e6665d22696c85",
"timestamp": "2022-03-11T02:00:47.948422Z",
"utsname_machine": "x86_64",
"utsname_release": "5.11.0-38-generic",
"utsname_sysname": "Linux",
"utsname_version": "#42~20.04.1-Ubuntu SMP Tue Sep 28 20:41:07 UTC 2021"
}</pre></p>