Bug #54750
crash: PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine>::my_context): abort
Status:
New
Priority:
Normal
Assignee:
-
Category:
-
Target version:
-
% Done:
0%
Source:
Telemetry
Tags:
Backport:
quincy,pacific
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
75608643bcbc3b81d4d72c0ae52f02590c25903bc362525b06fd89a97707e75d
9124409e46ab62be2213af274344fc1d757aa5230c94a7e135c8b2bddcbe8e75
Crash signature (v2):
Description
Assert condition: abort
Assert function: PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine>::my_context)
Sanitized backtrace:
PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)>::my_context) boost::statechart::simple_state<PeeringState::Started, PeeringState::PeeringMachine, PeeringState::Start, (boost::statechart::history_mode)>::react_impl(boost::statechart::event_base const&, void const*) boost::statechart::state_machine<PeeringState::PeeringMachine, PeeringState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&) PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, PeeringCtx&) OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&) ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&) OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) ShardedThreadPool::shardedthreadpool_worker(unsigned int) ShardedThreadPool::WorkThreadSharded::entry()
Crash dump sample:
{ "assert_condition": "abort", "assert_file": "osd/PeeringState.cc", "assert_func": "PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine>::my_context)", "assert_line": 4605, "assert_msg": "osd/PeeringState.cc: In function 'PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine>::my_context)' thread 7fd5d5036700 time 2021-12-23T06:08:48.423500+0100\nosd/PeeringState.cc: 4605: ceph_abort_msg(\"we got a bad state machine event\")", "assert_thread_name": "tp_osd_tp", "backtrace": [ "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7fd5f11ad140]", "gsignal()", "abort()", "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x18a) [0x55c5778b7196]", "(PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::my_context)+0xe0) [0x55c577c359b0]", "/usr/bin/ceph-osd(+0xe4f936) [0x55c577c49936]", "(boost::statechart::simple_state<PeeringState::Started, PeeringState::PeeringMachine, PeeringState::Start, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x145) [0x55c577c78e45]", "(boost::statechart::state_machine<PeeringState::PeeringMachine, PeeringState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x84) [0x55c577a4ab14]", "(PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, PeeringCtx&)+0xe8) [0x55c577a2f358]", "(OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x1b4) [0x55c577991e44]", "(ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x55) [0x55c577bef0b5]", "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xa27) [0x55c5779a4417]", "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41a) [0x55c57803e4ca]", "(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x55c578040830]", "/lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7fd5f11a1ea7]", "clone()" ], "ceph_version": "16.2.7", "crash_id": "2021-12-23T05:08:48.450814Z_f029daac-9f67-4232-b0aa-8c039ac1a4ae", "entity_name": "osd.8d96fca144057f26523df5e99f9ec660b2755058", "os_id": "11", "os_name": "Debian GNU/Linux 11 (bullseye)", "os_version": "11 (bullseye)", "os_version_id": "11", "process_name": "ceph-osd", "stack_sig": "75608643bcbc3b81d4d72c0ae52f02590c25903bc362525b06fd89a97707e75d", "timestamp": "2021-12-23T05:08:48.450814Z", "utsname_machine": "x86_64", "utsname_release": "5.13.19-2-pve", "utsname_sysname": "Linux", "utsname_version": "#1 SMP PVE 5.13.19-4 (Mon, 29 Nov 2021 12:10:09 +0100)" }
History
#1 Updated by Telemetry Bot about 2 years ago
#2 Updated by Laura Flores about 1 year ago
/a/yuriw-2023-02-22_20:55:15-rados-wip-yuri4-testing-2023-02-22-0817-quincy-distro-default-smithi/7184685
2023-02-23T07:52:05.305 INFO:tasks.ceph.osd.2.smithi111.stderr:/build/ceph-17.2.5-1072-gcbccb547/src/osd/PeeringState.cc: In function 'PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMac
hine>::my_context)' thread 7f6bbc1fe700 time 2023-02-23T07:52:05.202994+0000
2023-02-23T07:52:05.305 INFO:tasks.ceph.osd.2.smithi111.stderr:/build/ceph-17.2.5-1072-gcbccb547/src/osd/PeeringState.cc: 4544: ceph_abort_msg("we got a bad state machine event")
2023-02-23T07:52:05.305 INFO:tasks.ceph.osd.2.smithi111.stderr: ceph version 17.2.5-1072-gcbccb547 (cbccb547f47ec697c2e2ecf23392cc636ea19450) quincy (stable)
2023-02-23T07:52:05.305 INFO:tasks.ceph.osd.2.smithi111.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe0) [0x55c32f7725b3]
2023-02-23T07:52:05.305 INFO:tasks.ceph.osd.2.smithi111.stderr: 2: (PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_
::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mp
l_::na, mpl_::na>, (boost::statechart::history_mode)0>::my_context)+0xdf) [0x55c32fba789f]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 3: ceph-osd(+0x1076626) [0x55c32fbbd626]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 4: (boost::statechart::simple_state<PeeringState::Started, PeeringState::PeeringMachine, PeeringState::Start, (boost::statechart::history_mode)0>::react_impl(boost::statechar
t::event_base const&, void const*)+0x168) [0x55c32fbf3ba8]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 5: (boost::statechart::simple_state<PeeringState::Deleting, PeeringState::ToDelete, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl
_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::st
atechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xe3) [0x55c32fbe6b53]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 6: (boost::statechart::state_machine<PeeringState::PeeringMachine, PeeringState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translato
r>::process_event(boost::statechart::event_base const&)+0x74) [0x55c32f926d54]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 7: (PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, PeeringCtx&)+0x1aa) [0x55c32f912b3a]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 8: (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x34d) [0x55c32f88731d]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 9: (ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x56) [0x55c32fb5f3a6]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 10: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x935) [0x55c32f8701c5]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 11: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x403) [0x55c330000c43]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 12: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55c330003c84]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 13: /lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7f6bdbb8d609]
2023-02-23T07:52:05.306 INFO:tasks.ceph.osd.2.smithi111.stderr: 14: clone()
{
"crash_id": "2023-02-23T07:52:05.223818Z_c018c725-4158-4e11-8aa7-8f8bb275cf55",
"timestamp": "2023-02-23T07:52:05.223818Z",
"process_name": "ceph-osd",
"entity_name": "osd.2",
"ceph_version": "17.2.5-1072-gcbccb547",
"utsname_hostname": "smithi111",
"utsname_sysname": "Linux",
"utsname_release": "5.4.0-139-generic",
"utsname_version": "#156-Ubuntu SMP Fri Jan 20 17:27:18 UTC 2023",
"utsname_machine": "x86_64",
"os_name": "Ubuntu",
"os_id": "ubuntu",
"os_version_id": "20.04",
"os_version": "20.04.4 LTS (Focal Fossa)",
"assert_condition": "abort",
"assert_func": "PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine>::my_context)",
"assert_file": "/build/ceph-17.2.5-1072-gcbccb547/src/osd/PeeringState.cc",
"assert_line": 4544,
"assert_thread_name": "tp_osd_tp",
"assert_msg": "/build/ceph-17.2.5-1072-gcbccb547/src/osd/PeeringState.cc: In function 'PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine>::my_context)' thread 7f6bbc1fe700 time 2023-02-23T07:52:05.202994+0000\n/build/ceph-17.2.5-1072-gcbccb547/src/osd/PeeringState.cc: 4544: ceph_abort_msg(\"we got a bad state machine event\")\n",
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f6bdbb99420]",
"gsignal()",
"abort()",
"(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b0) [0x55c32f772683]",
"(PeeringState::Crashed::Crashed(boost::statechart::state<PeeringState::Crashed, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::my_context)+0xdf) [0x55c32fba789f]",
"ceph-osd(+0x1076626) [0x55c32fbbd626]",
"(boost::statechart::simple_state<PeeringState::Started, PeeringState::PeeringMachine, PeeringState::Start, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x168) [0x55c32fbf3ba8]",
"(boost::statechart::simple_state<PeeringState::Deleting, PeeringState::ToDelete, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xe3) [0x55c32fbe6b53]",
"(boost::statechart::state_machine<PeeringState::PeeringMachine, PeeringState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x74) [0x55c32f926d54]",
"(PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, PeeringCtx&)+0x1aa) [0x55c32f912b3a]",
"(OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x34d) [0x55c32f88731d]",
"(ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x56) [0x55c32fb5f3a6]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x935) [0x55c32f8701c5]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x403) [0x55c330000c43]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55c330003c84]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7f6bdbb8d609]",
"clone()"
]
}
#3 Updated by Laura Flores about 1 year ago
- Backport set to quincy,pacific
- Affected Versions v17.2.5 added
- Affected Versions deleted (
v16.2.7)
#4 Updated by Laura Flores about 1 year ago
- Affected Versions v16.2.7 added
- Affected Versions deleted (
v17.2.5)
#5 Updated by Telemetry Bot 11 months ago
- Crash signature (v1) updated (diff)
- Affected Versions v16.2.9, v17.2.5 added