Bug #54909
crash: virtual bool PrimaryLogPG::should_send_op(pg_shard_t, const hobject_t&): assert(is_backfill_target(peer))
Status:
New
Priority:
Normal
Assignee:
-
Category:
-
Target version:
-
% Done:
0%
Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
c7db9da3b09065c50965cca9ae1531df57ed6397017dec028f2e068c710db8ee
Crash signature (v2):
Description
Assert condition: is_backfill_target(peer)
Assert function: virtual bool PrimaryLogPG::should_send_op(pg_shard_t, const hobject_t&)
Sanitized backtrace:
PrimaryLogPG::should_send_op(pg_shard_t, hobject_t const&) ReplicatedBackend::generate_subop(hobject_t const&, eversion_t const&, unsigned long, osd_reqid_t, eversion_t, eversion_t, hobject_t, hobject_t, ceph::buffer::list const&, std::optional<pg_hit_set_history_t>&, ceph::os::Transaction&, pg_shard_t, pg_info_t const&) ReplicatedBackend::issue_op(hobject_t const&, eversion_t const&, unsigned long, osd_reqid_t, eversion_t, eversion_t, hobject_t, hobject_t, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, std::optional<pg_hit_set_history_t>&, ReplicatedBackend::InProgressOp*, ceph::os::Transaction&) ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> >&&, std::optional<pg_hit_set_history_t>&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>) PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*) PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >) PrimaryLogPG::AwaitAsyncWork::react(PrimaryLogPG::DoSnapWork const&) boost::statechart::simple_state<PrimaryLogPG::AwaitAsyncWork, PrimaryLogPG::Trimming, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)>::react_impl(boost::statechart::event_base const&, void const*) boost::statechart::state_machine<PrimaryLogPG::SnapTrimmer, PrimaryLogPG::NotTrimming, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&) PrimaryLogPG::snap_trimmer(unsigned int) ceph::osd::scheduler::PGSnapTrim::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&) OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) ShardedThreadPool::shardedthreadpool_worker(unsigned int) ShardedThreadPool::WorkThreadSharded::entry()
Crash dump sample:
{ "assert_condition": "is_backfill_target(peer)", "assert_file": "osd/PrimaryLogPG.cc", "assert_func": "virtual bool PrimaryLogPG::should_send_op(pg_shard_t, const hobject_t&)", "assert_line": 550, "assert_msg": "osd/PrimaryLogPG.cc: In function 'virtual bool PrimaryLogPG::should_send_op(pg_shard_t, const hobject_t&)' thread 7f0da0052700 time 2022-02-27T19:19:31.818439+0000\nosd/PrimaryLogPG.cc: 550: FAILED ceph_assert(is_backfill_target(peer))", "assert_thread_name": "tp_osd_tp", "backtrace": [ "/lib64/libpthread.so.0(+0x12c20) [0x7f0dc217ac20]", "gsignal()", "abort()", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x5577565edbcf]", "/usr/bin/ceph-osd(+0x56ad98) [0x5577565edd98]", "(PrimaryLogPG::should_send_op(pg_shard_t, hobject_t const&)+0x494) [0x5577567ec294]", "(ReplicatedBackend::generate_subop(hobject_t const&, eversion_t const&, unsigned long, osd_reqid_t, eversion_t, eversion_t, hobject_t, hobject_t, ceph::buffer::v15_2_0::list const&, std::optional<pg_hit_set_history_t>&, ceph::os::Transaction&, pg_shard_t, pg_info_t const&)+0x70e) [0x557756a7f52e]", "(ReplicatedBackend::issue_op(hobject_t const&, eversion_t const&, unsigned long, osd_reqid_t, eversion_t, eversion_t, hobject_t, hobject_t, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, std::optional<pg_hit_set_history_t>&, ReplicatedBackend::InProgressOp*, ceph::os::Transaction&)+0x4f8) [0x557756a80458]", "(ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> >&&, std::optional<pg_hit_set_history_t>&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>)+0xa94) [0x557756a82ef4]", "(PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0xcf0) [0x557756801000]", "(PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)+0x168) [0x5577568032a8]", "(PrimaryLogPG::AwaitAsyncWork::react(PrimaryLogPG::DoSnapWork const&)+0x727) [0x55775683add7]", "(boost::statechart::simple_state<PrimaryLogPG::AwaitAsyncWork, PrimaryLogPG::Trimming, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x82) [0x5577568c17e2]", "(boost::statechart::state_machine<PrimaryLogPG::SnapTrimmer, PrimaryLogPG::NotTrimming, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x5b) [0x55775688ec8b]", "(PrimaryLogPG::snap_trimmer(unsigned int)+0x17a) [0x5577567f859a]", "(ceph::osd::scheduler::PGSnapTrim::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x1a) [0x557756952dda]", "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xc28) [0x557756715d48]", "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x557756d875b4]", "(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x557756d8a254]", "/lib64/libpthread.so.0(+0x817a) [0x7f0dc217017a]", "clone()" ], "ceph_version": "16.2.7", "crash_id": "2022-02-27T19:19:31.838691Z_5f306b16-d3fc-46ca-a050-5e8d226d3f04", "entity_name": "osd.0555675fc3df70df5caf1682c7a1b9b41e63b0af", "os_id": "centos", "os_name": "CentOS Linux", "os_version": "8", "os_version_id": "8", "process_name": "ceph-osd", "stack_sig": "c7db9da3b09065c50965cca9ae1531df57ed6397017dec028f2e068c710db8ee", "timestamp": "2022-02-27T19:19:31.838691Z", "utsname_machine": "x86_64", "utsname_release": "5.4.140-1-pve", "utsname_sysname": "Linux", "utsname_version": "#1 SMP PVE 5.4.140-1 (Wed, 08 Sep 2021 16:21:59 +0200)" }