Ceph » crimson

#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=140347499583104) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (signo=6, threadid=140347499583104) at ./nptl/pthread_kill.c:78
#2  __GI___pthread_kill (threadid=140347499583104, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#3  0x00007fa5336bd476 in __GI_raise (sig=6) at ../sysdeps/posix/raise.c:26
#4  0x0000558bc11f381f in reraise_fatal (signum=6) at /home/xuxuehan/src/ceph/src/crimson/common/fatal_signal.cc:41
#5  0x0000558bc11f35ce in FatalSignal::signal_entry (signum=6, info=0x7fffdd8736f0) at /home/xuxuehan/src/ceph/src/crimson/common/fatal_signal.cc:62
#6  <signal handler called>
#7  __pthread_kill_implementation (no_tid=0, signo=6, threadid=140347499583104) at ./nptl/pthread_kill.c:44
#8  __pthread_kill_internal (signo=6, threadid=140347499583104) at ./nptl/pthread_kill.c:78
#9  __GI___pthread_kill (threadid=140347499583104, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#10 0x00007fa5336bd476 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#11 0x00007fa5336a37f3 in __GI_abort () at ./stdlib/abort.c:79
#12 0x0000558bc11d4e29 in ceph::__ceph_assert_fail (assertion=0x558bc3193766 "missing_loc.needs_recovery(oid)", file=0x558bc319050e "/home/xuxuehan/src/ceph/src/osd/PeeringState.cc", line=4255, 
    func=0x558bc3193706 "void PeeringState::recover_got(const hobject_t &, eversion_t, bool, ObjectStore::Transaction &)") at /home/xuxuehan/src/ceph/src/crimson/common/assert.cc:27
#13 0x0000558bc11d4d3f in ceph::__ceph_assert_fail (ctx=...) at /home/xuxuehan/src/ceph/src/crimson/common/assert.cc:14
#14 0x0000558bc0b3655b in PeeringState::recover_got (this=0x558bc59a5c88, oid=..., v=..., is_delete=false, t=...) at /home/xuxuehan/src/ceph/src/osd/PeeringState.cc:4255
#15 0x0000558bc094b849 in PGRecovery::on_local_recover (this=0x558bc57fd9a0, soid=..., recovery_info=..., is_delete=false, t=...) at /home/xuxuehan/src/ceph/src/crimson/osd/pg_recovery.cc:335
#16 0x0000558bc09d062c in ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}::operator()() const (this=0x558bc5ad0040)
    at /home/xuxuehan/src/ceph/src/crimson/osd/replicated_recovery_backend.cc:733
#17 0x0000558bc09d04e4 in seastar::futurize<bool>::invoke<ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}>(ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&) (func=...) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:2149
#18 0x0000558bc09d04b0 in seastar::futurize_invoke<ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}>(ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&) (func=...) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:2178
#19 0x0000558bc09d032d in crimson::interruptible::internal::call_with_interruption_impl<crimson::osd::IOInterruptCondition, ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}>(seastar::lw_shared_ptr<crimson::osd::IOInterruptCondition>, ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&) (interrupt_condition=..., func=...)
    at /home/xuxuehan/src/ceph/src/crimson/common/interruptible_future.h:200
#20 0x0000558bc09cff81 in crimson::interruptible::call_with_interruption<crimson::osd::IOInterruptCondition, ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}, bool>(seastar::lw_shared_ptr<crimson::osd::IOInterruptCondition>, ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&) (interrupt_condition=..., func=...)
    at /home/xuxuehan/src/ceph/src/crimson/common/interruptible_future.h:270
#21 0x0000558bc09cfe50 in crimson::interruptible::interruptible_future_detail<crimson::osd::IOInterruptCondition, seastar::future<void> >::then_interruptible<ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}>(ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&)::{lambda()#1}::operator()() (this=0x558bc5ad0040)
    at /home/xuxuehan/src/ceph/src/crimson/common/interruptible_future.h:461
#22 0x0000558bc09cfda4 in seastar::futurize<seastar::future<bool> >::invoke<crimson::interruptible::interruptible_future_detail<crimson::osd::IOInterruptCondition, seastar::future<void> >::then_interruptible<ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}>(ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&)::{lambda()#1}&>(ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&) (func=...) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:2147
#23 0x0000558bc09cfd70 in seastar::futurize_invoke<crimson::interruptible::interruptible_future_detail<crimson::osd::IOInterruptCondition, seastar::future<void> >::then_interruptible<ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}>(ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&)::{lambda()#1}&>(ReplicatedRecoveryBackend::_handle_pull_response(pg_shard_t, PushOp&, PullOp*, ceph::os::Transaction*)::$_23::operator()()::{lambda()#1}&&) (func=...) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:2178
#24 0x0000558bc09cfd33 in _ZZN7seastar6futureIvE4thenIZN7crimson13interruptible27interruptible_future_detailINS3_3osd20IOInterruptConditionES1_E18then_interruptibleIZZN25ReplicatedRecoveryBackend21_handle_pull_responseE10pg_shard_tR6PushOpP6PullOpPN4ceph2os11TransactionEEN4$_23clEvEUlvE_EEDaOT_EUlvE_NS0_IbEEEET0_SN_ENUlDpOT_E_clIJEEEDaST_ (this=0x558bc5ad0040) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:1539
#25 0x0000558bc09cfca0 in _ZN7seastar20noncopyable_functionIFNS_6futureIbEEvEE19indirect_vtable_forIZNS1_IvE4thenIZN7crimson13interruptible27interruptible_future_detailINS8_3osd20IOInterruptConditionES6_E18then_interruptibleIZZN25ReplicatedRecoveryBackend21_handle_pull_responseE10pg_shard_tR6PushOpP6PullOpPN4ceph2os11TransactionEEN4$_23clEvEUlvE_EEDaOT_EUlvE_S2_EET0_SS_EUlDpOT_E_E4callEPKS4_ (func=0x558bc6d542f0) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/util/noncopyable_function.hh:153
#26 0x0000558bc08dbea4 in seastar::noncopyable_function<seastar::future<bool> ()>::operator()() const (this=0x558bc6d542f0) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/util/noncopyable_function.hh:210
#27 0x0000558bc08dbe70 in std::__invoke_impl<seastar::future<bool>, seastar::noncopyable_function<seastar::future<bool> ()>&>(std::__invoke_other, seastar::noncopyable_function<seastar::future<bool> ()>&) (__f=...)
    at /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61
#28 0x0000558bc08dbe30 in std::__invoke<seastar::noncopyable_function<seastar::future<bool> ()>&>(seastar::noncopyable_function<seastar::future<bool> ()>&) (__fn=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96
#29 0x0000558bc08dbde0 in std::invoke<seastar::noncopyable_function<seastar::future<bool> ()>&>(seastar::noncopyable_function<seastar::future<bool> ()>&) (__fn=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/functional:97
#30 0x0000558bc08dbda4 in seastar::internal::future_invoke<seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::internal::monostate>(seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::internal::monostate&&) (func=..., v=...)
    at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:1221
#31 0x0000558bc08dbcd7 in seastar::future<void>::then_impl_nrvo<seastar::noncopyable_function<seastar::future<bool> ()>, seastar::future<bool> >(seastar::noncopyable_function<seastar::future<bool> ()>&&)::{lambda(seastar::internal::promise_base_with_type<bool>&&, seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::future_state<seastar::internal::monostate>&&)#1}::operator()(seastar::internal::promise_base_with_type<bool>&&, seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::future_state<seastar::internal::monostate>&&) const::{lambda()#1}::operator()() const (this=0x7fffdd875c50) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:1594
#32 0x0000558bc08dbc5d in seastar::futurize<seastar::future<bool> >::satisfy_with_result_of<seastar::future<void>::then_impl_nrvo<seastar::noncopyable_function<seastar::future<bool> ()>, seastar::future<bool> >(seastar::noncopyable_function<seastar::future<bool> ()>&&)::{lambda(seastar::internal::promise_base_with_type<bool>&&, seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::future_state<seastar::internal::monostate>&&)#1}::operator()(seastar::internal::promise_base_with_type<bool>&&, seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::future_state<seastar::internal::monostate>&&) const::{lambda()#1}>(seastar::internal::promise_base_with_type<bool>&&, seastar::noncopyable_function<seastar::future<bool> ()>&&) (pr=..., func=...)
    at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:2132
#33 0x0000558bc08dbbe5 in seastar::future<void>::then_impl_nrvo<seastar::noncopyable_function<seastar::future<bool> ()>, seastar::future<bool> >(seastar::noncopyable_function<seastar::future<bool> ()>&&)::{lambda(seastar::internal::promise_base_with_type<bool>&&, seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::future_state<seastar::internal::monostate>&&)#1}::operator()(seastar::internal::promise_base_with_type<bool>&&, seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::future_state<seastar::internal::monostate>&&) const (this=0x558bc6d542c0, pr=..., func=..., state=...) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:1587
#34 0x0000558bc08dbaad in seastar::continuation<seastar::internal::promise_base_with_type<bool>, seastar::noncopyable_function<seastar::future<bool> ()>, seastar::future<void>::then_impl_nrvo<seastar::noncopyable_function<seastar::future<bool> ()>, seastar::future<bool> >(seastar::noncopyable_function<seastar::future<bool> ()>&&)::{lambda(seastar::internal::promise_base_with_type<bool>&&, seastar::noncopyable_function<seastar::future<bool> ()>&, seastar::future_state<seastar::internal::monostate>&&)#1}, void>::run_and_dispose() (
    this=0x558bc6d542c0) at /home/xuxuehan/src/ceph/src/seastar/include/seastar/core/future.hh:781
#35 0x0000558bc2d68142 in seastar::reactor::run_tasks (this=0x558bc52a6600, tq=...) at /home/xuxuehan/src/ceph/src/seastar/src/core/reactor.cc:2517
#36 0x0000558bc2d6a0a3 in seastar::reactor::run_some_tasks (this=0x558bc52a6600) at /home/xuxuehan/src/ceph/src/seastar/src/core/reactor.cc:2954
#37 0x0000558bc2d6b6a3 in seastar::reactor::do_run (this=0x558bc52a6600) at /home/xuxuehan/src/ceph/src/seastar/src/core/reactor.cc:3123
#38 0x0000558bc2d6aa38 in seastar::reactor::run (this=0x558bc52a6600) at /home/xuxuehan/src/ceph/src/seastar/src/core/reactor.cc:3006
#39 0x0000558bc2cc8d6f in seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) (this=0x7fffdd876cd0, ac=5, av=0x558bc52952f0, func=...) at /home/xuxuehan/src/ceph/src/seastar/src/core/app-template.cc:266
#40 0x0000558bc2cc7fcb in seastar::app_template::run(int, char**, std::function<seastar::future<int> ()>&&) (this=0x7fffdd876cd0, ac=5, av=0x558bc52952f0, func=...) at /home/xuxuehan/src/ceph/src/seastar/src/core/app-template.cc:157
#41 0x0000558bc037e367 in main (argc=8, argv=0x7fffdd878cb8) at /home/xuxuehan/src/ceph/src/crimson/osd/main.cc:121

The reasone is that head/clone objects' versions in PGLog are not consistent with those in the object infos recorded in head/clone objects