Actions
Bug #47313
closedassert failure in crimson::osd::ObjectContext::drop_recovery_read()'
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Description
[Backtrace #0]
void seastar::backtrace<seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}>(seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}&&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/util/backtrace.hh:59
non-virtual thunk to std::_Nested_exception<std::runtime_error>::~_Nested_exception() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:730
(inlined by) print_with_backtrace at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:751
print_with_backtrace at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:758
non-virtual thunk to std::_Nested_exception<std::runtime_error>::~_Nested_exception() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:3517
(inlined by) operator() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:3499
(inlined by) _FUN at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:3495
.annobin_funlockfile.c_end at funlockfile.c:?
__GI___libc_sigaction at :?
__isoc99_fscanf.cold.0 at isoc99_fscanf.c:?
ceph::__ceph_assert_fail(char const*, char const*, int, char const*) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/common/assert.cc:27
ceph::__ceph_assert_fail(ceph::assert_data const&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/common/assert.cc:14
RecoveryBackend::clean_up(ceph::os::Transaction&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/object_context.h:215
(inlined by) RecoveryBackend::clean_up(ceph::os::Transaction&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/recovery_backend.cc:43
crimson::osd::PG::on_change(ceph::os::Transaction&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/recovery_backend.h:85
(inlined by) crimson::osd::PG::on_change(ceph::os::Transaction&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/pg.cc:940
PeeringState::start_peering_interval(boost::local_shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> > const&, int, std::vector<int, std::allocator<int> > const&, int, ceph::os::Transaction&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/osd/PeeringState.cc:691
PeeringState::Reset::react(PeeringState::AdvMap const&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/osd/PeeringState.cc:4366
boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/custom_reaction.hpp:41
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/simple_state.hpp:814
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/simple_state.hpp:850
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/simple_state.hpp:820
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/simple_state.hpp:850
(inlined by) boost::statechart::simple_state<PeeringState::Reset, PeeringState::PeeringMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/simple_state.hpp:488
PeeringState::advance_map(boost::local_shared_ptr<OSDMap const>, boost::local_shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, int, std::vector<int, std::allocator<int> >&, int, PeeringCtx&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/state_machine.hpp:86
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/null_exception_translator.hpp:33
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/state_machine.hpp:885
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/state_machine.hpp:910
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/build/boost/include/boost/statechart/state_machine.hpp:280
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/osd/PeeringState.h:1639
(inlined by) PeeringState::advance_map(boost::local_shared_ptr<OSDMap const>, boost::local_shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, int, std::vector<int, std::allocator<int> >&, int, PeeringCtx&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/osd/PeeringState.cc:439
crimson::osd::PG::handle_advance_map(boost::local_shared_ptr<OSDMap const>, PeeringCtx&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/pg.cc:470
operator() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd_operations/pg_advance_map.cc:72
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:95
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1684
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1694
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1995
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1568
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1214
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1490
(inlined by) operator() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd_operations/pg_advance_map.cc:73
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:2045
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:2129
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future-util.hh:611
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future-util.hh:644
(inlined by) operator() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd_operations/pg_advance_map.cc:66
crimson::osd::PGAdvanceMap::start() at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:96
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1685
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1696
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:2028
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1568
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1214
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1490
(inlined by) crimson::osd::PGAdvanceMap::start() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd_operations/pg_advance_map.cc:61
crimson::osd::OSD::consume_map(unsigned int) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/shard_services.h:100
(inlined by) operator()<std::pair<const spg_t, boost::intrusive_ptr<crimson::osd::PG> > > at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd.cc:1281
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:2045
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:2129
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future-util.hh:164
(inlined by) crimson::osd::OSD::consume_map(unsigned int) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd.cc:1278
operator() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd.cc:1045
crimson::osd::OSD::committed_osd_maps(unsigned long, unsigned long, boost::intrusive_ptr<MOSDMap>) at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:96
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1685
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1696
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:2028
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1568
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1214
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1490
(inlined by) crimson::osd::OSD::committed_osd_maps(unsigned long, unsigned long, boost::intrusive_ptr<MOSDMap>) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd.cc:1025
operator() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/osd.cc:1000
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:96
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1685
(inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1696
(inlined by) operator() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1551
(inlined by) ?? at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:2038
(inlined by) operator() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:1550
(inlined by) run_and_dispose at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/include/seastar/core/future.hh:648
seastar::reactor::run_tasks(seastar::reactor::task_queue&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:2192
non-virtual thunk to std::_Nested_exception<std::runtime_error>::~_Nested_exception() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:2608
non-virtual thunk to std::_Nested_exception<std::runtime_error>::~_Nested_exception() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:2591
(inlined by) seastar::reactor::run() at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/reactor.cc:2763
seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/seastar/src/core/app-template.cc:207
main at /usr/src/debug/ceph-16.0.0-5167.g0f0addb4d7.el8.x86_64/src/crimson/osd/main.cc:147
check_one_fd at check_fds.c:?
_start at ??:?
Files
Updated by Kefu Chai over 3 years ago
- File 47313.ceph-osd.1.log.bz2 47313.ceph-osd.1.log.bz2 added
log right before the stack trace
DEBUG 2020-09-05 09:29:23,053 [shard 0] osd - pg_epoch 41 pg[3.f( v 36'234 lc 26'179 (0'0,36'234] local-lis/les=35/36 n=0 ec=14/14 lis/c=35/30 les/c/f=36/31/0 sis=35 pruub=13.16620
7702s) [1,2] r=0 lpr=41 pi=[14,41)/3 lua=34'230 crt=36'234 mlcod 26'179 active pruub 71.880054758s@ noting past ([14,40] all_participants=0,1,2,3 intervals=([25,29] acting 0,3),([3
0,34] acting 0,2,3),([35,40] acting 1,2,3))
DEBUG 2020-09-05 09:29:23,053 [shard 0] osd - pg_epoch 41 pg[3.f( v 36'234 lc 26'179 (0'0,36'234] local-lis/les=35/36 n=0 ec=14/14 lis/c=35/30 les/c/f=36/31/0 sis=41 pruub=13.16620
7702s) [1,2] r=0 lpr=41 pi=[14,41)/3 lua=34'230 crt=36'234 mlcod 26'179 active pruub 71.880054758s@ on_new_interval
DEBUG 2020-09-05 09:29:23,053 [shard 0] osd - pg_epoch 41 pg[3.f( v 36'234 lc 26'179 (0'0,36'234] local-lis/les=35/36 n=0 ec=14/14 lis/c=35/30 les/c/f=36/31/0 sis=41 pruub=13.16620
7702s) [1,2] r=0 lpr=41 pi=[14,41)/3 lua=34'230 crt=36'234 mlcod 26'179 active pruub 71.880054758s@ on_new_interval upacting_features 0x3f01cfb8ffbdffff from {1, 2}+{1, 2}
DEBUG 2020-09-05 09:29:23,053 [shard 0] osd - pg_epoch 41 pg[3.f( v 36'234 lc 26'179 (0'0,36'234] local-lis/les=35/36 n=0 ec=14/14 lis/c=35/30 les/c/f=36/31/0 sis=41 pruub=13.16620
7702s) [1,2] r=0 lpr=41 pi=[14,41)/3 lua=34'230 crt=36'234 mlcod 26'179 active pruub 71.880054758s@ on_new_interval checking missing set deletes flag. missing = missing(14 may_inclu
de_deletes = 1)
DEBUG 2020-09-05 09:29:23,053 [shard 0] osd - pg_epoch 41 pg[3.f( v 36'234 lc 26'179 (0'0,36'234] local-lis/les=35/36 n=0 ec=14/14 lis/c=35/30 les/c/f=36/31/0 sis=41 pruub=13.16620
7702s) [1,2] r=0 lpr=41 pi=[14,41)/3 lua=34'230 crt=36'234 mlcod 26'179 active pruub 71.880054758s@ init_hb_stamps now {0x5571b5fe59c0}
DEBUG 2020-09-05 09:29:23,053 [shard 0] osd - pg_epoch 41 pg[3.f( v 36'234 lc 26'179 (0'0,36'234] local-lis/les=35/36 n=0 ec=14/14 lis/c=35/30 les/c/f=36/31/0 sis=41) [1,2] r=0 lpr
=41 pi=[14,41)/3 lua=34'230 crt=36'234 mlcod 26'179 active on_new_interval prior_readable_until_ub 0s (mnow 91.744713407s + 0s)
INFO 2020-09-05 09:29:23,053 [shard 0] osd - pg_epoch 41 pg[3.f( v 36'234 lc 26'179 (0'0,36'234] local-lis/les=35/36 n=0 ec=14/14 lis/c=35/30 les/c/f=36/31/0 sis=41) [1,2] r=0 lpr=41 pi=[14,41)/3 lua=34'230 crt=36'234 mlcod 26'179 active start_peering_interval up {1, 3, 2} -> {1, 2}, acting {1, 3, 2} -> {1, 2}, acting_primary 1 -> 1, up_primary 1 -> 1, role 0 -> 0, features acting 4540138292837744639 upacting 4540138292837744639
DEBUG 2020-09-05 09:29:23,053 [shard 0] osd - pg_epoch 41 pg[3.f( v 36'234 lc 26'179 (0'0,36'234] local-lis/les=35/36 n=0 ec=14/14 lis/c=35/30 les/c/f=36/31/0 sis=41) [1,2] r=0 lpr=41 pi=[14,41)/3 crt=36'234 mlcod 26'179 unknown clear_primary_state
ERROR 2020-09-05 09:29:23,053 [shard 0] none - /home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/16.0.0-5167-g0f0addb4d7/rpm/el8/BUILD/ceph-16.0.0-5167-g0f0addb4d7/src/crimson/osd/object_context.h:215 : In function 'void crimson::osd::ObjectContext::drop_recovery_read()', ceph_assert(%s)
rwstate.recovery_read_marker
Updated by Kefu Chai over 3 years ago
- Status changed from New to Fix Under Review
- Assignee set to Xuehan Xu
- Pull request ID set to 37007
Updated by Kefu Chai over 3 years ago
- Status changed from Fix Under Review to Resolved
Actions