Ceph » crimson

DEBUG 2020-08-25 13:21:55,885 [shard 0] filestore - do_transaction
DEBUG 2020-08-25 13:21:55,886 [shard 0] ms - [osd.3(cluster) v2:172.21.15.168:6805/18752@62952 >> osd.0 v2:172.21.15.144:6801/18864] GOT AckFrame: seq=692
DEBUG 2020-08-25 13:21:55,886 [shard 0] ms - [osd.3(cluster) v2:172.21.15.168:6805/18752@62952 >> osd.0 v2:172.21.15.144:6801/18864] <== #682 === osd_repop(client.4229.0:2380 3.2 e22/21) v3 (112)
DEBUG 2020-08-25 13:21:55,886 [shard 0] osd - replicated_request(id=1134, detail=RepRequest(from=0 req=osd_repop(client.4229.0:2380 3.2 e22/21 3:49de99f6:::smithi14420596-532:head v 22'99, mlcod=22'99) v3
)) start
DEBUG 2020-08-25 13:21:55,886 [shard 0] osd -  pg_epoch 22 pg[3.2( v 22'98 (0'0,22'98] local-lis/les=21/22 n=0 ec=14/14 lis/c=21/14 les/c/f=22/15/0 sis=21 pruub=11.052338892s) [0,3] r=1 lpr=21 pi=[14,21)/
1 luod=0'0 lua=19'97 crt=22'98 mlcod 22'98 active pruub 45.379686226s@ append_log log((0'0,22'98], crt=22'98) {22'99 (22'98) modify   3:49de99f6:::smithi14420596-532:head by client.4229.0:2380 2020-08-25T
13:21:54.095732+0000 0 ObjectCleanRegions clean_offsets: [0~1956409,2512586~18446744073707039029], clean_omap: 1, new_object: 0}
DEBUG 2020-08-25 13:21:55,886 [shard 0] osd -  pg_epoch 22 pg[3.2( v 22'99 (0'0,22'99] local-lis/les=21/22 n=0 ec=14/14 lis/c=21/14 les/c/f=22/15/0 sis=21 pruub=11.052338892s) [0,3] r=1 lpr=21 pi=[14,21)/
1 luod=0'0 lua=19'97 crt=22'98 lcod 22'98 mlcod 22'98 active pruub 45.379686226s@ add_log_entry 22'99 (22'98) modify   3:49de99f6:::smithi14420596-532:head by client.4229.0:2380 2020-08-25T13:21:54.095732
+0000 0 ObjectCleanRegions clean_offsets: [0~1956409,2512586~18446744073707039029], clean_omap: 1, new_object: 0
DEBUG 2020-08-25 13:21:55,886 [shard 0] osd -  pg_epoch 22 pg[3.2( v 22'99 (0'0,22'99] local-lis/les=21/22 n=0 ec=14/14 lis/c=21/14 les/c/f=22/15/0 sis=21 pruub=11.052338892s) [0,3] r=1 lpr=21 pi=[14,21)/
1 luod=0'0 lua=19'97 crt=22'99 lcod 22'98 mlcod 22'98 active pruub 45.379686226s@ append_log approx pg log length =  99
DEBUG 2020-08-25 13:21:55,886 [shard 0] osd -  pg_epoch 22 pg[3.2( v 22'99 (0'0,22'99] local-lis/les=21/22 n=0 ec=14/14 lis/c=21/14 les/c/f=22/15/0 sis=21 pruub=11.052338892s) [0,3] r=1 lpr=21 pi=[14,21)/
1 luod=0'0 lua=19'97 crt=22'99 lcod 22'98 mlcod 22'98 active pruub 45.379686226s@ append_log transaction_applied = 1
DEBUG 2020-08-25 13:21:55,887 [shard 0] osd - trim proposed trim_to = 0'0
DEBUG 2020-08-25 13:21:55,887 [shard 0] osd - write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 22'99, trimmed: , trimmed_dups: , clear_divergent_priors: 0
DEBUG 2020-08-25 13:21:55,887 [shard 0] filestore - do_transaction
DEBUG 2020-08-25 13:21:55,887 [shard 0] ms - [osd.3(cluster) v2:172.21.15.168:6805/18752@62952 >> osd.0 v2:172.21.15.144:6801/18864] <== #683 === MOSDPGPushReply(3.7 22/21 {PushReplyOp(3:eb464a4a:::smithi14420596-514:head)}) v3 (107)
DEBUG 2020-08-25 13:21:55,887 [shard 0] osd - background_recovery_sub(id=0, detail=MOSDPGPushReply(3.7 22/21 {PushReplyOp(3:eb464a4a:::smithi14420596-514:head)}) v3): start
DEBUG 2020-08-25 13:21:55,887 [shard 0] osd - handle_push_reply: MOSDPGPushReply(3.7 22/21 {PushReplyOp(3:eb464a4a:::smithi14420596-514:head)}) v3
DEBUG 2020-08-25 13:21:55,887 [shard 0] osd - _handle_push_reply, soid 3:eb464a4a:::smithi14420596-514:head, from 0
DEBUG 2020-08-25 13:21:55,887 [shard 0] osd - on_peer_recover: 3:eb464a4a:::smithi14420596-514:head, 19'98 on 0
ERROR 2020-08-25 13:21:55,887 [shard 0] none - /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/16.0.0-4670-ge307b9aefb4/rpm/el8/BUILD/ceph-16.0.0-4670-ge307b9aefb4/src/osd/osd_types.h:4816 : In function 'void pg_missing_set<TrackChanges>::got(const hobject_t&, eversion_t) [with bool TrackChanges = false]', ceph_assert(%s)
p->second.need <= v || p->second.is_delete()
Aborting on shard 0.
Backtrace:

Backtrace:
[Backtrace #0]
void seastar::backtrace<seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}>(seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}&&) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/util/backtrace.hh:59
non-virtual thunk to std::_Nested_exception<std::runtime_error>::~_Nested_exception() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:730
 (inlined by) print_with_backtrace at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:751
print_with_backtrace at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:758
non-virtual thunk to std::_Nested_exception<std::runtime_error>::~_Nested_exception() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:3517
 (inlined by) operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:3499
 (inlined by) _FUN at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:3495
.annobin_funlockfile.c_end at funlockfile.c:?
__GI___libc_sigaction at :?
__isoc99_fscanf.cold.0 at isoc99_fscanf.c:?
ceph::__ceph_assert_fail(char const*, char const*, int, char const*) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/common/assert.cc:27
ceph::__ceph_assert_fail(ceph::assert_data const&) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/common/assert.cc:14
PeeringState::on_peer_recover(pg_shard_t, hobject_t const&, eversion_t const&) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/osd/osd_types.h:4816
 (inlined by) PeeringState::on_peer_recover(pg_shard_t, hobject_t const&, eversion_t const&) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/osd/PeeringState.cc:3948
PGRecovery::on_peer_recover(pg_shard_t, hobject_t const&, ObjectRecoveryInfo const&) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/pg_recovery.cc:389
operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/replicated_recovery_backend.cc:845
 (inlined by) ReplicatedRecoveryBackend::_handle_push_reply(pg_shard_t, PushReplyOp const&, PushOp*) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/replicated_recovery_backend.cc:848
operator()<PushOp> at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/replicated_recovery_backend.cc:877
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:96
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1685
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1696
 (inlined by) do_with_impl<PushOp, ReplicatedRecoveryBackend::handle_push_reply(Ref<MOSDPGPushReply>)::<lambda(auto:125&)> > at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/do_with.hh:106
ReplicatedRecoveryBackend::handle_push_reply(boost::intrusive_ptr<MOSDPGPushReply>) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2045
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2129
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/do_with.hh:141
 (inlined by) ReplicatedRecoveryBackend::handle_push_reply(boost::intrusive_ptr<MOSDPGPushReply>) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/replicated_recovery_backend.cc:878
ReplicatedRecoveryBackend::handle_recovery_op(boost::intrusive_ptr<MOSDFastDispatchOp>) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/replicated_recovery_backend.cc:1091
operator()<boost::intrusive_ptr<crimson::osd::PG>, boost::intrusive_ptr<crimson::osd::RecoverySubRequest> > at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/osd_operations/recovery_subrequest.cc:24
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:96
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1685
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1696
 (inlined by) do_with_impl<boost::intrusive_ptr<crimson::osd::PG>, const boost::intrusive_ptr<crimson::osd::RecoverySubRequest>, crimson::osd::RecoverySubRequest::start()::<lambda(Ref<crimson::osd::PG>)>::<lambda(auto:84&, auto:85&)> > at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/do_with.hh:106
crimson::osd::RecoverySubRequest::start() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2045
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2129
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/do_with.hh:141
 (inlined by) operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/osd_operations/recovery_subrequest.cc:25
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:96
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1685
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1696
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2028
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1568
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1214
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1490
 (inlined by) crimson::osd::RecoverySubRequest::start() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/osd_operations/recovery_subrequest.cc:26
crimson::osd::OSD::handle_recovery_subreq(crimson::net::Connection*, boost::intrusive_ptr<MOSDFastDispatchOp>) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/shard_services.h:100
 (inlined by) crimson::osd::OSD::handle_recovery_subreq(crimson::net::Connection*, boost::intrusive_ptr<MOSDFastDispatchOp>) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/osd.cc:1152
operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/osd.cc:628
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2045
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2129
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/gate.hh:126
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/gate.hh:144
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/common/gated.h:37
 (inlined by) crimson::osd::OSD::ms_dispatch(crimson::net::Connection*, boost::intrusive_ptr<Message>) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/osd.cc:655
operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/net/chained_dispatchers.cc:9
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2045
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2129
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future-util.hh:611
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future-util.hh:668
 (inlined by) ChainedDispatchers::ms_dispatch(crimson::net::Connection*, boost::intrusive_ptr<Message>) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/net/chained_dispatchers.cc:10
operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/net/ProtocolV2.cc:1929
crimson::net::ProtocolV2::read_message(utime_t) at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:95
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1684
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1694
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1995
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1568
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1214
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1490
 (inlined by) crimson::net::ProtocolV2::read_message(utime_t) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/net/ProtocolV2.cc:1930
operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/net/ProtocolV2.cc:1978
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:96
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1685
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1696
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2028
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1568
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1214
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1490
 (inlined by) operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/net/ProtocolV2.cc:1979
crimson::net::ProtocolV2::read_message(utime_t) at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:60
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/bits/invoke.h:96
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1685
 (inlined by) ?? at /opt/rh/gcc-toolset-9/root/usr/include/c++/9/tuple:1696
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2028
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1568
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1214
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:1490
 (inlined by) operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/net/ProtocolV2.cc:2011
 (inlined by) operator() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future-util.hh:561
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2045
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future.hh:2129
 (inlined by) ?? at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/include/seastar/core/future-util.hh:262
seastar::reactor::run_tasks(seastar::reactor::task_queue&) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:2192
non-virtual thunk to std::_Nested_exception<std::runtime_error>::~_Nested_exception() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:2608
non-virtual thunk to std::_Nested_exception<std::runtime_error>::~_Nested_exception() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:2591
 (inlined by) seastar::reactor::run() at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/reactor.cc:2763
seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/seastar/src/core/app-template.cc:207
main at /usr/src/debug/ceph-16.0.0-4670.ge307b9aefb4.el8.x86_64/src/crimson/osd/main.cc:147
check_one_fd at check_fds.c:?
_start at ??:?

/a/kchai-2020-08-25_13:01:58-crimson-rados:thrash-wip-47031-kefu-distro-basic-smithi