Actions
Bug #38344
closedsrc/osd/PG.h: 1245: FAILED ceph_assert(prev >= 0)
Status:
Resolved
Priority:
High
Assignee:
David Zafman
Category:
-
Target version:
-
% Done:
0%
Source:
Development
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
http://pulpito.ceph.com/dzafman-2019-02-14_22:11:34-rados:thrash-master-distro-basic-smithi/3593928
2019-02-15 07:21:18.007 7f1611b42700 20 osd.1 424 share_map osd.6 v1:172.21.15.89:6805/11455 424
2019-02-15 07:21:18.007 7f1611b42700 20 osd.1 424 should_share_map osd.6 v1:172.21.15.89:6805/11455 424
2019-02-15 07:21:18.007 7f1611b42700 10 osd.1 pg_epoch: 424 pg[4.13( v 424'13715 (421'13711,424'13715] lb MIN (bitwise) local-lis/les=423/424 n=0 ec=309/277 lis/c 423/415 les/c/f 424/416/0 422/423/415) [6,4,1]/[6,4,5] r=-1 lpr=423 pi=[309,423)/2 luod=0'0 lua=421'13712 crt=424'13715 lcod 424'13714 active+remapped mbc={}] _handle_message: 0x557c06385c80
2019-02-15 07:21:18.007 7f1611b42700 10 osd.1 pg_epoch: 424 pg[4.13( v 424'13715 (421'13711,424'13715] lb MIN (bitwise) local-lis/les=423/424 n=0 ec=309/277 lis/c 423/415 les/c/f 424/416/0 422/423/415) [6,4,1]/[6,4,5] r=-1 lpr=423 pi=[309,423)/2 luod=0'0 lua=421'13712 crt=424'13715 lcod 424'13714 active+remapped mbc={}] handle_push ObjectRecoveryInfo(4:c829da80:::benchmark_data_smithi002_34250_object155360:head@328'8830, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{})ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false)
2019-02-15 07:21:18.007 7f1611b42700 10 bluestore(/var/lib/ceph/osd/ceph-1) stat 4.13_head #4:c829da80:::benchmark_data_smithi002_34250_object155360:head#
2019-02-15 07:21:18.059 7f1611b42700 -1 /build/ceph-14.0.1-3734-gb1b7174/src/osd/PG.h: In function 'void PG::add_local_num_bytes(int64_t)' thread 7f1611b42700 time 2019-02-15 07:21:18.011974
/build/ceph-14.0.1-3734-gb1b7174/src/osd/PG.h: 1245: FAILED ceph_assert(prev >= 0)
ceph version 14.0.1-3734-gb1b7174 (b1b7174cd698cf5e04319837183cdeb537127844) nautilus (dev)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x557bd506ed60]
2: (ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x557bd506ef3b]
3: (boost::statechart::event<PG::Backfilled, std::allocator<void> >::clone() const+0) [0x557bd537b340]
4: (ReplicatedBackend::submit_push_data(ObjectRecoveryInfo const&, bool, bool, bool, interval_set<unsigned long, std::map<unsigned long, unsigned long, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, unsigned long> > > > const&, ceph::buffer::list, ceph::buffer::list, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::list> > > const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::list> > > const&, ObjectStore::Transaction*)+0x5d7) [0x557bd54b7fe7]
5: (ReplicatedBackend::handle_push(pg_shard_t, PushOp const&, PushReplyOp*, ObjectStore::Transaction*)+0x193) [0x557bd54b8c93]
6: (ReplicatedBackend::_do_push(boost::intrusive_ptr<OpRequest>)+0x153) [0x557bd54b9033]
7: (ReplicatedBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x2a8) [0x557bd54c0668]
8: (PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x97) [0x557bd53c5c37]
9: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x705) [0x557bd5374d25]
10: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x1b3) [0x557bd51a08a3]
11: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x557bd544b032]
12: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xbf5) [0x557bd51be585]
13: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x4ac) [0x557bd57d9e2c]
14: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x557bd57dcf00]
15: (()+0x76db) [0x7f163928e6db]
16: (clone()+0x3f) [0x7f163802988f]
Actions