Bug #54684
crash: int64_t BlueFS::_read(BlueFS::FileReader*, uint64_t, size_t, ceph::bufferlist*, char*): assert(r == 0)
Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
0%
Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
2b5596d80e087b41d9c1c9a5e13a468393dcadfdbad9ebd4d4cbaaedb24d51b3
eb85c1fdea237931375f9ccc4b9f5e737cc5e9f15d68bd0236c5977f47cb0836
Crash signature (v2):
Description
Assert condition: r == 0
Assert function: int64_t BlueFS::_read(BlueFS::FileReader*, uint64_t, size_t, ceph::bufferlist*, char*)
Sanitized backtrace:
BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::list*, char*) BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long) rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*) rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock() rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward() rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next() rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*) rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*) rocksdb::DBIter::Next() RocksDBStore::RocksDBWholeSpaceIteratorImpl::next() BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*) BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*) PGBackend::objects_list_range(hobject_t const&, hobject_t const&, std::vector<hobject_t, std::allocator<hobject_t> >*, std::vector<ghobject_t, std::allocator<ghobject_t> >*) PgScrubber::build_scrub_map_chunk(ScrubMap&, ScrubMapBuilder&, hobject_t, hobject_t, bool) PgScrubber::build_replica_map_chunk() Scrub::ActiveReplica::react(Scrub::SchedReplica const&) boost::statechart::simple_state<Scrub::ActiveReplica, Scrub::ScrubMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)>::react_impl(boost::statechart::event_base const&, void const*) boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&) PgScrubber::send_start_replica(unsigned int) PG::forward_scrub_event(void (ScrubPgIF::*)(unsigned int), unsigned int) PG::replica_scrub(unsigned int, ThreadPool::TPHandle&) ceph::osd::scheduler::PGRepScrub::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&) OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) ShardedThreadPool::shardedthreadpool_worker(unsigned int) ShardedThreadPool::WorkThreadSharded::entry()
Crash dump sample:
{ "assert_condition": "r == 0", "assert_file": "os/bluestore/BlueFS.cc", "assert_func": "int64_t BlueFS::_read(BlueFS::FileReader*, uint64_t, size_t, ceph::bufferlist*, char*)", "assert_line": 2044, "assert_msg": "os/bluestore/BlueFS.cc: In function 'int64_t BlueFS::_read(BlueFS::FileReader*, uint64_t, size_t, ceph::bufferlist*, char*)' thread 7f9612b44700 time 2022-01-19T21:23:04.972504+0000\nos/bluestore/BlueFS.cc: 2044: FAILED ceph_assert(r == 0)", "assert_thread_name": "tp_osd_tp", "backtrace": [ "/lib64/libpthread.so.0(+0x12c20) [0x7f963537ac20]", "gsignal()", "abort()", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x564f4aa0bbcf]", "/usr/bin/ceph-osd(+0x56ad98) [0x564f4aa0bd98]", "(BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, char*)+0x7b0) [0x564f4b102f10]", "(BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)+0x1e) [0x564f4b13148e]", "(rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x564f4b5cc3ff]", "(rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const+0x2e) [0x564f4b72b1ee]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()+0x1a6) [0x564f4b7372a6]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()+0x1c6) [0x564f4b738126]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()+0x58) [0x564f4b738228]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)+0x16) [0x564f4b738266]", "/usr/bin/ceph-osd(+0x11f8d61) [0x564f4b699d61]", "(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x50) [0x564f4b7504f0]", "(rocksdb::DBIter::Next()+0x592) [0x564f4b627da2]", "(RocksDBStore::RocksDBWholeSpaceIteratorImpl::next()+0x31) [0x564f4b574a31]", "/usr/bin/ceph-osd(+0xb398ca) [0x564f4afda8ca]", "(BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0xf6b) [0x564f4b00dd0b]", "(BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0x2a6) [0x564f4b00f196]", "(PGBackend::objects_list_range(hobject_t const&, hobject_t const&, std::vector<hobject_t, std::allocator<hobject_t> >*, std::vector<ghobject_t, std::allocator<ghobject_t> >*)+0x41e) [0x564f4ace265e]", "(PgScrubber::build_scrub_map_chunk(ScrubMap&, ScrubMapBuilder&, hobject_t, hobject_t, bool)+0x59f) [0x564f4ae792af]", "(PgScrubber::build_replica_map_chunk()+0x1c2) [0x564f4ae79f22]", "(Scrub::ActiveReplica::react(Scrub::SchedReplica const&)+0xf1) [0x564f4ae8f301]", "(boost::statechart::simple_state<Scrub::ActiveReplica, Scrub::ScrubMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x45) [0x564f4ae93b45]", "(boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x167) [0x564f4ae887c7]", "(PgScrubber::send_start_replica(unsigned int)+0x3ef) [0x564f4ae70f1f]", "(PG::forward_scrub_event(void (ScrubPgIF::*)(unsigned int), unsigned int)+0x72) [0x564f4abc5072]", "(PG::replica_scrub(unsigned int, ThreadPool::TPHandle&)+0x146) [0x564f4abc5876]", "(ceph::osd::scheduler::PGRepScrub::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x16) [0x564f4ad70f86]", "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xc28) [0x564f4ab33d48]", "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x564f4b1a55b4]", "(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x564f4b1a8254]", "/lib64/libpthread.so.0(+0x817a) [0x7f963537017a]", "clone()" ], "ceph_version": "16.2.7", "crash_id": "2022-01-19T21:23:04.998624Z_ff276c5e-dc45-4ab5-b39d-ed5bc7978449", "entity_name": "osd.f9eaf370e2ea11f1384f0c6476cabede7a62f713", "os_id": "centos", "os_name": "CentOS Linux", "os_version": "8", "os_version_id": "8", "process_name": "ceph-osd", "stack_sig": "eb85c1fdea237931375f9ccc4b9f5e737cc5e9f15d68bd0236c5977f47cb0836", "timestamp": "2022-01-19T21:23:04.998624Z", "utsname_machine": "x86_64", "utsname_release": "5.4.0-94-generic", "utsname_sysname": "Linux", "utsname_version": "#106-Ubuntu SMP Thu Jan 6 23:58:14 UTC 2022" }