Project

General

Profile

Bug #54684

crash: int64_t BlueFS::_read(BlueFS::FileReader*, uint64_t, size_t, ceph::bufferlist*, char*): assert(r == 0)

Added by Telemetry Bot about 2 years ago. Updated about 2 years ago.

Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:

0%

Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):

2b5596d80e087b41d9c1c9a5e13a468393dcadfdbad9ebd4d4cbaaedb24d51b3
eb85c1fdea237931375f9ccc4b9f5e737cc5e9f15d68bd0236c5977f47cb0836


Description

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=5dfe479ba7b50b00e1cd1a3367ab16d39e2861d62b5ec7401582f262d5a9974e

Assert condition: r == 0
Assert function: int64_t BlueFS::_read(BlueFS::FileReader*, uint64_t, size_t, ceph::bufferlist*, char*)

Sanitized backtrace:

    BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::list*, char*)
    BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)
    rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)
    rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)
    rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)
    rocksdb::DBIter::Next()
    RocksDBStore::RocksDBWholeSpaceIteratorImpl::next()
    BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
    BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
    PGBackend::objects_list_range(hobject_t const&, hobject_t const&, std::vector<hobject_t, std::allocator<hobject_t> >*, std::vector<ghobject_t, std::allocator<ghobject_t> >*)
    PgScrubber::build_scrub_map_chunk(ScrubMap&, ScrubMapBuilder&, hobject_t, hobject_t, bool)
    PgScrubber::build_replica_map_chunk()
    Scrub::ActiveReplica::react(Scrub::SchedReplica const&)
    boost::statechart::simple_state<Scrub::ActiveReplica, Scrub::ScrubMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)>::react_impl(boost::statechart::event_base const&, void const*)
    boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)
    PgScrubber::send_start_replica(unsigned int)
    PG::forward_scrub_event(void (ScrubPgIF::*)(unsigned int), unsigned int)
    PG::replica_scrub(unsigned int, ThreadPool::TPHandle&)
    ceph::osd::scheduler::PGRepScrub::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
    OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
    ShardedThreadPool::shardedthreadpool_worker(unsigned int)
    ShardedThreadPool::WorkThreadSharded::entry()

Crash dump sample:
{
    "assert_condition": "r == 0",
    "assert_file": "os/bluestore/BlueFS.cc",
    "assert_func": "int64_t BlueFS::_read(BlueFS::FileReader*, uint64_t, size_t, ceph::bufferlist*, char*)",
    "assert_line": 2044,
    "assert_msg": "os/bluestore/BlueFS.cc: In function 'int64_t BlueFS::_read(BlueFS::FileReader*, uint64_t, size_t, ceph::bufferlist*, char*)' thread 7f9612b44700 time 2022-01-19T21:23:04.972504+0000\nos/bluestore/BlueFS.cc: 2044: FAILED ceph_assert(r == 0)",
    "assert_thread_name": "tp_osd_tp",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12c20) [0x7f963537ac20]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x564f4aa0bbcf]",
        "/usr/bin/ceph-osd(+0x56ad98) [0x564f4aa0bd98]",
        "(BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, char*)+0x7b0) [0x564f4b102f10]",
        "(BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)+0x1e) [0x564f4b13148e]",
        "(rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x564f4b5cc3ff]",
        "(rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const+0x2e) [0x564f4b72b1ee]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()+0x1a6) [0x564f4b7372a6]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()+0x1c6) [0x564f4b738126]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()+0x58) [0x564f4b738228]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)+0x16) [0x564f4b738266]",
        "/usr/bin/ceph-osd(+0x11f8d61) [0x564f4b699d61]",
        "(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x50) [0x564f4b7504f0]",
        "(rocksdb::DBIter::Next()+0x592) [0x564f4b627da2]",
        "(RocksDBStore::RocksDBWholeSpaceIteratorImpl::next()+0x31) [0x564f4b574a31]",
        "/usr/bin/ceph-osd(+0xb398ca) [0x564f4afda8ca]",
        "(BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0xf6b) [0x564f4b00dd0b]",
        "(BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0x2a6) [0x564f4b00f196]",
        "(PGBackend::objects_list_range(hobject_t const&, hobject_t const&, std::vector<hobject_t, std::allocator<hobject_t> >*, std::vector<ghobject_t, std::allocator<ghobject_t> >*)+0x41e) [0x564f4ace265e]",
        "(PgScrubber::build_scrub_map_chunk(ScrubMap&, ScrubMapBuilder&, hobject_t, hobject_t, bool)+0x59f) [0x564f4ae792af]",
        "(PgScrubber::build_replica_map_chunk()+0x1c2) [0x564f4ae79f22]",
        "(Scrub::ActiveReplica::react(Scrub::SchedReplica const&)+0xf1) [0x564f4ae8f301]",
        "(boost::statechart::simple_state<Scrub::ActiveReplica, Scrub::ScrubMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x45) [0x564f4ae93b45]",
        "(boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x167) [0x564f4ae887c7]",
        "(PgScrubber::send_start_replica(unsigned int)+0x3ef) [0x564f4ae70f1f]",
        "(PG::forward_scrub_event(void (ScrubPgIF::*)(unsigned int), unsigned int)+0x72) [0x564f4abc5072]",
        "(PG::replica_scrub(unsigned int, ThreadPool::TPHandle&)+0x146) [0x564f4abc5876]",
        "(ceph::osd::scheduler::PGRepScrub::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x16) [0x564f4ad70f86]",
        "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xc28) [0x564f4ab33d48]",
        "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x564f4b1a55b4]",
        "(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x564f4b1a8254]",
        "/lib64/libpthread.so.0(+0x817a) [0x7f963537017a]",
        "clone()" 
    ],
    "ceph_version": "16.2.7",
    "crash_id": "2022-01-19T21:23:04.998624Z_ff276c5e-dc45-4ab5-b39d-ed5bc7978449",
    "entity_name": "osd.f9eaf370e2ea11f1384f0c6476cabede7a62f713",
    "os_id": "centos",
    "os_name": "CentOS Linux",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-osd",
    "stack_sig": "eb85c1fdea237931375f9ccc4b9f5e737cc5e9f15d68bd0236c5977f47cb0836",
    "timestamp": "2022-01-19T21:23:04.998624Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.4.0-94-generic",
    "utsname_sysname": "Linux",
    "utsname_version": "#106-Ubuntu SMP Thu Jan 6 23:58:14 UTC 2022" 
}

History

#1 Updated by Telemetry Bot about 2 years ago

  • Crash signature (v1) updated (diff)
  • Crash signature (v2) updated (diff)
  • Affected Versions v16.2.6, v16.2.7 added

Also available in: Atom PDF