Ceph » bluestore

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=5a22a923304adbc924188e2c82cb70544d62babfce7e71baa4abb371fc2c1a27

Assert condition: (uint64_t)r == len
Assert function: virtual int KernelDevice::read(uint64_t, uint64_t, ceph::bufferlist*, IOContext*, bool)

Sanitized backtrace:

    KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)
    BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::list*, char*)
    BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)
    rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)
    rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)
    rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)
    rocksdb::DBIter::Next()
    CFIteratorImpl::next()
    BlueStore::OmapIteratorImpl::next()
    void PGLog::read_log_and_missing<pg_missing_set<true> >(ObjectStore*, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t, pg_info_t const&, PGLog::IndexedLog&, pg_missing_set<true>&, std::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >&, bool, bool*, DoutPrefixProvider const*, std::set<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >*, bool)
    PG::read_state(ObjectStore*)
    OSD::load_pgs()
    OSD::init()

{
    "assert_condition": "(uint64_t)r == len",
    "assert_file": "blk/kernel/KernelDevice.cc",
    "assert_func": "virtual int KernelDevice::read(uint64_t, uint64_t, ceph::bufferlist*, IOContext*, bool)",
    "assert_line": 1066,
    "assert_msg": "blk/kernel/KernelDevice.cc: In function 'virtual int KernelDevice::read(uint64_t, uint64_t, ceph::bufferlist*, IOContext*, bool)' thread 7f74ae1d2080 time 2022-03-08T02:29:28.530091-0600\nblk/kernel/KernelDevice.cc: 1066: FAILED ceph_assert((uint64_t)r == len)",
    "assert_thread_name": "ceph-osd",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12c20) [0x7f74ac174c20]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x5622bce4fbcf]",
        "/usr/bin/ceph-osd(+0x56ad98) [0x5622bce4fd98]",
        "(KernelDevice::read(unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, IOContext*, bool)+0x8a1) [0x5622bd9a66e1]",
        "(BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, char*)+0x79c) [0x5622bd546efc]",
        "(BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)+0x1e) [0x5622bd57548e]",
        "(rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x5622bda103ff]",
        "(rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const+0x2e) [0x5622bdb6f1ee]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()+0x1a6) [0x5622bdb7b2a6]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()+0x1c6) [0x5622bdb7c126]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()+0x58) [0x5622bdb7c228]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)+0x16) [0x5622bdb7c266]",
        "/usr/bin/ceph-osd(+0x11f8d61) [0x5622bdaddd61]",
        "(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x50) [0x5622bdb944f0]",
        "(rocksdb::DBIter::Next()+0x592) [0x5622bda6bda2]",
        "(CFIteratorImpl::next()+0x86) [0x5622bd9d3b56]",
        "(BlueStore::OmapIteratorImpl::next()+0x132) [0x5622bd439c72]",
        "(void PGLog::read_log_and_missing<pg_missing_set<true> >(ObjectStore*, boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t, pg_info_t const&, PGLog::IndexedLog&, pg_missing_set<true>&, std::__cxx11::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >&, bool, bool*, DoutPrefixProvider const*, std::set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >*, bool)+0x14d2) [0x5622bd0212c2]",
        "(PG::read_state(ObjectStore*)+0x13e7) [0x5622bd00e207]",
        "(OSD::load_pgs()+0xa47) [0x5622bcf5d947]",
        "(OSD::init()+0x26f7) [0x5622bcf89777]",
        "main()",
        "__libc_start_main()",
        "_start()" 
    ],
    "ceph_version": "16.2.7",
    "crash_id": "2022-03-08T08:29:28.561911Z_f99b144f-ae9b-4a38-af43-a983d9345c18",
    "entity_name": "osd.b0bd08d32c179cfd8631468c70524097aef68f76",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-osd",
    "stack_sig": "915540b0ab9cc702f7dd25f2bc634c7f77d0d67b105fbb42e23cda1220d0ac15",
    "timestamp": "2022-03-08T08:29:28.561911Z",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-358.el8.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Mon Jan 10 13:11:20 UTC 2022" 
}