Ceph » bluestore

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=cfcc233f2068b3489670f8574ee3e8bec5825e34fd5fa2676dfe6c1d0ecc8741

Assert condition: (uint64_t)r == len
Assert function: virtual int KernelDevice::read(uint64_t, uint64_t, ceph::bufferlist*, IOContext*, bool)

Sanitized backtrace:

    KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)
    BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::list*, char*)
    BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)
    rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)
    rocksdb::BlockBasedTable::PrefetchTail(rocksdb::ReadOptions const&, rocksdb::RandomAccessFileReader*, unsigned long, bool, rocksdb::TailPrefetchStats*, bool, bool, std::unique_ptr<rocksdb::FilePrefetchBuffer, std::default_delete<rocksdb::FilePrefetchBuffer> >*)
    rocksdb::BlockBasedTable::Open(rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::BlockBasedTableOptions const&, rocksdb::InternalKeyComparator const&, std::unique_ptr<rocksdb::RandomAccessFileReader, std::default_delete<rocksdb::RandomAccessFileReader> >&&, unsigned long, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, rocksdb::SliceTransform const*, bool, bool, int, bool, unsigned long, bool, rocksdb::TailPrefetchStats*, rocksdb::BlockCacheTracer*, unsigned long)
    rocksdb::BlockBasedTableFactory::NewTableReader(rocksdb::ReadOptions const&, rocksdb::TableReaderOptions const&, std::unique_ptr<rocksdb::RandomAccessFileReader, std::default_delete<rocksdb::RandomAccessFileReader> >&&, unsigned long, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, bool) const
    rocksdb::TableCache::GetTableReader(rocksdb::ReadOptions const&, rocksdb::FileOptions const&, rocksdb::InternalKeyComparator const&, rocksdb::FileDescriptor const&, bool, bool, rocksdb::HistogramImpl*, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, rocksdb::SliceTransform const*, bool, int, bool, unsigned long)
    rocksdb::TableCache::FindTable(rocksdb::ReadOptions const&, rocksdb::FileOptions const&, rocksdb::InternalKeyComparator const&, rocksdb::FileDescriptor const&, rocksdb::Cache::Handle**, rocksdb::SliceTransform const*, bool, bool, rocksdb::HistogramImpl*, bool, int, bool, unsigned long)
    rocksdb::VersionBuilder::Rep::LoadTableHandlers(rocksdb::InternalStats*, int, bool, bool, rocksdb::SliceTransform const*, unsigned long)::{lambda()#1}::operator()() const
    rocksdb::VersionBuilder::Rep::LoadTableHandlers(rocksdb::InternalStats*, int, bool, bool, rocksdb::SliceTransform const*, unsigned long)
    rocksdb::VersionBuilder::LoadTableHandlers(rocksdb::InternalStats*, int, bool, bool, rocksdb::SliceTransform const*, unsigned long)
    rocksdb::VersionEditHandler::LoadTables(rocksdb::ColumnFamilyData*, bool, bool)
    rocksdb::VersionEditHandler::CheckIterationResult(rocksdb::log::Reader const&, rocksdb::Status*)
    rocksdb::VersionEditHandlerBase::Iterate(rocksdb::log::Reader&, rocksdb::Status*)
    rocksdb::VersionSet::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, std::basic_string<char, std::char_traits<char>, std::allocator<char> >*)
    rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)
    rocksdb::DBImplReadOnly::OpenForReadOnlyWithoutCheck(rocksdb::DBOptions const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool)
    rocksdb::DB::OpenForReadOnly(rocksdb::DBOptions const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool)
    RocksDBStore::do_open(std::ostream&, bool, bool, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
    BlueStore::_open_db(bool, bool, bool)
    BlueStore::_open_db_and_around(bool, bool)
    BlueStore::_mount()
    OSD::init()

{
    "assert_condition": "(uint64_t)r == len",
    "assert_file": "blk/kernel/KernelDevice.cc",
    "assert_func": "virtual int KernelDevice::read(uint64_t, uint64_t, ceph::bufferlist*, IOContext*, bool)",
    "assert_line": 1255,
    "assert_msg": "blk/kernel/KernelDevice.cc: In function 'virtual int KernelDevice::read(uint64_t, uint64_t, ceph::bufferlist*, IOContext*, bool)' thread 7f907502e3c0 time 2022-12-27T04:21:22.757411+0000\nblk/kernel/KernelDevice.cc: 1255: FAILED ceph_assert((uint64_t)r == len)",
    "assert_thread_name": "ceph-osd",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12cf0) [0x7f9073231cf0]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x562017bb668b]",
        "/usr/bin/ceph-osd(+0x5977f7) [0x562017bb67f7]",
        "(KernelDevice::read(unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, IOContext*, bool)+0x9d7) [0x562018856f67]",
        "(BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, char*)+0x588) [0x562018379298]",
        "(BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)+0x1f) [0x5620183a148f]",
        "(rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x22) [0x5620188cdea2]",
        "(rocksdb::BlockBasedTable::PrefetchTail(rocksdb::ReadOptions const&, rocksdb::RandomAccessFileReader*, unsigned long, bool, rocksdb::TailPrefetchStats*, bool, bool, std::unique_ptr<rocksdb::FilePrefetchBuffer, std::default_delete<rocksdb::FilePrefetchBuffer> >*)+0x24b) [0x562018a43fbb]",
        "(rocksdb::BlockBasedTable::Open(rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::BlockBasedTableOptions const&, rocksdb::InternalKeyComparator const&, std::unique_ptr<rocksdb::RandomAccessFileReader, std::default_delete<rocksdb::RandomAccessFileReader> >&&, unsigned long, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, rocksdb::SliceTransform const*, bool, bool, int, bool, unsigned long, bool, rocksdb::TailPrefetchStats*, rocksdb::BlockCacheTracer*, unsigned long)+0x154) [0x562018a48484]",
        "(rocksdb::BlockBasedTableFactory::NewTableReader(rocksdb::ReadOptions const&, rocksdb::TableReaderOptions const&, std::unique_ptr<rocksdb::RandomAccessFileReader, std::default_delete<rocksdb::RandomAccessFileReader> >&&, unsigned long, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, bool) const+0x5d) [0x562018a3d95d]",
        "(rocksdb::TableCache::GetTableReader(rocksdb::ReadOptions const&, rocksdb::FileOptions const&, rocksdb::InternalKeyComparator const&, rocksdb::FileDescriptor const&, bool, bool, rocksdb::HistogramImpl*, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, rocksdb::SliceTransform const*, bool, int, bool, unsigned long)+0x42b) [0x56201898cbdb]",
        "(rocksdb::TableCache::FindTable(rocksdb::ReadOptions const&, rocksdb::FileOptions const&, rocksdb::InternalKeyComparator const&, rocksdb::FileDescriptor const&, rocksdb::Cache::Handle**, rocksdb::SliceTransform const*, bool, bool, rocksdb::HistogramImpl*, bool, int, bool, unsigned long)+0x32e) [0x56201898d58e]",
        "(rocksdb::VersionBuilder::Rep::LoadTableHandlers(rocksdb::InternalStats*, int, bool, bool, rocksdb::SliceTransform const*, unsigned long)::{lambda()#1}::operator()() const+0x1b3) [0x562018b19c63]",
        "(rocksdb::VersionBuilder::Rep::LoadTableHandlers(rocksdb::InternalStats*, int, bool, bool, rocksdb::SliceTransform const*, unsigned long)+0x414) [0x562018b226b4]",
        "(rocksdb::VersionBuilder::LoadTableHandlers(rocksdb::InternalStats*, int, bool, bool, rocksdb::SliceTransform const*, unsigned long)+0x23) [0x562018b197d3]",
        "(rocksdb::VersionEditHandler::LoadTables(rocksdb::ColumnFamilyData*, bool, bool)+0xb6) [0x562018b262f6]",
        "(rocksdb::VersionEditHandler::CheckIterationResult(rocksdb::log::Reader const&, rocksdb::Status*)+0x5db) [0x562018b269db]",
        "(rocksdb::VersionEditHandlerBase::Iterate(rocksdb::log::Reader&, rocksdb::Status*)+0x4e8) [0x562018b24f58]",
        "(rocksdb::VersionSet::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*)+0x3ff) [0x5620189bd57f]",
        "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0x1bf) [0x56201891f2df]",
        "(rocksdb::DBImplReadOnly::OpenForReadOnlyWithoutCheck(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool)+0x1ba) [0x5620189277ca]",
        "(rocksdb::DB::OpenForReadOnly(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool)+0x8b) [0x562018927dcb]",
        "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xdc4) [0x562018884204]",
        "(BlueStore::_open_db(bool, bool, bool)+0xbde) [0x562018299ede]",
        "(BlueStore::_open_db_and_around(bool, bool)+0x458) [0x5620182c7338]",
        "(BlueStore::_mount()+0x396) [0x5620182ca2f6]",
        "(OSD::init()+0x556) [0x562017cf66f6]",
        "main()",
        "__libc_start_main()" 
    ],
    "ceph_version": "17.2.5",
    "crash_id": "2022-12-27T04:21:22.774826Z_f29c5daa-0efb-43f2-bb7f-f6c6d2678089",
    "entity_name": "osd.4db69b34833a8a176744af32ed12c76e331f84b8",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-osd",
    "stack_sig": "e1faff1a51f5797747612f929c428ef4822975c37e08c19bdd6383cfb5c29573",
    "timestamp": "2022-12-27T04:21:22.774826Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.15.0-56-generic",
    "utsname_sysname": "Linux",
    "utsname_version": "#62-Ubuntu SMP Tue Nov 22 19:54:14 UTC 2022" 
}