Ceph » bluestore

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=7286bd203800f1a1a572445a412786db047ecffbfa32c775ca0cd0ac17331f49

Assert condition: p != added_maps_bl.end()
Assert function: void OSD::handle_osd_map(MOSDMap*)

Sanitized backtrace:

    KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)
    BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::list*, char*)
    BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)
    rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)
    rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)
    rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)
    rocksdb::DBIter::Next()
    ShardMergeIteratorImpl::next()
    BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
    BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
    PGBackend::objects_list_partial(hobject_t const&, int, int, std::vector<hobject_t, std::allocator<hobject_t> >*, hobject_t*)
    PrimaryLogPG::scan_range(int, int, BackfillInterval*, ThreadPool::TPHandle&)
    PrimaryLogPG::update_range(BackfillInterval*, ThreadPool::TPHandle&)
    PrimaryLogPG::recover_backfill(unsigned long, ThreadPool::TPHandle&, bool*)
    PrimaryLogPG::start_recovery_ops(unsigned long, ThreadPool::TPHandle&, unsigned long*)
    OSD::do_recovery(PG*, unsigned int, unsigned long, ThreadPool::TPHandle&)
    ceph::osd::scheduler::PGRecovery::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
    OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
    ShardedThreadPool::shardedthreadpool_worker(unsigned int)
    ShardedThreadPool::WorkThreadSharded::entry()

{
    "assert_condition": "p != added_maps_bl.end()",
    "assert_file": "osd/OSD.cc",
    "assert_func": "void OSD::handle_osd_map(MOSDMap*)",
    "assert_line": 8083,
    "assert_msg": "blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fb57e1e0700 time 2022-01-09T19:01:52.774690+0000\nblk/kernel/KernelDevice.cc: 600: ceph_abort_msg(\"Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!\")",
    "assert_thread_name": "ms_dispatch",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12c20) [0x7fb58b268c20]",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x56273640cbcf]",
        "ceph-osd(+0x56ad98) [0x56273640cd98]",
        "(KernelDevice::read(unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, IOContext*, bool)+0x8a1) [0x562736f636e1]",
        "(BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, char*)+0x79c) [0x562736b03efc]",
        "(BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)+0x1e) [0x562736b3248e]",
        "(rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x562736fcd3ff]",
        "(rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const+0x2e) [0x56273712c1ee]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()+0x1a6) [0x5627371382a6]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()+0x1c6) [0x562737139126]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()+0x58) [0x562737139228]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)+0x16) [0x562737139266]",
        "(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x50) [0x5627371514f0]",
        "(rocksdb::DBIter::Next()+0x592) [0x562737028da2]",
        "(ShardMergeIteratorImpl::next()+0x57) [0x562736f91ea7]",
        "ceph-osd(+0xb39c62) [0x5627369dbc62]",
        "(BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0xf6b) [0x562736a0ed0b]",
        "(BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0x2a6) [0x562736a10196]",
        "(PGBackend::objects_list_partial(hobject_t const&, int, int, std::vector<hobject_t, std::allocator<hobject_t> >*, hobject_t*)+0x8b4) [0x5627366e9904]",
        "(PrimaryLogPG::scan_range(int, int, BackfillInterval*, ThreadPool::TPHandle&)+0xea) [0x562736689b3a]",
        "(PrimaryLogPG::update_range(BackfillInterval*, ThreadPool::TPHandle&)+0x172) [0x56273668c4c2]",
        "(PrimaryLogPG::recover_backfill(unsigned long, ThreadPool::TPHandle&, bool*)+0x79a) [0x56273668dbca]",
        "(PrimaryLogPG::start_recovery_ops(unsigned long, ThreadPool::TPHandle&, unsigned long*)+0x10e3) [0x562736693ac3]",
        "(OSD::do_recovery(PG*, unsigned int, unsigned long, ThreadPool::TPHandle&)+0x2f5) [0x562736514205]",
        "(ceph::osd::scheduler::PGRecovery::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x1d) [0x562736771fed]",
        "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xc28) [0x562736534d48]",
        "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x562736ba65b4]",
        "(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x562736ba9254]",
        "/lib64/libpthread.so.0(+0x817a) [0x7fb58b25e17a]",
        "clone()" 
    ],
    "ceph_version": "16.2.7",
    "crash_id": "2022-01-09T19:01:52.896794Z_37ba1790-2751-4c1a-b264-d64d5db3dbde",
    "entity_name": "osd.d4dac8e4547ec02779ceb258a685c062afef07c0",
    "io_error": true,
    "io_error_code": -5,
    "io_error_devname": "dm-0",
    "io_error_length": 16384,
    "io_error_offset": 76412960768,
    "io_error_optype": 8,
    "io_error_path": "/var/lib/ceph/osd/ceph-1/block",
    "os_id": "centos",
    "os_name": "CentOS Linux",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-osd",
    "stack_sig": "57a93b2eeff5b0e472715133f53082bda67544ce5e0a74d84ee119a31cce8ae9",
    "timestamp": "2022-01-09T19:01:52.896794Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.4.0-92-generic",
    "utsname_sysname": "Linux",
    "utsname_version": "#103-Ubuntu SMP Fri Nov 26 16:13:00 UTC 2021" 
}