Ceph » bluestore

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=e3ba3712874b2b22682a00f451fda676857e3f8e24b9377cdc9bdf42fdc094b4

Sanitized backtrace:

    tcmalloc::CentralFreeList::FetchFromOneSpans(int, void**, void**)
    tcmalloc::CentralFreeList::RemoveRange(void**, void**, int)
    tcmalloc::ThreadCache::FetchFromCentralCache(unsigned int, int, void* (*)(unsigned long))
    rocksdb::AllocateBlock(unsigned long, rocksdb::MemoryAllocator*)
    rocksdb::BlockFetcher::PrepareBufferForBlockFromFile()
    rocksdb::BlockFetcher::ReadBlockContents()
    rocksdb::Status rocksdb::BlockBasedTable::MaybeReadBlockAndLoadToCache<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::BlockContents*) const
    rocksdb::Status rocksdb::BlockBasedTable::RetrieveBlock<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, bool, bool) const
    rocksdb::DataBlockIter* rocksdb::BlockBasedTable::NewDataBlockIterator<rocksdb::DataBlockIter>(rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::DataBlockIter*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::Status, rocksdb::FilePrefetchBuffer*, bool) const
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindKeyForward()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()
    rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)
    rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()
    rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()
    rocksdb::MergingIterator::Next()
    rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)
    rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()
    rocksdb::DBIter::Next()
    rocksdb::ArenaWrappedDBIter::Next()
    RocksDBStore::RocksDBWholeSpaceIteratorImpl::next()
    BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
    BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
    PGBackend::objects_list_partial(hobject_t const&, int, int, std::vector<hobject_t, std::allocator<hobject_t> >*, hobject_t*)
    PgScrubber::select_range()
    Scrub::NewChunk::NewChunk(boost::statechart::state<Scrub::NewChunk, Scrub::ActiveScrubbing, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)>::my_context)
    boost::statechart::simple_state<Scrub::PendingTimer, Scrub::ActiveScrubbing, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)>::react_impl(boost::statechart::event_base const&, void const*)
    boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)
    PgScrubber::send_scrub_resched(unsigned int)
    ceph::osd::scheduler::PGScrubResched::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
    OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
    ShardedThreadPool::shardedthreadpool_worker(unsigned int)
    ShardedThreadPool::WorkThreadSharded::entry()

{
    "backtrace": [
        "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7fdb2e203140]",
        "(tcmalloc::CentralFreeList::FetchFromOneSpans(int, void**, void**)+0x3c) [0x7fdb2e54ab4c]",
        "(tcmalloc::CentralFreeList::RemoveRange(void**, void**, int)+0xdd) [0x7fdb2e54af5d]",
        "(tcmalloc::ThreadCache::FetchFromCentralCache(unsigned int, int, void* (*)(unsigned long))+0x73) [0x7fdb2e54e863]",
        "(rocksdb::AllocateBlock(unsigned long, rocksdb::MemoryAllocator*)+0x75) [0x5653e80dddf9]",
        "(rocksdb::BlockFetcher::PrepareBufferForBlockFromFile()+0x109) [0x5653e8113d97]",
        "(rocksdb::BlockFetcher::ReadBlockContents()+0x10e) [0x5653e8111416]",
        "(rocksdb::Status rocksdb::BlockBasedTable::MaybeReadBlockAndLoadToCache<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::BlockContents*) const+0x45e) [0x5653e80e6068]",
        "(rocksdb::Status rocksdb::BlockBasedTable::RetrieveBlock<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, bool, bool) const+0x10d) [0x5653e80e3c2d]",
        "(rocksdb::DataBlockIter* rocksdb::BlockBasedTable::NewDataBlockIterator<rocksdb::DataBlockIter>(rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::DataBlockIter*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::Status, rocksdb::FilePrefetchBuffer*, bool) const+0x2db) [0x5653e80e71a3]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()+0x4a1) [0x5653e80fc351]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()+0x2c2) [0x5653e80fde70]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindKeyForward()+0x90) [0x5653e80fc8ee]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()+0x8a) [0x5653e80fa214]",
        "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)+0x1d) [0x5653e80fa243]",
        "(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x62) [0x5653e7f4c2de]",
        "/usr/bin/ceph-osd(+0x17ef049) [0x5653e7fe3049]",
        "/usr/bin/ceph-osd(+0x17efa51) [0x5653e7fe3a51]",
        "(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x62) [0x5653e7f4c2de]",
        "(rocksdb::MergingIterator::Next()+0xaf) [0x5653e811c51b]",
        "(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x28) [0x5653e811c656]",
        "(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x62) [0x5653e7f4c2de]",
        "(rocksdb::DBIter::Next()+0x1dc) [0x5653e7f43d4c]",
        "(rocksdb::ArenaWrappedDBIter::Next()+0x1c) [0x5653e81aec84]",
        "(RocksDBStore::RocksDBWholeSpaceIteratorImpl::next()+0x4a) [0x5653e7e24ada]",
        "/usr/bin/ceph-osd(+0x1077bad) [0x5653e786bbad]",
        "(BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0xfde) [0x5653e7897ebe]",
        "(BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0xc7) [0x5653e7899467]",
        "(PGBackend::objects_list_partial(hobject_t const&, int, int, std::vector<hobject_t, std::allocator<hobject_t> >*, hobject_t*)+0xc4d) [0x5653e75624bd]",
        "(PgScrubber::select_range()+0x374) [0x5653e76ebd74]",
        "(Scrub::NewChunk::NewChunk(boost::statechart::state<Scrub::NewChunk, Scrub::ActiveScrubbing, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::my_context)+0x128) [0x5653e7707638]",
        "(boost::statechart::simple_state<Scrub::PendingTimer, Scrub::ActiveScrubbing, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x127) [0x5653e7713b97]",
        "(boost::statechart::state_machine<Scrub::ScrubMachine, Scrub::NotActive, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x84) [0x5653e77039d4]",
        "(PgScrubber::send_scrub_resched(unsigned int)+0xb1) [0x5653e76ff1f1]",
        "(ceph::osd::scheduler::PGScrubResched::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x1a) [0x5653e75e8ada]",
        "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xa27) [0x5653e739e417]",
        "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41a) [0x5653e7a384ca]",
        "(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5653e7a3a830]",
        "/lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7fdb2e1f7ea7]",
        "clone()" 
    ],
    "ceph_version": "16.2.7",
    "crash_id": "2022-01-07T07:04:17.348059Z_d15470d4-1680-4ad1-9612-b1b099f947a6",
    "entity_name": "osd.4f380c829dd0d6fb28f110a25ef6a446f41b9cb0",
    "os_id": "11",
    "os_name": "Debian GNU/Linux 11 (bullseye)",
    "os_version": "11 (bullseye)",
    "os_version_id": "11",
    "process_name": "ceph-osd",
    "stack_sig": "0ce1cd02fad149161b6dee39abfe97056f7f4ea6e237b46fd14316158690f339",
    "timestamp": "2022-01-07T07:04:17.348059Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.11.22-7-pve",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP PVE 5.11.22-12 (Sun, 07 Nov 2021 21:46:36 +0100)" 
}