Bug #54809
crash: rocksdb::SyncPoint::GetInstance()
Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
0%
Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
ee8e84dd61fe471dbd4dc7aff14fb2780243a1dba6d9cc13d56a6ae0ace3867d
Crash signature (v2):
Description
Sanitized backtrace:
rocksdb::SyncPoint::GetInstance() rocksdb::Block::Block(rocksdb::BlockContents&&, unsigned long, unsigned long, rocksdb::Statistics*) rocksdb::BlocklikeTraits<rocksdb::Block>::Create(rocksdb::BlockContents&&, unsigned long, unsigned long, rocksdb::Statistics*, bool, rocksdb::FilterPolicy const*) rocksdb::Status rocksdb::BlockBasedTable::PutDataBlockToCache<rocksdb::Block>(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockContents*, rocksdb::CompressionType, rocksdb::UncompressionDict const&, unsigned long, rocksdb::MemoryAllocator*, rocksdb::BlockType, rocksdb::GetContext*) const rocksdb::Status rocksdb::BlockBasedTable::MaybeReadBlockAndLoadToCache<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::BlockContents*) const rocksdb::Status rocksdb::BlockBasedTable::RetrieveBlock<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, bool, bool) const rocksdb::DataBlockIter* rocksdb::BlockBasedTable::NewDataBlockIterator<rocksdb::DataBlockIter>(rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::DataBlockIter*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::Status, rocksdb::FilePrefetchBuffer*, bool) const rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock() rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward() rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindKeyForward() rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next() rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*) rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next() rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next() rocksdb::MergingIterator::Next() rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*) rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next() rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*) rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*) rocksdb::DBIter::Seek(rocksdb::Slice const&) rocksdb::ArenaWrappedDBIter::Seek(rocksdb::Slice const&) RocksDBStore::RocksDBTransactionImpl::rm_range_keys(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) BlueStore::_omap_rmkey_range(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*) BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*) ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*) OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr<OSDMap const>, ThreadPool::TPHandle*) OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&) ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&) OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) ShardedThreadPool::shardedthreadpool_worker(unsigned int) ShardedThreadPool::WorkThreadSharded::entry()
Crash dump sample:
{ "backtrace": [ "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7f16f9bdf140]", "(rocksdb::SyncPoint::GetInstance()+0) [0x5575e6396808]", "(rocksdb::Block::Block(rocksdb::BlockContents&&, unsigned long, unsigned long, rocksdb::Statistics*)+0xa9) [0x5575e647a05f]", "(rocksdb::BlocklikeTraits<rocksdb::Block>::Create(rocksdb::BlockContents&&, unsigned long, unsigned long, rocksdb::Statistics*, bool, rocksdb::FilterPolicy const*)+0x61) [0x5575e63360ef]", "(rocksdb::Status rocksdb::BlockBasedTable::PutDataBlockToCache<rocksdb::Block>(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockContents*, rocksdb::CompressionType, rocksdb::UncompressionDict const&, unsigned long, rocksdb::MemoryAllocator*, rocksdb::BlockType, rocksdb::GetContext*) const+0x344) [0x5575e63408ba]", "(rocksdb::Status rocksdb::BlockBasedTable::MaybeReadBlockAndLoadToCache<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::BlockContents*) const+0x579) [0x5575e633c183]", "(rocksdb::Status rocksdb::BlockBasedTable::RetrieveBlock<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, bool, bool) const+0x10d) [0x5575e6339c2d]", "(rocksdb::DataBlockIter* rocksdb::BlockBasedTable::NewDataBlockIterator<rocksdb::DataBlockIter>(rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::DataBlockIter*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, rocksdb::Status, rocksdb::FilePrefetchBuffer*, bool) const+0x2db) [0x5575e633d1a3]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::InitDataBlock()+0x4a1) [0x5575e6352351]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindBlockForward()+0x2c2) [0x5575e6353e70]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::FindKeyForward()+0x90) [0x5575e63528ee]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::Next()+0x8a) [0x5575e6350214]", "(rocksdb::BlockBasedTableIterator<rocksdb::DataBlockIter, rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)+0x1d) [0x5575e6350243]", "(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x62) [0x5575e61a22de]", "/usr/bin/ceph-osd(+0x17ef049) [0x5575e6239049]", "/usr/bin/ceph-osd(+0x17efa51) [0x5575e6239a51]", "(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x62) [0x5575e61a22de]", "(rocksdb::MergingIterator::Next()+0xaf) [0x5575e637251b]", "(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x28) [0x5575e6372656]", "(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x62) [0x5575e61a22de]", "(rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*)+0xd7b) [0x5575e619ad87]", "(rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*)+0x70) [0x5575e6199fd8]", "(rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x39b) [0x5575e619e55f]", "(rocksdb::ArenaWrappedDBIter::Seek(rocksdb::Slice const&)+0x27) [0x5575e6404c3b]", "(RocksDBStore::RocksDBTransactionImpl::rm_range_keys(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xf1) [0x5575e60936a1]", "(BlueStore::_omap_rmkey_range(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x17b) [0x5575e5ae55fb]", "(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x9bb) [0x5575e5b430cb]", "(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2e0) [0x5575e5b44bb0]", "(ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x83) [0x5575e562cb83]", "(OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr<OSDMap const>, ThreadPool::TPHandle*)+0x101) [0x5575e55c1c71]", "(OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x20c) [0x5575e55e1e9c]", "(ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x55) [0x5575e583f0b5]", "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xa27) [0x5575e55f4417]", "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41a) [0x5575e5c8e4ca]", "(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5575e5c90830]", "/lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7f16f9bd3ea7]", "clone()" ], "ceph_version": "16.2.7", "crash_id": "2022-01-31T01:06:13.205794Z_f1554193-537a-4c08-8f82-fcef18bcf2d0", "entity_name": "osd.356980760f837dcd9efbd536835450a1cdd69b2e", "os_id": "11", "os_name": "Debian GNU/Linux 11 (bullseye)", "os_version": "11 (bullseye)", "os_version_id": "11", "process_name": "ceph-osd", "stack_sig": "ee8e84dd61fe471dbd4dc7aff14fb2780243a1dba6d9cc13d56a6ae0ace3867d", "timestamp": "2022-01-31T01:06:13.205794Z", "utsname_machine": "x86_64", "utsname_release": "5.13.19-3-pve", "utsname_sysname": "Linux", "utsname_version": "#1 SMP PVE 5.13.19-7 (Thu, 20 Jan 2022 16:37:56 +0100)" }