Bug #56361
crash: virtual int KernelDevice::flush(): abort
Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
0%
Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
47ffebe233759f02b250df9ace7807cd5ea41749de498017a85ecc51c2d9fae0
Crash signature (v2):
Description
Assert condition: abort
Assert function: virtual int KernelDevice::flush()
Sanitized backtrace:
KernelDevice::flush() BlueFS::flush_bdev(std::array<bool, 5ul>&) BlueFS::_flush_bdev_safely(BlueFS::FileWriter*) BlueFS::_flush_and_sync_log(std::unique_lock<std::mutex>&, unsigned long, unsigned long) BlueFS::_fsync(BlueFS::FileWriter*, std::unique_lock<std::mutex>&) BlueRocksWritableFile::Sync() rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*) rocksdb::WritableFileWriter::SyncInternal(bool) rocksdb::WritableFileWriter::Sync(bool) rocksdb::BuildTable(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::FileSystem*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::FileOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint, unsigned long) rocksdb::FlushJob::WriteLevel0Table() rocksdb::FlushJob::Run(rocksdb::LogsWithPrepTracker*, rocksdb::FileMetaData*) rocksdb::DBImpl::FlushMemTableToOutputFile(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, bool*, rocksdb::JobContext*, rocksdb::SuperVersionContext*, std::vector<unsigned long, std::allocator<unsigned long> >&, unsigned long, rocksdb::SnapshotChecker*, rocksdb::LogBuffer*, rocksdb::Env::Priority) rocksdb::DBImpl::FlushMemTablesToOutputFiles(rocksdb::autovector<rocksdb::DBImpl::BGFlushArg, 8ul> const&, bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::Env::Priority) rocksdb::DBImpl::BackgroundFlush(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::FlushReason*, rocksdb::Env::Priority) rocksdb::DBImpl::BackgroundCallFlush(rocksdb::Env::Priority) rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)
Crash dump sample:
{ "archived": "2022-06-13 11:04:53.042604", "assert_condition": "abort", "assert_file": "blk/kernel/KernelDevice.cc", "assert_func": "virtual int KernelDevice::flush()", "assert_line": 460, "assert_msg": "blk/kernel/KernelDevice.cc: In function 'virtual int KernelDevice::flush()' thread 7f228d7c4700 time 2022-06-13T10:50:51.894857+0100\nblk/kernel/KernelDevice.cc: 460: ceph_abort_msg(\"abort() called\")", "assert_thread_name": "rocksdb:high0", "backtrace": [ "/lib64/libpthread.so.0(+0x12ce0) [0x7f229d7d5ce0]", "gsignal()", "abort()", "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x557aab796f6d]", "(KernelDevice::flush()+0x847) [0x557aac2f02c7]", "(BlueFS::flush_bdev(std::array<bool, 5ul>&)+0x9d) [0x557aabe9a51d]", "(BlueFS::_flush_bdev_safely(BlueFS::FileWriter*)+0x11a) [0x557aabe9a75a]", "(BlueFS::_flush_and_sync_log(std::unique_lock<std::mutex>&, unsigned long, unsigned long)+0xa81) [0x557aabe9df21]", "(BlueFS::_fsync(BlueFS::FileWriter*, std::unique_lock<std::mutex>&)+0x322) [0x557aabe9fb62]", "(BlueRocksWritableFile::Sync()+0x6c) [0x557aabec73dc]", "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x557aac3621cf]", "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x402) [0x557aac473942]", "(rocksdb::WritableFileWriter::Sync(bool)+0x88) [0x557aac474f88]", "(rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::FileSystem*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::FileOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint, unsigned long)+0x2ddb) [0x557aac5420eb]", "(rocksdb::FlushJob::WriteLevel0Table()+0x1003) [0x557aac3d8563]", "(rocksdb::FlushJob::Run(rocksdb::LogsWithPrepTracker*, rocksdb::FileMetaData*)+0xcd) [0x557aac3da5fd]", "(rocksdb::DBImpl::FlushMemTableToOutputFile(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, bool*, rocksdb::JobContext*, rocksdb::SuperVersionContext*, std::vector<unsigned long, std::allocator<unsigned long> >&, unsigned long, rocksdb::SnapshotChecker*, rocksdb::LogBuffer*, rocksdb::Env::Priority)+0x3c1) [0x557aac385111]", "(rocksdb::DBImpl::FlushMemTablesToOutputFiles(rocksdb::autovector<rocksdb::DBImpl::BGFlushArg, 8ul> const&, bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::Env::Priority)+0x49b) [0x557aac387e4b]", "(rocksdb::DBImpl::BackgroundFlush(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::FlushReason*, rocksdb::Env::Priority)+0xaf5) [0x557aac388f15]", "(rocksdb::DBImpl::BackgroundCallFlush(rocksdb::Env::Priority)+0xc8) [0x557aac38f5c8]", "(rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x24a) [0x557aac5198ca]", "(rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x5d) [0x557aac519a4d]", "/lib64/libstdc++.so.6(+0xc2ba3) [0x7f229ce1eba3]", "/lib64/libpthread.so.0(+0x81cf) [0x7f229d7cb1cf]", "clone()" ], "ceph_version": "16.2.9", "crash_id": "2022-06-13T09:50:51.924721Z_878f285e-c373-4c2d-b36c-80fad9cbd6dd", "entity_name": "osd.249f31f7b408773a543c8ef1083a48b6465b300f", "os_id": "rocky", "os_name": "Rocky Linux", "os_version": "8.6 (Green Obsidian)", "os_version_id": "8.6", "process_name": "ceph-osd", "stack_sig": "47ffebe233759f02b250df9ace7807cd5ea41749de498017a85ecc51c2d9fae0", "timestamp": "2022-06-13T09:50:51.924721Z", "utsname_machine": "x86_64", "utsname_release": "4.18.0-372.9.1.el8.x86_64", "utsname_sysname": "Linux", "utsname_version": "#1 SMP Tue May 10 14:48:47 UTC 2022" }