Bug #54734
crash: void KernelDevice::_aio_thread(): abort
Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
0%
Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
2f38660fbbd580de8d4d3ec0d8a03f8ae2a83f16d3c0962e52091118aa29586f
3ed453666db10ce6e905eba3250e363d9e37461f578a0a645626435fa8cc1ee5
62786ad51027c0ab015bde163098b7c6ffc8b910e674b0d4642e2a034141fcaa
Crash signature (v2):
Description
Assert condition: abort
Assert function: void KernelDevice::_aio_thread()
Sanitized backtrace:
KernelDevice::flush() BlueFS::flush_bdev(std::array<bool, 5ul>&) BlueFS::_flush_bdev_safely(BlueFS::FileWriter*) BlueFS::_fsync(BlueFS::FileWriter*, std::unique_lock<std::mutex>&) BlueRocksWritableFile::Sync() rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*) rocksdb::WritableFileWriter::SyncInternal(bool) rocksdb::WritableFileWriter::Sync(bool) rocksdb::DBImpl::WriteToWAL(rocksdb::WriteThread::WriteGroup const&, rocksdb::log::Writer*, unsigned long*, bool, bool, unsigned long) rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*) rocksdb::DBImpl::Write(rocksdb::WriteOptions const&, rocksdb::WriteBatch*) RocksDBStore::submit_common(rocksdb::WriteOptions&, std::shared_ptr<KeyValueDB::TransactionImpl>) RocksDBStore::submit_transaction_sync(std::shared_ptr<KeyValueDB::TransactionImpl>) BlueStore::_kv_sync_thread() BlueStore::KVSyncThread::entry()
Crash dump sample:
{ "assert_condition": "abort", "assert_file": "blk/kernel/KernelDevice.cc", "assert_func": "void KernelDevice::_aio_thread()", "assert_line": 600, "assert_msg": "blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7f0c02703700 time 2021-12-28T11:14:19.256027+0000\nblk/kernel/KernelDevice.cc: 600: ceph_abort_msg(\"Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!\")", "assert_thread_name": "bstore_aio", "backtrace": [ "/lib64/libpthread.so.0(+0x12c20) [0x7f0c0df88c20]", "gsignal()", "abort()", "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x560d4876b38f]", "(KernelDevice::flush()+0x847) [0x560d492b98f7]", "(BlueFS::flush_bdev(std::array<bool, 5ul>&)+0x9d) [0x560d48e646ed]", "(BlueFS::_flush_bdev_safely(BlueFS::FileWriter*)+0x11a) [0x560d48e6492a]", "(BlueFS::_fsync(BlueFS::FileWriter*, std::unique_lock<std::mutex>&)+0x180) [0x560d48e69b90]", "(BlueRocksWritableFile::Sync()+0x6c) [0x560d48e915ac]", "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x560d4932ab6f]", "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x402) [0x560d4943c2e2]", "(rocksdb::WritableFileWriter::Sync(bool)+0x88) [0x560d4943d928]", "(rocksdb::DBImpl::WriteToWAL(rocksdb::WriteThread::WriteGroup const&, rocksdb::log::Writer*, unsigned long*, bool, bool, unsigned long)+0x309) [0x560d4933e2c9]", "(rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*)+0x2629) [0x560d49346e69]", "(rocksdb::DBImpl::Write(rocksdb::WriteOptions const&, rocksdb::WriteBatch*)+0x21) [0x560d49347061]", "(RocksDBStore::submit_common(rocksdb::WriteOptions&, std::shared_ptr<KeyValueDB::TransactionImpl>)+0x84) [0x560d492e6ba4]", "(RocksDBStore::submit_transaction_sync(std::shared_ptr<KeyValueDB::TransactionImpl>)+0x9a) [0x560d492e75aa]", "(BlueStore::_kv_sync_thread()+0x30d8) [0x560d48dd6998]", "(BlueStore::KVSyncThread::entry()+0x11) [0x560d48dfd881]", "/lib64/libpthread.so.0(+0x817a) [0x7f0c0df7e17a]", "clone()" ], "ceph_version": "16.2.7", "crash_id": "2021-12-28T11:14:19.267817Z_6c0c2310-1c36-48c1-bbba-5274c302cb6a", "entity_name": "osd.49e49dc53bacd5d00ab78b1213d22c047242459d", "io_error": true, "io_error_code": -5, "io_error_devname": "dm-2", "io_error_length": 4096, "io_error_offset": 12550090752, "io_error_optype": 8, "io_error_path": "/var/lib/ceph/osd/ceph-18/block", "os_id": "centos", "os_name": "CentOS Linux", "os_version": "8", "os_version_id": "8", "process_name": "ceph-osd", "stack_sig": "3ed453666db10ce6e905eba3250e363d9e37461f578a0a645626435fa8cc1ee5", "timestamp": "2021-12-28T11:14:19.267817Z", "utsname_machine": "x86_64", "utsname_release": "5.14.14-200.fc34.x86_64", "utsname_sysname": "Linux", "utsname_version": "#1 SMP Wed Oct 20 16:15:12 UTC 2021" }