Actions
Bug #54762
opencrash: void KernelDevice::_aio_thread(): abort
Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
0%
Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
7910e40824bda6f4d45862782435c8f28cfc6a99edd32e7a5ee5f155b1afcb40
Crash signature (v2):
Description
Assert condition: abort
Assert function: void KernelDevice::_aio_thread()
Sanitized backtrace:
KernelDevice::flush() BlueFS::flush_bdev(std::array<bool, 5ul>&) BlueFS::_flush_bdev_safely(BlueFS::FileWriter*) BlueFS::_flush_and_sync_log(std::unique_lock<std::mutex>&, unsigned long, unsigned long) BlueFS::_fsync(BlueFS::FileWriter*, std::unique_lock<std::mutex>&) BlueRocksWritableFile::Sync() rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*) rocksdb::WritableFileWriter::SyncInternal(bool) rocksdb::WritableFileWriter::Sync(bool) rocksdb::DBImpl::WriteToWAL(rocksdb::WriteThread::WriteGroup const&, rocksdb::log::Writer*, unsigned long*, bool, bool, unsigned long) rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*) rocksdb::DBImpl::Write(rocksdb::WriteOptions const&, rocksdb::WriteBatch*) RocksDBStore::submit_common(rocksdb::WriteOptions&, std::shared_ptr<KeyValueDB::TransactionImpl>) RocksDBStore::submit_transaction_sync(std::shared_ptr<KeyValueDB::TransactionImpl>) BlueStore::_kv_sync_thread() BlueStore::KVSyncThread::entry()
Crash dump sample:
{ "assert_condition": "abort", "assert_file": "blk/kernel/KernelDevice.cc", "assert_func": "void KernelDevice::_aio_thread()", "assert_line": 600, "assert_msg": "blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7f04e73f3700 time 2021-12-28T11:14:19.256082+0000\nblk/kernel/KernelDevice.cc: 600: ceph_abort_msg(\"Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!\")", "assert_thread_name": "bstore_aio", "backtrace": [ "/lib64/libpthread.so.0(+0x12c20) [0x7f04f2c78c20]", "gsignal()", "abort()", "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x556aa6b6b38f]", "(KernelDevice::flush()+0x847) [0x556aa76b98f7]", "(BlueFS::flush_bdev(std::array<bool, 5ul>&)+0x9d) [0x556aa72646ed]", "(BlueFS::_flush_bdev_safely(BlueFS::FileWriter*)+0x11a) [0x556aa726492a]", "(BlueFS::_flush_and_sync_log(std::unique_lock<std::mutex>&, unsigned long, unsigned long)+0xa81) [0x556aa72680f1]", "(BlueFS::_fsync(BlueFS::FileWriter*, std::unique_lock<std::mutex>&)+0x322) [0x556aa7269d32]", "(BlueRocksWritableFile::Sync()+0x6c) [0x556aa72915ac]", "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x556aa772ab6f]", "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x402) [0x556aa783c2e2]", "(rocksdb::WritableFileWriter::Sync(bool)+0x88) [0x556aa783d928]", "(rocksdb::DBImpl::WriteToWAL(rocksdb::WriteThread::WriteGroup const&, rocksdb::log::Writer*, unsigned long*, bool, bool, unsigned long)+0x309) [0x556aa773e2c9]", "(rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*)+0x2629) [0x556aa7746e69]", "(rocksdb::DBImpl::Write(rocksdb::WriteOptions const&, rocksdb::WriteBatch*)+0x21) [0x556aa7747061]", "(RocksDBStore::submit_common(rocksdb::WriteOptions&, std::shared_ptr<KeyValueDB::TransactionImpl>)+0x84) [0x556aa76e6ba4]", "(RocksDBStore::submit_transaction_sync(std::shared_ptr<KeyValueDB::TransactionImpl>)+0x9a) [0x556aa76e75aa]", "(BlueStore::_kv_sync_thread()+0x30d8) [0x556aa71d6998]", "(BlueStore::KVSyncThread::entry()+0x11) [0x556aa71fd881]", "/lib64/libpthread.so.0(+0x817a) [0x7f04f2c6e17a]", "clone()" ], "ceph_version": "16.2.7", "crash_id": "2021-12-28T11:14:19.268448Z_b4cca7ae-9a7f-4279-a509-166e92f13813", "entity_name": "osd.498d8fe5e2c8fe21a98cd9a97bedfd856c800508", "io_error": true, "io_error_code": -5, "io_error_devname": "dm-3", "io_error_length": 65536, "io_error_offset": 10648748032, "io_error_optype": 8, "io_error_path": "/var/lib/ceph/osd/ceph-19/block", "os_id": "centos", "os_name": "CentOS Linux", "os_version": "8", "os_version_id": "8", "process_name": "ceph-osd", "stack_sig": "7910e40824bda6f4d45862782435c8f28cfc6a99edd32e7a5ee5f155b1afcb40", "timestamp": "2021-12-28T11:14:19.268448Z", "utsname_machine": "x86_64", "utsname_release": "5.14.14-200.fc34.x86_64", "utsname_sysname": "Linux", "utsname_version": "#1 SMP Wed Oct 20 16:15:12 UTC 2021" }
Updated by Telemetry Bot about 2 years ago
Actions