Project

General

Profile

Bug #54734

crash: void KernelDevice::_aio_thread(): abort

Added by Telemetry Bot about 2 years ago.

Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:

0%

Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):

2f38660fbbd580de8d4d3ec0d8a03f8ae2a83f16d3c0962e52091118aa29586f
3ed453666db10ce6e905eba3250e363d9e37461f578a0a645626435fa8cc1ee5
62786ad51027c0ab015bde163098b7c6ffc8b910e674b0d4642e2a034141fcaa


Description

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=400773dcbaa54cd9edc55f72af0f441af3b9be9493e3866d75a3a055a3a638b4

Assert condition: abort
Assert function: void KernelDevice::_aio_thread()

Sanitized backtrace:

    KernelDevice::flush()
    BlueFS::flush_bdev(std::array<bool, 5ul>&)
    BlueFS::_flush_bdev_safely(BlueFS::FileWriter*)
    BlueFS::_fsync(BlueFS::FileWriter*, std::unique_lock<std::mutex>&)
    BlueRocksWritableFile::Sync()
    rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)
    rocksdb::WritableFileWriter::SyncInternal(bool)
    rocksdb::WritableFileWriter::Sync(bool)
    rocksdb::DBImpl::WriteToWAL(rocksdb::WriteThread::WriteGroup const&, rocksdb::log::Writer*, unsigned long*, bool, bool, unsigned long)
    rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*)
    rocksdb::DBImpl::Write(rocksdb::WriteOptions const&, rocksdb::WriteBatch*)
    RocksDBStore::submit_common(rocksdb::WriteOptions&, std::shared_ptr<KeyValueDB::TransactionImpl>)
    RocksDBStore::submit_transaction_sync(std::shared_ptr<KeyValueDB::TransactionImpl>)
    BlueStore::_kv_sync_thread()
    BlueStore::KVSyncThread::entry()

Crash dump sample:
{
    "assert_condition": "abort",
    "assert_file": "blk/kernel/KernelDevice.cc",
    "assert_func": "void KernelDevice::_aio_thread()",
    "assert_line": 600,
    "assert_msg": "blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7f0c02703700 time 2021-12-28T11:14:19.256027+0000\nblk/kernel/KernelDevice.cc: 600: ceph_abort_msg(\"Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!\")",
    "assert_thread_name": "bstore_aio",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12c20) [0x7f0c0df88c20]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x560d4876b38f]",
        "(KernelDevice::flush()+0x847) [0x560d492b98f7]",
        "(BlueFS::flush_bdev(std::array<bool, 5ul>&)+0x9d) [0x560d48e646ed]",
        "(BlueFS::_flush_bdev_safely(BlueFS::FileWriter*)+0x11a) [0x560d48e6492a]",
        "(BlueFS::_fsync(BlueFS::FileWriter*, std::unique_lock<std::mutex>&)+0x180) [0x560d48e69b90]",
        "(BlueRocksWritableFile::Sync()+0x6c) [0x560d48e915ac]",
        "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x560d4932ab6f]",
        "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x402) [0x560d4943c2e2]",
        "(rocksdb::WritableFileWriter::Sync(bool)+0x88) [0x560d4943d928]",
        "(rocksdb::DBImpl::WriteToWAL(rocksdb::WriteThread::WriteGroup const&, rocksdb::log::Writer*, unsigned long*, bool, bool, unsigned long)+0x309) [0x560d4933e2c9]",
        "(rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*)+0x2629) [0x560d49346e69]",
        "(rocksdb::DBImpl::Write(rocksdb::WriteOptions const&, rocksdb::WriteBatch*)+0x21) [0x560d49347061]",
        "(RocksDBStore::submit_common(rocksdb::WriteOptions&, std::shared_ptr<KeyValueDB::TransactionImpl>)+0x84) [0x560d492e6ba4]",
        "(RocksDBStore::submit_transaction_sync(std::shared_ptr<KeyValueDB::TransactionImpl>)+0x9a) [0x560d492e75aa]",
        "(BlueStore::_kv_sync_thread()+0x30d8) [0x560d48dd6998]",
        "(BlueStore::KVSyncThread::entry()+0x11) [0x560d48dfd881]",
        "/lib64/libpthread.so.0(+0x817a) [0x7f0c0df7e17a]",
        "clone()" 
    ],
    "ceph_version": "16.2.7",
    "crash_id": "2021-12-28T11:14:19.267817Z_6c0c2310-1c36-48c1-bbba-5274c302cb6a",
    "entity_name": "osd.49e49dc53bacd5d00ab78b1213d22c047242459d",
    "io_error": true,
    "io_error_code": -5,
    "io_error_devname": "dm-2",
    "io_error_length": 4096,
    "io_error_offset": 12550090752,
    "io_error_optype": 8,
    "io_error_path": "/var/lib/ceph/osd/ceph-18/block",
    "os_id": "centos",
    "os_name": "CentOS Linux",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-osd",
    "stack_sig": "3ed453666db10ce6e905eba3250e363d9e37461f578a0a645626435fa8cc1ee5",
    "timestamp": "2021-12-28T11:14:19.267817Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.14.14-200.fc34.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Wed Oct 20 16:15:12 UTC 2021" 
}

History

#1 Updated by Telemetry Bot about 2 years ago

  • Crash signature (v1) updated (diff)
  • Crash signature (v2) updated (diff)
  • Affected Versions v16.2.0, v16.2.6, v16.2.7 added

Also available in: Atom PDF