Project

General

Profile

Bug #55187

ceph_abort_msg(\"bluefs enospc\")

Added by Aishwarya Mathuria 6 months ago. Updated 2 months ago.

Status:
Need More Info
Priority:
Normal
Assignee:
Target version:
-
% Done:

0%

Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):

26b10e6a73998f3943e92eec28c9f88f78d70cd727c267977a9a51aebf221938
8399067597cb57a8aae13abe0f112a83b5cf7cc7d53b85d56fd1afb99a6c5bed


Description

from osd crash info in gibba cluster:


$ sudo ceph crash info 2022-04-05T02:08:50.176782Z_e45030ee-e34b-46a4-bdde-3bcb3e8005fa
{
    "assert_condition": "abort",
    "assert_file": "/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc",
    "assert_func": "int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)",
    "assert_line": 3137,
    "assert_msg": "/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7f10c8ce23c0 time 2022-04-05T02:08:50.161549+0000\n/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc: 3137: ceph_abort_msg(\"bluefs enospc\")\n",
    "assert_thread_name": "ceph-osd",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12ce0) [0x7f10c6ee7ce0]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b8) [0x56488be01c6c]",
        "(BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)+0x943) [0x56488c5697f3]",
        "(BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)+0xa9) [0x56488c5699d9]",
        "(BlueFS::fsync(BlueFS::FileWriter*)+0x19e) [0x56488c58620e]",
        "(BlueRocksWritableFile::Sync()+0x18) [0x56488c595f88]",
        "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x56488cab96bf]",
        "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x662) [0x56488cbe9e92]",
        "(rocksdb::WritableFileWriter::Sync(bool)+0xf8) [0x56488cbeb858]",
        "(rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)+0x11d) [0x56488cbe2d5d]",
        "(rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x181c) [0x56488cba9aac]",
        "(rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)+0xad1) [0x56488cbab711]",
        "(rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x1c6) [0x56488cac4ba6]",
        "(rocksdb::DBImpl::DeleteUnreferencedSstFiles()+0x99a) [0x56488caf4d1a]",
        "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0x1269) [0x56488cb035d9]",
        "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x5a3) [0x56488cafc3c3]",
        "(rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x56488cafda75]",
        "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x10a1) [0x56488ca70541]",
        "(BlueStore::_open_db(bool, bool, bool)+0x68c) [0x56488c47100c]",
        "(BlueStore::_open_db_and_around(bool, bool)+0x34b) [0x56488c4ba23b]",
        "(BlueStore::_mount()+0x1ae) [0x56488c4bd38e]",
        "(OSD::init()+0x403) [0x56488bf3f513]",
        "main()",
        "__libc_start_main()",
        "_start()" 
    ],
    "ceph_version": "17.1.0-163-g4e244311",
    "crash_id": "2022-04-05T02:08:50.176782Z_e45030ee-e34b-46a4-bdde-3bcb3e8005fa",
    "entity_name": "osd.652",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-osd",
    "stack_sig": "8399067597cb57a8aae13abe0f112a83b5cf7cc7d53b85d56fd1afb99a6c5bed",
    "timestamp": "2022-04-05T02:08:50.176782Z",
    "utsname_hostname": "gibba024",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-301.1.el8.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Tue Apr 13 16:24:22 UTC 2021" 
}

History

#1 Updated by Aishwarya Mathuria 6 months ago

  • Subject changed from quincy: ceph_abort_msg(\"bluefs enospc\") to ceph_abort_msg(\"bluefs enospc\")

#2 Updated by Igor Fedotov 6 months ago

@Aishwarya could you please share the link to the relevant teuthology job where the issue occurred?

#3 Updated by Igor Fedotov 6 months ago

And this ticket (along with related ones) might be relevant:
https://tracker.ceph.com/issues/53466

#4 Updated by Neha Ojha 6 months ago

Igor, this issue was seen in the gibba cluster, while upgrading to 4e244311a4a30b157b41694d9cd9c9a9ecef285f. Currently, osd.652 is down, you can log into gibba001 and check it out.

#5 Updated by Vikhyat Umrao 6 months ago

  • Description updated (diff)

#6 Updated by Adam Kupczyk 5 months ago

  • Status changed from New to Need More Info

#7 Updated by Telemetry Bot 2 months ago

  • Crash signature (v1) updated (diff)
  • Crash signature (v2) updated (diff)
  • Affected Versions v17.1.0, v17.2.1 added

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=80c8aa5fc17d3e51500a97e14c20b473d4bdc7ea8153a950702e6c6f425be6b4

Assert condition: abort
Assert function: int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)

Sanitized backtrace:

    BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)
    BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)
    BlueFS::fsync(BlueFS::FileWriter*)
    BlueRocksWritableFile::Sync()
    rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)
    rocksdb::WritableFileWriter::SyncInternal(bool)
    rocksdb::WritableFileWriter::Sync(bool)
    rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)
    rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)
    rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)
    rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)
    rocksdb::DBImpl::DeleteUnreferencedSstFiles()
    rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)
    rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)
    rocksdb::DB::Open(rocksdb::DBOptions const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)
    RocksDBStore::do_open(std::ostream&, bool, bool, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
    BlueStore::_open_db(bool, bool, bool)
    BlueStore::_open_db_and_around(bool, bool)
    BlueStore::_mount()
    OSD::init()

Crash dump sample:
{
    "assert_condition": "abort",
    "assert_file": "os/bluestore/BlueFS.cc",
    "assert_func": "int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)",
    "assert_line": 3137,
    "assert_msg": "os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7f8ef6dfb240 time 2022-07-22T13:50:14.151115+0500\nos/bluestore/BlueFS.cc: 3137: ceph_abort_msg(\"bluefs enospc\")",
    "assert_thread_name": "ceph-osd",
    "backtrace": [
        "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7f8ef7452140]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x18c) [0x55c6acfa0128]",
        "(BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)+0x71b) [0x55c6ad6c0abb]",
        "(BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)+0x9a) [0x55c6ad6c0c8a]",
        "(BlueFS::fsync(BlueFS::FileWriter*)+0x87) [0x55c6ad6d2ac7]",
        "(BlueRocksWritableFile::Sync()+0x15) [0x55c6ad6e7215]",
        "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x3f) [0x55c6adbebc8b]",
        "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x257) [0x55c6addf38e7]",
        "(rocksdb::WritableFileWriter::Sync(bool)+0x192) [0x55c6addf32aa]",
        "(rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)+0x144) [0x55c6adde96a2]",
        "(rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x210f) [0x55c6add6eb1f]",
        "(rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)+0x917) [0x55c6add70e85]",
        "(rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x162) [0x55c6adbe7ef6]",
        "(rocksdb::DBImpl::DeleteUnreferencedSstFiles()+0x9e2) [0x55c6adc7ea4e]",
        "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0xe95) [0x55c6adc898cb]",
        "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x657) [0x55c6adc900ab]",
        "(rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x52) [0x55c6adc8f414]",
        "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1076) [0x55c6adb99136]",
        "(BlueStore::_open_db(bool, bool, bool)+0xa1e) [0x55c6ad5c8c9e]",
        "(BlueStore::_open_db_and_around(bool, bool)+0x397) [0x55c6ad616017]",
        "(BlueStore::_mount()+0x377) [0x55c6ad6187e7]",
        "(OSD::init()+0x54b) [0x55c6ad08ddfb]",
        "main()",
        "__libc_start_main()",
        "_start()" 
    ],
    "ceph_version": "17.2.1",
    "crash_id": "2022-07-22T08:50:14.197629Z_86cb5b3b-bde1-451d-b176-c2974d9f2ad7",
    "entity_name": "osd.d05cb7524a4d873e08ff38e52d1a11efeaf17111",
    "os_id": "11",
    "os_name": "Debian GNU/Linux 11 (bullseye)",
    "os_version": "11 (bullseye)",
    "os_version_id": "11",
    "process_name": "ceph-osd",
    "stack_sig": "26b10e6a73998f3943e92eec28c9f88f78d70cd727c267977a9a51aebf221938",
    "timestamp": "2022-07-22T08:50:14.197629Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.15.39-1-pve",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP PVE 5.15.39-1 (Wed, 22 Jun 2022 17:22:00 +0200)" 
}

Also available in: Atom PDF