Project

General

Profile

Bug #55187

ceph_abort_msg(\"bluefs enospc\")

Added by Aishwarya Mathuria 10 months ago. Updated 6 months ago.

Status:
Need More Info
Priority:
Normal
Assignee:
Target version:
-
% Done:

0%

Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):

26b10e6a73998f3943e92eec28c9f88f78d70cd727c267977a9a51aebf221938
8399067597cb57a8aae13abe0f112a83b5cf7cc7d53b85d56fd1afb99a6c5bed


Description

from osd crash info in gibba cluster:


$ sudo ceph crash info 2022-04-05T02:08:50.176782Z_e45030ee-e34b-46a4-bdde-3bcb3e8005fa
{
    "assert_condition": "abort",
    "assert_file": "/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc",
    "assert_func": "int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)",
    "assert_line": 3137,
    "assert_msg": "/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7f10c8ce23c0 time 2022-04-05T02:08:50.161549+0000\n/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc: 3137: ceph_abort_msg(\"bluefs enospc\")\n",
    "assert_thread_name": "ceph-osd",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12ce0) [0x7f10c6ee7ce0]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b8) [0x56488be01c6c]",
        "(BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)+0x943) [0x56488c5697f3]",
        "(BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)+0xa9) [0x56488c5699d9]",
        "(BlueFS::fsync(BlueFS::FileWriter*)+0x19e) [0x56488c58620e]",
        "(BlueRocksWritableFile::Sync()+0x18) [0x56488c595f88]",
        "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x56488cab96bf]",
        "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x662) [0x56488cbe9e92]",
        "(rocksdb::WritableFileWriter::Sync(bool)+0xf8) [0x56488cbeb858]",
        "(rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)+0x11d) [0x56488cbe2d5d]",
        "(rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x181c) [0x56488cba9aac]",
        "(rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)+0xad1) [0x56488cbab711]",
        "(rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x1c6) [0x56488cac4ba6]",
        "(rocksdb::DBImpl::DeleteUnreferencedSstFiles()+0x99a) [0x56488caf4d1a]",
        "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0x1269) [0x56488cb035d9]",
        "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x5a3) [0x56488cafc3c3]",
        "(rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x56488cafda75]",
        "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x10a1) [0x56488ca70541]",
        "(BlueStore::_open_db(bool, bool, bool)+0x68c) [0x56488c47100c]",
        "(BlueStore::_open_db_and_around(bool, bool)+0x34b) [0x56488c4ba23b]",
        "(BlueStore::_mount()+0x1ae) [0x56488c4bd38e]",
        "(OSD::init()+0x403) [0x56488bf3f513]",
        "main()",
        "__libc_start_main()",
        "_start()" 
    ],
    "ceph_version": "17.1.0-163-g4e244311",
    "crash_id": "2022-04-05T02:08:50.176782Z_e45030ee-e34b-46a4-bdde-3bcb3e8005fa",
    "entity_name": "osd.652",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-osd",
    "stack_sig": "8399067597cb57a8aae13abe0f112a83b5cf7cc7d53b85d56fd1afb99a6c5bed",
    "timestamp": "2022-04-05T02:08:50.176782Z",
    "utsname_hostname": "gibba024",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-301.1.el8.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Tue Apr 13 16:24:22 UTC 2021" 
}

History

#1 Updated by Aishwarya Mathuria 10 months ago

  • Subject changed from quincy: ceph_abort_msg(\"bluefs enospc\") to ceph_abort_msg(\"bluefs enospc\")

#2 Updated by Igor Fedotov 10 months ago

@Aishwarya could you please share the link to the relevant teuthology job where the issue occurred?

#3 Updated by Igor Fedotov 10 months ago

And this ticket (along with related ones) might be relevant:
https://tracker.ceph.com/issues/53466

#4 Updated by Neha Ojha 10 months ago

Igor, this issue was seen in the gibba cluster, while upgrading to 4e244311a4a30b157b41694d9cd9c9a9ecef285f. Currently, osd.652 is down, you can log into gibba001 and check it out.

#5 Updated by Vikhyat Umrao 10 months ago

  • Description updated (diff)

#6 Updated by Adam Kupczyk 9 months ago

  • Status changed from New to Need More Info

#7 Updated by Telemetry Bot 6 months ago

  • Crash signature (v1) updated (diff)
  • Crash signature (v2) updated (diff)
  • Affected Versions v17.1.0, v17.2.1 added

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=80c8aa5fc17d3e51500a97e14c20b473d4bdc7ea8153a950702e6c6f425be6b4

Assert condition: abort
Assert function: int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)

Sanitized backtrace:

    BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)
    BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)
    BlueFS::fsync(BlueFS::FileWriter*)
    BlueRocksWritableFile::Sync()
    rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)
    rocksdb::WritableFileWriter::SyncInternal(bool)
    rocksdb::WritableFileWriter::Sync(bool)
    rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)
    rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)
    rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)
    rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)
    rocksdb::DBImpl::DeleteUnreferencedSstFiles()
    rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)
    rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)
    rocksdb::DB::Open(rocksdb::DBOptions const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)
    RocksDBStore::do_open(std::ostream&, bool, bool, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
    BlueStore::_open_db(bool, bool, bool)
    BlueStore::_open_db_and_around(bool, bool)
    BlueStore::_mount()
    OSD::init()

Crash dump sample:
{
    "assert_condition": "abort",
    "assert_file": "os/bluestore/BlueFS.cc",
    "assert_func": "int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)",
    "assert_line": 3137,
    "assert_msg": "os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7f8ef6dfb240 time 2022-07-22T13:50:14.151115+0500\nos/bluestore/BlueFS.cc: 3137: ceph_abort_msg(\"bluefs enospc\")",
    "assert_thread_name": "ceph-osd",
    "backtrace": [
        "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7f8ef7452140]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x18c) [0x55c6acfa0128]",
        "(BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)+0x71b) [0x55c6ad6c0abb]",
        "(BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)+0x9a) [0x55c6ad6c0c8a]",
        "(BlueFS::fsync(BlueFS::FileWriter*)+0x87) [0x55c6ad6d2ac7]",
        "(BlueRocksWritableFile::Sync()+0x15) [0x55c6ad6e7215]",
        "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x3f) [0x55c6adbebc8b]",
        "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x257) [0x55c6addf38e7]",
        "(rocksdb::WritableFileWriter::Sync(bool)+0x192) [0x55c6addf32aa]",
        "(rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)+0x144) [0x55c6adde96a2]",
        "(rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x210f) [0x55c6add6eb1f]",
        "(rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)+0x917) [0x55c6add70e85]",
        "(rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x162) [0x55c6adbe7ef6]",
        "(rocksdb::DBImpl::DeleteUnreferencedSstFiles()+0x9e2) [0x55c6adc7ea4e]",
        "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0xe95) [0x55c6adc898cb]",
        "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x657) [0x55c6adc900ab]",
        "(rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x52) [0x55c6adc8f414]",
        "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1076) [0x55c6adb99136]",
        "(BlueStore::_open_db(bool, bool, bool)+0xa1e) [0x55c6ad5c8c9e]",
        "(BlueStore::_open_db_and_around(bool, bool)+0x397) [0x55c6ad616017]",
        "(BlueStore::_mount()+0x377) [0x55c6ad6187e7]",
        "(OSD::init()+0x54b) [0x55c6ad08ddfb]",
        "main()",
        "__libc_start_main()",
        "_start()" 
    ],
    "ceph_version": "17.2.1",
    "crash_id": "2022-07-22T08:50:14.197629Z_86cb5b3b-bde1-451d-b176-c2974d9f2ad7",
    "entity_name": "osd.d05cb7524a4d873e08ff38e52d1a11efeaf17111",
    "os_id": "11",
    "os_name": "Debian GNU/Linux 11 (bullseye)",
    "os_version": "11 (bullseye)",
    "os_version_id": "11",
    "process_name": "ceph-osd",
    "stack_sig": "26b10e6a73998f3943e92eec28c9f88f78d70cd727c267977a9a51aebf221938",
    "timestamp": "2022-07-22T08:50:14.197629Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.15.39-1-pve",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP PVE 5.15.39-1 (Wed, 22 Jun 2022 17:22:00 +0200)" 
}

Also available in: Atom PDF