Bug #55187
openceph_abort_msg(\"bluefs enospc\")
0%
26b10e6a73998f3943e92eec28c9f88f78d70cd727c267977a9a51aebf221938
8399067597cb57a8aae13abe0f112a83b5cf7cc7d53b85d56fd1afb99a6c5bed
377b349ab599c018b343be11822ce470211583cc6818d78236570d98ff370612
c304f31a21263ed6b30172009d58f9ff6cba17becc3b4967832582db1895377c
dacb5fd5dc12dec4f6aa2b730486660d594b82a637a1c3dc093505c238186d2b
e988ba94c1d72faff8cfd6aaad780b94d209970c4417fb7093dcb250637e187f
Description
from osd crash info in gibba cluster:
$ sudo ceph crash info 2022-04-05T02:08:50.176782Z_e45030ee-e34b-46a4-bdde-3bcb3e8005fa { "assert_condition": "abort", "assert_file": "/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc", "assert_func": "int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)", "assert_line": 3137, "assert_msg": "/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7f10c8ce23c0 time 2022-04-05T02:08:50.161549+0000\n/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc: 3137: ceph_abort_msg(\"bluefs enospc\")\n", "assert_thread_name": "ceph-osd", "backtrace": [ "/lib64/libpthread.so.0(+0x12ce0) [0x7f10c6ee7ce0]", "gsignal()", "abort()", "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b8) [0x56488be01c6c]", "(BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)+0x943) [0x56488c5697f3]", "(BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)+0xa9) [0x56488c5699d9]", "(BlueFS::fsync(BlueFS::FileWriter*)+0x19e) [0x56488c58620e]", "(BlueRocksWritableFile::Sync()+0x18) [0x56488c595f88]", "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x56488cab96bf]", "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x662) [0x56488cbe9e92]", "(rocksdb::WritableFileWriter::Sync(bool)+0xf8) [0x56488cbeb858]", "(rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)+0x11d) [0x56488cbe2d5d]", "(rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x181c) [0x56488cba9aac]", "(rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)+0xad1) [0x56488cbab711]", "(rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x1c6) [0x56488cac4ba6]", "(rocksdb::DBImpl::DeleteUnreferencedSstFiles()+0x99a) [0x56488caf4d1a]", "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0x1269) [0x56488cb035d9]", "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x5a3) [0x56488cafc3c3]", "(rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x56488cafda75]", "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x10a1) [0x56488ca70541]", "(BlueStore::_open_db(bool, bool, bool)+0x68c) [0x56488c47100c]", "(BlueStore::_open_db_and_around(bool, bool)+0x34b) [0x56488c4ba23b]", "(BlueStore::_mount()+0x1ae) [0x56488c4bd38e]", "(OSD::init()+0x403) [0x56488bf3f513]", "main()", "__libc_start_main()", "_start()" ], "ceph_version": "17.1.0-163-g4e244311", "crash_id": "2022-04-05T02:08:50.176782Z_e45030ee-e34b-46a4-bdde-3bcb3e8005fa", "entity_name": "osd.652", "os_id": "centos", "os_name": "CentOS Stream", "os_version": "8", "os_version_id": "8", "process_name": "ceph-osd", "stack_sig": "8399067597cb57a8aae13abe0f112a83b5cf7cc7d53b85d56fd1afb99a6c5bed", "timestamp": "2022-04-05T02:08:50.176782Z", "utsname_hostname": "gibba024", "utsname_machine": "x86_64", "utsname_release": "4.18.0-301.1.el8.x86_64", "utsname_sysname": "Linux", "utsname_version": "#1 SMP Tue Apr 13 16:24:22 UTC 2021" }
Updated by Aishwarya Mathuria about 2 years ago
- Subject changed from quincy: ceph_abort_msg(\"bluefs enospc\") to ceph_abort_msg(\"bluefs enospc\")
Updated by Igor Fedotov about 2 years ago
@Aishwarya could you please share the link to the relevant teuthology job where the issue occurred?
Updated by Igor Fedotov about 2 years ago
And this ticket (along with related ones) might be relevant:
https://tracker.ceph.com/issues/53466
Updated by Neha Ojha about 2 years ago
Igor, this issue was seen in the gibba cluster, while upgrading to 4e244311a4a30b157b41694d9cd9c9a9ecef285f. Currently, osd.652 is down, you can log into gibba001 and check it out.
Updated by Adam Kupczyk almost 2 years ago
- Status changed from New to Need More Info
Updated by Telemetry Bot over 1 year ago
- Crash signature (v1) updated (diff)
- Crash signature (v2) updated (diff)
- Affected Versions v17.1.0, v17.2.1 added
Assert condition: abort
Assert function: int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)
Sanitized backtrace:
BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long) BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*) BlueFS::fsync(BlueFS::FileWriter*) BlueRocksWritableFile::Sync() rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*) rocksdb::WritableFileWriter::SyncInternal(bool) rocksdb::WritableFileWriter::Sync(bool) rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*) rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*) rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&) rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*) rocksdb::DBImpl::DeleteUnreferencedSstFiles() rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*) rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool) rocksdb::DB::Open(rocksdb::DBOptions const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**) RocksDBStore::do_open(std::ostream&, bool, bool, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) BlueStore::_open_db(bool, bool, bool) BlueStore::_open_db_and_around(bool, bool) BlueStore::_mount() OSD::init()
Crash dump sample:
{ "assert_condition": "abort", "assert_file": "os/bluestore/BlueFS.cc", "assert_func": "int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)", "assert_line": 3137, "assert_msg": "os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7f8ef6dfb240 time 2022-07-22T13:50:14.151115+0500\nos/bluestore/BlueFS.cc: 3137: ceph_abort_msg(\"bluefs enospc\")", "assert_thread_name": "ceph-osd", "backtrace": [ "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7f8ef7452140]", "gsignal()", "abort()", "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x18c) [0x55c6acfa0128]", "(BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)+0x71b) [0x55c6ad6c0abb]", "(BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)+0x9a) [0x55c6ad6c0c8a]", "(BlueFS::fsync(BlueFS::FileWriter*)+0x87) [0x55c6ad6d2ac7]", "(BlueRocksWritableFile::Sync()+0x15) [0x55c6ad6e7215]", "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x3f) [0x55c6adbebc8b]", "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x257) [0x55c6addf38e7]", "(rocksdb::WritableFileWriter::Sync(bool)+0x192) [0x55c6addf32aa]", "(rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)+0x144) [0x55c6adde96a2]", "(rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x210f) [0x55c6add6eb1f]", "(rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)+0x917) [0x55c6add70e85]", "(rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x162) [0x55c6adbe7ef6]", "(rocksdb::DBImpl::DeleteUnreferencedSstFiles()+0x9e2) [0x55c6adc7ea4e]", "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0xe95) [0x55c6adc898cb]", "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x657) [0x55c6adc900ab]", "(rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x52) [0x55c6adc8f414]", "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1076) [0x55c6adb99136]", "(BlueStore::_open_db(bool, bool, bool)+0xa1e) [0x55c6ad5c8c9e]", "(BlueStore::_open_db_and_around(bool, bool)+0x397) [0x55c6ad616017]", "(BlueStore::_mount()+0x377) [0x55c6ad6187e7]", "(OSD::init()+0x54b) [0x55c6ad08ddfb]", "main()", "__libc_start_main()", "_start()" ], "ceph_version": "17.2.1", "crash_id": "2022-07-22T08:50:14.197629Z_86cb5b3b-bde1-451d-b176-c2974d9f2ad7", "entity_name": "osd.d05cb7524a4d873e08ff38e52d1a11efeaf17111", "os_id": "11", "os_name": "Debian GNU/Linux 11 (bullseye)", "os_version": "11 (bullseye)", "os_version_id": "11", "process_name": "ceph-osd", "stack_sig": "26b10e6a73998f3943e92eec28c9f88f78d70cd727c267977a9a51aebf221938", "timestamp": "2022-07-22T08:50:14.197629Z", "utsname_machine": "x86_64", "utsname_release": "5.15.39-1-pve", "utsname_sysname": "Linux", "utsname_version": "#1 SMP PVE 5.15.39-1 (Wed, 22 Jun 2022 17:22:00 +0200)" }
Updated by Telemetry Bot 12 months ago
- Crash signature (v1) updated (diff)
- Affected Versions v17.2.3, v17.2.4, v17.2.5 added