Bug #55187
Updated by Vikhyat Umrao about 2 years ago
from osd crash info in gibba cluster: <pre> $ sudo ceph crash info 2022-04-05T02:08:50.176782Z_e45030ee-e34b-46a4-bdde-3bcb3e8005fa { "assert_condition": "abort", "assert_file": "/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc", "assert_func": "int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)", "assert_line": 3137, "assert_msg": "/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range_F(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7f10c8ce23c0 time 2022-04-05T02:08:50.161549+0000\n/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.1.0-163-g4e244311/rpm/el8/BUILD/ceph-17.1.0-163-g4e244311/src/os/bluestore/BlueFS.cc: 3137: ceph_abort_msg(\"bluefs enospc\")\n", "assert_thread_name": "ceph-osd", "backtrace": [ "/lib64/libpthread.so.0(+0x12ce0) [0x7f10c6ee7ce0]", "gsignal()", "abort()", "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b8) [0x56488be01c6c]", "(BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)+0x943) [0x56488c5697f3]", "(BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)+0xa9) [0x56488c5699d9]", "(BlueFS::fsync(BlueFS::FileWriter*)+0x19e) [0x56488c58620e]", "(BlueRocksWritableFile::Sync()+0x18) [0x56488c595f88]", "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x1f) [0x56488cab96bf]", "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x662) [0x56488cbe9e92]", "(rocksdb::WritableFileWriter::Sync(bool)+0xf8) [0x56488cbeb858]", "(rocksdb::SyncManifest(rocksdb::Env*, rocksdb::ImmutableDBOptions const*, rocksdb::WritableFileWriter*)+0x11d) [0x56488cbe2d5d]", "(rocksdb::VersionSet::ProcessManifestWrites(std::deque<rocksdb::VersionSet::ManifestWriter, std::allocator<rocksdb::VersionSet::ManifestWriter> >&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x181c) [0x56488cba9aac]", "(rocksdb::VersionSet::LogAndApply(rocksdb::autovector<rocksdb::ColumnFamilyData*, 8ul> const&, rocksdb::autovector<rocksdb::MutableCFOptions const*, 8ul> const&, rocksdb::autovector<rocksdb::autovector<rocksdb::VersionEdit*, 8ul>, 8ul> const&, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*, std::vector<std::function<void (rocksdb::Status const&)>, std::allocator<std::function<void (rocksdb::Status const&)> > > const&)+0xad1) [0x56488cbab711]", "(rocksdb::VersionSet::LogAndApply(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, rocksdb::VersionEdit*, rocksdb::InstrumentedMutex*, rocksdb::FSDirectory*, bool, rocksdb::ColumnFamilyOptions const*)+0x1c6) [0x56488cac4ba6]", "(rocksdb::DBImpl::DeleteUnreferencedSstFiles()+0x99a) [0x56488caf4d1a]", "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0x1269) [0x56488cb035d9]", "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x5a3) [0x56488cafc3c3]", "(rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x56488cafda75]", "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x10a1) [0x56488ca70541]", "(BlueStore::_open_db(bool, bool, bool)+0x68c) [0x56488c47100c]", "(BlueStore::_open_db_and_around(bool, bool)+0x34b) [0x56488c4ba23b]", "(BlueStore::_mount()+0x1ae) [0x56488c4bd38e]", "(OSD::init()+0x403) [0x56488bf3f513]", "main()", "__libc_start_main()", "_start()" ], "ceph_version": "17.1.0-163-g4e244311", "crash_id": "2022-04-05T02:08:50.176782Z_e45030ee-e34b-46a4-bdde-3bcb3e8005fa", "entity_name": "osd.652", "os_id": "centos", "os_name": "CentOS Stream", "os_version": "8", "os_version_id": "8", "process_name": "ceph-osd", "stack_sig": "8399067597cb57a8aae13abe0f112a83b5cf7cc7d53b85d56fd1afb99a6c5bed", "timestamp": "2022-04-05T02:08:50.176782Z", "utsname_hostname": "gibba024", "utsname_machine": "x86_64", "utsname_release": "4.18.0-301.1.el8.x86_64", "utsname_sysname": "Linux", "utsname_version": "#1 SMP Tue Apr 13 16:24:22 UTC 2021" } </pre>