Ceph » bluestore

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=a33947a3ddbb26d8cf02c253b4c14d5b7774345571dc5a9078bada3a6c106d3d

Assert condition: abort
Assert function: int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)

Sanitized backtrace:

    BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)
    BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)
    BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)
    BlueRocksWritableFile::Append(rocksdb::Slice const&)
    rocksdb::LegacyWritableFileWrapper::Append(rocksdb::Slice const&, rocksdb::IOOptions const&, rocksdb::IODebugContext*)
    rocksdb::WritableFileWriter::WriteBuffered(char const*, unsigned long)
    rocksdb::WritableFileWriter::Append(rocksdb::Slice const&)
    rocksdb::BlockBasedTableBuilder::WriteRawBlock(rocksdb::Slice const&, rocksdb::CompressionType, rocksdb::BlockHandle*, bool)
    rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*, bool)
    rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::BlockBuilder*, rocksdb::BlockHandle*, bool)
    rocksdb::BlockBasedTableBuilder::Flush()
    rocksdb::BlockBasedTableBuilder::Add(rocksdb::Slice const&, rocksdb::Slice const&)
    rocksdb::BuildTable(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::FileSystem*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::FileOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint, unsigned long)
    rocksdb::FlushJob::WriteLevel0Table()
    rocksdb::FlushJob::Run(rocksdb::LogsWithPrepTracker*, rocksdb::FileMetaData*)
    rocksdb::DBImpl::FlushMemTableToOutputFile(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, bool*, rocksdb::JobContext*, rocksdb::SuperVersionContext*, std::vector<unsigned long, std::allocator<unsigned long> >&, unsigned long, rocksdb::SnapshotChecker*, rocksdb::LogBuffer*, rocksdb::Env::Priority)
    rocksdb::DBImpl::FlushMemTablesToOutputFiles(rocksdb::autovector<rocksdb::DBImpl::BGFlushArg, 8ul> const&, bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::Env::Priority)
    rocksdb::DBImpl::BackgroundFlush(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::FlushReason*, rocksdb::Env::Priority)
    rocksdb::DBImpl::BackgroundCallFlush(rocksdb::Env::Priority)
    rocksdb::DBImpl::BGWorkFlush(void*)
    void std::__invoke_impl<void, void (*&)(void*), void*&>(std::__invoke_other, void (*&)(void*), void*&)
    std::__invoke_result<void (*&)(void*), void*&>::type std::__invoke<void (*&)(void*), void*&>(void (*&)(void*), void*&)
    void std::_Bind<void (*(void*))(void*)>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>)
    void std::_Bind<void (*(void*))(void*)>::operator()<, void>()
    void std::__invoke_impl<void, std::_Bind<void (*(void*))(void*)>&>(std::__invoke_other, std::_Bind<void (*(void*))(void*)>&)
    std::enable_if<std::__and_<std::is_void<void>, std::__is_invocable<std::_Bind<void (*(void*))(void*)>&> >::value, void>::type std::__invoke_r<void, std::_Bind<void (*(void*))(void*)>&>(std::_Bind<void (*(void*))(void*)>&)
    rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)
    rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)
    void std::__invoke_impl<void, void (*)(void*), rocksdb::BGThreadMetadata*>(std::__invoke_other, void (*&&)(void*), rocksdb::BGThreadMetadata*&&)
    std::__invoke_result<void (*)(void*), rocksdb::BGThreadMetadata*>::type std::__invoke<void (*)(void*), rocksdb::BGThreadMetadata*>(void (*&&)(void*), rocksdb::BGThreadMetadata*&&)
    void std::thread::_Invoker<std::tuple<void (*)(void*), rocksdb::BGThreadMetadata*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>)
    std::thread::_Invoker<std::tuple<void (*)(void*), rocksdb::BGThreadMetadata*> >::operator()()

{
    "archived": "2022-07-25 22:08:01.876037",
    "assert_condition": "abort",
    "assert_file": "os/bluestore/BlueFS.cc",
    "assert_func": "int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)",
    "assert_line": 2768,
    "assert_msg": "os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7f9791643700 time 2022-07-25T15:01:42.463701-0700\nos/bluestore/BlueFS.cc: 2768: ceph_abort_msg(\"bluefs enospc\")",
    "assert_thread_name": "rocksdb:high0",
    "backtrace": [
        "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7f979cf94140]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x18a) [0x56457b0364de]",
        "(BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x9bd) [0x56457b728cbd]",
        "(BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x9a) [0x56457b7292ca]",
        "(BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x2f) [0x56457b73a55f]",
        "(BlueRocksWritableFile::Append(rocksdb::Slice const&)+0x100) [0x56457b7528d0]",
        "(rocksdb::LegacyWritableFileWrapper::Append(rocksdb::Slice const&, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x48) [0x56457bc1ae8e]",
        "(rocksdb::WritableFileWriter::WriteBuffered(char const*, unsigned long)+0x338) [0x56457bdf5958]",
        "(rocksdb::WritableFileWriter::Append(rocksdb::Slice const&)+0x5d7) [0x56457bdf3edb]",
        "(rocksdb::BlockBasedTableBuilder::WriteRawBlock(rocksdb::Slice const&, rocksdb::CompressionType, rocksdb::BlockHandle*, bool)+0x11d) [0x56457bfbdf17]",
        "(rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*, bool)+0x7d0) [0x56457bfbdcfe]",
        "(rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::BlockBuilder*, rocksdb::BlockHandle*, bool)+0x48) [0x56457bfbd51a]",
        "(rocksdb::BlockBasedTableBuilder::Flush()+0x9a) [0x56457bfbd4ca]",
        "(rocksdb::BlockBasedTableBuilder::Add(rocksdb::Slice const&, rocksdb::Slice const&)+0x197) [0x56457bfbcfff]",
        "(rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::FileSystem*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::FileOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint, unsigned long)+0x782) [0x56457bf40372]",
        "(rocksdb::FlushJob::WriteLevel0Table()+0xb68) [0x56457bcf5b22]",
        "(rocksdb::FlushJob::Run(rocksdb::LogsWithPrepTracker*, rocksdb::FileMetaData*)+0x275) [0x56457bcf46f7]",
        "(rocksdb::DBImpl::FlushMemTableToOutputFile(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, bool*, rocksdb::JobContext*, rocksdb::SuperVersionContext*, std::vector<unsigned long, std::allocator<unsigned long> >&, unsigned long, rocksdb::SnapshotChecker*, rocksdb::LogBuffer*, rocksdb::Env::Priority)+0x4c4) [0x56457bc76bc4]",
        "(rocksdb::DBImpl::FlushMemTablesToOutputFiles(rocksdb::autovector<rocksdb::DBImpl::BGFlushArg, 8ul> const&, bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::Env::Priority)+0x1b6) [0x56457bc7731c]",
        "(rocksdb::DBImpl::BackgroundFlush(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::FlushReason*, rocksdb::Env::Priority)+0x524) [0x56457bc82b2a]",
        "(rocksdb::DBImpl::BackgroundCallFlush(rocksdb::Env::Priority)+0x1d4) [0x56457bc83072]",
        "(rocksdb::DBImpl::BGWorkFlush(void*)+0xc8) [0x56457bc82002]",
        "(void std::__invoke_impl<void, void (*&)(void*), void*&>(std::__invoke_other, void (*&)(void*), void*&)+0x34) [0x56457bef6152]",
        "(std::__invoke_result<void (*&)(void*), void*&>::type std::__invoke<void (*&)(void*), void*&>(void (*&)(void*), void*&)+0x37) [0x56457bef5ded]",
        "(void std::_Bind<void (*(void*))(void*)>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>)+0x48) [0x56457bef570e]",
        "(void std::_Bind<void (*(void*))(void*)>::operator()<, void>()+0x24) [0x56457bef4e52]",
        "(void std::__invoke_impl<void, std::_Bind<void (*(void*))(void*)>&>(std::__invoke_other, std::_Bind<void (*(void*))(void*)>&)+0x20) [0x56457bef4617]",
        "(std::enable_if<std::__and_<std::is_void<void>, std::__is_invocable<std::_Bind<void (*(void*))(void*)>&> >::value, void>::type std::__invoke_r<void, std::_Bind<void (*(void*))(void*)>&>(std::_Bind<void (*(void*))(void*)>&)+0x20) [0x56457bef38ee]",
        "(std::_Function_handler<void (), std::_Bind<void (*(void*))(void*)> >::_M_invoke(std::_Any_data const&)+0x20) [0x56457bef2ae4]",
        "(rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x269) [0x56457beeedd1]",
        "(rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x10d) [0x56457beeef37]",
        "(void std::__invoke_impl<void, void (*)(void*), rocksdb::BGThreadMetadata*>(std::__invoke_other, void (*&&)(void*), rocksdb::BGThreadMetadata*&&)+0x34) [0x56457bef668d]",
        "(std::__invoke_result<void (*)(void*), rocksdb::BGThreadMetadata*>::type std::__invoke<void (*)(void*), rocksdb::BGThreadMetadata*>(void (*&&)(void*), rocksdb::BGThreadMetadata*&&)+0x37) [0x56457bef6608]",
        "(void std::thread::_Invoker<std::tuple<void (*)(void*), rocksdb::BGThreadMetadata*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>)+0x43) [0x56457bef657b]",
        "(std::thread::_Invoker<std::tuple<void (*)(void*), rocksdb::BGThreadMetadata*> >::operator()()+0x18) [0x56457bef6534]",
        "(std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(void*), rocksdb::BGThreadMetadata*> > >::_M_run()+0x1c) [0x56457bef6518]",
        "/usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xceed0) [0x7f979ce17ed0]",
        "/lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7f979cf88ea7]",
        "clone()" 
    ],
    "ceph_version": "16.2.9",
    "crash_id": "2022-07-25T22:01:42.607744Z_b1133988-b8c0-40a0-acb6-50811e3f2b08",
    "entity_name": "osd.4916906f7ef77f94d953c4532a872a2179a54fe1",
    "os_id": "11",
    "os_name": "Debian GNU/Linux 11 (bullseye)",
    "os_version": "11 (bullseye)",
    "os_version_id": "11",
    "process_name": "ceph-osd",
    "stack_sig": "17df610107b443b1095c1a96590445e4410d10b32957a52b86737cd4bfa3b877",
    "timestamp": "2022-07-25T22:01:42.607744Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.15.35-2-pve",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP PVE 5.15.35-5 (Wed, 08 Jun 2022 15:02:51 +0200)" 
}