ceph-osd[7526]: *** Caught signal (Segmentation fault) ** ceph-osd[7526]: in thread 7f66eb3e3700 thread_name:tp_osd_tp ceph-osd[7526]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[7526]: 1: (()+0xa542b4) [0xdc6af302b4] ceph-osd[7526]: 2: (()+0x11390) [0x7f6708c56390] ceph-osd[7526]: 3: (()+0x1f8af) [0x7f670a5838af] ceph-osd[7526]: 4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc6b2fc1e9] ceph-osd[7526]: 5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry*, bool)+0x3b7) [0xdc6b2fdbc7] ceph-osd[7526]: 6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc6b2fdf0c] ceph-osd[7526]: 7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc6b306687] ceph-osd[7526]: 8: (()+0xe5594e) [0xdc6b33194e] ceph-osd[7526]: 9: (()+0xe55a16) [0xdc6b331a16] ceph-osd[7526]: 10: (rocksdb::MergingIterator::Next()+0x449) [0xdc6b314ce9] ceph-osd[7526]: 11: (rocksdb::DBIter::FindNextUserEntryInternal(bool, bool)+0x182) [0xdc6b3b20b2] ceph-osd[7526]: 12: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x314) [0xdc6b3b3454] ceph-osd[7526]: 13: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x46) [0xdc6ae73036] ceph-osd[7526]: 14: (BlueStore::_omap_rmkey_range(BlueStore::TransContext*, boost::intrusive_ptr&, boost::intrusive_ptr&, std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x14e) [0xdc6adcda3e] ceph-osd[7526]: 15: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x1e50) [0xdc6ae2f930] ceph-osd[7526]: 16: (BlueStore::queue_transactions(ObjectStore::Sequencer*, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x52e) [0xdc6ae3079e] ceph-osd[7526]: 17: (PrimaryLogPG::queue_transactions(std::vector >&, boost::intrusive_ptr)+0x66) [0xdc6ab5ab86] ceph-osd[7526]: 18: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr >&&, eversion_t const&, eversion_t const&, std::vector > const&, boost::optional&, Context*, Context*, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr)+0xcb8) [0xdc6ac8a838] ceph-osd[7526]: 19: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0x9fa) [0xdc6aaf7c2a] ceph-osd[7526]: 20: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x139b) [0xdc6ab4026b] ceph-osd[7526]: 21: (PrimaryLogPG::do_op(boost::intrusive_ptr&)+0x308b) [0xdc6ab43c8b] ceph-osd[7526]: 22: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0xe73) [0xdc6aafdd03] ceph-osd[7526]: 23: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x3a9) [0xdc6a981fd9] ceph-osd[7526]: 24: (PGQueueable::RunVis::operator()(boost::intrusive_ptr const&)+0x57) [0xdc6ac193f7] ceph-osd[7526]: 25: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x130e) [0xdc6a9a957e] ceph-osd[7526]: 26: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x884) [0xdc6af77e34] ceph-osd[7526]: 27: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xdc6af7ae70] ceph-osd[7526]: 28: (()+0x76ba) [0x7f6708c4c6ba] ceph-osd[7526]: 29: (clone()+0x6d) [0x7f6707cc33dd] ceph-osd[7526]: 2017-08-27 15:41:50.172556 7f66eb3e3700 -1 *** Caught signal (Segmentation fault) ** ceph-osd[7526]: in thread 7f66eb3e3700 thread_name:tp_osd_tp ceph-osd[7526]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[7526]: 1: (()+0xa542b4) [0xdc6af302b4] ceph-osd[7526]: 2: (()+0x11390) [0x7f6708c56390] ceph-osd[7526]: 3: (()+0x1f8af) [0x7f670a5838af] ceph-osd[7526]: 4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc6b2fc1e9] ceph-osd[7526]: 5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry*, bool)+0x3b7) [0xdc6b2fdbc7] ceph-osd[7526]: 6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc6b2fdf0c] ceph-osd[7526]: 7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc6b306687] ceph-osd[7526]: 8: (()+0xe5594e) [0xdc6b33194e] ceph-osd[7526]: 9: (()+0xe55a16) [0xdc6b331a16] ceph-osd[7526]: 10: (rocksdb::MergingIterator::Next()+0x449) [0xdc6b314ce9] ceph-osd[7526]: 11: (rocksdb::DBIter::FindNextUserEntryInternal(bool, bool)+0x182) [0xdc6b3b20b2] ceph-osd[7526]: 12: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x314) [0xdc6b3b3454] ceph-osd[7526]: 13: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x46) [0xdc6ae73036] ceph-osd[7526]: 14: (BlueStore::_omap_rmkey_range(BlueStore::TransContext*, boost::intrusive_ptr&, boost::intrusive_ptr&, std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x14e) [0xdc6adcda3e] ceph-osd[7526]: 15: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x1e50) [0xdc6ae2f930] ceph-osd[7526]: 16: (BlueStore::queue_transactions(ObjectStore::Sequencer*, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x52e) [0xdc6ae3079e] ceph-osd[7526]: 17: (PrimaryLogPG::queue_transactions(std::vector >&, boost::intrusive_ptr)+0x66) [0xdc6ab5ab86] ceph-osd[7526]: 18: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr >&&, eversion_t const&, eversion_t const&, std::vector > const&, boost::optional&, Context*, Context*, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr)+0xcb8) [0xdc6ac8a838] ceph-osd[7526]: 19: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0x9fa) [0xdc6aaf7c2a] ceph-osd[7526]: 20: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x139b) [0xdc6ab4026b] ceph-osd[7526]: 21: (PrimaryLogPG::do_op(boost::intrusive_ptr&)+0x308b) [0xdc6ab43c8b] ceph-osd[7526]: 22: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0xe73) [0xdc6aafdd03] ceph-osd[7526]: 23: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x3a9) [0xdc6a981fd9] ceph-osd[7526]: 24: (PGQueueable::RunVis::operator()(boost::intrusive_ptr const&)+0x57) [0xdc6ac193f7] ceph-osd[7526]: 25: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x130e) [0xdc6a9a957e] ceph-osd[7526]: 26: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x884) [0xdc6af77e34] ceph-osd[7526]: 27: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xdc6af7ae70] ceph-osd[7526]: 28: (()+0x76ba) [0x7f6708c4c6ba] ceph-osd[7526]: 29: (clone()+0x6d) [0x7f6707cc33dd] ceph-osd[7526]: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. ceph-osd[7526]: 0> 2017-08-27 15:41:50.172556 7f66eb3e3700 -1 *** Caught signal (Segmentation fault) ** ceph-osd[7526]: in thread 7f66eb3e3700 thread_name:tp_osd_tp ceph-osd[7526]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[7526]: 1: (()+0xa542b4) [0xdc6af302b4] ceph-osd[7526]: 2: (()+0x11390) [0x7f6708c56390] ceph-osd[7526]: 3: (()+0x1f8af) [0x7f670a5838af] ceph-osd[7526]: 4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc6b2fc1e9] ceph-osd[7526]: 5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry*, bool)+0x3b7) [0xdc6b2fdbc7] ceph-osd[7526]: 6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc6b2fdf0c] ceph-osd[7526]: 7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc6b306687] ceph-osd[7526]: 8: (()+0xe5594e) [0xdc6b33194e] ceph-osd[7526]: 9: (()+0xe55a16) [0xdc6b331a16] ceph-osd[7526]: 10: (rocksdb::MergingIterator::Next()+0x449) [0xdc6b314ce9] ceph-osd[7526]: 11: (rocksdb::DBIter::FindNextUserEntryInternal(bool, bool)+0x182) [0xdc6b3b20b2] ceph-osd[7526]: 12: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x314) [0xdc6b3b3454] ceph-osd[7526]: 13: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x46) [0xdc6ae73036] ceph-osd[7526]: 14: (BlueStore::_omap_rmkey_range(BlueStore::TransContext*, boost::intrusive_ptr&, boost::intrusive_ptr&, std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x14e) [0xdc6adcda3e] ceph-osd[7526]: 15: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x1e50) [0xdc6ae2f930] ceph-osd[7526]: 16: (BlueStore::queue_transactions(ObjectStore::Sequencer*, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x52e) [0xdc6ae3079e] ceph-osd[7526]: 17: (PrimaryLogPG::queue_transactions(std::vector >&, boost::intrusive_ptr)+0x66) [0xdc6ab5ab86] ceph-osd[7526]: 18: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr >&&, eversion_t const&, eversion_t const&, std::vector > const&, boost::optional&, Context*, Context*, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr)+0xcb8) [0xdc6ac8a838] ceph-osd[7526]: 19: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0x9fa) [0xdc6aaf7c2a] ceph-osd[7526]: 20: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x139b) [0xdc6ab4026b] ceph-osd[7526]: 21: (PrimaryLogPG::do_op(boost::intrusive_ptr&)+0x308b) [0xdc6ab43c8b] ceph-osd[7526]: 22: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0xe73) [0xdc6aafdd03] ceph-osd[7526]: 23: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x3a9) [0xdc6a981fd9] ceph-osd[7526]: 24: (PGQueueable::RunVis::operator()(boost::intrusive_ptr const&)+0x57) [0xdc6ac193f7] ceph-osd[7526]: 25: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x130e) [0xdc6a9a957e] ceph-osd[7526]: 26: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x884) [0xdc6af77e34] ceph-osd[7526]: 27: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xdc6af7ae70] ceph-osd[7526]: 28: (()+0x76ba) [0x7f6708c4c6ba] ceph-osd[7526]: 29: (clone()+0x6d) [0x7f6707cc33dd] ceph-osd[7526]: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. systemd[1]: ceph-osd@13.service: Main process exited, code=killed, status=11/SEGV systemd[1]: ceph-osd@13.service: Unit entered failed state. systemd[1]: ceph-osd@13.service: Failed with result 'signal'. systemd[1]: ceph-osd@13.service: Service hold-off time over, scheduling restart. systemd[1]: Stopped Ceph object storage daemon osd.13. systemd[1]: Starting Ceph object storage daemon osd.13... systemd[1]: Started Ceph object storage daemon osd.13. ceph-osd[26349]: starting osd.13 at - osd_data /var/lib/ceph/osd/ceph-13 /var/lib/ceph/osd/ceph-13/journal ceph-osd[26349]: *** Caught signal (Segmentation fault) ** ceph-osd[26349]: in thread 7f9c14bfd700 thread_name:rocksdb:bg0 ceph-osd[26349]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[26349]: 1: (()+0xa542b4) [0x759eb7a2b4] ceph-osd[26349]: 2: (()+0x11390) [0x7f9c2a433390] ceph-osd[26349]: 3: (()+0x1f8af) [0x7f9c2bd608af] ceph-osd[26349]: 4: (rocksdb::Arena::AllocateNewBlock(unsigned long)+0x7c) [0x759ef7c60c] ceph-osd[26349]: 5: (rocksdb::Arena::AllocateFallback(unsigned long, bool)+0x45) [0x759ef7c785] ceph-osd[26349]: 6: (rocksdb::Arena::AllocateAligned(unsigned long, unsigned long, rocksdb::Logger*)+0x100) [0x759ef7c910] ceph-osd[26349]: 7: (rocksdb::DoGenerateLevelFilesBrief(rocksdb::LevelFilesBrief*, std::vector > const&, rocksdb::Arena*)+0x18b) [0x759eefcb3b] ceph-osd[26349]: 8: (rocksdb::Compaction::Compaction(rocksdb::VersionStorageInfo*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, std::vector >, int, unsigned long, unsigned long, unsigned int, rocksdb::CompressionType, std::vector >, bool, double, bool, rocksdb::CompactionReason)+0xc29) [0x759efb43c9] ceph-osd[26349]: 9: (rocksdb::LevelCompactionPicker::PickCompaction(std::__cxx11::basic_string, std::allocator > const&, rocksdb::MutableCFOptions const&, rocksdb::VersionStorageInfo*, rocksdb::LogBuffer*)+0xccd) [0x759efbe9cd] ceph-osd[26349]: 10: (rocksdb::ColumnFamilyData::PickCompaction(rocksdb::MutableCFOptions const&, rocksdb::LogBuffer*)+0x2c) [0x759efaadcc] ceph-osd[26349]: 11: (rocksdb::DBImpl::BackgroundCompaction(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, void*)+0x1a68) [0x759efedba8] ceph-osd[26349]: 12: (rocksdb::DBImpl::BackgroundCallCompaction(void*)+0x127) [0x759efeef87] ceph-osd[26349]: 13: (rocksdb::DBImpl::BGWorkCompaction(void*)+0x90) [0x759efef540] ceph-osd[26349]: 14: (rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x24e) [0x759f06143e] ceph-osd[26349]: 15: (rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x3d) [0x759f06161d] ceph-osd[26349]: 16: (()+0xb8c80) [0x7f9c29d3ac80] ceph-osd[26349]: 17: (()+0x76ba) [0x7f9c2a4296ba] ceph-osd[26349]: 18: (clone()+0x6d) [0x7f9c294a03dd] ceph-osd[26349]: 2017-08-27 15:42:15.410852 7f9c14bfd700 -1 *** Caught signal (Segmentation fault) ** ceph-osd[26349]: in thread 7f9c14bfd700 thread_name:rocksdb:bg0 ceph-osd[26349]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[26349]: 1: (()+0xa542b4) [0x759eb7a2b4] ceph-osd[26349]: 2: (()+0x11390) [0x7f9c2a433390] ceph-osd[26349]: 3: (()+0x1f8af) [0x7f9c2bd608af] ceph-osd[26349]: 4: (rocksdb::Arena::AllocateNewBlock(unsigned long)+0x7c) [0x759ef7c60c] ceph-osd[26349]: 5: (rocksdb::Arena::AllocateFallback(unsigned long, bool)+0x45) [0x759ef7c785] ceph-osd[26349]: 6: (rocksdb::Arena::AllocateAligned(unsigned long, unsigned long, rocksdb::Logger*)+0x100) [0x759ef7c910] ceph-osd[26349]: 7: (rocksdb::DoGenerateLevelFilesBrief(rocksdb::LevelFilesBrief*, std::vector > const&, rocksdb::Arena*)+0x18b) [0x759eefcb3b] ceph-osd[26349]: 8: (rocksdb::Compaction::Compaction(rocksdb::VersionStorageInfo*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, std::vector >, int, unsigned long, unsigned long, unsigned int, rocksdb::CompressionType, std::vector >, bool, double, bool, rocksdb::CompactionReason)+0xc29) [0x759efb43c9] ceph-osd[26349]: 9: (rocksdb::LevelCompactionPicker::PickCompaction(std::__cxx11::basic_string, std::allocator > const&, rocksdb::MutableCFOptions const&, rocksdb::VersionStorageInfo*, rocksdb::LogBuffer*)+0xccd) [0x759efbe9cd] ceph-osd[26349]: 10: (rocksdb::ColumnFamilyData::PickCompaction(rocksdb::MutableCFOptions const&, rocksdb::LogBuffer*)+0x2c) [0x759efaadcc] ceph-osd[26349]: 11: (rocksdb::DBImpl::BackgroundCompaction(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, void*)+0x1a68) [0x759efedba8] ceph-osd[26349]: 12: (rocksdb::DBImpl::BackgroundCallCompaction(void*)+0x127) [0x759efeef87] ceph-osd[26349]: 13: (rocksdb::DBImpl::BGWorkCompaction(void*)+0x90) [0x759efef540] ceph-osd[26349]: 14: (rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x24e) [0x759f06143e] ceph-osd[26349]: 15: (rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x3d) [0x759f06161d] ceph-osd[26349]: 16: (()+0xb8c80) [0x7f9c29d3ac80] ceph-osd[26349]: 17: (()+0x76ba) [0x7f9c2a4296ba] ceph-osd[26349]: 18: (clone()+0x6d) [0x7f9c294a03dd] ceph-osd[26349]: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. ceph-osd[26349]: 0> 2017-08-27 15:42:15.410852 7f9c14bfd700 -1 *** Caught signal (Segmentation fault) ** ceph-osd[26349]: in thread 7f9c14bfd700 thread_name:rocksdb:bg0 ceph-osd[26349]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[26349]: 1: (()+0xa542b4) [0x759eb7a2b4] ceph-osd[26349]: 2: (()+0x11390) [0x7f9c2a433390] ceph-osd[26349]: 3: (()+0x1f8af) [0x7f9c2bd608af] ceph-osd[26349]: 4: (rocksdb::Arena::AllocateNewBlock(unsigned long)+0x7c) [0x759ef7c60c] ceph-osd[26349]: 5: (rocksdb::Arena::AllocateFallback(unsigned long, bool)+0x45) [0x759ef7c785] ceph-osd[26349]: 6: (rocksdb::Arena::AllocateAligned(unsigned long, unsigned long, rocksdb::Logger*)+0x100) [0x759ef7c910] ceph-osd[26349]: 7: (rocksdb::DoGenerateLevelFilesBrief(rocksdb::LevelFilesBrief*, std::vector > const&, rocksdb::Arena*)+0x18b) [0x759eefcb3b] ceph-osd[26349]: 8: (rocksdb::Compaction::Compaction(rocksdb::VersionStorageInfo*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, std::vector >, int, unsigned long, unsigned long, unsigned int, rocksdb::CompressionType, std::vector >, bool, double, bool, rocksdb::CompactionReason)+0xc29) [0x759efb43c9] ceph-osd[26349]: 9: (rocksdb::LevelCompactionPicker::PickCompaction(std::__cxx11::basic_string, std::allocator > const&, rocksdb::MutableCFOptions const&, rocksdb::VersionStorageInfo*, rocksdb::LogBuffer*)+0xccd) [0x759efbe9cd] ceph-osd[26349]: 10: (rocksdb::ColumnFamilyData::PickCompaction(rocksdb::MutableCFOptions const&, rocksdb::LogBuffer*)+0x2c) [0x759efaadcc] ceph-osd[26349]: 11: (rocksdb::DBImpl::BackgroundCompaction(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, void*)+0x1a68) [0x759efedba8] ceph-osd[26349]: 12: (rocksdb::DBImpl::BackgroundCallCompaction(void*)+0x127) [0x759efeef87] ceph-osd[26349]: 13: (rocksdb::DBImpl::BGWorkCompaction(void*)+0x90) [0x759efef540] ceph-osd[26349]: 14: (rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x24e) [0x759f06143e] ceph-osd[26349]: 15: (rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x3d) [0x759f06161d] ceph-osd[26349]: 16: (()+0xb8c80) [0x7f9c29d3ac80] ceph-osd[26349]: 17: (()+0x76ba) [0x7f9c2a4296ba] ceph-osd[26349]: 18: (clone()+0x6d) [0x7f9c294a03dd] ceph-osd[26349]: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. systemd[1]: ceph-osd@13.service: Main process exited, code=killed, status=11/SEGV systemd[1]: ceph-osd@13.service: Unit entered failed state. systemd[1]: ceph-osd@13.service: Failed with result 'signal'. systemd[1]: ceph-osd@13.service: Service hold-off time over, scheduling restart. systemd[1]: Stopped Ceph object storage daemon osd.13. systemd[1]: Starting Ceph object storage daemon osd.13... systemd[1]: Started Ceph object storage daemon osd.13. ceph-osd[26755]: starting osd.13 at - osd_data /var/lib/ceph/osd/ceph-13 /var/lib/ceph/osd/ceph-13/journal ceph-osd[26755]: *** Caught signal (Segmentation fault) ** ceph-osd[26755]: in thread 7f2d71f04e40 thread_name:ceph-osd ceph-osd[26755]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[26755]: 1: (()+0xa542b4) [0xdc0ada32b4] ceph-osd[26755]: 2: (()+0x11390) [0x7f2d701b5390] ceph-osd[26755]: 3: (()+0x1f8af) [0x7f2d71ae28af] ceph-osd[26755]: 4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc0b16f1e9] ceph-osd[26755]: 5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry*, bool)+0x3b7) [0xdc0b170bc7] ceph-osd[26755]: 6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc0b170f0c] ceph-osd[26755]: 7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc0b179687] ceph-osd[26755]: 8: (()+0xe5594e) [0xdc0b1a494e] ceph-osd[26755]: 9: (()+0xe55ec6) [0xdc0b1a4ec6] ceph-osd[26755]: 10: (rocksdb::MergingIterator::Seek(rocksdb::Slice const&)+0x126) [0xdc0b186026] ceph-osd[26755]: 11: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x20a) [0xdc0b22634a] ceph-osd[26755]: 12: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x46) [0xdc0ace6036] ceph-osd[26755]: 13: (BitmapFreelistManager::init()+0x2d6) [0xdc0ad4b626] ceph-osd[26755]: 14: (BlueStore::_open_fm(bool)+0x77f) [0xdc0ac3bdff] ceph-osd[26755]: 15: (BlueStore::_mount(bool)+0x3bc) [0xdc0aca72fc] ceph-osd[26755]: 16: (OSD::init()+0x3df) [0xdc0a822a8f] ceph-osd[26755]: 17: (main()+0x2eb8) [0xdc0a735138] ceph-osd[26755]: 18: (__libc_start_main()+0xf0) [0x7f2d6f13b830] ceph-osd[26755]: 19: (_start()+0x29) [0xdc0a7c0a69] ceph-osd[26755]: 2017-08-27 15:42:36.685509 7f2d71f04e40 -1 *** Caught signal (Segmentation fault) ** ceph-osd[26755]: in thread 7f2d71f04e40 thread_name:ceph-osd ceph-osd[26755]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[26755]: 1: (()+0xa542b4) [0xdc0ada32b4] ceph-osd[26755]: 2: (()+0x11390) [0x7f2d701b5390] ceph-osd[26755]: 3: (()+0x1f8af) [0x7f2d71ae28af] ceph-osd[26755]: 4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc0b16f1e9] ceph-osd[26755]: 5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry*, bool)+0x3b7) [0xdc0b170bc7] ceph-osd[26755]: 6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc0b170f0c] ceph-osd[26755]: 7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc0b179687] ceph-osd[26755]: 8: (()+0xe5594e) [0xdc0b1a494e] ceph-osd[26755]: 9: (()+0xe55ec6) [0xdc0b1a4ec6] ceph-osd[26755]: 10: (rocksdb::MergingIterator::Seek(rocksdb::Slice const&)+0x126) [0xdc0b186026] ceph-osd[26755]: 11: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x20a) [0xdc0b22634a] ceph-osd[26755]: 12: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x46) [0xdc0ace6036] ceph-osd[26755]: 13: (BitmapFreelistManager::init()+0x2d6) [0xdc0ad4b626] ceph-osd[26755]: 14: (BlueStore::_open_fm(bool)+0x77f) [0xdc0ac3bdff] ceph-osd[26755]: 15: (BlueStore::_mount(bool)+0x3bc) [0xdc0aca72fc] ceph-osd[26755]: 16: (OSD::init()+0x3df) [0xdc0a822a8f] ceph-osd[26755]: 17: (main()+0x2eb8) [0xdc0a735138] ceph-osd[26755]: 18: (__libc_start_main()+0xf0) [0x7f2d6f13b830] ceph-osd[26755]: 19: (_start()+0x29) [0xdc0a7c0a69] ceph-osd[26755]: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. ceph-osd[26755]: 0> 2017-08-27 15:42:36.685509 7f2d71f04e40 -1 *** Caught signal (Segmentation fault) ** ceph-osd[26755]: in thread 7f2d71f04e40 thread_name:ceph-osd ceph-osd[26755]: ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc) ceph-osd[26755]: 1: (()+0xa542b4) [0xdc0ada32b4] ceph-osd[26755]: 2: (()+0x11390) [0x7f2d701b5390] ceph-osd[26755]: 3: (()+0x1f8af) [0x7f2d71ae28af] ceph-osd[26755]: 4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc0b16f1e9] ceph-osd[26755]: 5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry*, bool)+0x3b7) [0xdc0b170bc7] ceph-osd[26755]: 6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc0b170f0c] ceph-osd[26755]: 7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc0b179687] ceph-osd[26755]: 8: (()+0xe5594e) [0xdc0b1a494e] ceph-osd[26755]: 9: (()+0xe55ec6) [0xdc0b1a4ec6] ceph-osd[26755]: 10: (rocksdb::MergingIterator::Seek(rocksdb::Slice const&)+0x126) [0xdc0b186026] ceph-osd[26755]: 11: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x20a) [0xdc0b22634a] ceph-osd[26755]: 12: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string, std::allocator > const&, std::__cxx11::basic_string, std::allocator > const&)+0x46) [0xdc0ace6036] ceph-osd[26755]: 13: (BitmapFreelistManager::init()+0x2d6) [0xdc0ad4b626] ceph-osd[26755]: 14: (BlueStore::_open_fm(bool)+0x77f) [0xdc0ac3bdff] ceph-osd[26755]: 15: (BlueStore::_mount(bool)+0x3bc) [0xdc0aca72fc] ceph-osd[26755]: 16: (OSD::init()+0x3df) [0xdc0a822a8f] ceph-osd[26755]: 17: (main()+0x2eb8) [0xdc0a735138] ceph-osd[26755]: 18: (__libc_start_main()+0xf0) [0x7f2d6f13b830] ceph-osd[26755]: 19: (_start()+0x29) [0xdc0a7c0a69] ceph-osd[26755]: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. systemd[1]: ceph-osd@13.service: Main process exited, code=dumped, status=11/SEGV systemd[1]: ceph-osd@13.service: Unit entered failed state. systemd[1]: ceph-osd@13.service: Failed with result 'core-dump'. systemd[1]: ceph-osd@13.service: Service hold-off time over, scheduling restart. systemd[1]: Stopped Ceph object storage daemon osd.13. systemd[1]: Starting Ceph object storage daemon osd.13... systemd[1]: Started Ceph object storage daemon osd.13. ceph-osd[27156]: starting osd.13 at - osd_data /var/lib/ceph/osd/ceph-13 /var/lib/ceph/osd/ceph-13/journal ceph-osd[27156]: 2017-08-27 15:43:13.118701 7fe42a521e40 -1 osd.13 6034 log_to_monitors {default=true}