Project

General

Profile

osd.13-segfault.txt

SSD - Mikko Tanner, 08/27/2017 02:05 PM

Download (30.3 KB)

 
1
ceph-osd[7526]: *** Caught signal (Segmentation fault) **
2
ceph-osd[7526]:  in thread 7f66eb3e3700 thread_name:tp_osd_tp
3
ceph-osd[7526]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
4
ceph-osd[7526]:  1: (()+0xa542b4) [0xdc6af302b4]
5
ceph-osd[7526]:  2: (()+0x11390) [0x7f6708c56390]
6
ceph-osd[7526]:  3: (()+0x1f8af) [0x7f670a5838af]
7
ceph-osd[7526]:  4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc6b2fc1e9]
8
ceph-osd[7526]:  5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, bool)+0x3b7) [0xdc6b2fdbc7]
9
ceph-osd[7526]:  6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc6b2fdf0c]
10
ceph-osd[7526]:  7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc6b306687]
11
ceph-osd[7526]:  8: (()+0xe5594e) [0xdc6b33194e]
12
ceph-osd[7526]:  9: (()+0xe55a16) [0xdc6b331a16]
13
ceph-osd[7526]:  10: (rocksdb::MergingIterator::Next()+0x449) [0xdc6b314ce9]
14
ceph-osd[7526]:  11: (rocksdb::DBIter::FindNextUserEntryInternal(bool, bool)+0x182) [0xdc6b3b20b2]
15
ceph-osd[7526]:  12: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x314) [0xdc6b3b3454]
16
ceph-osd[7526]:  13: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x46) [0xdc6ae73036]
17
ceph-osd[7526]:  14: (BlueStore::_omap_rmkey_range(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x14e) [0xdc6adcda3e]
18
ceph-osd[7526]:  15: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x1e50) [0xdc6ae2f930]
19
ceph-osd[7526]:  16: (BlueStore::queue_transactions(ObjectStore::Sequencer*, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x52e) [0xdc6ae3079e]
20
ceph-osd[7526]:  17: (PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x66) [0xdc6ab5ab86]
21
ceph-osd[7526]:  18: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, boost::optional<pg_hit_set_history_t>&, Context*, Context*, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>)+0xcb8) [0xdc6ac8a838]
22
ceph-osd[7526]:  19: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0x9fa) [0xdc6aaf7c2a]
23
ceph-osd[7526]:  20: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x139b) [0xdc6ab4026b]
24
ceph-osd[7526]:  21: (PrimaryLogPG::do_op(boost::intrusive_ptr<OpRequest>&)+0x308b) [0xdc6ab43c8b]
25
ceph-osd[7526]:  22: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0xe73) [0xdc6aafdd03]
26
ceph-osd[7526]:  23: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3a9) [0xdc6a981fd9]
27
ceph-osd[7526]:  24: (PGQueueable::RunVis::operator()(boost::intrusive_ptr<OpRequest> const&)+0x57) [0xdc6ac193f7]
28
ceph-osd[7526]:  25: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x130e) [0xdc6a9a957e]
29
ceph-osd[7526]:  26: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x884) [0xdc6af77e34]
30
ceph-osd[7526]:  27: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xdc6af7ae70]
31
ceph-osd[7526]:  28: (()+0x76ba) [0x7f6708c4c6ba]
32
ceph-osd[7526]:  29: (clone()+0x6d) [0x7f6707cc33dd]
33

    
34
ceph-osd[7526]: 2017-08-27 15:41:50.172556 7f66eb3e3700 -1 *** Caught signal (Segmentation fault) **
35
ceph-osd[7526]:  in thread 7f66eb3e3700 thread_name:tp_osd_tp
36
ceph-osd[7526]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
37
ceph-osd[7526]:  1: (()+0xa542b4) [0xdc6af302b4]
38
ceph-osd[7526]:  2: (()+0x11390) [0x7f6708c56390]
39
ceph-osd[7526]:  3: (()+0x1f8af) [0x7f670a5838af]
40
ceph-osd[7526]:  4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc6b2fc1e9]
41
ceph-osd[7526]:  5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, bool)+0x3b7) [0xdc6b2fdbc7]
42
ceph-osd[7526]:  6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc6b2fdf0c]
43
ceph-osd[7526]:  7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc6b306687]
44
ceph-osd[7526]:  8: (()+0xe5594e) [0xdc6b33194e]
45
ceph-osd[7526]:  9: (()+0xe55a16) [0xdc6b331a16]
46
ceph-osd[7526]:  10: (rocksdb::MergingIterator::Next()+0x449) [0xdc6b314ce9]
47
ceph-osd[7526]:  11: (rocksdb::DBIter::FindNextUserEntryInternal(bool, bool)+0x182) [0xdc6b3b20b2]
48
ceph-osd[7526]:  12: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x314) [0xdc6b3b3454]
49
ceph-osd[7526]:  13: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x46) [0xdc6ae73036]
50
ceph-osd[7526]:  14: (BlueStore::_omap_rmkey_range(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x14e) [0xdc6adcda3e]
51
ceph-osd[7526]:  15: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x1e50) [0xdc6ae2f930]
52
ceph-osd[7526]:  16: (BlueStore::queue_transactions(ObjectStore::Sequencer*, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x52e) [0xdc6ae3079e]
53
ceph-osd[7526]:  17: (PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x66) [0xdc6ab5ab86]
54
ceph-osd[7526]:  18: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, boost::optional<pg_hit_set_history_t>&, Context*, Context*, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>)+0xcb8) [0xdc6ac8a838]
55
ceph-osd[7526]:  19: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0x9fa) [0xdc6aaf7c2a]
56
ceph-osd[7526]:  20: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x139b) [0xdc6ab4026b]
57
ceph-osd[7526]:  21: (PrimaryLogPG::do_op(boost::intrusive_ptr<OpRequest>&)+0x308b) [0xdc6ab43c8b]
58
ceph-osd[7526]:  22: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0xe73) [0xdc6aafdd03]
59
ceph-osd[7526]:  23: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3a9) [0xdc6a981fd9]
60
ceph-osd[7526]:  24: (PGQueueable::RunVis::operator()(boost::intrusive_ptr<OpRequest> const&)+0x57) [0xdc6ac193f7]
61
ceph-osd[7526]:  25: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x130e) [0xdc6a9a957e]
62
ceph-osd[7526]:  26: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x884) [0xdc6af77e34]
63
ceph-osd[7526]:  27: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xdc6af7ae70]
64
ceph-osd[7526]:  28: (()+0x76ba) [0x7f6708c4c6ba]
65
ceph-osd[7526]:  29: (clone()+0x6d) [0x7f6707cc33dd]
66
ceph-osd[7526]:  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
67

    
68
ceph-osd[7526]:      0> 2017-08-27 15:41:50.172556 7f66eb3e3700 -1 *** Caught signal (Segmentation fault) **
69
ceph-osd[7526]:  in thread 7f66eb3e3700 thread_name:tp_osd_tp
70
ceph-osd[7526]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
71
ceph-osd[7526]:  1: (()+0xa542b4) [0xdc6af302b4]
72
ceph-osd[7526]:  2: (()+0x11390) [0x7f6708c56390]
73
ceph-osd[7526]:  3: (()+0x1f8af) [0x7f670a5838af]
74
ceph-osd[7526]:  4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc6b2fc1e9]
75
ceph-osd[7526]:  5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, bool)+0x3b7) [0xdc6b2fdbc7]
76
ceph-osd[7526]:  6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc6b2fdf0c]
77
ceph-osd[7526]:  7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc6b306687]
78
ceph-osd[7526]:  8: (()+0xe5594e) [0xdc6b33194e]
79
ceph-osd[7526]:  9: (()+0xe55a16) [0xdc6b331a16]
80
ceph-osd[7526]:  10: (rocksdb::MergingIterator::Next()+0x449) [0xdc6b314ce9]
81
ceph-osd[7526]:  11: (rocksdb::DBIter::FindNextUserEntryInternal(bool, bool)+0x182) [0xdc6b3b20b2]
82
ceph-osd[7526]:  12: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x314) [0xdc6b3b3454]
83
ceph-osd[7526]:  13: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x46) [0xdc6ae73036]
84
ceph-osd[7526]:  14: (BlueStore::_omap_rmkey_range(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x14e) [0xdc6adcda3e]
85
ceph-osd[7526]:  15: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x1e50) [0xdc6ae2f930]
86
ceph-osd[7526]:  16: (BlueStore::queue_transactions(ObjectStore::Sequencer*, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x52e) [0xdc6ae3079e]
87
ceph-osd[7526]:  17: (PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x66) [0xdc6ab5ab86]
88
ceph-osd[7526]:  18: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, boost::optional<pg_hit_set_history_t>&, Context*, Context*, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>)+0xcb8) [0xdc6ac8a838]
89
ceph-osd[7526]:  19: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0x9fa) [0xdc6aaf7c2a]
90
ceph-osd[7526]:  20: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x139b) [0xdc6ab4026b]
91
ceph-osd[7526]:  21: (PrimaryLogPG::do_op(boost::intrusive_ptr<OpRequest>&)+0x308b) [0xdc6ab43c8b]
92
ceph-osd[7526]:  22: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0xe73) [0xdc6aafdd03]
93
ceph-osd[7526]:  23: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3a9) [0xdc6a981fd9]
94
ceph-osd[7526]:  24: (PGQueueable::RunVis::operator()(boost::intrusive_ptr<OpRequest> const&)+0x57) [0xdc6ac193f7]
95
ceph-osd[7526]:  25: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x130e) [0xdc6a9a957e]
96
ceph-osd[7526]:  26: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x884) [0xdc6af77e34]
97
ceph-osd[7526]:  27: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xdc6af7ae70]
98
ceph-osd[7526]:  28: (()+0x76ba) [0x7f6708c4c6ba]
99
ceph-osd[7526]:  29: (clone()+0x6d) [0x7f6707cc33dd]
100
ceph-osd[7526]:  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
101

    
102
systemd[1]: ceph-osd@13.service: Main process exited, code=killed, status=11/SEGV
103
systemd[1]: ceph-osd@13.service: Unit entered failed state.
104
systemd[1]: ceph-osd@13.service: Failed with result 'signal'.
105
systemd[1]: ceph-osd@13.service: Service hold-off time over, scheduling restart.
106
systemd[1]: Stopped Ceph object storage daemon osd.13.
107
systemd[1]: Starting Ceph object storage daemon osd.13...
108
systemd[1]: Started Ceph object storage daemon osd.13.
109
ceph-osd[26349]: starting osd.13 at - osd_data /var/lib/ceph/osd/ceph-13 /var/lib/ceph/osd/ceph-13/journal
110

    
111
ceph-osd[26349]: *** Caught signal (Segmentation fault) **
112
ceph-osd[26349]:  in thread 7f9c14bfd700 thread_name:rocksdb:bg0
113
ceph-osd[26349]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
114
ceph-osd[26349]:  1: (()+0xa542b4) [0x759eb7a2b4]
115
ceph-osd[26349]:  2: (()+0x11390) [0x7f9c2a433390]
116
ceph-osd[26349]:  3: (()+0x1f8af) [0x7f9c2bd608af]
117
ceph-osd[26349]:  4: (rocksdb::Arena::AllocateNewBlock(unsigned long)+0x7c) [0x759ef7c60c]
118
ceph-osd[26349]:  5: (rocksdb::Arena::AllocateFallback(unsigned long, bool)+0x45) [0x759ef7c785]
119
ceph-osd[26349]:  6: (rocksdb::Arena::AllocateAligned(unsigned long, unsigned long, rocksdb::Logger*)+0x100) [0x759ef7c910]
120
ceph-osd[26349]:  7: (rocksdb::DoGenerateLevelFilesBrief(rocksdb::LevelFilesBrief*, std::vector<rocksdb::FileMetaData*, std::allocator<rocksdb::FileMetaData*> > const&, rocksdb::Arena*)+0x18b) [0x759eefcb3b]
121
ceph-osd[26349]:  8: (rocksdb::Compaction::Compaction(rocksdb::VersionStorageInfo*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, std::vector<rocksdb::CompactionInputFiles, std::allocator<rocksdb::CompactionInputFiles> >, int, unsigned long, unsigned long, unsigned int, rocksdb::CompressionType, std::vector<rocksdb::FileMetaData*, std::allocator<rocksdb::FileMetaData*> >, bool, double, bool, rocksdb::CompactionReason)+0xc29) [0x759efb43c9]
122
ceph-osd[26349]:  9: (rocksdb::LevelCompactionPicker::PickCompaction(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::MutableCFOptions const&, rocksdb::VersionStorageInfo*, rocksdb::LogBuffer*)+0xccd) [0x759efbe9cd]
123
ceph-osd[26349]:  10: (rocksdb::ColumnFamilyData::PickCompaction(rocksdb::MutableCFOptions const&, rocksdb::LogBuffer*)+0x2c) [0x759efaadcc]
124
ceph-osd[26349]:  11: (rocksdb::DBImpl::BackgroundCompaction(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, void*)+0x1a68) [0x759efedba8]
125
ceph-osd[26349]:  12: (rocksdb::DBImpl::BackgroundCallCompaction(void*)+0x127) [0x759efeef87]
126
ceph-osd[26349]:  13: (rocksdb::DBImpl::BGWorkCompaction(void*)+0x90) [0x759efef540]
127
ceph-osd[26349]:  14: (rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x24e) [0x759f06143e]
128
ceph-osd[26349]:  15: (rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x3d) [0x759f06161d]
129
ceph-osd[26349]:  16: (()+0xb8c80) [0x7f9c29d3ac80]
130
ceph-osd[26349]:  17: (()+0x76ba) [0x7f9c2a4296ba]
131
ceph-osd[26349]:  18: (clone()+0x6d) [0x7f9c294a03dd]
132

    
133
ceph-osd[26349]: 2017-08-27 15:42:15.410852 7f9c14bfd700 -1 *** Caught signal (Segmentation fault) **
134
ceph-osd[26349]:  in thread 7f9c14bfd700 thread_name:rocksdb:bg0
135
ceph-osd[26349]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
136
ceph-osd[26349]:  1: (()+0xa542b4) [0x759eb7a2b4]
137
ceph-osd[26349]:  2: (()+0x11390) [0x7f9c2a433390]
138
ceph-osd[26349]:  3: (()+0x1f8af) [0x7f9c2bd608af]
139
ceph-osd[26349]:  4: (rocksdb::Arena::AllocateNewBlock(unsigned long)+0x7c) [0x759ef7c60c]
140
ceph-osd[26349]:  5: (rocksdb::Arena::AllocateFallback(unsigned long, bool)+0x45) [0x759ef7c785]
141
ceph-osd[26349]:  6: (rocksdb::Arena::AllocateAligned(unsigned long, unsigned long, rocksdb::Logger*)+0x100) [0x759ef7c910]
142
ceph-osd[26349]:  7: (rocksdb::DoGenerateLevelFilesBrief(rocksdb::LevelFilesBrief*, std::vector<rocksdb::FileMetaData*, std::allocator<rocksdb::FileMetaData*> > const&, rocksdb::Arena*)+0x18b) [0x759eefcb3b]
143
ceph-osd[26349]:  8: (rocksdb::Compaction::Compaction(rocksdb::VersionStorageInfo*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, std::vector<rocksdb::CompactionInputFiles, std::allocator<rocksdb::CompactionInputFiles> >, int, unsigned long, unsigned long, unsigned int, rocksdb::CompressionType, std::vector<rocksdb::FileMetaData*, std::allocator<rocksdb::FileMetaData*> >, bool, double, bool, rocksdb::CompactionReason)+0xc29) [0x759efb43c9]
144
ceph-osd[26349]:  9: (rocksdb::LevelCompactionPicker::PickCompaction(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::MutableCFOptions const&, rocksdb::VersionStorageInfo*, rocksdb::LogBuffer*)+0xccd) [0x759efbe9cd]
145
ceph-osd[26349]:  10: (rocksdb::ColumnFamilyData::PickCompaction(rocksdb::MutableCFOptions const&, rocksdb::LogBuffer*)+0x2c) [0x759efaadcc]
146
ceph-osd[26349]:  11: (rocksdb::DBImpl::BackgroundCompaction(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, void*)+0x1a68) [0x759efedba8]
147
ceph-osd[26349]:  12: (rocksdb::DBImpl::BackgroundCallCompaction(void*)+0x127) [0x759efeef87]
148
ceph-osd[26349]:  13: (rocksdb::DBImpl::BGWorkCompaction(void*)+0x90) [0x759efef540]
149
ceph-osd[26349]:  14: (rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x24e) [0x759f06143e]
150
ceph-osd[26349]:  15: (rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x3d) [0x759f06161d]
151
ceph-osd[26349]:  16: (()+0xb8c80) [0x7f9c29d3ac80]
152
ceph-osd[26349]:  17: (()+0x76ba) [0x7f9c2a4296ba]
153
ceph-osd[26349]:  18: (clone()+0x6d) [0x7f9c294a03dd]
154
ceph-osd[26349]:  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
155

    
156
ceph-osd[26349]:      0> 2017-08-27 15:42:15.410852 7f9c14bfd700 -1 *** Caught signal (Segmentation fault) **
157
ceph-osd[26349]:  in thread 7f9c14bfd700 thread_name:rocksdb:bg0
158
ceph-osd[26349]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
159
ceph-osd[26349]:  1: (()+0xa542b4) [0x759eb7a2b4]
160
ceph-osd[26349]:  2: (()+0x11390) [0x7f9c2a433390]
161
ceph-osd[26349]:  3: (()+0x1f8af) [0x7f9c2bd608af]
162
ceph-osd[26349]:  4: (rocksdb::Arena::AllocateNewBlock(unsigned long)+0x7c) [0x759ef7c60c]
163
ceph-osd[26349]:  5: (rocksdb::Arena::AllocateFallback(unsigned long, bool)+0x45) [0x759ef7c785]
164
ceph-osd[26349]:  6: (rocksdb::Arena::AllocateAligned(unsigned long, unsigned long, rocksdb::Logger*)+0x100) [0x759ef7c910]
165
ceph-osd[26349]:  7: (rocksdb::DoGenerateLevelFilesBrief(rocksdb::LevelFilesBrief*, std::vector<rocksdb::FileMetaData*, std::allocator<rocksdb::FileMetaData*> > const&, rocksdb::Arena*)+0x18b) [0x759eefcb3b]
166
ceph-osd[26349]:  8: (rocksdb::Compaction::Compaction(rocksdb::VersionStorageInfo*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, std::vector<rocksdb::CompactionInputFiles, std::allocator<rocksdb::CompactionInputFiles> >, int, unsigned long, unsigned long, unsigned int, rocksdb::CompressionType, std::vector<rocksdb::FileMetaData*, std::allocator<rocksdb::FileMetaData*> >, bool, double, bool, rocksdb::CompactionReason)+0xc29) [0x759efb43c9]
167
ceph-osd[26349]:  9: (rocksdb::LevelCompactionPicker::PickCompaction(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::MutableCFOptions const&, rocksdb::VersionStorageInfo*, rocksdb::LogBuffer*)+0xccd) [0x759efbe9cd]
168
ceph-osd[26349]:  10: (rocksdb::ColumnFamilyData::PickCompaction(rocksdb::MutableCFOptions const&, rocksdb::LogBuffer*)+0x2c) [0x759efaadcc]
169
ceph-osd[26349]:  11: (rocksdb::DBImpl::BackgroundCompaction(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, void*)+0x1a68) [0x759efedba8]
170
ceph-osd[26349]:  12: (rocksdb::DBImpl::BackgroundCallCompaction(void*)+0x127) [0x759efeef87]
171
ceph-osd[26349]:  13: (rocksdb::DBImpl::BGWorkCompaction(void*)+0x90) [0x759efef540]
172
ceph-osd[26349]:  14: (rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x24e) [0x759f06143e]
173
ceph-osd[26349]:  15: (rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x3d) [0x759f06161d]
174
ceph-osd[26349]:  16: (()+0xb8c80) [0x7f9c29d3ac80]
175
ceph-osd[26349]:  17: (()+0x76ba) [0x7f9c2a4296ba]
176
ceph-osd[26349]:  18: (clone()+0x6d) [0x7f9c294a03dd]
177
ceph-osd[26349]:  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
178

    
179
systemd[1]: ceph-osd@13.service: Main process exited, code=killed, status=11/SEGV
180
systemd[1]: ceph-osd@13.service: Unit entered failed state.
181
systemd[1]: ceph-osd@13.service: Failed with result 'signal'.
182
systemd[1]: ceph-osd@13.service: Service hold-off time over, scheduling restart.
183
systemd[1]: Stopped Ceph object storage daemon osd.13.
184
systemd[1]: Starting Ceph object storage daemon osd.13...
185
systemd[1]: Started Ceph object storage daemon osd.13.
186
ceph-osd[26755]: starting osd.13 at - osd_data /var/lib/ceph/osd/ceph-13 /var/lib/ceph/osd/ceph-13/journal
187

    
188
ceph-osd[26755]: *** Caught signal (Segmentation fault) **
189
ceph-osd[26755]:  in thread 7f2d71f04e40 thread_name:ceph-osd
190
ceph-osd[26755]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
191
ceph-osd[26755]:  1: (()+0xa542b4) [0xdc0ada32b4]
192
ceph-osd[26755]:  2: (()+0x11390) [0x7f2d701b5390]
193
ceph-osd[26755]:  3: (()+0x1f8af) [0x7f2d71ae28af]
194
ceph-osd[26755]:  4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc0b16f1e9]
195
ceph-osd[26755]:  5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, bool)+0x3b7) [0xdc0b170bc7]
196
ceph-osd[26755]:  6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc0b170f0c]
197
ceph-osd[26755]:  7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc0b179687]
198
ceph-osd[26755]:  8: (()+0xe5594e) [0xdc0b1a494e]
199
ceph-osd[26755]:  9: (()+0xe55ec6) [0xdc0b1a4ec6]
200
ceph-osd[26755]:  10: (rocksdb::MergingIterator::Seek(rocksdb::Slice const&)+0x126) [0xdc0b186026]
201
ceph-osd[26755]:  11: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x20a) [0xdc0b22634a]
202
ceph-osd[26755]:  12: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x46) [0xdc0ace6036]
203
ceph-osd[26755]:  13: (BitmapFreelistManager::init()+0x2d6) [0xdc0ad4b626]
204
ceph-osd[26755]:  14: (BlueStore::_open_fm(bool)+0x77f) [0xdc0ac3bdff]
205
ceph-osd[26755]:  15: (BlueStore::_mount(bool)+0x3bc) [0xdc0aca72fc]
206
ceph-osd[26755]:  16: (OSD::init()+0x3df) [0xdc0a822a8f]
207
ceph-osd[26755]:  17: (main()+0x2eb8) [0xdc0a735138]
208
ceph-osd[26755]:  18: (__libc_start_main()+0xf0) [0x7f2d6f13b830]
209
ceph-osd[26755]:  19: (_start()+0x29) [0xdc0a7c0a69]
210

    
211
ceph-osd[26755]: 2017-08-27 15:42:36.685509 7f2d71f04e40 -1 *** Caught signal (Segmentation fault) **
212
ceph-osd[26755]:  in thread 7f2d71f04e40 thread_name:ceph-osd
213
ceph-osd[26755]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
214
ceph-osd[26755]:  1: (()+0xa542b4) [0xdc0ada32b4]
215
ceph-osd[26755]:  2: (()+0x11390) [0x7f2d701b5390]
216
ceph-osd[26755]:  3: (()+0x1f8af) [0x7f2d71ae28af]
217
ceph-osd[26755]:  4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc0b16f1e9]
218
ceph-osd[26755]:  5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, bool)+0x3b7) [0xdc0b170bc7]
219
ceph-osd[26755]:  6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc0b170f0c]
220
ceph-osd[26755]:  7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc0b179687]
221
ceph-osd[26755]:  8: (()+0xe5594e) [0xdc0b1a494e]
222
ceph-osd[26755]:  9: (()+0xe55ec6) [0xdc0b1a4ec6]
223
ceph-osd[26755]:  10: (rocksdb::MergingIterator::Seek(rocksdb::Slice const&)+0x126) [0xdc0b186026]
224
ceph-osd[26755]:  11: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x20a) [0xdc0b22634a]
225
ceph-osd[26755]:  12: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x46) [0xdc0ace6036]
226
ceph-osd[26755]:  13: (BitmapFreelistManager::init()+0x2d6) [0xdc0ad4b626]
227
ceph-osd[26755]:  14: (BlueStore::_open_fm(bool)+0x77f) [0xdc0ac3bdff]
228
ceph-osd[26755]:  15: (BlueStore::_mount(bool)+0x3bc) [0xdc0aca72fc]
229
ceph-osd[26755]:  16: (OSD::init()+0x3df) [0xdc0a822a8f]
230
ceph-osd[26755]:  17: (main()+0x2eb8) [0xdc0a735138]
231
ceph-osd[26755]:  18: (__libc_start_main()+0xf0) [0x7f2d6f13b830]
232
ceph-osd[26755]:  19: (_start()+0x29) [0xdc0a7c0a69]
233
ceph-osd[26755]:  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
234

    
235
ceph-osd[26755]:      0> 2017-08-27 15:42:36.685509 7f2d71f04e40 -1 *** Caught signal (Segmentation fault) **
236
ceph-osd[26755]:  in thread 7f2d71f04e40 thread_name:ceph-osd
237
ceph-osd[26755]:  ceph version 12.1.4 (a5f84b37668fc8e03165aaf5cbb380c78e4deba4) luminous (rc)
238
ceph-osd[26755]:  1: (()+0xa542b4) [0xdc0ada32b4]
239
ceph-osd[26755]:  2: (()+0x11390) [0x7f2d701b5390]
240
ceph-osd[26755]:  3: (()+0x1f8af) [0x7f2d71ae28af]
241
ceph-osd[26755]:  4: (rocksdb::BlockBasedTable::PutDataBlockToCache(rocksdb::Slice const&, rocksdb::Slice const&, rocksdb::Cache*, rocksdb::Cache*, rocksdb::ReadOptions const&, rocksdb::ImmutableCFOptions const&, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, rocksdb::Block*, unsigned int, rocksdb::Slice const&, unsigned long, bool, rocksdb::Cache::Priority)+0x1d9) [0xdc0b16f1e9]
242
ceph-osd[26755]:  5: (rocksdb::BlockBasedTable::MaybeLoadDataBlockToCache(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::Slice, rocksdb::BlockBasedTable::CachableEntry<rocksdb::Block>*, bool)+0x3b7) [0xdc0b170bc7]
243
ceph-osd[26755]:  6: (rocksdb::BlockBasedTable::NewDataBlockIterator(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::BlockIter*, bool, rocksdb::Status)+0x2ac) [0xdc0b170f0c]
244
ceph-osd[26755]:  7: (rocksdb::BlockBasedTable::BlockEntryIteratorState::NewSecondaryIterator(rocksdb::Slice const&)+0x97) [0xdc0b179687]
245
ceph-osd[26755]:  8: (()+0xe5594e) [0xdc0b1a494e]
246
ceph-osd[26755]:  9: (()+0xe55ec6) [0xdc0b1a4ec6]
247
ceph-osd[26755]:  10: (rocksdb::MergingIterator::Seek(rocksdb::Slice const&)+0x126) [0xdc0b186026]
248
ceph-osd[26755]:  11: (rocksdb::DBIter::Seek(rocksdb::Slice const&)+0x20a) [0xdc0b22634a]
249
ceph-osd[26755]:  12: (RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x46) [0xdc0ace6036]
250
ceph-osd[26755]:  13: (BitmapFreelistManager::init()+0x2d6) [0xdc0ad4b626]
251
ceph-osd[26755]:  14: (BlueStore::_open_fm(bool)+0x77f) [0xdc0ac3bdff]
252
ceph-osd[26755]:  15: (BlueStore::_mount(bool)+0x3bc) [0xdc0aca72fc]
253
ceph-osd[26755]:  16: (OSD::init()+0x3df) [0xdc0a822a8f]
254
ceph-osd[26755]:  17: (main()+0x2eb8) [0xdc0a735138]
255
ceph-osd[26755]:  18: (__libc_start_main()+0xf0) [0x7f2d6f13b830]
256
ceph-osd[26755]:  19: (_start()+0x29) [0xdc0a7c0a69]
257

    
258
ceph-osd[26755]:  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
259
systemd[1]: ceph-osd@13.service: Main process exited, code=dumped, status=11/SEGV
260
systemd[1]: ceph-osd@13.service: Unit entered failed state.
261
systemd[1]: ceph-osd@13.service: Failed with result 'core-dump'.
262
systemd[1]: ceph-osd@13.service: Service hold-off time over, scheduling restart.
263
systemd[1]: Stopped Ceph object storage daemon osd.13.
264
systemd[1]: Starting Ceph object storage daemon osd.13...
265
systemd[1]: Started Ceph object storage daemon osd.13.
266

    
267
ceph-osd[27156]: starting osd.13 at - osd_data /var/lib/ceph/osd/ceph-13 /var/lib/ceph/osd/ceph-13/journal
268
ceph-osd[27156]: 2017-08-27 15:43:13.118701 7fe42a521e40 -1 osd.13 6034 log_to_monitors {default=true}