Project

General

Profile

osd4-kvstore.txt

Tobias Fischer, 10/28/2019 03:26 PM

Download (33.1 KB)

 
1
2019-10-28 16:23:27.947 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _mount path /var/lib/ceph/osd/ceph-4/
2
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block type kernel
3
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057e700 /var/lib/ceph/osd/ceph-4//block) open path /var/lib/ceph/osd/ceph-4//block
4
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057e700 /var/lib/ceph/osd/ceph-4//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
5
2019-10-28 16:23:27.947 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2
6
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block.db type kernel
7
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057ee00 /var/lib/ceph/osd/ceph-4//block.db) open path /var/lib/ceph/osd/ceph-4//block.db
8
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057ee00 /var/lib/ceph/osd/ceph-4//block.db) open size 28989980672 (0x6bff00000, 27 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
9
2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-4//block.db size 27 GiB
10
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block type kernel
11
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f180 /var/lib/ceph/osd/ceph-4//block) open path /var/lib/ceph/osd/ceph-4//block
12
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f180 /var/lib/ceph/osd/ceph-4//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
13
2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 2 path /var/lib/ceph/osd/ceph-4//block size 279 GiB
14
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block.wal type kernel
15
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f500 /var/lib/ceph/osd/ceph-4//block.wal) open path /var/lib/ceph/osd/ceph-4//block.wal
16
2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f500 /var/lib/ceph/osd/ceph-4//block.wal) open size 1073741824 (0x40000000, 1 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
17
2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 0 path /var/lib/ceph/osd/ceph-4//block.wal size 1 GiB
18
2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs mount
19
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compaction_readahead_size = 2097152
20
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compression = kNoCompression
21
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_background_compactions = 2
22
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_write_buffer_number = 4
23
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option min_write_buffer_number_to_merge = 1
24
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option recycle_log_file_num = 4
25
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option writable_file_max_buffer_size = 0
26
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option write_buffer_size = 268435456
27
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compaction_readahead_size = 2097152
28
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compression = kNoCompression
29
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_background_compactions = 2
30
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_write_buffer_number = 4
31
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option min_write_buffer_number_to_merge = 1
32
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option recycle_log_file_num = 4
33
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option writable_file_max_buffer_size = 0
34
2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option write_buffer_size = 268435456
35
2019-10-28 16:23:27.963 7fb8ba257180  1 rocksdb: do_open column families: [default]
36
2019-10-28 16:23:29.255 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2
37
2019-10-28 16:23:29.283 7fb8ba257180  1 freelist init
38
2019-10-28 16:23:29.291 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_alloc opening allocation metadata
39
2019-10-28 16:23:29.295 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_alloc loaded 0 B in 0 extents
40
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: In function 'void AllocatorLevel02<T>::_mark_allocated(uint64_t, uint64_t) [with L1 = AllocatorLevel01Loose; uint64_t = long unsigned int]' thread 7fb8ba257180 time 2019-10-28 16:23:29.297245
41
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: 750: FAILED ceph_assert(available >= allocated)
42
 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
43
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x7fb8bb4e460e]
44
 2: (()+0x2807e6) [0x7fb8bb4e47e6]
45
 3: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x55e80cb6c163]
46
 4: (BlueStore::_open_alloc()+0x1f5) [0x55e80c992f95]
47
 5: (BlueStore::_open_db_and_around(bool)+0xa2) [0x55e80c9ab3f2]
48
 6: (BlueStore::_mount(bool, bool)+0x58a) [0x55e80c9f162a]
49
 7: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x55e80c962f22]
50
 8: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x55e80c965f56]
51
 9: (main()+0x28e) [0x55e80c93de4e]
52
 10: (__libc_start_main()+0xeb) [0x7fb8ba76409b]
53
 11: (_start()+0x2a) [0x55e80c962daa]
54
2019-10-28 16:23:29.295 7fb8ba257180 -1 /data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: In function 'void AllocatorLevel02<T>::_mark_allocated(uint64_t, uint64_t) [with L1 = AllocatorLevel01Loose; uint64_t = long unsigned int]' thread 7fb8ba257180 time 2019-10-28 16:23:29.297245
55
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: 750: FAILED ceph_assert(available >= allocated)
56

    
57
 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
58
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x7fb8bb4e460e]
59
 2: (()+0x2807e6) [0x7fb8bb4e47e6]
60
 3: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x55e80cb6c163]
61
 4: (BlueStore::_open_alloc()+0x1f5) [0x55e80c992f95]
62
 5: (BlueStore::_open_db_and_around(bool)+0xa2) [0x55e80c9ab3f2]
63
 6: (BlueStore::_mount(bool, bool)+0x58a) [0x55e80c9f162a]
64
 7: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x55e80c962f22]
65
 8: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x55e80c965f56]
66
 9: (main()+0x28e) [0x55e80c93de4e]
67
 10: (__libc_start_main()+0xeb) [0x7fb8ba76409b]
68
 11: (_start()+0x2a) [0x55e80c962daa]
69

    
70
*** Caught signal (Aborted) **
71
 in thread 7fb8ba257180 thread_name:ceph-kvstore-to
72
 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
73
 1: (()+0x12730) [0x7fb8bacbd730]
74
 2: (gsignal()+0x10b) [0x7fb8ba7777bb]
75
 3: (abort()+0x121) [0x7fb8ba762535]
76
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7fb8bb4e465f]
77
 5: (()+0x2807e6) [0x7fb8bb4e47e6]
78
 6: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x55e80cb6c163]
79
 7: (BlueStore::_open_alloc()+0x1f5) [0x55e80c992f95]
80
 8: (BlueStore::_open_db_and_around(bool)+0xa2) [0x55e80c9ab3f2]
81
 9: (BlueStore::_mount(bool, bool)+0x58a) [0x55e80c9f162a]
82
 10: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x55e80c962f22]
83
 11: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x55e80c965f56]
84
 12: (main()+0x28e) [0x55e80c93de4e]
85
 13: (__libc_start_main()+0xeb) [0x7fb8ba76409b]
86
 14: (_start()+0x2a) [0x55e80c962daa]
87
2019-10-28 16:23:29.295 7fb8ba257180 -1 *** Caught signal (Aborted) **
88
 in thread 7fb8ba257180 thread_name:ceph-kvstore-to
89

    
90
 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
91
 1: (()+0x12730) [0x7fb8bacbd730]
92
 2: (gsignal()+0x10b) [0x7fb8ba7777bb]
93
 3: (abort()+0x121) [0x7fb8ba762535]
94
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7fb8bb4e465f]
95
 5: (()+0x2807e6) [0x7fb8bb4e47e6]
96
 6: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x55e80cb6c163]
97
 7: (BlueStore::_open_alloc()+0x1f5) [0x55e80c992f95]
98
 8: (BlueStore::_open_db_and_around(bool)+0xa2) [0x55e80c9ab3f2]
99
 9: (BlueStore::_mount(bool, bool)+0x58a) [0x55e80c9f162a]
100
 10: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x55e80c962f22]
101
 11: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x55e80c965f56]
102
 12: (main()+0x28e) [0x55e80c93de4e]
103
 13: (__libc_start_main()+0xeb) [0x7fb8ba76409b]
104
 14: (_start()+0x2a) [0x55e80c962daa]
105
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
106

    
107
--- begin dump of recent events ---
108
   -68> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command assert hook 0x55e80f8ca090
109
   -67> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command abort hook 0x55e80f8ca090
110
   -66> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perfcounters_dump hook 0x55e80f8ca090
111
   -65> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command 1 hook 0x55e80f8ca090
112
   -64> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf dump hook 0x55e80f8ca090
113
   -63> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perfcounters_schema hook 0x55e80f8ca090
114
   -62> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf histogram dump hook 0x55e80f8ca090
115
   -61> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command 2 hook 0x55e80f8ca090
116
   -60> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf schema hook 0x55e80f8ca090
117
   -59> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf histogram schema hook 0x55e80f8ca090
118
   -58> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf reset hook 0x55e80f8ca090
119
   -57> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config show hook 0x55e80f8ca090
120
   -56> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config help hook 0x55e80f8ca090
121
   -55> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config set hook 0x55e80f8ca090
122
   -54> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config unset hook 0x55e80f8ca090
123
   -53> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config get hook 0x55e80f8ca090
124
   -52> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config diff hook 0x55e80f8ca090
125
   -51> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config diff get hook 0x55e80f8ca090
126
   -50> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command log flush hook 0x55e80f8ca090
127
   -49> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command log dump hook 0x55e80f8ca090
128
   -48> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command log reopen hook 0x55e80f8ca090
129
   -47> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command dump_mempools hook 0x55e80f9aa068
130
   -46> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _mount path /var/lib/ceph/osd/ceph-4/
131
   -45> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block type kernel
132
   -44> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057e700 /var/lib/ceph/osd/ceph-4//block) open path /var/lib/ceph/osd/ceph-4//block
133
   -43> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057e700 /var/lib/ceph/osd/ceph-4//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
134
   -42> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2
135
   -41> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block.db type kernel
136
   -40> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057ee00 /var/lib/ceph/osd/ceph-4//block.db) open path /var/lib/ceph/osd/ceph-4//block.db
137
   -39> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057ee00 /var/lib/ceph/osd/ceph-4//block.db) open size 28989980672 (0x6bff00000, 27 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
138
   -38> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-4//block.db size 27 GiB
139
   -37> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block type kernel
140
   -36> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f180 /var/lib/ceph/osd/ceph-4//block) open path /var/lib/ceph/osd/ceph-4//block
141
   -35> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f180 /var/lib/ceph/osd/ceph-4//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
142
   -34> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 2 path /var/lib/ceph/osd/ceph-4//block size 279 GiB
143
   -33> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block.wal type kernel
144
   -32> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f500 /var/lib/ceph/osd/ceph-4//block.wal) open path /var/lib/ceph/osd/ceph-4//block.wal
145
   -31> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f500 /var/lib/ceph/osd/ceph-4//block.wal) open size 1073741824 (0x40000000, 1 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
146
   -30> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 0 path /var/lib/ceph/osd/ceph-4//block.wal size 1 GiB
147
   -29> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs mount
148
   -28> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compaction_readahead_size = 2097152
149
   -27> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compression = kNoCompression
150
   -26> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_background_compactions = 2
151
   -25> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_write_buffer_number = 4
152
   -24> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option min_write_buffer_number_to_merge = 1
153
   -23> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option recycle_log_file_num = 4
154
   -22> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option writable_file_max_buffer_size = 0
155
   -21> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option write_buffer_size = 268435456
156
   -20> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compaction_readahead_size = 2097152
157
   -19> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compression = kNoCompression
158
   -18> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_background_compactions = 2
159
   -17> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_write_buffer_number = 4
160
   -16> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option min_write_buffer_number_to_merge = 1
161
   -15> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option recycle_log_file_num = 4
162
   -14> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option writable_file_max_buffer_size = 0
163
   -13> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option write_buffer_size = 268435456
164
   -12> 2019-10-28 16:23:27.963 7fb8ba257180  1 rocksdb: do_open column families: [default]
165
   -11> 2019-10-28 16:23:29.255 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2
166
   -10> 2019-10-28 16:23:29.283 7fb8ba257180  1 get compressor snappy = 0
167
    -9> 2019-10-28 16:23:29.283 7fb8ba257180  1 load compressor snappy
168
    -8> 2019-10-28 16:23:29.283 7fb8ba257180  1 add compressor snappy 0x55e80f98adb0
169
    -7> 2019-10-28 16:23:29.283 7fb8ba257180  1 get compressor snappy = 0x55e80f98adb0
170
    -6> 2019-10-28 16:23:29.283 7fb8ba257180  1 load: compressor snappy loaded and registered
171
    -5> 2019-10-28 16:23:29.283 7fb8ba257180  1 get compressor snappy = 0x55e80f98adb0
172
    -4> 2019-10-28 16:23:29.283 7fb8ba257180  1 freelist init
173
    -3> 2019-10-28 16:23:29.291 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_alloc opening allocation metadata
174
    -2> 2019-10-28 16:23:29.295 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_alloc loaded 0 B in 0 extents
175
    -1> 2019-10-28 16:23:29.295 7fb8ba257180 -1 /data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: In function 'void AllocatorLevel02<T>::_mark_allocated(uint64_t, uint64_t) [with L1 = AllocatorLevel01Loose; uint64_t = long unsigned int]' thread 7fb8ba257180 time 2019-10-28 16:23:29.297245
176
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: 750: FAILED ceph_assert(available >= allocated)
177

    
178
 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
179
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x7fb8bb4e460e]
180
 2: (()+0x2807e6) [0x7fb8bb4e47e6]
181
 3: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x55e80cb6c163]
182
 4: (BlueStore::_open_alloc()+0x1f5) [0x55e80c992f95]
183
 5: (BlueStore::_open_db_and_around(bool)+0xa2) [0x55e80c9ab3f2]
184
 6: (BlueStore::_mount(bool, bool)+0x58a) [0x55e80c9f162a]
185
 7: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x55e80c962f22]
186
 8: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x55e80c965f56]
187
 9: (main()+0x28e) [0x55e80c93de4e]
188
 10: (__libc_start_main()+0xeb) [0x7fb8ba76409b]
189
 11: (_start()+0x2a) [0x55e80c962daa]
190

    
191
     0> 2019-10-28 16:23:29.295 7fb8ba257180 -1 *** Caught signal (Aborted) **
192
 in thread 7fb8ba257180 thread_name:ceph-kvstore-to
193

    
194
 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
195
 1: (()+0x12730) [0x7fb8bacbd730]
196
 2: (gsignal()+0x10b) [0x7fb8ba7777bb]
197
 3: (abort()+0x121) [0x7fb8ba762535]
198
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7fb8bb4e465f]
199
 5: (()+0x2807e6) [0x7fb8bb4e47e6]
200
 6: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x55e80cb6c163]
201
 7: (BlueStore::_open_alloc()+0x1f5) [0x55e80c992f95]
202
 8: (BlueStore::_open_db_and_around(bool)+0xa2) [0x55e80c9ab3f2]
203
 9: (BlueStore::_mount(bool, bool)+0x58a) [0x55e80c9f162a]
204
 10: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x55e80c962f22]
205
 11: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x55e80c965f56]
206
 12: (main()+0x28e) [0x55e80c93de4e]
207
 13: (__libc_start_main()+0xeb) [0x7fb8ba76409b]
208
 14: (_start()+0x2a) [0x55e80c962daa]
209
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
210

    
211
--- logging levels ---
212
   0/ 5 none
213
   0/ 1 lockdep
214
   0/ 1 context
215
   1/ 1 crush
216
   1/ 5 mds
217
   1/ 5 mds_balancer
218
   1/ 5 mds_locker
219
   1/ 5 mds_log
220
   1/ 5 mds_log_expire
221
   1/ 5 mds_migrator
222
   0/ 1 buffer
223
   0/ 1 timer
224
   0/ 1 filer
225
   0/ 1 striper
226
   0/ 1 objecter
227
   0/ 5 rados
228
   0/ 5 rbd
229
   0/ 5 rbd_mirror
230
   0/ 5 rbd_replay
231
   0/ 5 journaler
232
   0/ 5 objectcacher
233
   0/ 5 client
234
   1/ 5 osd
235
   0/ 5 optracker
236
   0/ 5 objclass
237
   1/ 3 filestore
238
   1/ 3 journal
239
   0/ 0 ms
240
   1/ 5 mon
241
   0/10 monc
242
   1/ 5 paxos
243
   0/ 5 tp
244
   1/ 5 auth
245
   1/ 5 crypto
246
   1/ 1 finisher
247
   1/ 1 reserver
248
   1/ 5 heartbeatmap
249
   1/ 5 perfcounter
250
   1/ 5 rgw
251
   1/ 5 rgw_sync
252
   1/10 civetweb
253
   1/ 5 javaclient
254
   1/ 5 asok
255
   1/ 1 throttle
256
   0/ 0 refs
257
   1/ 5 xio
258
   1/ 5 compressor
259
   1/ 5 bluestore
260
   1/ 5 bluefs
261
   1/ 3 bdev
262
   1/ 5 kstore
263
   2/ 2 rocksdb
264
   4/ 5 leveldb
265
   4/ 5 memdb
266
   1/ 5 kinetic
267
   1/ 5 fuse
268
   1/ 5 mgr
269
   1/ 5 mgrc
270
   1/ 5 dpdk
271
   1/ 5 eventtrace
272
  -2/-2 (syslog threshold)
273
  99/99 (stderr threshold)
274
  max_recent       500
275
  max_new         1000
276
  log_file 
277
--- end dump of recent events ---
278
--- begin dump of recent events ---
279
   -68> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command assert hook 0x55e80f8ca090
280
   -67> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command abort hook 0x55e80f8ca090
281
   -66> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perfcounters_dump hook 0x55e80f8ca090
282
   -65> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command 1 hook 0x55e80f8ca090
283
   -64> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf dump hook 0x55e80f8ca090
284
   -63> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perfcounters_schema hook 0x55e80f8ca090
285
   -62> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf histogram dump hook 0x55e80f8ca090
286
   -61> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command 2 hook 0x55e80f8ca090
287
   -60> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf schema hook 0x55e80f8ca090
288
   -59> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf histogram schema hook 0x55e80f8ca090
289
   -58> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command perf reset hook 0x55e80f8ca090
290
   -57> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config show hook 0x55e80f8ca090
291
   -56> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config help hook 0x55e80f8ca090
292
   -55> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config set hook 0x55e80f8ca090
293
   -54> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config unset hook 0x55e80f8ca090
294
   -53> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config get hook 0x55e80f8ca090
295
   -52> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config diff hook 0x55e80f8ca090
296
   -51> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command config diff get hook 0x55e80f8ca090
297
   -50> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command log flush hook 0x55e80f8ca090
298
   -49> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command log dump hook 0x55e80f8ca090
299
   -48> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command log reopen hook 0x55e80f8ca090
300
   -47> 2019-10-28 16:23:27.939 7fb8ba257180  5 asok(0x55e80fab4000) register_command dump_mempools hook 0x55e80f9aa068
301
   -46> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _mount path /var/lib/ceph/osd/ceph-4/
302
   -45> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block type kernel
303
   -44> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057e700 /var/lib/ceph/osd/ceph-4//block) open path /var/lib/ceph/osd/ceph-4//block
304
   -43> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057e700 /var/lib/ceph/osd/ceph-4//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
305
   -42> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2
306
   -41> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block.db type kernel
307
   -40> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057ee00 /var/lib/ceph/osd/ceph-4//block.db) open path /var/lib/ceph/osd/ceph-4//block.db
308
   -39> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057ee00 /var/lib/ceph/osd/ceph-4//block.db) open size 28989980672 (0x6bff00000, 27 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
309
   -38> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-4//block.db size 27 GiB
310
   -37> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block type kernel
311
   -36> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f180 /var/lib/ceph/osd/ceph-4//block) open path /var/lib/ceph/osd/ceph-4//block
312
   -35> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f180 /var/lib/ceph/osd/ceph-4//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
313
   -34> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 2 path /var/lib/ceph/osd/ceph-4//block size 279 GiB
314
   -33> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev create path /var/lib/ceph/osd/ceph-4//block.wal type kernel
315
   -32> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f500 /var/lib/ceph/osd/ceph-4//block.wal) open path /var/lib/ceph/osd/ceph-4//block.wal
316
   -31> 2019-10-28 16:23:27.947 7fb8ba257180  1 bdev(0x55e81057f500 /var/lib/ceph/osd/ceph-4//block.wal) open size 1073741824 (0x40000000, 1 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
317
   -30> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs add_block_device bdev 0 path /var/lib/ceph/osd/ceph-4//block.wal size 1 GiB
318
   -29> 2019-10-28 16:23:27.947 7fb8ba257180  1 bluefs mount
319
   -28> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compaction_readahead_size = 2097152
320
   -27> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compression = kNoCompression
321
   -26> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_background_compactions = 2
322
   -25> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_write_buffer_number = 4
323
   -24> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option min_write_buffer_number_to_merge = 1
324
   -23> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option recycle_log_file_num = 4
325
   -22> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option writable_file_max_buffer_size = 0
326
   -21> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option write_buffer_size = 268435456
327
   -20> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compaction_readahead_size = 2097152
328
   -19> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option compression = kNoCompression
329
   -18> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_background_compactions = 2
330
   -17> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option max_write_buffer_number = 4
331
   -16> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option min_write_buffer_number_to_merge = 1
332
   -15> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option recycle_log_file_num = 4
333
   -14> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option writable_file_max_buffer_size = 0
334
   -13> 2019-10-28 16:23:27.959 7fb8ba257180  0  set rocksdb option write_buffer_size = 268435456
335
   -12> 2019-10-28 16:23:27.963 7fb8ba257180  1 rocksdb: do_open column families: [default]
336
   -11> 2019-10-28 16:23:29.255 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2
337
   -10> 2019-10-28 16:23:29.283 7fb8ba257180  1 get compressor snappy = 0
338
    -9> 2019-10-28 16:23:29.283 7fb8ba257180  1 load compressor snappy
339
    -8> 2019-10-28 16:23:29.283 7fb8ba257180  1 add compressor snappy 0x55e80f98adb0
340
    -7> 2019-10-28 16:23:29.283 7fb8ba257180  1 get compressor snappy = 0x55e80f98adb0
341
    -6> 2019-10-28 16:23:29.283 7fb8ba257180  1 load: compressor snappy loaded and registered
342
    -5> 2019-10-28 16:23:29.283 7fb8ba257180  1 get compressor snappy = 0x55e80f98adb0
343
    -4> 2019-10-28 16:23:29.283 7fb8ba257180  1 freelist init
344
    -3> 2019-10-28 16:23:29.291 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_alloc opening allocation metadata
345
    -2> 2019-10-28 16:23:29.295 7fb8ba257180  1 bluestore(/var/lib/ceph/osd/ceph-4/) _open_alloc loaded 0 B in 0 extents
346
    -1> 2019-10-28 16:23:29.295 7fb8ba257180 -1 /data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: In function 'void AllocatorLevel02<T>::_mark_allocated(uint64_t, uint64_t) [with L1 = AllocatorLevel01Loose; uint64_t = long unsigned int]' thread 7fb8ba257180 time 2019-10-28 16:23:29.297245
347
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: 750: FAILED ceph_assert(available >= allocated)
348

    
349
 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
350
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x7fb8bb4e460e]
351
 2: (()+0x2807e6) [0x7fb8bb4e47e6]
352
 3: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x55e80cb6c163]
353
 4: (BlueStore::_open_alloc()+0x1f5) [0x55e80c992f95]
354
 5: (BlueStore::_open_db_and_around(bool)+0xa2) [0x55e80c9ab3f2]
355
 6: (BlueStore::_mount(bool, bool)+0x58a) [0x55e80c9f162a]
356
 7: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x55e80c962f22]
357
 8: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x55e80c965f56]
358
 9: (main()+0x28e) [0x55e80c93de4e]
359
 10: (__libc_start_main()+0xeb) [0x7fb8ba76409b]
360
 11: (_start()+0x2a) [0x55e80c962daa]
361

    
362
     0> 2019-10-28 16:23:29.295 7fb8ba257180 -1 *** Caught signal (Aborted) **
363
 in thread 7fb8ba257180 thread_name:ceph-kvstore-to
364

    
365
 ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
366
 1: (()+0x12730) [0x7fb8bacbd730]
367
 2: (gsignal()+0x10b) [0x7fb8ba7777bb]
368
 3: (abort()+0x121) [0x7fb8ba762535]
369
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7fb8bb4e465f]
370
 5: (()+0x2807e6) [0x7fb8bb4e47e6]
371
 6: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x55e80cb6c163]
372
 7: (BlueStore::_open_alloc()+0x1f5) [0x55e80c992f95]
373
 8: (BlueStore::_open_db_and_around(bool)+0xa2) [0x55e80c9ab3f2]
374
 9: (BlueStore::_mount(bool, bool)+0x58a) [0x55e80c9f162a]
375
 10: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x55e80c962f22]
376
 11: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x55e80c965f56]
377
 12: (main()+0x28e) [0x55e80c93de4e]
378
 13: (__libc_start_main()+0xeb) [0x7fb8ba76409b]
379
 14: (_start()+0x2a) [0x55e80c962daa]
380
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
381

    
382
--- logging levels ---
383
   0/ 5 none
384
   0/ 1 lockdep
385
   0/ 1 context
386
   1/ 1 crush
387
   1/ 5 mds
388
   1/ 5 mds_balancer
389
   1/ 5 mds_locker
390
   1/ 5 mds_log
391
   1/ 5 mds_log_expire
392
   1/ 5 mds_migrator
393
   0/ 1 buffer
394
   0/ 1 timer
395
   0/ 1 filer
396
   0/ 1 striper
397
   0/ 1 objecter
398
   0/ 5 rados
399
   0/ 5 rbd
400
   0/ 5 rbd_mirror
401
   0/ 5 rbd_replay
402
   0/ 5 journaler
403
   0/ 5 objectcacher
404
   0/ 5 client
405
   1/ 5 osd
406
   0/ 5 optracker
407
   0/ 5 objclass
408
   1/ 3 filestore
409
   1/ 3 journal
410
   0/ 0 ms
411
   1/ 5 mon
412
   0/10 monc
413
   1/ 5 paxos
414
   0/ 5 tp
415
   1/ 5 auth
416
   1/ 5 crypto
417
   1/ 1 finisher
418
   1/ 1 reserver
419
   1/ 5 heartbeatmap
420
   1/ 5 perfcounter
421
   1/ 5 rgw
422
   1/ 5 rgw_sync
423
   1/10 civetweb
424
   1/ 5 javaclient
425
   1/ 5 asok
426
   1/ 1 throttle
427
   0/ 0 refs
428
   1/ 5 xio
429
   1/ 5 compressor
430
   1/ 5 bluestore
431
   1/ 5 bluefs
432
   1/ 3 bdev
433
   1/ 5 kstore
434
   2/ 2 rocksdb
435
   4/ 5 leveldb
436
   4/ 5 memdb
437
   1/ 5 kinetic
438
   1/ 5 fuse
439
   1/ 5 mgr
440
   1/ 5 mgrc
441
   1/ 5 dpdk
442
   1/ 5 eventtrace
443
  -2/-2 (syslog threshold)
444
  99/99 (stderr threshold)
445
  max_recent       500
446
  max_new         1000
447
  log_file /var/lib/ceph/crash/2019-10-28_15:23:29.299670Z_9b7c305a-8772-43d0-8a4b-d42b5f121362/log
448
--- end dump of recent events ---