Project

General

Profile

ceph-osd.0.log

debug output of: ceph-volume lvm prepare --bluestore --data /dev/sdg - Rainer Krienke, 03/05/2019 11:45 AM

Download (43.3 KB)

 
1
2019-03-05 12:37:26.922 7fa4af337240  0 set uid:gid to 64045:64045 (ceph:ceph)
2
2019-03-05 12:37:26.922 7fa4af337240  0 ceph version 13.2.4 (b10be4d44915a4d78a8e06aa31919e74927b142e) mimic (stable), process ceph-osd, pid 71549
3
2019-03-05 12:37:26.922 7fa4af337240  0 pidfile_write: ignore empty --pid-file
4
2019-03-05 12:37:26.926 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0/) set_cache_shards 1
5
2019-03-05 12:37:26.926 7fa4af337240  1 bluestore(/var/lib/ceph/osd/ceph-0/) mkfs path /var/lib/ceph/osd/ceph-0/
6
2019-03-05 12:37:26.926 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label
7
2019-03-05 12:37:26.926 7fa4af337240  2 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label unable to decode label at offset 102: buffer::malformed_input: void bluestore_bdev_label_t::decode(ceph::buffer::list::iterator&) decode past end of struct encoding
8
2019-03-05 12:37:26.926 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label
9
2019-03-05 12:37:26.926 7fa4af337240  2 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label unable to decode label at offset 102: buffer::malformed_input: void bluestore_bdev_label_t::decode(ceph::buffer::list::iterator&) decode past end of struct encoding
10
2019-03-05 12:37:26.926 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label
11
2019-03-05 12:37:26.930 7fa4af337240  2 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label unable to decode label at offset 102: buffer::malformed_input: void bluestore_bdev_label_t::decode(ceph::buffer::list::iterator&) decode past end of struct encoding
12
2019-03-05 12:37:26.930 7fa4af337240 -1 bluestore(/var/lib/ceph/osd/ceph-0/) _read_fsid unparsable uuid 
13
2019-03-05 12:37:26.930 7fa4af337240  1 bluestore(/var/lib/ceph/osd/ceph-0/) mkfs using provided fsid 889f02eb-fe94-4829-a11c-b87ed81c5936
14
2019-03-05 12:37:26.930 7fa4af337240 20 bluestore(/var/lib/ceph/osd/ceph-0/) _setup_block_symlink_or_file name block path  size 10737418240 create=1
15
2019-03-05 12:37:26.930 7fa4af337240 20 bluestore(/var/lib/ceph/osd/ceph-0/) _setup_block_symlink_or_file name block.wal path  size 100663296 create=0
16
2019-03-05 12:37:26.930 7fa4af337240 20 bluestore(/var/lib/ceph/osd/ceph-0/) _setup_block_symlink_or_file name block.db path  size 0 create=0
17
2019-03-05 12:37:26.930 7fa4af337240  1 bdev create path /var/lib/ceph/osd/ceph-0//block type kernel
18
2019-03-05 12:37:26.930 7fa4af337240  1 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) open path /var/lib/ceph/osd/ceph-0//block
19
2019-03-05 12:37:26.930 7fa4af337240 20 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) open devname dm-0
20
2019-03-05 12:37:26.930 7fa4af337240 20 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) _detect_vdo no VDO volume maps to dm-0
21
2019-03-05 12:37:26.930 7fa4af337240 10 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) _aio_start
22
2019-03-05 12:37:26.930 7fa4af337240  1 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) open size 4000783007744 (0x3a381400000, 3.6 TiB) block_size 4096 (4 KiB) rotational
23
2019-03-05 12:37:26.930 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0//block) _write_bdev_label path /var/lib/ceph/osd/ceph-0//block label bdev(osd_uuid 889f02eb-fe94-4829-a11c-b87ed81c5936, size 0x3a381400000, btime 2019-03-05 12:37:26.933971, desc main, 0 meta)
24
2019-03-05 12:37:26.930 7fa4a0a89700 10 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) _aio_thread start
25
2019-03-05 12:37:26.930 7fa49eaaf700 20 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) _discard_thread sleep
26
2019-03-05 12:37:26.934 7fa4af337240  1 bluestore(/var/lib/ceph/osd/ceph-0/) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2
27
2019-03-05 12:37:26.934 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0/) _open_db kv_backend = rocksdb
28
2019-03-05 12:37:26.934 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0/) _open_db do_bluefs = 1
29
2019-03-05 12:37:26.934 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0/) _open_db initializing bluefs
30
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0//block
31
2019-03-05 12:37:26.934 7fa4af337240  1 bdev create path /var/lib/ceph/osd/ceph-0//block type kernel
32
2019-03-05 12:37:26.934 7fa4af337240  1 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) open path /var/lib/ceph/osd/ceph-0//block
33
2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) open devname dm-0
34
2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _detect_vdo no VDO volume maps to dm-0
35
2019-03-05 12:37:26.934 7fa4af337240 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_start
36
2019-03-05 12:37:26.934 7fa4af337240  1 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) open size 4000783007744 (0x3a381400000, 3.6 TiB) block_size 4096 (4 KiB) rotational
37
2019-03-05 12:37:26.934 7fa4af337240  1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0//block size 3.6 TiB
38
2019-03-05 12:37:26.934 7fa4af337240  1 bluefs add_block_extent bdev 1 0x1bf1f500000~2542a00000
39
2019-03-05 12:37:26.934 7fa49e2ae700 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_thread start
40
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs add_block_extent done
41
2019-03-05 12:37:26.934 7fa4af337240  1 bluefs mkfs osd_uuid 889f02eb-fe94-4829-a11c-b87ed81c5936
42
2019-03-05 12:37:26.934 7fa4af337240 20 bluefs _init_alloc
43
2019-03-05 12:37:26.934 7fa4af337240 10 stupidalloc 0x0x556d7e91baa0 init_add_free 0x1bf1f500000~2542a00000
44
2019-03-05 12:37:26.934 7fa49daad700 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _discard_thread sleep
45
2019-03-05 12:37:26.934 7fa4af337240  1 bluefs mkfs uuid 06421aba-924a-4dd8-8320-bde13e4b9ca4
46
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _allocate len 0x400000 from 0
47
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _allocate len 0x400000 from 1
48
2019-03-05 12:37:26.934 7fa4af337240 10 stupidalloc 0x0x556d7e91baa0 reserve need 0x400000 num_free 0x2542a00000 num_reserved 0x0
49
2019-03-05 12:37:26.934 7fa4af337240 10 stupidalloc 0x0x556d7e91baa0 allocate_int want_size 0x400000 alloc_unit 0x100000 hint 0x0
50
2019-03-05 12:37:26.934 7fa4af337240 20 bluefs mkfs op_alloc_add 1 0x1bf1f500000~2542a00000
51
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _flush_and_sync_log txn(seq 1 len 0x13 crc 0x7ded50fd)
52
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _pad_bl padding with 0xfc7 zeros
53
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _flush 0x556d7e94a8c0 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
54
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _flush_range 0x556d7e94a8c0 pos 0x0 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
55
2019-03-05 12:37:26.934 7fa4af337240 20 bluefs _flush_range file now file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
56
2019-03-05 12:37:26.934 7fa4af337240 20 bluefs _flush_range in 1:0x1bf1f500000+400000 x_off 0x0
57
2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) aio_write 0x1bf1f500000~1000 (direct)
58
2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) aio_write rebuilding buffer to be aligned
59
2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_start 0x1bf1f500000~1000
60
2019-03-05 12:37:26.934 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) aio_write 0x1bf1f500000~1000 aio 0x556d7e94b2d0
61
2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) aio_submit ioc 0x556d7ed88490 pending 1 running 0
62
2019-03-05 12:37:26.934 7fa4af337240 20 bluefs _flush_range h 0x556d7e94a8c0 pos now 0x1000
63
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _claim_completed_aios got 1 aios
64
2019-03-05 12:37:26.934 7fa4af337240 10 bluefs wait_for_aio 0x556d7e94a8c0
65
2019-03-05 12:37:26.934 7fa4af337240 10 bdev aio_wait 0x556d7ed88490 waiting for 1 aios to complete
66
2019-03-05 12:37:26.946 7fa49e2ae700 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_finish 1 0x1bf1f500000~1000
67
2019-03-05 12:37:26.946 7fa49e2ae700 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_thread finished aio 0x556d7e94b2d0 r 4096 ioc 0x556d7ed88490 with 0 aios left
68
2019-03-05 12:37:26.946 7fa4af337240 20 bdev aio_wait 0x556d7ed88490 done
69
2019-03-05 12:37:26.946 7fa4af337240 10 bluefs wait_for_aio 0x556d7e94a8c0 done in 0.010733
70
2019-03-05 12:37:26.946 7fa4af337240 20 bluefs flush_bdev
71
2019-03-05 12:37:26.946 7fa4af337240 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) flush start
72
2019-03-05 12:37:26.946 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) flush in 0.000005
73
2019-03-05 12:37:26.946 7fa4af337240 20 bluefs _flush_and_sync_log log_seq_stable 1
74
2019-03-05 12:37:26.946 7fa4af337240 10 bluefs _write_super super block length(encoded): 89
75
2019-03-05 12:37:26.946 7fa4af337240 10 bluefs _write_super superblock 1
76
2019-03-05 12:37:26.946 7fa4af337240 10 bluefs _write_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
77
2019-03-05 12:37:26.946 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) write 0x1000~1000 (direct)
78
2019-03-05 12:37:26.946 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) write rebuilding buffer to be aligned
79
2019-03-05 12:37:26.946 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _sync_write 0x1000~1000 buffered
80
2019-03-05 12:37:26.970 7fa4af337240 20 bluefs _write_super v 1 crc 0xbf92795f offset 0x1000
81
2019-03-05 12:37:26.970 7fa4af337240 20 bluefs flush_bdev
82
2019-03-05 12:37:26.970 7fa4af337240 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) flush start
83
2019-03-05 12:37:26.970 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) flush in 0.000006
84
2019-03-05 12:37:26.970 7fa4af337240 10 bluefs _close_writer 0x556d7e94a8c0 type 0
85
2019-03-05 12:37:26.970 7fa4af337240 20 bdev aio_wait 0x556d7ed88490 done
86
2019-03-05 12:37:26.970 7fa4af337240 20 bluefs _stop_alloc
87
2019-03-05 12:37:26.970 7fa4af337240 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) discard_drain
88
2019-03-05 12:37:26.970 7fa4af337240  1 stupidalloc 0x0x556d7e91baa0 shutdown
89
2019-03-05 12:37:26.970 7fa4af337240 10 bluefs mkfs success
90
2019-03-05 12:37:26.970 7fa4af337240  1 bluefs mount
91
2019-03-05 12:37:26.970 7fa4af337240 10 bluefs _open_super
92
2019-03-05 12:37:26.970 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) read 0x1000~1000 (direct)
93
2019-03-05 12:37:26.970 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_start 0x1000~1000
94
2019-03-05 12:37:26.974 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_finish 1 0x1000~1000
95
2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _open_super superblock 1
96
2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
97
2019-03-05 12:37:26.974 7fa4af337240 20 bluefs _init_alloc
98
2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _replay
99
2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
100
2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _read h 0x556d7e914f80 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
101
2019-03-05 12:37:26.974 7fa4af337240 20 bluefs _read fetching 0x0~100000 of 1:0x1bf1f500000+400000
102
2019-03-05 12:37:26.974 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) read 0x1bf1f500000~100000 (buffered)
103
2019-03-05 12:37:26.974 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_start 0x1bf1f500000~100000
104
2019-03-05 12:37:26.986 7fa4af337240 -1 /build/ceph-13.2.4/src/os/bluestore/KernelDevice.cc: In function 'virtual int KernelDevice::read(uint64_t, uint64_t, ceph::bufferlist*, IOContext*, bool)' thread 7fa4af337240 time 2019-03-05 12:37:26.989832
105
/build/ceph-13.2.4/src/os/bluestore/KernelDevice.cc: 821: FAILED assert((uint64_t)r == len)
106

    
107
 ceph version 13.2.4 (b10be4d44915a4d78a8e06aa31919e74927b142e) mimic (stable)
108
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x102) [0x7fa4a67573e2]
109
 2: (()+0x26d5a7) [0x7fa4a67575a7]
110
 3: (KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)+0x4a7) [0x556d7bd43817]
111
 4: (BlueFS::_read(BlueFS::FileReader*, BlueFS::FileReaderBuffer*, unsigned long, unsigned long, ceph::buffer::list*, char*)+0x435) [0x556d7bd035c5]
112
 5: (BlueFS::_replay(bool, bool)+0x214) [0x556d7bd09434]
113
 6: (BlueFS::mount()+0x1f1) [0x556d7bd0dc81]
114
 7: (BlueStore::_open_db(bool, bool)+0x17cd) [0x556d7bc3404d]
115
 8: (BlueStore::mkfs()+0x805) [0x556d7bc64fe5]
116
 9: (OSD::mkfs(CephContext*, ObjectStore*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, uuid_d, int)+0x1b0) [0x556d7b80d480]
117
 10: (main()+0x4222) [0x556d7b6f4462]
118
 11: (__libc_start_main()+0xe7) [0x7fa4a3d14b97]
119
 12: (_start()+0x2a) [0x556d7b7bd95a]
120
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
121

    
122
--- begin dump of recent events ---
123
  -191> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command perfcounters_dump hook 0x556d7e8d4310
124
  -190> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command 1 hook 0x556d7e8d4310
125
  -189> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command perf dump hook 0x556d7e8d4310
126
  -188> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command perfcounters_schema hook 0x556d7e8d4310
127
  -187> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command perf histogram dump hook 0x556d7e8d4310
128
  -186> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command 2 hook 0x556d7e8d4310
129
  -185> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command perf schema hook 0x556d7e8d4310
130
  -184> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command perf histogram schema hook 0x556d7e8d4310
131
  -183> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command perf reset hook 0x556d7e8d4310
132
  -182> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command config show hook 0x556d7e8d4310
133
  -181> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command config help hook 0x556d7e8d4310
134
  -180> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command config set hook 0x556d7e8d4310
135
  -179> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command config unset hook 0x556d7e8d4310
136
  -178> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command config get hook 0x556d7e8d4310
137
  -177> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command config diff hook 0x556d7e8d4310
138
  -176> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command config diff get hook 0x556d7e8d4310
139
  -175> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command log flush hook 0x556d7e8d4310
140
  -174> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command log dump hook 0x556d7e8d4310
141
  -173> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command log reopen hook 0x556d7e8d4310
142
  -172> 2019-03-05 12:37:26.910 7fa4af337240  5 asok(0x556d7e8d6900) register_command dump_mempools hook 0x556d7ec98068
143
  -171> 2019-03-05 12:37:26.914 7fa4af337240 10 monclient: get_monmap_and_config
144
  -170> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient: build_initial_monmap
145
  -169> 2019-03-05 12:37:26.918 7fa4a0a89700  2 Event(0x556d7e8d6e00 nevent=5000 time_id=1).set_owner idx=0 owner=140345046767360
146
  -168> 2019-03-05 12:37:26.918 7fa4a0288700  2 Event(0x556d7e8d7040 nevent=5000 time_id=1).set_owner idx=1 owner=140345038374656
147
  -167> 2019-03-05 12:37:26.918 7fa49fa87700  2 Event(0x556d7e8d7280 nevent=5000 time_id=1).set_owner idx=2 owner=140345029981952
148
  -166> 2019-03-05 12:37:26.918 7fa4af337240  1  Processor -- start
149
  -165> 2019-03-05 12:37:26.918 7fa4af337240  1 -- - start start
150
  -164> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient: init
151
  -163> 2019-03-05 12:37:26.918 7fa4af337240  5 adding auth protocol: cephx
152
  -162> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient: auth_supported 2 method cephx
153
  -161> 2019-03-05 12:37:26.918 7fa4af337240  2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-0//keyring
154
  -160> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient: _reopen_session rank -1
155
  -159> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient(hunting): picked mon.ceph2 con 0x556d7ed6e700 addr 141.26.70.211:6789/0
156
  -158> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient(hunting): picked mon.ceph1 con 0x556d7ed6ee00 addr 141.26.70.210:6789/0
157
  -157> 2019-03-05 12:37:26.918 7fa4af337240  1 -- - --> 141.26.70.210:6789/0 -- auth(proto 0 26 bytes epoch 2) v1 -- 0x556d7ece0780 con 0
158
  -156> 2019-03-05 12:37:26.918 7fa4af337240  1 -- - --> 141.26.70.211:6789/0 -- auth(proto 0 26 bytes epoch 2) v1 -- 0x556d7ece0a00 con 0
159
  -155> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient(hunting): _renew_subs
160
  -154> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient(hunting): authenticate will time out at 2019-03-05 12:42:26.923126
161
  -153> 2019-03-05 12:37:26.918 7fa49fa87700  1 -- 141.26.70.210:0/1189472781 learned_addr learned my addr 141.26.70.210:0/1189472781
162
  -152> 2019-03-05 12:37:26.918 7fa49fa87700  2 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:6789/0 conn(0x556d7ed6ee00 :-1 s=STATE_CONNECTING_WAIT_ACK_SEQ pgs=0 cs=0 l=0)._process_connection got newly_acked_seq 0 vs out_seq 0
163
  -151> 2019-03-05 12:37:26.918 7fa4a0288700  2 -- 141.26.70.210:0/1189472781 >> 141.26.70.211:6789/0 conn(0x556d7ed6e700 :-1 s=STATE_CONNECTING_WAIT_ACK_SEQ pgs=0 cs=0 l=0)._process_connection got newly_acked_seq 0 vs out_seq 0
164
  -150> 2019-03-05 12:37:26.918 7fa49fa87700  5 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:6789/0 conn(0x556d7ed6ee00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=54 cs=1 l=1). rx mon.0 seq 1 0x556d7ece0780 auth_reply(proto 2 0 (0) Success) v1
165
  -149> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 <== mon.0 141.26.70.210:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (606083615 0 0) 0x556d7ece0780 con 0x556d7ed6ee00
166
  -148> 2019-03-05 12:37:26.918 7fa49f286700 10 monclient(hunting): my global_id is 14115
167
  -147> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 --> 141.26.70.210:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- 0x556d7ece0f00 con 0
168
  -146> 2019-03-05 12:37:26.918 7fa4a0288700  5 -- 141.26.70.210:0/1189472781 >> 141.26.70.211:6789/0 conn(0x556d7ed6e700 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=4417 cs=1 l=1). rx mon.1 seq 1 0x556d7ece0a00 auth_reply(proto 2 0 (0) Success) v1
169
  -145> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 <== mon.1 141.26.70.211:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (1859929057 0 0) 0x556d7ece0a00 con 0x556d7ed6e700
170
  -144> 2019-03-05 12:37:26.918 7fa49f286700 10 monclient(hunting): my global_id is 14137
171
  -143> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 --> 141.26.70.211:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- 0x556d7ece0780 con 0
172
  -142> 2019-03-05 12:37:26.918 7fa49fa87700  5 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:6789/0 conn(0x556d7ed6ee00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=54 cs=1 l=1). rx mon.0 seq 2 0x556d7ece0f00 auth_reply(proto 2 0 (0) Success) v1
173
  -141> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 <== mon.0 141.26.70.210:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (3926786789 0 0) 0x556d7ece0f00 con 0x556d7ed6ee00
174
  -140> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 >> 141.26.70.211:6789/0 conn(0x556d7ed6e700 :-1 s=STATE_OPEN pgs=4417 cs=1 l=1).mark_down
175
  -139> 2019-03-05 12:37:26.918 7fa49f286700  2 -- 141.26.70.210:0/1189472781 >> 141.26.70.211:6789/0 conn(0x556d7ed6e700 :-1 s=STATE_OPEN pgs=4417 cs=1 l=1)._stop
176
  -138> 2019-03-05 12:37:26.918 7fa49f286700  1 monclient: found mon.ceph1
177
  -137> 2019-03-05 12:37:26.918 7fa49f286700 10 monclient: _send_mon_message to mon.ceph1 at 141.26.70.210:6789/0
178
  -136> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 --> 141.26.70.210:6789/0 -- mon_subscribe({config=0+,monmap=3+}) v3 -- 0x556d7e8d7440 con 0
179
  -135> 2019-03-05 12:37:26.918 7fa49f286700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-03-05 12:36:56.924395)
180
  -134> 2019-03-05 12:37:26.918 7fa49f286700 10 monclient: _send_mon_message to mon.ceph1 at 141.26.70.210:6789/0
181
  -133> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 --> 141.26.70.210:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- 0x556d7ece0a00 con 0
182
  -132> 2019-03-05 12:37:26.918 7fa4af337240  5 monclient: authenticate success, global_id 14115
183
  -131> 2019-03-05 12:37:26.918 7fa49fa87700  5 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:6789/0 conn(0x556d7ed6ee00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=54 cs=1 l=1). rx mon.0 seq 3 0x556d7e8d7440 config(0 keys) v1
184
  -130> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 <== mon.0 141.26.70.210:6789/0 3 ==== config(0 keys) v1 ==== 4+0+0 (0 0 0) 0x556d7e8d7440 con 0x556d7ed6ee00
185
  -129> 2019-03-05 12:37:26.918 7fa49f286700 10 monclient: handle_config config(0 keys) v1
186
  -128> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient: get_monmap_and_config success
187
  -127> 2019-03-05 12:37:26.918 7fa4af337240 10 monclient: shutdown
188
  -126> 2019-03-05 12:37:26.918 7fa4af337240  1 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:6789/0 conn(0x556d7ed6ee00 :-1 s=STATE_OPEN pgs=54 cs=1 l=1).mark_down
189
  -125> 2019-03-05 12:37:26.918 7fa49fa87700  5 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:6789/0 conn(0x556d7ed6ee00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=54 cs=1 l=1). rx mon.0 seq 4 0x556d7ece0a00 auth_reply(proto 2 0 (0) Success) v1
190
  -124> 2019-03-05 12:37:26.918 7fa4af337240  2 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:6789/0 conn(0x556d7ed6ee00 :-1 s=STATE_OPEN pgs=54 cs=1 l=1)._stop
191
  -123> 2019-03-05 12:37:26.918 7fa49da83700  4 set_mon_vals no callback set
192
  -122> 2019-03-05 12:37:26.918 7fa49f286700  1 -- 141.26.70.210:0/1189472781 <== mon.0 141.26.70.210:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (3157836970 0 0) 0x556d7ece0a00 con 0x556d7ed6ee00
193
  -121> 2019-03-05 12:37:26.918 7fa49f286700 10 monclient: discarding stray monitor message auth_reply(proto 2 0 (0) Success) v1
194
  -120> 2019-03-05 12:37:26.922 7fa4af337240  1 -- 141.26.70.210:0/1189472781 shutdown_connections 
195
  -119> 2019-03-05 12:37:26.922 7fa4af337240  5 -- 141.26.70.210:0/1189472781 shutdown_connections mark down 141.26.70.210:6789/0 0x556d7ed6ee00
196
  -118> 2019-03-05 12:37:26.922 7fa4af337240  5 -- 141.26.70.210:0/1189472781 shutdown_connections mark down 141.26.70.211:6789/0 0x556d7ed6e700
197
  -117> 2019-03-05 12:37:26.922 7fa4af337240  5 -- 141.26.70.210:0/1189472781 shutdown_connections delete 0x556d7ed6e700
198
  -116> 2019-03-05 12:37:26.922 7fa4af337240  5 -- 141.26.70.210:0/1189472781 shutdown_connections delete 0x556d7ed6ee00
199
  -115> 2019-03-05 12:37:26.922 7fa4af337240  1 -- 141.26.70.210:0/1189472781 shutdown_connections 
200
  -114> 2019-03-05 12:37:26.922 7fa4af337240  1 -- 141.26.70.210:0/1189472781 wait complete.
201
  -113> 2019-03-05 12:37:26.922 7fa4af337240  1 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:0/1189472781 conn(0x556d7ed6e000 :-1 s=STATE_NONE pgs=0 cs=0 l=0).mark_down
202
  -112> 2019-03-05 12:37:26.922 7fa4af337240  2 -- 141.26.70.210:0/1189472781 >> 141.26.70.210:0/1189472781 conn(0x556d7ed6e000 :-1 s=STATE_NONE pgs=0 cs=0 l=0)._stop
203
  -111> 2019-03-05 12:37:26.922 7fa4af337240  0 set uid:gid to 64045:64045 (ceph:ceph)
204
  -110> 2019-03-05 12:37:26.922 7fa4af337240  0 ceph version 13.2.4 (b10be4d44915a4d78a8e06aa31919e74927b142e) mimic (stable), process ceph-osd, pid 71549
205
  -109> 2019-03-05 12:37:26.922 7fa4af337240  0 pidfile_write: ignore empty --pid-file
206
  -108> 2019-03-05 12:37:26.926 7fa4af337240  5 asok(0x556d7e8d6900) init /var/run/ceph/ceph-osd.0.asok
207
  -107> 2019-03-05 12:37:26.926 7fa4af337240  5 asok(0x556d7e8d6900) bind_and_listen /var/run/ceph/ceph-osd.0.asok
208
  -106> 2019-03-05 12:37:26.926 7fa4af337240  5 asok(0x556d7e8d6900) register_command 0 hook 0x556d7e8d3550
209
  -105> 2019-03-05 12:37:26.926 7fa4af337240  5 asok(0x556d7e8d6900) register_command version hook 0x556d7e8d3550
210
  -104> 2019-03-05 12:37:26.926 7fa4af337240  5 asok(0x556d7e8d6900) register_command git_version hook 0x556d7e8d3550
211
  -103> 2019-03-05 12:37:26.926 7fa4af337240  5 asok(0x556d7e8d6900) register_command help hook 0x556d7e8d44b0
212
  -102> 2019-03-05 12:37:26.926 7fa4af337240  5 asok(0x556d7e8d6900) register_command get_command_descriptions hook 0x556d7e8d44c0
213
  -101> 2019-03-05 12:37:26.926 7fa4a0288700  5 asok(0x556d7e8d6900) entry start
214
  -100> 2019-03-05 12:37:26.926 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0/) set_cache_shards 1
215
   -99> 2019-03-05 12:37:26.926 7fa4af337240  1 bluestore(/var/lib/ceph/osd/ceph-0/) mkfs path /var/lib/ceph/osd/ceph-0/
216
   -98> 2019-03-05 12:37:26.926 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label
217
   -97> 2019-03-05 12:37:26.926 7fa4af337240  2 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label unable to decode label at offset 102: buffer::malformed_input: void bluestore_bdev_label_t::decode(ceph::buffer::list::iterator&) decode past end of struct encoding
218
   -96> 2019-03-05 12:37:26.926 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label
219
   -95> 2019-03-05 12:37:26.926 7fa4af337240  2 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label unable to decode label at offset 102: buffer::malformed_input: void bluestore_bdev_label_t::decode(ceph::buffer::list::iterator&) decode past end of struct encoding
220
   -94> 2019-03-05 12:37:26.926 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label
221
   -93> 2019-03-05 12:37:26.930 7fa4af337240  2 bluestore(/var/lib/ceph/osd/ceph-0//block) _read_bdev_label unable to decode label at offset 102: buffer::malformed_input: void bluestore_bdev_label_t::decode(ceph::buffer::list::iterator&) decode past end of struct encoding
222
   -92> 2019-03-05 12:37:26.930 7fa4af337240 -1 bluestore(/var/lib/ceph/osd/ceph-0/) _read_fsid unparsable uuid 
223
   -91> 2019-03-05 12:37:26.930 7fa4af337240  1 bluestore(/var/lib/ceph/osd/ceph-0/) mkfs using provided fsid 889f02eb-fe94-4829-a11c-b87ed81c5936
224
   -90> 2019-03-05 12:37:26.930 7fa4af337240 20 bluestore(/var/lib/ceph/osd/ceph-0/) _setup_block_symlink_or_file name block path  size 10737418240 create=1
225
   -89> 2019-03-05 12:37:26.930 7fa4af337240 20 bluestore(/var/lib/ceph/osd/ceph-0/) _setup_block_symlink_or_file name block.wal path  size 100663296 create=0
226
   -88> 2019-03-05 12:37:26.930 7fa4af337240 20 bluestore(/var/lib/ceph/osd/ceph-0/) _setup_block_symlink_or_file name block.db path  size 0 create=0
227
   -87> 2019-03-05 12:37:26.930 7fa4af337240  1 bdev create path /var/lib/ceph/osd/ceph-0//block type kernel
228
   -86> 2019-03-05 12:37:26.930 7fa4af337240  1 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) open path /var/lib/ceph/osd/ceph-0//block
229
   -85> 2019-03-05 12:37:26.930 7fa4af337240 20 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) open devname dm-0
230
   -84> 2019-03-05 12:37:26.930 7fa4af337240 20 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) _detect_vdo no VDO volume maps to dm-0
231
   -83> 2019-03-05 12:37:26.930 7fa4af337240 10 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) _aio_start
232
   -82> 2019-03-05 12:37:26.930 7fa4af337240  1 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) open size 4000783007744 (0x3a381400000, 3.6 TiB) block_size 4096 (4 KiB) rotational
233
   -81> 2019-03-05 12:37:26.930 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0//block) _write_bdev_label path /var/lib/ceph/osd/ceph-0//block label bdev(osd_uuid 889f02eb-fe94-4829-a11c-b87ed81c5936, size 0x3a381400000, btime 2019-03-05 12:37:26.933971, desc main, 0 meta)
234
   -80> 2019-03-05 12:37:26.930 7fa4a0a89700 10 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) _aio_thread start
235
   -79> 2019-03-05 12:37:26.930 7fa49eaaf700 20 bdev(0x556d7ed7e700 /var/lib/ceph/osd/ceph-0//block) _discard_thread sleep
236
   -78> 2019-03-05 12:37:26.934 7fa4af337240  1 bluestore(/var/lib/ceph/osd/ceph-0/) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2
237
   -77> 2019-03-05 12:37:26.934 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0/) _open_db kv_backend = rocksdb
238
   -76> 2019-03-05 12:37:26.934 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0/) _open_db do_bluefs = 1
239
   -75> 2019-03-05 12:37:26.934 7fa4af337240 10 bluestore(/var/lib/ceph/osd/ceph-0/) _open_db initializing bluefs
240
   -74> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0//block
241
   -73> 2019-03-05 12:37:26.934 7fa4af337240  1 bdev create path /var/lib/ceph/osd/ceph-0//block type kernel
242
   -72> 2019-03-05 12:37:26.934 7fa4af337240  1 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) open path /var/lib/ceph/osd/ceph-0//block
243
   -71> 2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) open devname dm-0
244
   -70> 2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _detect_vdo no VDO volume maps to dm-0
245
   -69> 2019-03-05 12:37:26.934 7fa4af337240 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_start
246
   -68> 2019-03-05 12:37:26.934 7fa4af337240  1 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) open size 4000783007744 (0x3a381400000, 3.6 TiB) block_size 4096 (4 KiB) rotational
247
   -67> 2019-03-05 12:37:26.934 7fa4af337240  1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-0//block size 3.6 TiB
248
   -66> 2019-03-05 12:37:26.934 7fa4af337240  1 bluefs add_block_extent bdev 1 0x1bf1f500000~2542a00000
249
   -65> 2019-03-05 12:37:26.934 7fa49e2ae700 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_thread start
250
   -64> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs add_block_extent done
251
   -63> 2019-03-05 12:37:26.934 7fa4af337240  1 bluefs mkfs osd_uuid 889f02eb-fe94-4829-a11c-b87ed81c5936
252
   -62> 2019-03-05 12:37:26.934 7fa4af337240 20 bluefs _init_alloc
253
   -61> 2019-03-05 12:37:26.934 7fa4af337240 10 stupidalloc 0x0x556d7e91baa0 init_add_free 0x1bf1f500000~2542a00000
254
   -60> 2019-03-05 12:37:26.934 7fa49daad700 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _discard_thread sleep
255
   -59> 2019-03-05 12:37:26.934 7fa4af337240  1 bluefs mkfs uuid 06421aba-924a-4dd8-8320-bde13e4b9ca4
256
   -58> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _allocate len 0x400000 from 0
257
   -57> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _allocate len 0x400000 from 1
258
   -56> 2019-03-05 12:37:26.934 7fa4af337240 10 stupidalloc 0x0x556d7e91baa0 reserve need 0x400000 num_free 0x2542a00000 num_reserved 0x0
259
   -55> 2019-03-05 12:37:26.934 7fa4af337240 10 stupidalloc 0x0x556d7e91baa0 allocate_int want_size 0x400000 alloc_unit 0x100000 hint 0x0
260
   -54> 2019-03-05 12:37:26.934 7fa4af337240 20 bluefs mkfs op_alloc_add 1 0x1bf1f500000~2542a00000
261
   -53> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _flush_and_sync_log txn(seq 1 len 0x13 crc 0x7ded50fd)
262
   -52> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _pad_bl padding with 0xfc7 zeros
263
   -51> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _flush 0x556d7e94a8c0 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
264
   -50> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _flush_range 0x556d7e94a8c0 pos 0x0 0x0~1000 to file(ino 1 size 0x0 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
265
   -49> 2019-03-05 12:37:26.934 7fa4af337240 20 bluefs _flush_range file now file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
266
   -48> 2019-03-05 12:37:26.934 7fa4af337240 20 bluefs _flush_range in 1:0x1bf1f500000+400000 x_off 0x0
267
   -47> 2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) aio_write 0x1bf1f500000~1000 (direct)
268
   -46> 2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) aio_write rebuilding buffer to be aligned
269
   -45> 2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_start 0x1bf1f500000~1000
270
   -44> 2019-03-05 12:37:26.934 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) aio_write 0x1bf1f500000~1000 aio 0x556d7e94b2d0
271
   -43> 2019-03-05 12:37:26.934 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) aio_submit ioc 0x556d7ed88490 pending 1 running 0
272
   -42> 2019-03-05 12:37:26.934 7fa4af337240 20 bluefs _flush_range h 0x556d7e94a8c0 pos now 0x1000
273
   -41> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs _claim_completed_aios got 1 aios
274
   -40> 2019-03-05 12:37:26.934 7fa4af337240 10 bluefs wait_for_aio 0x556d7e94a8c0
275
   -39> 2019-03-05 12:37:26.934 7fa4af337240 10 bdev aio_wait 0x556d7ed88490 waiting for 1 aios to complete
276
   -38> 2019-03-05 12:37:26.946 7fa49e2ae700 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_finish 1 0x1bf1f500000~1000
277
   -37> 2019-03-05 12:37:26.946 7fa49e2ae700 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_thread finished aio 0x556d7e94b2d0 r 4096 ioc 0x556d7ed88490 with 0 aios left
278
   -36> 2019-03-05 12:37:26.946 7fa4af337240 20 bdev aio_wait 0x556d7ed88490 done
279
   -35> 2019-03-05 12:37:26.946 7fa4af337240 10 bluefs wait_for_aio 0x556d7e94a8c0 done in 0.010733
280
   -34> 2019-03-05 12:37:26.946 7fa4af337240 20 bluefs flush_bdev
281
   -33> 2019-03-05 12:37:26.946 7fa4af337240 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) flush start
282
   -32> 2019-03-05 12:37:26.946 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) flush in 0.000005
283
   -31> 2019-03-05 12:37:26.946 7fa4af337240 20 bluefs _flush_and_sync_log log_seq_stable 1
284
   -30> 2019-03-05 12:37:26.946 7fa4af337240 10 bluefs _write_super super block length(encoded): 89
285
   -29> 2019-03-05 12:37:26.946 7fa4af337240 10 bluefs _write_super superblock 1
286
   -28> 2019-03-05 12:37:26.946 7fa4af337240 10 bluefs _write_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
287
   -27> 2019-03-05 12:37:26.946 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) write 0x1000~1000 (direct)
288
   -26> 2019-03-05 12:37:26.946 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) write rebuilding buffer to be aligned
289
   -25> 2019-03-05 12:37:26.946 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _sync_write 0x1000~1000 buffered
290
   -24> 2019-03-05 12:37:26.970 7fa4af337240 20 bluefs _write_super v 1 crc 0xbf92795f offset 0x1000
291
   -23> 2019-03-05 12:37:26.970 7fa4af337240 20 bluefs flush_bdev
292
   -22> 2019-03-05 12:37:26.970 7fa4af337240 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) flush start
293
   -21> 2019-03-05 12:37:26.970 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) flush in 0.000006
294
   -20> 2019-03-05 12:37:26.970 7fa4af337240 10 bluefs _close_writer 0x556d7e94a8c0 type 0
295
   -19> 2019-03-05 12:37:26.970 7fa4af337240 20 bdev aio_wait 0x556d7ed88490 done
296
   -18> 2019-03-05 12:37:26.970 7fa4af337240 20 bluefs _stop_alloc
297
   -17> 2019-03-05 12:37:26.970 7fa4af337240 10 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) discard_drain
298
   -16> 2019-03-05 12:37:26.970 7fa4af337240  1 stupidalloc 0x0x556d7e91baa0 shutdown
299
   -15> 2019-03-05 12:37:26.970 7fa4af337240 10 bluefs mkfs success
300
   -14> 2019-03-05 12:37:26.970 7fa4af337240  1 bluefs mount
301
   -13> 2019-03-05 12:37:26.970 7fa4af337240 10 bluefs _open_super
302
   -12> 2019-03-05 12:37:26.970 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) read 0x1000~1000 (direct)
303
   -11> 2019-03-05 12:37:26.970 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_start 0x1000~1000
304
   -10> 2019-03-05 12:37:26.974 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_finish 1 0x1000~1000
305
    -9> 2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _open_super superblock 1
306
    -8> 2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _open_super log_fnode file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
307
    -7> 2019-03-05 12:37:26.974 7fa4af337240 20 bluefs _init_alloc
308
    -6> 2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _replay
309
    -5> 2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _replay log_fnode file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
310
    -4> 2019-03-05 12:37:26.974 7fa4af337240 10 bluefs _read h 0x556d7e914f80 0x0~1000 from file(ino 1 size 0x1000 mtime 0.000000 bdev 0 allocated 400000 extents [1:0x1bf1f500000+400000])
311
    -3> 2019-03-05 12:37:26.974 7fa4af337240 20 bluefs _read fetching 0x0~100000 of 1:0x1bf1f500000+400000
312
    -2> 2019-03-05 12:37:26.974 7fa4af337240  5 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) read 0x1bf1f500000~100000 (buffered)
313
    -1> 2019-03-05 12:37:26.974 7fa4af337240 20 bdev(0x556d7ed7ea80 /var/lib/ceph/osd/ceph-0//block) _aio_log_start 0x1bf1f500000~100000
314
     0> 2019-03-05 12:37:26.986 7fa4af337240 -1 /build/ceph-13.2.4/src/os/bluestore/KernelDevice.cc: In function 'virtual int KernelDevice::read(uint64_t, uint64_t, ceph::bufferlist*, IOContext*, bool)' thread 7fa4af337240 time 2019-03-05 12:37:26.989832
315
/build/ceph-13.2.4/src/os/bluestore/KernelDevice.cc: 821: FAILED assert((uint64_t)r == len)
316

    
317
 ceph version 13.2.4 (b10be4d44915a4d78a8e06aa31919e74927b142e) mimic (stable)
318
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x102) [0x7fa4a67573e2]
319
 2: (()+0x26d5a7) [0x7fa4a67575a7]
320
 3: (KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)+0x4a7) [0x556d7bd43817]
321
 4: (BlueFS::_read(BlueFS::FileReader*, BlueFS::FileReaderBuffer*, unsigned long, unsigned long, ceph::buffer::list*, char*)+0x435) [0x556d7bd035c5]
322
 5: (BlueFS::_replay(bool, bool)+0x214) [0x556d7bd09434]
323
 6: (BlueFS::mount()+0x1f1) [0x556d7bd0dc81]
324
 7: (BlueStore::_open_db(bool, bool)+0x17cd) [0x556d7bc3404d]
325
 8: (BlueStore::mkfs()+0x805) [0x556d7bc64fe5]
326
 9: (OSD::mkfs(CephContext*, ObjectStore*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, uuid_d, int)+0x1b0) [0x556d7b80d480]
327
 10: (main()+0x4222) [0x556d7b6f4462]
328
 11: (__libc_start_main()+0xe7) [0x7fa4a3d14b97]
329
 12: (_start()+0x2a) [0x556d7b7bd95a]
330
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
331

    
332
--- logging levels ---
333
   0/ 5 none
334
   0/ 1 lockdep
335
   0/ 1 context
336
   1/ 1 crush
337
   1/ 5 mds
338
   1/ 5 mds_balancer
339
   1/ 5 mds_locker
340
   1/ 5 mds_log
341
   1/ 5 mds_log_expire
342
   1/ 5 mds_migrator
343
   0/ 1 buffer
344
   0/ 1 timer
345
   0/ 1 filer
346
   0/ 1 striper
347
   0/ 1 objecter
348
   0/ 5 rados
349
   0/ 5 rbd
350
   0/ 5 rbd_mirror
351
   0/ 5 rbd_replay
352
   0/ 5 journaler
353
   0/ 5 objectcacher
354
   0/ 5 client
355
   1/ 5 osd
356
   0/ 5 optracker
357
   0/ 5 objclass
358
   1/ 3 filestore
359
   1/ 3 journal
360
   0/ 5 ms
361
   1/ 5 mon
362
   0/10 monc
363
   1/ 5 paxos
364
   0/ 5 tp
365
   1/ 5 auth
366
   1/ 5 crypto
367
   1/ 1 finisher
368
   1/ 1 reserver
369
   1/ 5 heartbeatmap
370
   1/ 5 perfcounter
371
   1/ 5 rgw
372
   1/ 5 rgw_sync
373
   1/10 civetweb
374
   1/ 5 javaclient
375
   1/ 5 asok
376
   1/ 1 throttle
377
   0/ 0 refs
378
   1/ 5 xio
379
   1/ 5 compressor
380
  20/20 bluestore
381
  20/20 bluefs
382
  20/20 bdev
383
   1/ 5 kstore
384
   4/ 5 rocksdb
385
   4/ 5 leveldb
386
   4/ 5 memdb
387
   1/ 5 kinetic
388
   1/ 5 fuse
389
   1/ 5 mgr
390
   1/ 5 mgrc
391
   1/ 5 dpdk
392
   1/ 5 eventtrace
393
  -2/-2 (syslog threshold)
394
  -1/-1 (stderr threshold)
395
  max_recent     10000
396
  max_new         1000
397
  log_file /var/log/ceph/ceph-osd.0.log
398
--- end dump of recent events ---
399
2019-03-05 12:37:26.990 7fa4af337240 -1 *** Caught signal (Aborted) **
400
 in thread 7fa4af337240 thread_name:ceph-osd
401

    
402
 ceph version 13.2.4 (b10be4d44915a4d78a8e06aa31919e74927b142e) mimic (stable)
403
 1: (()+0x92aa40) [0x556d7bd54a40]
404
 2: (()+0x12890) [0x7fa4a4e5b890]
405
 3: (gsignal()+0xc7) [0x7fa4a3d31e97]
406
 4: (abort()+0x141) [0x7fa4a3d33801]
407
 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x250) [0x7fa4a6757530]
408
 6: (()+0x26d5a7) [0x7fa4a67575a7]
409
 7: (KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)+0x4a7) [0x556d7bd43817]
410
 8: (BlueFS::_read(BlueFS::FileReader*, BlueFS::FileReaderBuffer*, unsigned long, unsigned long, ceph::buffer::list*, char*)+0x435) [0x556d7bd035c5]
411
 9: (BlueFS::_replay(bool, bool)+0x214) [0x556d7bd09434]
412
 10: (BlueFS::mount()+0x1f1) [0x556d7bd0dc81]
413
 11: (BlueStore::_open_db(bool, bool)+0x17cd) [0x556d7bc3404d]
414
 12: (BlueStore::mkfs()+0x805) [0x556d7bc64fe5]
415
 13: (OSD::mkfs(CephContext*, ObjectStore*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, uuid_d, int)+0x1b0) [0x556d7b80d480]
416
 14: (main()+0x4222) [0x556d7b6f4462]
417
 15: (__libc_start_main()+0xe7) [0x7fa4a3d14b97]
418
 16: (_start()+0x2a) [0x556d7b7bd95a]
419
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
420

    
421
--- begin dump of recent events ---
422
     0> 2019-03-05 12:37:26.990 7fa4af337240 -1 *** Caught signal (Aborted) **
423
 in thread 7fa4af337240 thread_name:ceph-osd
424

    
425
 ceph version 13.2.4 (b10be4d44915a4d78a8e06aa31919e74927b142e) mimic (stable)
426
 1: (()+0x92aa40) [0x556d7bd54a40]
427
 2: (()+0x12890) [0x7fa4a4e5b890]
428
 3: (gsignal()+0xc7) [0x7fa4a3d31e97]
429
 4: (abort()+0x141) [0x7fa4a3d33801]
430
 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x250) [0x7fa4a6757530]
431
 6: (()+0x26d5a7) [0x7fa4a67575a7]
432
 7: (KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)+0x4a7) [0x556d7bd43817]
433
 8: (BlueFS::_read(BlueFS::FileReader*, BlueFS::FileReaderBuffer*, unsigned long, unsigned long, ceph::buffer::list*, char*)+0x435) [0x556d7bd035c5]
434
 9: (BlueFS::_replay(bool, bool)+0x214) [0x556d7bd09434]
435
 10: (BlueFS::mount()+0x1f1) [0x556d7bd0dc81]
436
 11: (BlueStore::_open_db(bool, bool)+0x17cd) [0x556d7bc3404d]
437
 12: (BlueStore::mkfs()+0x805) [0x556d7bc64fe5]
438
 13: (OSD::mkfs(CephContext*, ObjectStore*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, uuid_d, int)+0x1b0) [0x556d7b80d480]
439
 14: (main()+0x4222) [0x556d7b6f4462]
440
 15: (__libc_start_main()+0xe7) [0x7fa4a3d14b97]
441
 16: (_start()+0x2a) [0x556d7b7bd95a]
442
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
443

    
444
--- logging levels ---
445
   0/ 5 none
446
   0/ 1 lockdep
447
   0/ 1 context
448
   1/ 1 crush
449
   1/ 5 mds
450
   1/ 5 mds_balancer
451
   1/ 5 mds_locker
452
   1/ 5 mds_log
453
   1/ 5 mds_log_expire
454
   1/ 5 mds_migrator
455
   0/ 1 buffer
456
   0/ 1 timer
457
   0/ 1 filer
458
   0/ 1 striper
459
   0/ 1 objecter
460
   0/ 5 rados
461
   0/ 5 rbd
462
   0/ 5 rbd_mirror
463
   0/ 5 rbd_replay
464
   0/ 5 journaler
465
   0/ 5 objectcacher
466
   0/ 5 client
467
   1/ 5 osd
468
   0/ 5 optracker
469
   0/ 5 objclass
470
   1/ 3 filestore
471
   1/ 3 journal
472
   0/ 5 ms
473
   1/ 5 mon
474
   0/10 monc
475
   1/ 5 paxos
476
   0/ 5 tp
477
   1/ 5 auth
478
   1/ 5 crypto
479
   1/ 1 finisher
480
   1/ 1 reserver
481
   1/ 5 heartbeatmap
482
   1/ 5 perfcounter
483
   1/ 5 rgw
484
   1/ 5 rgw_sync
485
   1/10 civetweb
486
   1/ 5 javaclient
487
   1/ 5 asok
488
   1/ 1 throttle
489
   0/ 0 refs
490
   1/ 5 xio
491
   1/ 5 compressor
492
  20/20 bluestore
493
  20/20 bluefs
494
  20/20 bdev
495
   1/ 5 kstore
496
   4/ 5 rocksdb
497
   4/ 5 leveldb
498
   4/ 5 memdb
499
   1/ 5 kinetic
500
   1/ 5 fuse
501
   1/ 5 mgr
502
   1/ 5 mgrc
503
   1/ 5 dpdk
504
   1/ 5 eventtrace
505
  -2/-2 (syslog threshold)
506
  -1/-1 (stderr threshold)
507
  max_recent     10000
508
  max_new         1000
509
  log_file /var/log/ceph/ceph-osd.0.log
510
--- end dump of recent events ---