Project

General

Profile

Bug #52079 » bluefs mount failed to replay log (5) on h3.txt

Viktor Svecov, 08/06/2021 05:09 AM

 
root@h3:~# /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph-osd --privileged --group-add=disk --init --name ceph-4c2ac23c-46d9-11eb-8b27-eb6197542e4f-osd.2 -e CONTAINER_IMAGE=ceph/ceph@sha256:15b15fb7a708970f1b734285ac08aef45dcd76e86866af37412d041e00853743 -e NODE_NAME=h3 -e CEPH_USE_RANDOM_NONCE=1 -v /var/run/ceph/4c2ac23c-46d9-11eb-8b27-eb6197542e4f:/var/run/ceph:z -v /var/log/ceph/4c2ac23c-46d9-11eb-8b27-eb6197542e4f:/var/log/ceph:z -v /var/lib/ceph/4c2ac23c-46d9-11eb-8b27-eb6197542e4f/crash:/var/lib/ceph/crash:z -v /var/lib/ceph/4c2ac23c-46d9-11eb-8b27-eb6197542e4f/osd.2:/var/lib/ceph/osd/ceph-2:z -v /var/lib/ceph/4c2ac23c-46d9-11eb-8b27-eb6197542e4f/osd.2/config:/etc/ceph/ceph.conf:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm ceph/ceph@sha256:15b15fb7a708970f1b734285ac08aef45dcd76e86866af37412d041e00853743 -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true '--default-log-stderr-prefix=debug '
debug 2021-08-06T04:38:52.166+0000 7f139f28e080 0 set uid:gid to 167:167 (ceph:ceph)
debug 2021-08-06T04:38:52.166+0000 7f139f28e080 0 ceph version 16.2.1 (afb9061ab4117f798c858c741efa6390e48ccf10) pacific (stable), process ceph-osd, pid 8
debug 2021-08-06T04:38:52.166+0000 7f139f28e080 0 pidfile_write: ignore empty --pid-file
debug 2021-08-06T04:38:52.174+0000 7f139f28e080 1 bdev(0x55898d26c800 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:52.174+0000 7f139f28e080 1 bdev(0x55898d26c800 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:52.174+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:52.174+0000 7f139f28e080 1 bdev(0x55898d26cc00 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:52.174+0000 7f139f28e080 1 bdev(0x55898d26cc00 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:52.174+0000 7f139f28e080 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-2/block size 931 GiB
debug 2021-08-06T04:38:52.174+0000 7f139f28e080 1 bdev(0x55898d26cc00 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:52.466+0000 7f139f28e080 1 bdev(0x55898d26c800 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:52.706+0000 7f139f28e080 0 starting osd.2 osd_data /var/lib/ceph/osd/ceph-2 /var/lib/ceph/osd/ceph-2/journal
debug 2021-08-06T04:38:52.726+0000 7f139f28e080 0 load: jerasure load: lrc load: isa
debug 2021-08-06T04:38:52.726+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:52.726+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:52.726+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:52.726+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:53.138+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:53.138+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:53.138+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:53.138+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:53.426+0000 7f139f28e080 0 osd.2:0.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2021-08-06T04:38:53.426+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:53.426+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:53.426+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:53.426+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:53.706+0000 7f139f28e080 0 osd.2:1.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2021-08-06T04:38:53.706+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:53.706+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:53.706+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:53.706+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:53.986+0000 7f139f28e080 0 osd.2:2.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2021-08-06T04:38:53.986+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:53.986+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:53.986+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:53.986+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:54.266+0000 7f139f28e080 0 osd.2:3.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2021-08-06T04:38:54.266+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:54.266+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:54.266+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:54.266+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:54.546+0000 7f139f28e080 0 osd.2:4.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2021-08-06T04:38:54.546+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:54.546+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:54.546+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:54.546+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:54.826+0000 7f139f28e080 0 osd.2:5.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2021-08-06T04:38:54.826+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:54.826+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:54.826+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:54.826+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:55.114+0000 7f139f28e080 0 osd.2:6.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2021-08-06T04:38:55.114+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:55.114+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:55.114+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:55.114+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:55.402+0000 7f139f28e080 0 osd.2:7.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196)
debug 2021-08-06T04:38:55.402+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _mount path /var/lib/ceph/osd/ceph-2
debug 2021-08-06T04:38:55.402+0000 7f139f28e080 0 bluestore(/var/lib/ceph/osd/ceph-2) _open_db_and_around read-only:0 repair:0
debug 2021-08-06T04:38:55.402+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:55.402+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:55.406+0000 7f139f28e080 1 bluestore(/var/lib/ceph/osd/ceph-2) _set_cache_sizes cache_size 3221225472 meta 0.45 kv 0.45 data 0.06
debug 2021-08-06T04:38:55.406+0000 7f139f28e080 1 bdev(0x55898df4e800 /var/lib/ceph/osd/ceph-2/block) open path /var/lib/ceph/osd/ceph-2/block
debug 2021-08-06T04:38:55.406+0000 7f139f28e080 1 bdev(0x55898df4e800 /var/lib/ceph/osd/ceph-2/block) open size 1000077262848 (0xe8d9400000, 931 GiB) block_size 4096 (4 KiB) non-rotational discard supported
debug 2021-08-06T04:38:55.406+0000 7f139f28e080 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-2/block size 931 GiB
debug 2021-08-06T04:38:55.406+0000 7f139f28e080 1 bluefs mount
debug 2021-08-06T04:38:55.406+0000 7f139f28e080 1 bluefs _init_alloc shared, id 1, capacity 0xe8d9400000, block size 0x10000
debug 2021-08-06T04:38:55.526+0000 7f139f28e080 -1 bluefs _replay 0xc07000: stop: failed to decode: bad crc 3449997429 expected 0: Malformed input
debug 2021-08-06T04:38:55.526+0000 7f139f28e080 -1 bluefs mount failed to replay log: (5) Input/output error
debug 2021-08-06T04:38:55.526+0000 7f139f28e080 -1 bluestore(/var/lib/ceph/osd/ceph-2) _open_bluefs failed bluefs mount: (5) Input/output error
debug 2021-08-06T04:38:55.526+0000 7f139f28e080 -1 bluestore(/var/lib/ceph/osd/ceph-2) _open_db failed to prepare db environment:
debug 2021-08-06T04:38:55.526+0000 7f139f28e080 1 bdev(0x55898df4e400 /var/lib/ceph/osd/ceph-2/block) close
debug 2021-08-06T04:38:55.686+0000 7f139f28e080 -1 osd.2 0 OSD:init: unable to mount object store
debug 2021-08-06T04:38:55.686+0000 7f139f28e080 -1 ** ERROR: osd init failed: (5) Input/output error
(2-2/4)