=============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-MN1:~$ sudo journalctl -u ceph-osd@85.service -e ... Apr 19 15:06:32 P20B-SR4-R2-CEPH-DS01 systemd[1]: ceph-osd@85.service: Service hold-off time over, scheduling restart. Apr 19 15:06:32 P20B-SR4-R2-CEPH-DS01 systemd[1]: Stopped Ceph object storage daemon osd.85. Apr 19 15:06:32 P20B-SR4-R2-CEPH-DS01 systemd[1]: Starting Ceph object storage daemon osd.85... Apr 19 15:06:32 P20B-SR4-R2-CEPH-DS01 systemd[1]: Started Ceph object storage daemon osd.85. Apr 19 15:06:32 P20B-SR4-R2-CEPH-DS01 ceph-osd[218088]: 2018-04-19 15:06:32.429301 7fe675a7de00 0 set uid:gid to 64045:64045 (ceph:ceph) Apr 19 15:06:32 P20B-SR4-R2-CEPH-DS01 ceph-osd[218088]: 2018-04-19 15:06:32.429313 7fe675a7de00 0 ceph version 12.2.2-4 (b28f8fe5d26502807ee9b486787f023fafa02b65) luminous (stable), process (unknown), pid 218088 Apr 19 15:06:33 P20B-SR4-R2-CEPH-DS01 ceph-osd[218088]: 2018-04-19 15:06:33.281809 7fe675a7de00 -1 bluestore(/var/lib/ceph/osd/ceph-85/block) _read_bdev_label failed to read from /var/lib/ceph/osd/ceph-85/block: (5) Input/output error Apr 19 15:06:33 P20B-SR4-R2-CEPH-DS01 ceph-osd[218088]: 2018-04-19 15:06:33.281857 7fe675a7de00 -1 ** ERROR: unable to open OSD superblock on /var/lib/ceph/osd/ceph-85: (2) No such file or directory Apr 19 15:06:33 P20B-SR4-R2-CEPH-DS01 ceph-osd[218088]: 2018-04-19 15:06:33.281809 7fe675a7de00 -1 bluestore(/var/lib/ceph/osd/ceph-85/block) _read_bdev_label failed to read from /var/lib/ceph/osd/ceph-85/block: (5) Input/output error Apr 19 15:06:33 P20B-SR4-R2-CEPH-DS01 ceph-osd[218088]: 2018-04-19 15:06:33.281857 7fe675a7de00 -1 ** ERROR: unable to open OSD superblock on /var/lib/ceph/osd/ceph-85: (2) No such file or directory Apr 19 15:06:33 P20B-SR4-R2-CEPH-DS01 systemd[1]: ceph-osd@85.service: Main process exited, code=exited, status=1/FAILURE Apr 19 15:06:33 P20B-SR4-R2-CEPH-DS01 systemd[1]: ceph-osd@85.service: Unit entered failed state. Apr 19 15:06:33 P20B-SR4-R2-CEPH-DS01 systemd[1]: ceph-osd@85.service: Failed with result 'exit-code'. ... =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-MN1:~$ dmesg -T ... [Thu Apr 19 15:09:30 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:09:42 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:10:03 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:10:05 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:10:24 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:10:39 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:10:45 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:11:01 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:11:06 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:11:27 2018] Buffer I/O error on dev dm-2, logical block 0, async page read [Thu Apr 19 15:11:32 2018] Buffer I/O error on dev dm-2, logical block 0, async page read ... =============================================================================================================================================================== ceph osd metadata 85 { "id": 85, "arch": "x86_64", "back_addr": "10.12.0.1:6814/84739", "back_iface": "internal", "bluefs": "1", "bluefs_db_access_mode": "blk", "bluefs_db_block_size": "4096", "bluefs_db_dev": "253:2", "bluefs_db_dev_node": "dm-2", "bluefs_db_dm": "1", "bluefs_db_dm_name": "ceph--fda7bc3b--0047--45c8--8f16--5a8764664a9f-osd--block--52032eb1--698d--409c--94ee--385c76825638", "bluefs_db_driver": "KernelDevice", "bluefs_db_partition_path": "/dev/dm-2", "bluefs_db_physical_disk_ID_MODEL": "WDC_WD1004FBYZ-01YCBB1", "bluefs_db_physical_disk_ID_REVISION": "RR03", "bluefs_db_physical_disk_ID_SERIAL": "WDC_WD1004FBYZ-01YCBB1_WD-WMC6M0D3AZMS", "bluefs_db_physical_disk_ID_SERIAL_SHORT": "WD-WMC6M0D3AZMS", "bluefs_db_physical_disk_dev": "8:160", "bluefs_db_physical_disk_dev_node": "sdk", "bluefs_db_physical_disk_model": "WDC WD1004FBYZ-0", "bluefs_db_rotational": "1", "bluefs_db_size": "1000203091968", "bluefs_db_slaves_count": "\u0001", "bluefs_db_type": "hdd", "bluefs_single_shared_device": "1", "bluestore_bdev_access_mode": "blk", "bluestore_bdev_block_size": "4096", "bluestore_bdev_dev": "253:2", "bluestore_bdev_dev_node": "dm-2", "bluestore_bdev_dm": "1", "bluestore_bdev_dm_name": "ceph--fda7bc3b--0047--45c8--8f16--5a8764664a9f-osd--block--52032eb1--698d--409c--94ee--385c76825638", "bluestore_bdev_driver": "KernelDevice", "bluestore_bdev_partition_path": "/dev/dm-2", "bluestore_bdev_physical_disk_ID_MODEL": "WDC_WD1004FBYZ-01YCBB1", "bluestore_bdev_physical_disk_ID_REVISION": "RR03", "bluestore_bdev_physical_disk_ID_SERIAL": "WDC_WD1004FBYZ-01YCBB1_WD-WMC6M0D3AZMS", "bluestore_bdev_physical_disk_ID_SERIAL_SHORT": "WD-WMC6M0D3AZMS", "bluestore_bdev_physical_disk_dev": "8:160", "bluestore_bdev_physical_disk_dev_node": "sdk", "bluestore_bdev_physical_disk_model": "WDC WD1004FBYZ-0", "bluestore_bdev_rotational": "1", "bluestore_bdev_size": "1000203091968", "bluestore_bdev_slaves_count": "\u0001", "bluestore_bdev_type": "hdd", "ceph_version": "ceph version 12.2.2-4 (b28f8fe5d26502807ee9b486787f023fafa02b65) luminous (stable)", "cpu": "Intel(R) Xeon(R) CPU E3-1220 v5 @ 3.00GHz", "default_device_class": "hdd", "distro": "ubuntu", "distro_description": "Ubuntu 16.04.4 LTS", "distro_version": "16.04", "front_addr": "10.8.0.1:6814/84739", "front_iface": "internal", "hb_back_addr": "10.12.0.1:6815/84739", "hb_front_addr": "10.8.0.1:6815/84739", "hostname": "P20B-SR4-R2-CEPH-DS01", "journal_rotational": "1", "kernel_description": "#201803311811 SMP Fri Apr 6 14:02:49 UTC 2018", "kernel_version": "4.14.32-041432-generic", "mem_swap_kb": "0", "mem_total_kb": "65525748", "os": "Linux", "osd_data": "/var/lib/ceph/osd/ceph-85", "osd_objectstore": "bluestore", "rotational": "1" } =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo ls /sys/block/dm-2/slaves/ -lah total 0 drwxr-xr-x 2 root root 0 Apr 19 15:02 . drwxr-xr-x 9 root root 0 Apr 19 09:42 .. lrwxrwxrwx 1 root root 0 Apr 19 15:09 sdl -> ../../../../pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:14/end_device-0:14/target0:0:14/0:0:14:0/block/sdl =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sdf 8:80 0 931.5G 0 disk └─ceph--71ca75f3--abcb--4ccf--b770--ad0f8a682cc6-osd--block--5662bcdd--92d7--48b8--bf6a--c9a656f06a87 253:4 0 931.5G 0 lvm sdd 8:48 0 931.5G 0 disk └─ceph--8752ef58--7c53--4617--8fda--910fe3fc7965-osd--block--f0d7cffd--f2c1--4f69--9914--fb6617b1f2de 253:1 0 931.5G 0 lvm sdb 8:16 0 931.5G 0 disk └─ceph--9ecdbeef--8b0c--4a99--87c8--63247823062b-osd--block--4d30f4e6--ee83--42c4--87be--95940f2be156 253:5 0 931.5G 0 lvm sdi 8:128 0 223.6G 0 disk └─sdi1 8:129 0 223.6G 0 part / sdg 8:96 0 931.5G 0 disk └─ceph--b7b65f3f--261a--4134--b068--c32c17ecf99a-osd--block--df8587e6--dcce--4247--8fe5--0eb75e9bf8ef 253:6 0 931.5G 0 lvm sde 8:64 0 931.5G 0 disk └─ceph--547d0f23--4eba--41bc--8dd0--2d088c21cba3-osd--block--321ed806--1817--4ad1--859c--edc28931431d 253:0 0 931.5G 0 lvm sdc 8:32 0 931.5G 0 disk └─ceph--41d790ec--1b55--4bac--b17c--235232834c89-osd--block--d1d88ae4--84d3--4291--ab5d--4d691c141def 253:8 0 931.5G 0 lvm sda 8:0 0 931.5G 0 disk └─ceph--a029f071--69eb--4b52--8a22--70881bfe09b6-osd--block--6137d463--aaae--47c1--a8d7--9652fdc5a44e 253:3 0 931.5G 0 lvm sdj 8:144 0 931.5G 0 disk sdh 8:112 0 931.5G 0 disk └─ceph--c72158ae--b87d--466c--9a06--bf8e508ba82c-osd--block--63bc5048--7faf--455c--b589--ffdb90347b31 253:7 0 931.5G 0 lvm =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ ls /sys/block/ -lah total 0 drwxr-xr-x 2 root root 0 Apr 19 15:04 . dr-xr-xr-x 13 root root 0 Apr 19 09:42 .. lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-0 -> ../devices/virtual/block/dm-0 lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-1 -> ../devices/virtual/block/dm-1 lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-2 -> ../devices/virtual/block/dm-2 lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-3 -> ../devices/virtual/block/dm-3 lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-4 -> ../devices/virtual/block/dm-4 lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-5 -> ../devices/virtual/block/dm-5 lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-6 -> ../devices/virtual/block/dm-6 lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-7 -> ../devices/virtual/block/dm-7 lrwxrwxrwx 1 root root 0 Apr 19 14:13 dm-8 -> ../devices/virtual/block/dm-8 lrwxrwxrwx 1 root root 0 Apr 19 14:13 loop0 -> ../devices/virtual/block/loop0 lrwxrwxrwx 1 root root 0 Apr 19 14:13 loop1 -> ../devices/virtual/block/loop1 lrwxrwxrwx 1 root root 0 Apr 19 14:13 loop2 -> ../devices/virtual/block/loop2 lrwxrwxrwx 1 root root 0 Apr 19 14:13 loop3 -> ../devices/virtual/block/loop3 lrwxrwxrwx 1 root root 0 Apr 19 14:13 loop4 -> ../devices/virtual/block/loop4 lrwxrwxrwx 1 root root 0 Apr 19 14:13 loop5 -> ../devices/virtual/block/loop5 lrwxrwxrwx 1 root root 0 Apr 19 14:13 loop6 -> ../devices/virtual/block/loop6 lrwxrwxrwx 1 root root 0 Apr 19 14:13 loop7 -> ../devices/virtual/block/loop7 lrwxrwxrwx 1 root root 0 Apr 19 14:13 sda -> ../devices/pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:0/end_device-0:0/target0:0:0/0:0:0:0/block/sda lrwxrwxrwx 1 root root 0 Apr 19 14:13 sdb -> ../devices/pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:1/end_device-0:1/target0:0:1/0:0:1:0/block/sdb lrwxrwxrwx 1 root root 0 Apr 19 14:13 sdc -> ../devices/pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:2/end_device-0:2/target0:0:2/0:0:2:0/block/sdc lrwxrwxrwx 1 root root 0 Apr 19 14:13 sdd -> ../devices/pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:3/end_device-0:3/target0:0:3/0:0:3:0/block/sdd lrwxrwxrwx 1 root root 0 Apr 19 14:13 sde -> ../devices/pci0000:00/0000:00:17.0/ata6/host6/target6:0:0/6:0:0:0/block/sde lrwxrwxrwx 1 root root 0 Apr 19 15:07 sdf -> ../devices/pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:15/end_device-0:15/target0:0:15/0:0:15:0/block/sdf lrwxrwxrwx 1 root root 0 Apr 19 15:07 sdg -> ../devices/pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:13/end_device-0:13/target0:0:13/0:0:13:0/block/sdg lrwxrwxrwx 1 root root 0 Apr 19 14:13 sdh -> ../devices/pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:7/end_device-0:7/target0:0:7/0:0:7:0/block/sdh lrwxrwxrwx 1 root root 0 Apr 19 14:13 sdi -> ../devices/pci0000:00/0000:00:17.0/ata4/host4/target4:0:0/4:0:0:0/block/sdi lrwxrwxrwx 1 root root 0 Apr 19 15:07 sdj -> ../devices/pci0000:00/0000:00:1c.0/0000:03:00.0/host0/port-0:16/end_device-0:16/target0:0:16/0:0:16:0/block/sdj =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ ls -lah /dev/dm-* brw-rw---- 1 ceph ceph 253, 0 Apr 19 15:08 /dev/dm-0 brw-rw---- 1 ceph ceph 253, 1 Apr 19 15:08 /dev/dm-1 brw-rw---- 1 ceph ceph 253, 2 Apr 19 15:03 /dev/dm-2 brw-rw---- 1 ceph ceph 253, 3 Apr 19 15:08 /dev/dm-3 brw-rw---- 1 ceph ceph 253, 4 Apr 19 15:08 /dev/dm-4 brw-rw---- 1 ceph ceph 253, 5 Apr 19 15:08 /dev/dm-5 brw-rw---- 1 ceph ceph 253, 6 Apr 19 15:08 /dev/dm-6 brw-rw---- 1 ceph ceph 253, 7 Apr 19 15:08 /dev/dm-7 brw-rw---- 1 ceph ceph 253, 8 Apr 19 15:08 /dev/dm-8 =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ ls -lah /dev/sd* brw-rw---- 1 root disk 8, 0 Apr 19 09:42 /dev/sda brw-rw---- 1 root disk 8, 16 Apr 19 09:42 /dev/sdb brw-rw---- 1 root disk 8, 32 Apr 19 09:42 /dev/sdc brw-rw---- 1 root disk 8, 48 Apr 19 09:42 /dev/sdd brw-rw---- 1 root disk 8, 64 Apr 19 10:27 /dev/sde brw-rw---- 1 root disk 8, 80 Apr 19 15:02 /dev/sdf brw-rw---- 1 root disk 8, 96 Apr 19 15:02 /dev/sdg brw-rw---- 1 root disk 8, 112 Apr 19 09:42 /dev/sdh brw-rw---- 1 root disk 8, 128 Apr 19 09:42 /dev/sdi brw-rw---- 1 root disk 8, 129 Apr 19 09:42 /dev/sdi1 brw-rw---- 1 root disk 8, 144 Apr 19 15:04 /dev/sdj =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo ceph-volume lvm list ====== osd.24 ====== [block] /dev/ceph-8752ef58-7c53-4617-8fda-910fe3fc7965/osd-block-f0d7cffd-f2c1-4f69-9914-fb6617b1f2de type block osd id 24 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid f0d7cffd-f2c1-4f69-9914-fb6617b1f2de encrypted 0 cephx lockbox secret block uuid ObbrAE-DvU3-rMln-iPjo-Neq2-saGU-LTR948 block device /dev/ceph-8752ef58-7c53-4617-8fda-910fe3fc7965/osd-block-f0d7cffd-f2c1-4f69-9914-fb6617b1f2de crush device class None ====== osd.47 ====== [block] /dev/ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b/osd-block-4d30f4e6-ee83-42c4-87be-95940f2be156 type block osd id 47 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 4d30f4e6-ee83-42c4-87be-95940f2be156 block uuid ki0U15-X07x-djoq-TeVU-tcYO-toFX-mrFD1H block device /dev/ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b/osd-block-4d30f4e6-ee83-42c4-87be-95940f2be156 ====== osd.60 ====== [block] /dev/ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3/osd-block-321ed806-1817-4ad1-859c-edc28931431d type block osd id 60 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 321ed806-1817-4ad1-859c-edc28931431d block uuid aA5xfp-Va1v-feYR-Isoc-bRaA-qE9R-5Z4euS block device /dev/ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3/osd-block-321ed806-1817-4ad1-859c-edc28931431d ====== osd.75 ====== [block] /dev/ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6/osd-block-5662bcdd-92d7-48b8-bf6a-c9a656f06a87 type block osd id 75 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 5662bcdd-92d7-48b8-bf6a-c9a656f06a87 block uuid rVX9PU-Lo1I-zZw8-Qbv0-RiLF-Eu35-ym7VPz block device /dev/ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6/osd-block-5662bcdd-92d7-48b8-bf6a-c9a656f06a87 ===== osd.113 ====== [block] /dev/ceph-a029f071-69eb-4b52-8a22-70881bfe09b6/osd-block-6137d463-aaae-47c1-a8d7-9652fdc5a44e type block osd id 113 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 6137d463-aaae-47c1-a8d7-9652fdc5a44e block uuid 2hh5rL-LEXT-p1UZ-e3Kj-2fJU-Lx7e-znJsPQ block device /dev/ceph-a029f071-69eb-4b52-8a22-70881bfe09b6/osd-block-6137d463-aaae-47c1-a8d7-9652fdc5a44e ===== osd.129 ====== [block] /dev/ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c/osd-block-63bc5048-7faf-455c-b589-ffdb90347b31 type block osd id 129 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 63bc5048-7faf-455c-b589-ffdb90347b31 block uuid kIY39U-1Tm4-j87u-Rnro-U1CI-7Vju-FBWfKo block device /dev/ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c/osd-block-63bc5048-7faf-455c-b589-ffdb90347b31 ====== osd.8 ======= [block] /dev/ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a/osd-block-df8587e6-dcce-4247-8fe5-0eb75e9bf8ef type block osd id 8 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid df8587e6-dcce-4247-8fe5-0eb75e9bf8ef block uuid ujxTpI-Ya8G-nsXe-Nv1t-0rjs-2XpG-Pg6Y1o block device /dev/ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a/osd-block-df8587e6-dcce-4247-8fe5-0eb75e9bf8ef ====== osd.93 ====== [block] /dev/ceph-41d790ec-1b55-4bac-b17c-235232834c89/osd-block-d1d88ae4-84d3-4291-ab5d-4d691c141def type block osd id 93 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid d1d88ae4-84d3-4291-ab5d-4d691c141def block uuid d2uwyJ-gVnZ-rC15-Zzp7-sCeu-cJ7w-pyyhj0 block device /dev/ceph-41d790ec-1b55-4bac-b17c-235232834c89/osd-block-d1d88ae4-84d3-4291-ab5d-4d691c141def =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo lvdisplay --- Logical volume --- LV Path /dev/ceph-a029f071-69eb-4b52-8a22-70881bfe09b6/osd-block-6137d463-aaae-47c1-a8d7-9652fdc5a44e LV Name osd-block-6137d463-aaae-47c1-a8d7-9652fdc5a44e VG Name ceph-a029f071-69eb-4b52-8a22-70881bfe09b6 LV UUID 2hh5rL-LEXT-p1UZ-e3Kj-2fJU-Lx7e-znJsPQ LV Write Access read/write LV Creation host, time P20B-SR4-R2-CEPH-DS01, 2018-02-12 14:39:36 +0000 LV Status available # open 4 LV Size 931.51 GiB Current LE 238467 Segments 1 Allocation inherit Read ahead sectors auto - currently set to 16384 Block device 253:3 --- Logical volume --- LV Path /dev/ceph-41d790ec-1b55-4bac-b17c-235232834c89/osd-block-d1d88ae4-84d3-4291-ab5d-4d691c141def LV Name osd-block-d1d88ae4-84d3-4291-ab5d-4d691c141def VG Name ceph-41d790ec-1b55-4bac-b17c-235232834c89 LV UUID d2uwyJ-gVnZ-rC15-Zzp7-sCeu-cJ7w-pyyhj0 LV Write Access read/write LV Creation host, time P20B-SR4-R2-CEPH-DS01, 2018-02-12 14:39:26 +0000 LV Status available # open 4 LV Size 931.51 GiB Current LE 238467 Segments 1 Allocation inherit Read ahead sectors auto - currently set to 16384 Block device 253:8 --- Logical volume --- LV Path /dev/ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a/osd-block-df8587e6-dcce-4247-8fe5-0eb75e9bf8ef LV Name osd-block-df8587e6-dcce-4247-8fe5-0eb75e9bf8ef VG Name ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a LV UUID ujxTpI-Ya8G-nsXe-Nv1t-0rjs-2XpG-Pg6Y1o LV Write Access read/write LV Creation host, time P20B-SR4-R2-CEPH-DS01, 2018-02-12 14:39:02 +0000 LV Status available # open 4 LV Size 931.51 GiB Current LE 238467 Segments 1 Allocation inherit Read ahead sectors auto - currently set to 16384 Block device 253:6 --- Logical volume --- LV Path /dev/ceph-8752ef58-7c53-4617-8fda-910fe3fc7965/osd-block-f0d7cffd-f2c1-4f69-9914-fb6617b1f2de LV Name osd-block-f0d7cffd-f2c1-4f69-9914-fb6617b1f2de VG Name ceph-8752ef58-7c53-4617-8fda-910fe3fc7965 LV UUID ObbrAE-DvU3-rMln-iPjo-Neq2-saGU-LTR948 LV Write Access read/write LV Creation host, time P20B-SR4-R2-CEPH-DS01, 2018-03-28 07:02:45 +0000 LV Status available # open 4 LV Size 931.51 GiB Current LE 238467 Segments 1 Allocation inherit Read ahead sectors auto - currently set to 16384 Block device 253:1 --- Logical volume --- LV Path /dev/ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c/osd-block-63bc5048-7faf-455c-b589-ffdb90347b31 LV Name osd-block-63bc5048-7faf-455c-b589-ffdb90347b31 VG Name ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c LV UUID kIY39U-1Tm4-j87u-Rnro-U1CI-7Vju-FBWfKo LV Write Access read/write LV Creation host, time P20B-SR4-R2-CEPH-DS01, 2018-02-12 14:39:44 +0000 LV Status available # open 4 LV Size 931.51 GiB Current LE 238467 Segments 1 Allocation inherit Read ahead sectors auto - currently set to 16384 Block device 253:7 --- Logical volume --- LV Path /dev/ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6/osd-block-5662bcdd-92d7-48b8-bf6a-c9a656f06a87 LV Name osd-block-5662bcdd-92d7-48b8-bf6a-c9a656f06a87 VG Name ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6 LV UUID rVX9PU-Lo1I-zZw8-Qbv0-RiLF-Eu35-ym7VPz LV Write Access read/write LV Creation host, time P20B-SR4-R2-CEPH-DS01, 2018-02-12 14:39:18 +0000 LV Status available # open 4 LV Size 931.51 GiB Current LE 238467 Segments 1 Allocation inherit Read ahead sectors auto - currently set to 16384 Block device 253:4 --- Logical volume --- LV Path /dev/ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b/osd-block-4d30f4e6-ee83-42c4-87be-95940f2be156 LV Name osd-block-4d30f4e6-ee83-42c4-87be-95940f2be156 VG Name ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b LV UUID ki0U15-X07x-djoq-TeVU-tcYO-toFX-mrFD1H LV Write Access read/write LV Creation host, time P20B-SR4-R2-CEPH-DS01, 2018-02-12 14:39:11 +0000 LV Status available # open 4 LV Size 931.51 GiB Current LE 238467 Segments 1 Allocation inherit Read ahead sectors auto - currently set to 16384 Block device 253:5 --- Logical volume --- LV Path /dev/ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3/osd-block-321ed806-1817-4ad1-859c-edc28931431d LV Name osd-block-321ed806-1817-4ad1-859c-edc28931431d VG Name ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3 LV UUID aA5xfp-Va1v-feYR-Isoc-bRaA-qE9R-5Z4euS LV Write Access read/write LV Creation host, time P20B-SR4-R2-CEPH-DS01, 2018-02-12 14:39:14 +0000 LV Status available # open 4 LV Size 931.51 GiB Current LE 238467 Segments 1 Allocation inherit Read ahead sectors auto - currently set to 16384 Block device 253:0 =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo vgdisplay --- Volume group --- VG Name ceph-a029f071-69eb-4b52-8a22-70881bfe09b6 System ID Format lvm2 Metadata Areas 1 Metadata Sequence No 13 VG Access read/write VG Status resizable MAX LV 0 Cur LV 1 Open LV 1 Max PV 0 Cur PV 1 Act PV 1 VG Size 931.51 GiB PE Size 4.00 MiB Total PE 238467 Alloc PE / Size 238467 / 931.51 GiB Free PE / Size 0 / 0 VG UUID YI8UhQ-r35a-92JF-0Zsh-Hfcy-nbEF-bt8PeG --- Volume group --- VG Name ceph-41d790ec-1b55-4bac-b17c-235232834c89 System ID Format lvm2 Metadata Areas 1 Metadata Sequence No 13 VG Access read/write VG Status resizable MAX LV 0 Cur LV 1 Open LV 1 Max PV 0 Cur PV 1 Act PV 1 VG Size 931.51 GiB PE Size 4.00 MiB Total PE 238467 Alloc PE / Size 238467 / 931.51 GiB Free PE / Size 0 / 0 VG UUID 56lMQt-haNO-POx4-fdWF-xOF7-CdLQ-aEctnW --- Volume group --- VG Name ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a System ID Format lvm2 Metadata Areas 1 Metadata Sequence No 13 VG Access read/write VG Status resizable MAX LV 0 Cur LV 1 Open LV 1 Max PV 0 Cur PV 1 Act PV 1 VG Size 931.51 GiB PE Size 4.00 MiB Total PE 238467 Alloc PE / Size 238467 / 931.51 GiB Free PE / Size 0 / 0 VG UUID kRJSiI-gQEx-oed6-2FA7-fcZo-Japi-yBOXRK --- Volume group --- VG Name ceph-8752ef58-7c53-4617-8fda-910fe3fc7965 System ID Format lvm2 Metadata Areas 1 Metadata Sequence No 16 VG Access read/write VG Status resizable MAX LV 0 Cur LV 1 Open LV 1 Max PV 0 Cur PV 1 Act PV 1 VG Size 931.51 GiB PE Size 4.00 MiB Total PE 238467 Alloc PE / Size 238467 / 931.51 GiB Free PE / Size 0 / 0 VG UUID tGvgJ0-XSCz-uNi9-KCX0-GDfJ-q1W5-zyOc8p --- Volume group --- VG Name ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c System ID Format lvm2 Metadata Areas 1 Metadata Sequence No 13 VG Access read/write VG Status resizable MAX LV 0 Cur LV 1 Open LV 1 Max PV 0 Cur PV 1 Act PV 1 VG Size 931.51 GiB PE Size 4.00 MiB Total PE 238467 Alloc PE / Size 238467 / 931.51 GiB Free PE / Size 0 / 0 VG UUID ICG0tP-hNZJ-t1p6-sZwN-05Pl-xNCJ-GQsPyG --- Volume group --- VG Name ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6 System ID Format lvm2 Metadata Areas 1 Metadata Sequence No 13 VG Access read/write VG Status resizable MAX LV 0 Cur LV 1 Open LV 1 Max PV 0 Cur PV 1 Act PV 1 VG Size 931.51 GiB PE Size 4.00 MiB Total PE 238467 Alloc PE / Size 238467 / 931.51 GiB Free PE / Size 0 / 0 VG UUID A00X7i-PSuX-Ddlq-fACF-eSWg-qAXi-pbdVsc --- Volume group --- VG Name ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b System ID Format lvm2 Metadata Areas 1 Metadata Sequence No 13 VG Access read/write VG Status resizable MAX LV 0 Cur LV 1 Open LV 1 Max PV 0 Cur PV 1 Act PV 1 VG Size 931.51 GiB PE Size 4.00 MiB Total PE 238467 Alloc PE / Size 238467 / 931.51 GiB Free PE / Size 0 / 0 VG UUID kRhu0f-t2dT-HxfH-mxdU-tkaE-jF3E-PwGeFA --- Volume group --- VG Name ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3 System ID Format lvm2 Metadata Areas 1 Metadata Sequence No 13 VG Access read/write VG Status resizable MAX LV 0 Cur LV 1 Open LV 1 Max PV 0 Cur PV 1 Act PV 1 VG Size 931.51 GiB PE Size 4.00 MiB Total PE 238467 Alloc PE / Size 238467 / 931.51 GiB Free PE / Size 0 / 0 VG UUID mj2CJA-E723-g1dP-FNzc-9VBA-wI7f-JueOYu =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo pvdisplay /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638: read failed after 0 of 4096 at 0: Input/output error /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638: read failed after 0 of 4096 at 1000203026432: Input/output error /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638: read failed after 0 of 4096 at 1000203083776: Input/output error /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638: read failed after 0 of 4096 at 4096: Input/output error --- Physical volume --- PV Name /dev/sda VG Name ceph-a029f071-69eb-4b52-8a22-70881bfe09b6 PV Size 931.51 GiB / not usable 1.71 MiB Allocatable yes (but full) PE Size 4.00 MiB Total PE 238467 Free PE 0 Allocated PE 238467 PV UUID FgADkr-PAtz-vnct-bb8y-1291-MT9Q-GDrnts --- Physical volume --- PV Name /dev/sdc VG Name ceph-41d790ec-1b55-4bac-b17c-235232834c89 PV Size 931.51 GiB / not usable 1.71 MiB Allocatable yes (but full) PE Size 4.00 MiB Total PE 238467 Free PE 0 Allocated PE 238467 PV UUID JT74uF-s5XE-dlNG-BXRh-fnkJ-iRxM-58Dm68 --- Physical volume --- PV Name /dev/sdg VG Name ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a PV Size 931.51 GiB / not usable 1.71 MiB Allocatable yes (but full) PE Size 4.00 MiB Total PE 238467 Free PE 0 Allocated PE 238467 PV UUID zzCAsV-IO66-9zUP-cNtY-Hu76-zklt-SuGym9 --- Physical volume --- PV Name /dev/sdd VG Name ceph-8752ef58-7c53-4617-8fda-910fe3fc7965 PV Size 931.51 GiB / not usable 1.71 MiB Allocatable yes (but full) PE Size 4.00 MiB Total PE 238467 Free PE 0 Allocated PE 238467 PV UUID HHBi2v-8ZaU-K9Yl-qV54-CduT-doEk-nzsGTl --- Physical volume --- PV Name /dev/sdh VG Name ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c PV Size 931.51 GiB / not usable 1.71 MiB Allocatable yes (but full) PE Size 4.00 MiB Total PE 238467 Free PE 0 Allocated PE 238467 PV UUID NX14bH-sFCv-5Yfv-SLOW-MlUB-YMdG-VeR377 --- Physical volume --- PV Name /dev/sdf VG Name ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6 PV Size 931.51 GiB / not usable 1.71 MiB Allocatable yes (but full) PE Size 4.00 MiB Total PE 238467 Free PE 0 Allocated PE 238467 PV UUID 9WfHgO-d5eU-isgY-fJ1o-BcRj-zdZ0-vwKNoU --- Physical volume --- PV Name /dev/sdb VG Name ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b PV Size 931.51 GiB / not usable 1.71 MiB Allocatable yes (but full) PE Size 4.00 MiB Total PE 238467 Free PE 0 Allocated PE 238467 PV UUID gFpiGl-2DCu-vR7n-ZoNm-6Fx3-Mw9b-sndpFm --- Physical volume --- PV Name /dev/sde VG Name ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3 PV Size 931.51 GiB / not usable 1.71 MiB Allocatable yes (but full) PE Size 4.00 MiB Total PE 238467 Free PE 0 Allocated PE 238467 PV UUID tfsPtd-asXF-aCjK-skFw-hz7i-Gled-ZX2n0k =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo hdparm -I /dev/sd* /dev/sda: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D5WDNT Firmware Revision: RR03 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdb: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D1JDSM Firmware Revision: RR03 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdc: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D342A3 Firmware Revision: RR03 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdd: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D8SMY8 Firmware Revision: RR03 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sde: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D7V9TZ Firmware Revision: RR04 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdf: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D20JLW Firmware Revision: RR03 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdg: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D1FA4N Firmware Revision: RR03 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdh: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D4Z0DM Firmware Revision: RR03 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdi: ATA device, with non-removable media Model Number: INTEL SSDSC2KW240H6 Serial Number: CVLT61660BAC240CGN Firmware Revision: LSBG200 Transport: Serial, ATA8-AST, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdi1: ATA device, with non-removable media Model Number: INTEL SSDSC2KW240H6 Serial Number: CVLT61660BAC240CGN Firmware Revision: LSBG200 Transport: Serial, ATA8-AST, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... /dev/sdj: ATA device, with non-removable media Model Number: WDC WD1004FBYZ-01YCBB1 Serial Number: WD-WMC6M0D3AZMS Firmware Revision: RR03 Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6, SATA Rev 3.0 ... =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo vgscan Reading all physical volumes. This may take a while... Found volume group "ceph-a029f071-69eb-4b52-8a22-70881bfe09b6" using metadata type lvm2 Found volume group "ceph-41d790ec-1b55-4bac-b17c-235232834c89" using metadata type lvm2 Found volume group "ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a" using metadata type lvm2 Found volume group "ceph-8752ef58-7c53-4617-8fda-910fe3fc7965" using metadata type lvm2 Found volume group "ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c" using metadata type lvm2 Found volume group "ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6" using metadata type lvm2 Found volume group "ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b" using metadata type lvm2 Found volume group "ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3" using metadata type lvm2 =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo vgscan --cache /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638: read failed after 0 of 4096 at 0: Input/output error /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638: read failed after 0 of 4096 at 1000203026432: Input/output error /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638: read failed after 0 of 4096 at 1000203083776: Input/output error /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638: read failed after 0 of 4096 at 4096: Input/output error Reading all physical volumes. This may take a while... Found volume group "ceph-a029f071-69eb-4b52-8a22-70881bfe09b6" using metadata type lvm2 Found volume group "ceph-41d790ec-1b55-4bac-b17c-235232834c89" using metadata type lvm2 Found volume group "ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a" using metadata type lvm2 Found volume group "ceph-8752ef58-7c53-4617-8fda-910fe3fc7965" using metadata type lvm2 Found volume group "ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c" using metadata type lvm2 Found volume group "ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6" using metadata type lvm2 Found volume group "ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b" using metadata type lvm2 Found volume group "ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3" using metadata type lvm2 Found volume group "ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f" using metadata type lvm2 =============================================================================================================================================================== FIX NEXT =============================================================================================================================================================== sudo dmsetup remove /dev/dm-2 echo 1 | sudo tee /sys/block/sdj/device/delete echo "- - -" | sudo tee /sys/class/scsi_host/host*/scan sudo dmesg -T [Thu Apr 19 16:04:17 2018] sd 0:0:16:0: [sdj] Synchronizing SCSI cache [Thu Apr 19 16:04:23 2018] scsi 0:0:16:0: Direct-Access ATA WDC WD1004FBYZ-0 RR03 PQ: 0 ANSI: 6 [Thu Apr 19 16:04:23 2018] scsi 0:0:16:0: SATA: handle(0x000d), sas_addr(0x4433221104000000), phy(4), device_name(0x50014ee00431aed7) [Thu Apr 19 16:04:23 2018] scsi 0:0:16:0: SATA: enclosure_logical_id(0x500605b00984efd0), slot(7) [Thu Apr 19 16:04:23 2018] scsi 0:0:16:0: atapi(n), ncq(y), asyn_notify(n), smart(y), fua(y), sw_preserve(y) [Thu Apr 19 16:04:23 2018] sd 0:0:16:0: Attached scsi generic sg5 type 0 [Thu Apr 19 16:04:23 2018] ata1: hard resetting link [Thu Apr 19 16:04:23 2018] sd 0:0:16:0: [sdj] 1953525168 512-byte logical blocks: (1.00 TB/932 GiB) [Thu Apr 19 16:04:23 2018] sd 0:0:16:0: [sdj] Write Protect is off [Thu Apr 19 16:04:23 2018] sd 0:0:16:0: [sdj] Mode Sense: 7f 00 10 08 [Thu Apr 19 16:04:23 2018] sd 0:0:16:0: [sdj] Write cache: enabled, read cache: enabled, supports DPO and FUA [Thu Apr 19 16:04:24 2018] sd 0:0:16:0: [sdj] Attached SCSI disk =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sdf 8:80 0 931.5G 0 disk └─ceph--71ca75f3--abcb--4ccf--b770--ad0f8a682cc6-osd--block--5662bcdd--92d7--48b8--bf6a--c9a656f06a87 253:4 0 931.5G 0 lvm sdd 8:48 0 931.5G 0 disk └─ceph--8752ef58--7c53--4617--8fda--910fe3fc7965-osd--block--f0d7cffd--f2c1--4f69--9914--fb6617b1f2de 253:1 0 931.5G 0 lvm sdb 8:16 0 931.5G 0 disk └─ceph--9ecdbeef--8b0c--4a99--87c8--63247823062b-osd--block--4d30f4e6--ee83--42c4--87be--95940f2be156 253:5 0 931.5G 0 lvm sdi 8:128 0 223.6G 0 disk └─sdi1 8:129 0 223.6G 0 part / sdg 8:96 0 931.5G 0 disk └─ceph--b7b65f3f--261a--4134--b068--c32c17ecf99a-osd--block--df8587e6--dcce--4247--8fe5--0eb75e9bf8ef 253:6 0 931.5G 0 lvm sde 8:64 0 931.5G 0 disk └─ceph--547d0f23--4eba--41bc--8dd0--2d088c21cba3-osd--block--321ed806--1817--4ad1--859c--edc28931431d 253:0 0 931.5G 0 lvm sdc 8:32 0 931.5G 0 disk └─ceph--41d790ec--1b55--4bac--b17c--235232834c89-osd--block--d1d88ae4--84d3--4291--ab5d--4d691c141def 253:8 0 931.5G 0 lvm sda 8:0 0 931.5G 0 disk └─ceph--a029f071--69eb--4b52--8a22--70881bfe09b6-osd--block--6137d463--aaae--47c1--a8d7--9652fdc5a44e 253:3 0 931.5G 0 lvm sdj 8:144 0 931.5G 0 disk └─ceph--fda7bc3b--0047--45c8--8f16--5a8764664a9f-osd--block--52032eb1--698d--409c--94ee--385c76825638 253:2 0 931.5G 0 lvm sdh 8:112 0 931.5G 0 disk └─ceph--c72158ae--b87d--466c--9a06--bf8e508ba82c-osd--block--63bc5048--7faf--455c--b589--ffdb90347b31 253:7 0 931.5G 0 lvm =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo umount /var/lib/ceph/osd/ceph-85 ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo ceph-volume lvm list ====== osd.24 ====== [block] /dev/ceph-8752ef58-7c53-4617-8fda-910fe3fc7965/osd-block-f0d7cffd-f2c1-4f69-9914-fb6617b1f2de type block osd id 24 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid f0d7cffd-f2c1-4f69-9914-fb6617b1f2de encrypted 0 cephx lockbox secret block uuid ObbrAE-DvU3-rMln-iPjo-Neq2-saGU-LTR948 block device /dev/ceph-8752ef58-7c53-4617-8fda-910fe3fc7965/osd-block-f0d7cffd-f2c1-4f69-9914-fb6617b1f2de crush device class None ====== osd.47 ====== [block] /dev/ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b/osd-block-4d30f4e6-ee83-42c4-87be-95940f2be156 type block osd id 47 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 4d30f4e6-ee83-42c4-87be-95940f2be156 block uuid ki0U15-X07x-djoq-TeVU-tcYO-toFX-mrFD1H block device /dev/ceph-9ecdbeef-8b0c-4a99-87c8-63247823062b/osd-block-4d30f4e6-ee83-42c4-87be-95940f2be156 ====== osd.60 ====== [block] /dev/ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3/osd-block-321ed806-1817-4ad1-859c-edc28931431d type block osd id 60 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 321ed806-1817-4ad1-859c-edc28931431d block uuid aA5xfp-Va1v-feYR-Isoc-bRaA-qE9R-5Z4euS block device /dev/ceph-547d0f23-4eba-41bc-8dd0-2d088c21cba3/osd-block-321ed806-1817-4ad1-859c-edc28931431d ====== osd.75 ====== [block] /dev/ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6/osd-block-5662bcdd-92d7-48b8-bf6a-c9a656f06a87 type block osd id 75 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 5662bcdd-92d7-48b8-bf6a-c9a656f06a87 block uuid rVX9PU-Lo1I-zZw8-Qbv0-RiLF-Eu35-ym7VPz block device /dev/ceph-71ca75f3-abcb-4ccf-b770-ad0f8a682cc6/osd-block-5662bcdd-92d7-48b8-bf6a-c9a656f06a87 ===== osd.113 ====== [block] /dev/ceph-a029f071-69eb-4b52-8a22-70881bfe09b6/osd-block-6137d463-aaae-47c1-a8d7-9652fdc5a44e type block osd id 113 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 6137d463-aaae-47c1-a8d7-9652fdc5a44e block uuid 2hh5rL-LEXT-p1UZ-e3Kj-2fJU-Lx7e-znJsPQ block device /dev/ceph-a029f071-69eb-4b52-8a22-70881bfe09b6/osd-block-6137d463-aaae-47c1-a8d7-9652fdc5a44e ===== osd.129 ====== [block] /dev/ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c/osd-block-63bc5048-7faf-455c-b589-ffdb90347b31 type block osd id 129 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 63bc5048-7faf-455c-b589-ffdb90347b31 block uuid kIY39U-1Tm4-j87u-Rnro-U1CI-7Vju-FBWfKo block device /dev/ceph-c72158ae-b87d-466c-9a06-bf8e508ba82c/osd-block-63bc5048-7faf-455c-b589-ffdb90347b31 ====== osd.8 ======= [block] /dev/ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a/osd-block-df8587e6-dcce-4247-8fe5-0eb75e9bf8ef type block osd id 8 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid df8587e6-dcce-4247-8fe5-0eb75e9bf8ef block uuid ujxTpI-Ya8G-nsXe-Nv1t-0rjs-2XpG-Pg6Y1o block device /dev/ceph-b7b65f3f-261a-4134-b068-c32c17ecf99a/osd-block-df8587e6-dcce-4247-8fe5-0eb75e9bf8ef ====== osd.93 ====== [block] /dev/ceph-41d790ec-1b55-4bac-b17c-235232834c89/osd-block-d1d88ae4-84d3-4291-ab5d-4d691c141def type block osd id 93 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid d1d88ae4-84d3-4291-ab5d-4d691c141def block uuid d2uwyJ-gVnZ-rC15-Zzp7-sCeu-cJ7w-pyyhj0 block device /dev/ceph-41d790ec-1b55-4bac-b17c-235232834c89/osd-block-d1d88ae4-84d3-4291-ab5d-4d691c141def ====== osd.85 ====== [block] /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638 type block osd id 85 cluster fsid b7b65f3f-261a-4134-b068-c32c17ecf99a cluster name ceph osd fsid 52032eb1-698d-409c-94ee-385c76825638 block uuid FObhrP-1ODQ-ofUH-Lbpc-urn9-pvMm-siuchN block device /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638 =============================================================================================================================================================== ubuntu@P20B-SR4-R2-CEPH-DS01:~$ sudo ceph-volume lvm trigger 85-52032eb1-698d-409c-94ee-385c76825638 Running command: mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-85 Running command: ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638 --path /var/lib/ceph/osd/ceph-85 Running command: ln -snf /dev/ceph-fda7bc3b-0047-45c8-8f16-5a8764664a9f/osd-block-52032eb1-698d-409c-94ee-385c76825638 /var/lib/ceph/osd/ceph-85/block Running command: chown -R ceph:ceph /dev/dm-2 Running command: chown -R ceph:ceph /var/lib/ceph/osd/ceph-85 Running command: systemctl enable ceph-volume@lvm-85-52032eb1-698d-409c-94ee-385c76825638 Running command: systemctl start ceph-osd@85 --> ceph-volume lvm activate successful for osd ID: 85