1. init status [root@ln-ceph-rpm build]# ceph -s cluster: id: afec64f8-d9ee-4262-9410-fcf907807e2c health: HEALTH_WARN nobackfill flag(s) set services: mon: 3 daemons, quorum a,b,c (age 2m) mgr: x(active, since 2m) ioa: 0 daemonsno daemons active osd: 8 osds: 8 up (since 2m), 8 in (since 2m) flags nobackfill data: pools: 2 pools, 33 pgs objects: 261 objects, 1.0 GiB usage: 20 GiB used, 788 GiB / 808 GiB avail pgs: 33 active+clean [root@ln-ceph-rpm build]# ceph osd df tree ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME -3 0 - 0 B 0 B 0 B 0 B 0 B 0 B 0 0 - root myroot -1 0.78870 - 808 GiB 20 GiB 3.5 GiB 7.1 MiB 8.0 GiB 788 GiB 2.42 1.00 - root default -11 0.19717 - 202 GiB 4.8 GiB 843 MiB 1.6 MiB 2.0 GiB 197 GiB 2.39 0.99 - host ln-ceph-rpm 6 hdd 0.09859 1.00000 101 GiB 2.5 GiB 505 MiB 980 KiB 1023 MiB 99 GiB 2.47 1.02 14 up osd.6 7 hdd 0.09859 1.00000 101 GiB 2.3 GiB 337 MiB 698 KiB 1023 MiB 99 GiB 2.31 0.96 9 up osd.7 -5 0.19717 - 202 GiB 4.8 GiB 858 MiB 1.3 MiB 2.0 GiB 197 GiB 2.40 0.99 - host node1 0 0.09859 1.00000 101 GiB 2.3 GiB 345 MiB 732 KiB 1023 MiB 99 GiB 2.32 0.96 9 up osd.0 1 hdd 0.09859 1.00000 101 GiB 2.5 GiB 513 MiB 646 KiB 1023 MiB 98 GiB 2.48 1.03 16 up osd.1 -6 0.19717 - 202 GiB 4.9 GiB 903 MiB 2.2 MiB 2.0 GiB 197 GiB 2.42 1.00 - host node2 2 hdd 0.09859 1.00000 101 GiB 2.4 GiB 417 MiB 1.2 MiB 1023 MiB 99 GiB 2.39 0.99 12 up osd.2 3 hdd 0.09859 1.00000 101 GiB 2.5 GiB 485 MiB 983 KiB 1023 MiB 99 GiB 2.45 1.01 14 up osd.3 -9 0.19717 - 202 GiB 5.0 GiB 990 MiB 2.0 MiB 2.0 GiB 197 GiB 2.46 1.02 - host node3 4 hdd 0.09859 1.00000 101 GiB 2.5 GiB 541 MiB 1011 KiB 1023 MiB 98 GiB 2.51 1.04 13 up osd.4 5 hdd 0.09859 1.00000 101 GiB 2.4 GiB 449 MiB 994 KiB 1023 MiB 99 GiB 2.42 1.00 12 up osd.5 TOTAL 808 GiB 20 GiB 3.5 GiB 7.1 MiB 8.0 GiB 788 GiB 2.42 MIN/MAX VAR: 0.96/1.04 STDDEV: 0.07 [root@ln-ceph-rpm build]# ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 707 GiB 690 GiB 10 GiB 17 GiB 2.43 TOTAL 808 GiB 788 GiB 12 GiB 20 GiB 2.42 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 260 GiB test 2 32 1.0 GiB 261 3.0 GiB 0.38 260 GiB 2. kill osd.0 down root 1580290 6.4 0.6 1019672 203772 ? Ssl 17:45 0:15 /root/thinkfs-ssh/build/bin/ceph-osd -i 0 -c /root/thinkfs-ssh/build/ceph.conf [root@ln-ceph-rpm build]# kill -9 1580290 [root@ln-ceph-rpm build]# ceph -s cluster: id: afec64f8-d9ee-4262-9410-fcf907807e2c health: HEALTH_WARN nobackfill flag(s) set 1 osds down Degraded data redundancy: 72/783 objects degraded (9.195%), 8 pgs degraded services: mon: 3 daemons, quorum a,b,c (age 4m) mgr: x(active, since 4m) ioa: 0 daemonsno daemons active osd: 8 osds: 7 up (since 16s), 8 in (since 3m) flags nobackfill data: pools: 2 pools, 33 pgs objects: 261 objects, 1.0 GiB usage: 20 GiB used, 788 GiB / 808 GiB avail pgs: 72/783 objects degraded (9.195%) 24 active+clean 8 active+undersized+degraded 1 active+undersized [root@ln-ceph-rpm build]# ceph osd df tree ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME -3 0 - 0 B 0 B 0 B 0 B 0 B 0 B 0 0 - root myroot -1 0.78870 - 808 GiB 20 GiB 3.5 GiB 7.1 MiB 8.0 GiB 788 GiB 2.41 1.00 - root default -11 0.19717 - 202 GiB 4.8 GiB 840 MiB 1.6 MiB 2.0 GiB 197 GiB 2.39 0.99 - host ln-ceph-rpm 6 hdd 0.09859 1.00000 101 GiB 2.5 GiB 504 MiB 980 KiB 1023 MiB 99 GiB 2.47 1.02 14 up osd.6 7 hdd 0.09859 1.00000 101 GiB 2.3 GiB 336 MiB 698 KiB 1023 MiB 99 GiB 2.31 0.95 9 up osd.7 -5 0.19717 - 202 GiB 4.8 GiB 857 MiB 1.3 MiB 2.0 GiB 197 GiB 2.40 0.99 - host node1 0 0.09859 1.00000 101 GiB 2.3 GiB 345 MiB 732 KiB 1023 MiB 99 GiB 2.32 0.96 0 down osd.0 1 hdd 0.09859 1.00000 101 GiB 2.5 GiB 512 MiB 646 KiB 1023 MiB 98 GiB 2.48 1.03 16 up osd.1 -6 0.19717 - 202 GiB 4.9 GiB 900 MiB 2.2 MiB 2.0 GiB 197 GiB 2.42 1.00 - host node2 2 hdd 0.09859 1.00000 101 GiB 2.4 GiB 416 MiB 1.2 MiB 1023 MiB 99 GiB 2.38 0.99 12 up osd.2 3 hdd 0.09859 1.00000 101 GiB 2.5 GiB 484 MiB 983 KiB 1023 MiB 99 GiB 2.45 1.01 14 up osd.3 -9 0.19717 - 202 GiB 5.0 GiB 988 MiB 2.0 MiB 2.0 GiB 197 GiB 2.46 1.02 - host node3 4 hdd 0.09859 1.00000 101 GiB 2.5 GiB 540 MiB 1011 KiB 1023 MiB 98 GiB 2.50 1.04 13 up osd.4 5 hdd 0.09859 1.00000 101 GiB 2.4 GiB 448 MiB 994 KiB 1023 MiB 99 GiB 2.41 1.00 12 up osd.5 TOTAL 808 GiB 20 GiB 3.5 GiB 7.1 MiB 8.0 GiB 788 GiB 2.41 MIN/MAX VAR: 0.95/1.04 STDDEV: 0.07 [root@ln-ceph-rpm build]# ceph df detail --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 707 GiB 690 GiB 10 GiB 17 GiB 2.43 TOTAL 808 GiB 788 GiB 12 GiB 20 GiB 2.41 --- POOLS --- POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 227 GiB N/A N/A 0 0 B 0 B test 2 32 1.0 GiB 1.0 GiB 566 KiB 261 3.0 GiB 3.0 GiB 1.7 MiB 0.44 227 GiB N/A N/A 261 0 B 0 B 2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.0 is_down=1 is_out=0 2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.1 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.2 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.3 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.4 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.5 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.6 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.7 sum=0.875 before_percentage=0.125 after_percentage=0.142857 3. osd.0 out but not backfill [root@ln-ceph-rpm build]# ceph -s cluster: id: afec64f8-d9ee-4262-9410-fcf907807e2c health: HEALTH_WARN nobackfill flag(s) set Degraded data redundancy: 64/783 objects degraded (8.174%), 7 pgs degraded services: mon: 3 daemons, quorum a,b,c (age 6m) mgr: x(active, since 6m) ioa: 0 daemonsno daemons active osd: 8 osds: 7 up (since 2m), 7 in (since 31s); 7 remapped pgs flags nobackfill data: pools: 2 pools, 33 pgs objects: 261 objects, 1.0 GiB usage: 17 GiB used, 690 GiB / 707 GiB avail pgs: 64/783 objects degraded (8.174%) 26 active+clean 4 active+undersized+degraded+remapped+backfill_wait 3 active+undersized+degraded+remapped+backfilling [root@ln-ceph-rpm build]# ceph osd df tree ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME -3 0 - 0 B 0 B 0 B 0 B 0 B 0 B 0 0 - root myroot -1 0.78870 - 707 GiB 17 GiB 3.2 GiB 6.4 MiB 7.0 GiB 690 GiB 2.43 1.00 - root default -11 0.19717 - 202 GiB 4.9 GiB 874 MiB 1.6 MiB 2.0 GiB 197 GiB 2.40 0.99 - host ln-ceph-rpm 6 hdd 0.09859 1.00000 101 GiB 2.5 GiB 505 MiB 980 KiB 1023 MiB 99 GiB 2.47 1.01 14 up osd.6 7 hdd 0.09859 1.00000 101 GiB 2.4 GiB 369 MiB 698 KiB 1023 MiB 99 GiB 2.34 0.96 11 up osd.7 -5 0.19717 - 101 GiB 2.5 GiB 513 MiB 646 KiB 1023 MiB 98 GiB 2.48 1.02 - host node1 0 0.09859 0 0 B 0 B 0 B 0 B 0 B 0 B 0 0 0 down osd.0 1 hdd 0.09859 1.00000 101 GiB 2.5 GiB 513 MiB 646 KiB 1023 MiB 98 GiB 2.48 1.02 16 up osd.1 -6 0.19717 - 202 GiB 4.9 GiB 902 MiB 2.2 MiB 2.0 GiB 197 GiB 2.42 0.99 - host node2 2 hdd 0.09859 1.00000 101 GiB 2.4 GiB 417 MiB 1.2 MiB 1023 MiB 99 GiB 2.38 0.98 12 up osd.2 3 hdd 0.09859 1.00000 101 GiB 2.5 GiB 485 MiB 983 KiB 1023 MiB 99 GiB 2.45 1.01 14 up osd.3 -9 0.19717 - 202 GiB 5.0 GiB 989 MiB 2.0 MiB 2.0 GiB 197 GiB 2.46 1.01 - host node3 4 hdd 0.09859 1.00000 101 GiB 2.5 GiB 541 MiB 1011 KiB 1023 MiB 98 GiB 2.50 1.03 13 up osd.4 5 hdd 0.09859 1.00000 101 GiB 2.4 GiB 449 MiB 994 KiB 1023 MiB 99 GiB 2.41 0.99 12 up osd.5 TOTAL 707 GiB 17 GiB 3.2 GiB 6.4 MiB 7.0 GiB 690 GiB 2.43 MIN/MAX VAR: 0.96/1.03 STDDEV: 0.05 [root@ln-ceph-rpm build]# ceph df detail --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 707 GiB 690 GiB 10 GiB 17 GiB 2.43 TOTAL 707 GiB 690 GiB 10 GiB 17 GiB 2.43 --- POOLS --- POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 227 GiB N/A N/A 0 0 B 0 B test 2 32 1.0 GiB 1.0 GiB 566 KiB 261 3.0 GiB 3.0 GiB 1.7 MiB 0.44 227 GiB N/A N/A 261 0 B 0 B 2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.0 is_down=1 is_out=1 2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.1 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.2 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.3 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.4 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:53:32.279+0800 7f54b5669700 0 filter_down_out_osd osd.5 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:53:32.279+0800 7f54b5669700 0 filter_down_out_osd osd.6 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:53:32.279+0800 7f54b5669700 0 filter_down_out_osd osd.7 sum=0.875 before_percentage=0.125 after_percentage=0.142857 4. osd.0 out and recovery HEALTH_OK [root@ln-ceph-rpm build]# ceph -s cluster: id: afec64f8-d9ee-4262-9410-fcf907807e2c health: HEALTH_OK services: mon: 3 daemons, quorum a,b,c (age 9m) mgr: x(active, since 9m) ioa: 0 daemonsno daemons active osd: 8 osds: 7 up (since 5m), 7 in (since 3m) data: pools: 2 pools, 33 pgs objects: 261 objects, 1.0 GiB usage: 17 GiB used, 690 GiB / 707 GiB avail pgs: 33 active+clean [root@ln-ceph-rpm build]# ceph osd df tree ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME -3 0 - 0 B 0 B 0 B 0 B 0 B 0 B 0 0 - root myroot -1 0.78870 - 707 GiB 17 GiB 3.4 GiB 6.4 MiB 7.0 GiB 690 GiB 2.47 1.00 - root default -11 0.19717 - 202 GiB 4.9 GiB 959 MiB 1.6 MiB 2.0 GiB 197 GiB 2.44 0.99 - host ln-ceph-rpm 6 hdd 0.09859 1.00000 101 GiB 2.6 GiB 590 MiB 980 KiB 1023 MiB 98 GiB 2.55 1.03 16 up osd.6 7 hdd 0.09859 1.00000 101 GiB 2.4 GiB 369 MiB 698 KiB 1023 MiB 99 GiB 2.34 0.95 11 up osd.7 -5 0.19717 - 101 GiB 2.5 GiB 561 MiB 646 KiB 1023 MiB 98 GiB 2.52 1.02 - host node1 0 0.09859 0 0 B 0 B 0 B 0 B 0 B 0 B 0 0 0 down osd.0 1 hdd 0.09859 1.00000 101 GiB 2.5 GiB 561 MiB 646 KiB 1023 MiB 98 GiB 2.52 1.02 18 up osd.1 -6 0.19717 - 202 GiB 5.0 GiB 975 MiB 2.2 MiB 2.0 GiB 197 GiB 2.45 0.99 - host node2 2 hdd 0.09859 1.00000 101 GiB 2.5 GiB 490 MiB 1.2 MiB 1023 MiB 99 GiB 2.45 0.99 14 up osd.2 3 hdd 0.09859 1.00000 101 GiB 2.5 GiB 485 MiB 983 KiB 1023 MiB 99 GiB 2.45 0.99 14 up osd.3 -9 0.19717 - 202 GiB 5.0 GiB 1.0 GiB 2.0 MiB 2.0 GiB 197 GiB 2.48 1.01 - host node3 4 hdd 0.09859 1.00000 101 GiB 2.5 GiB 541 MiB 1011 KiB 1023 MiB 98 GiB 2.50 1.01 13 up osd.4 5 hdd 0.09859 1.00000 101 GiB 2.5 GiB 493 MiB 994 KiB 1023 MiB 99 GiB 2.46 1.00 13 up osd.5 TOTAL 707 GiB 17 GiB 3.4 GiB 6.4 MiB 7.0 GiB 690 GiB 2.47 MIN/MAX VAR: 0.95/1.03 STDDEV: 0.06 [root@ln-ceph-rpm build]# ceph df detail --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 707 GiB 690 GiB 10 GiB 17 GiB 2.47 TOTAL 707 GiB 690 GiB 10 GiB 17 GiB 2.47 --- POOLS --- POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 227 GiB N/A N/A 0 0 B 0 B test 2 32 1.1 GiB 1.1 GiB 566 KiB 261 3.3 GiB 3.3 GiB 1.7 MiB 0.48 227 GiB N/A N/A 261 0 B 0 B 2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.0 is_down=1 is_out=1 2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.1 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.2 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.3 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.4 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.5 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.6 sum=0.875 before_percentage=0.125 after_percentage=0.142857 2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.7 sum=0.875 before_percentage=0.125 after_percentage=0.142857