|
|
|
|
|
1. init status
|
|
|
|
[root@ln-ceph-rpm build]# ceph -s
|
|
cluster:
|
|
id: afec64f8-d9ee-4262-9410-fcf907807e2c
|
|
health: HEALTH_WARN
|
|
nobackfill flag(s) set
|
|
|
|
services:
|
|
mon: 3 daemons, quorum a,b,c (age 2m)
|
|
mgr: x(active, since 2m)
|
|
ioa: 0 daemonsno daemons active
|
|
osd: 8 osds: 8 up (since 2m), 8 in (since 2m)
|
|
flags nobackfill
|
|
|
|
data:
|
|
pools: 2 pools, 33 pgs
|
|
objects: 261 objects, 1.0 GiB
|
|
usage: 20 GiB used, 788 GiB / 808 GiB avail
|
|
pgs: 33 active+clean
|
|
|
|
|
|
[root@ln-ceph-rpm build]# ceph osd df tree
|
|
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME
|
|
-3 0 - 0 B 0 B 0 B 0 B 0 B 0 B 0 0 - root myroot
|
|
-1 0.78870 - 808 GiB 20 GiB 3.5 GiB 7.1 MiB 8.0 GiB 788 GiB 2.42 1.00 - root default
|
|
-11 0.19717 - 202 GiB 4.8 GiB 843 MiB 1.6 MiB 2.0 GiB 197 GiB 2.39 0.99 - host ln-ceph-rpm
|
|
6 hdd 0.09859 1.00000 101 GiB 2.5 GiB 505 MiB 980 KiB 1023 MiB 99 GiB 2.47 1.02 14 up osd.6
|
|
7 hdd 0.09859 1.00000 101 GiB 2.3 GiB 337 MiB 698 KiB 1023 MiB 99 GiB 2.31 0.96 9 up osd.7
|
|
-5 0.19717 - 202 GiB 4.8 GiB 858 MiB 1.3 MiB 2.0 GiB 197 GiB 2.40 0.99 - host node1
|
|
0 0.09859 1.00000 101 GiB 2.3 GiB 345 MiB 732 KiB 1023 MiB 99 GiB 2.32 0.96 9 up osd.0
|
|
1 hdd 0.09859 1.00000 101 GiB 2.5 GiB 513 MiB 646 KiB 1023 MiB 98 GiB 2.48 1.03 16 up osd.1
|
|
-6 0.19717 - 202 GiB 4.9 GiB 903 MiB 2.2 MiB 2.0 GiB 197 GiB 2.42 1.00 - host node2
|
|
2 hdd 0.09859 1.00000 101 GiB 2.4 GiB 417 MiB 1.2 MiB 1023 MiB 99 GiB 2.39 0.99 12 up osd.2
|
|
3 hdd 0.09859 1.00000 101 GiB 2.5 GiB 485 MiB 983 KiB 1023 MiB 99 GiB 2.45 1.01 14 up osd.3
|
|
-9 0.19717 - 202 GiB 5.0 GiB 990 MiB 2.0 MiB 2.0 GiB 197 GiB 2.46 1.02 - host node3
|
|
4 hdd 0.09859 1.00000 101 GiB 2.5 GiB 541 MiB 1011 KiB 1023 MiB 98 GiB 2.51 1.04 13 up osd.4
|
|
5 hdd 0.09859 1.00000 101 GiB 2.4 GiB 449 MiB 994 KiB 1023 MiB 99 GiB 2.42 1.00 12 up osd.5
|
|
TOTAL 808 GiB 20 GiB 3.5 GiB 7.1 MiB 8.0 GiB 788 GiB 2.42
|
|
MIN/MAX VAR: 0.96/1.04 STDDEV: 0.07
|
|
|
|
[root@ln-ceph-rpm build]# ceph df
|
|
--- RAW STORAGE ---
|
|
CLASS SIZE AVAIL USED RAW USED %RAW USED
|
|
hdd 707 GiB 690 GiB 10 GiB 17 GiB 2.43
|
|
TOTAL 808 GiB 788 GiB 12 GiB 20 GiB 2.42
|
|
|
|
--- POOLS ---
|
|
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
|
|
device_health_metrics 1 1 0 B 0 0 B 0 260 GiB
|
|
test 2 32 1.0 GiB 261 3.0 GiB 0.38 260 GiB
|
|
|
|
|
|
|
|
|
|
2. kill osd.0 down
|
|
root 1580290 6.4 0.6 1019672 203772 ? Ssl 17:45 0:15 /root/thinkfs-ssh/build/bin/ceph-osd -i 0 -c /root/thinkfs-ssh/build/ceph.conf
|
|
[root@ln-ceph-rpm build]# kill -9 1580290
|
|
|
|
|
|
[root@ln-ceph-rpm build]# ceph -s
|
|
cluster:
|
|
id: afec64f8-d9ee-4262-9410-fcf907807e2c
|
|
health: HEALTH_WARN
|
|
nobackfill flag(s) set
|
|
1 osds down
|
|
Degraded data redundancy: 72/783 objects degraded (9.195%), 8 pgs degraded
|
|
|
|
services:
|
|
mon: 3 daemons, quorum a,b,c (age 4m)
|
|
mgr: x(active, since 4m)
|
|
ioa: 0 daemonsno daemons active
|
|
osd: 8 osds: 7 up (since 16s), 8 in (since 3m)
|
|
flags nobackfill
|
|
|
|
data:
|
|
pools: 2 pools, 33 pgs
|
|
objects: 261 objects, 1.0 GiB
|
|
usage: 20 GiB used, 788 GiB / 808 GiB avail
|
|
pgs: 72/783 objects degraded (9.195%)
|
|
24 active+clean
|
|
8 active+undersized+degraded
|
|
1 active+undersized
|
|
|
|
|
|
[root@ln-ceph-rpm build]# ceph osd df tree
|
|
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME
|
|
-3 0 - 0 B 0 B 0 B 0 B 0 B 0 B 0 0 - root myroot
|
|
-1 0.78870 - 808 GiB 20 GiB 3.5 GiB 7.1 MiB 8.0 GiB 788 GiB 2.41 1.00 - root default
|
|
-11 0.19717 - 202 GiB 4.8 GiB 840 MiB 1.6 MiB 2.0 GiB 197 GiB 2.39 0.99 - host ln-ceph-rpm
|
|
6 hdd 0.09859 1.00000 101 GiB 2.5 GiB 504 MiB 980 KiB 1023 MiB 99 GiB 2.47 1.02 14 up osd.6
|
|
7 hdd 0.09859 1.00000 101 GiB 2.3 GiB 336 MiB 698 KiB 1023 MiB 99 GiB 2.31 0.95 9 up osd.7
|
|
-5 0.19717 - 202 GiB 4.8 GiB 857 MiB 1.3 MiB 2.0 GiB 197 GiB 2.40 0.99 - host node1
|
|
0 0.09859 1.00000 101 GiB 2.3 GiB 345 MiB 732 KiB 1023 MiB 99 GiB 2.32 0.96 0 down osd.0
|
|
1 hdd 0.09859 1.00000 101 GiB 2.5 GiB 512 MiB 646 KiB 1023 MiB 98 GiB 2.48 1.03 16 up osd.1
|
|
-6 0.19717 - 202 GiB 4.9 GiB 900 MiB 2.2 MiB 2.0 GiB 197 GiB 2.42 1.00 - host node2
|
|
2 hdd 0.09859 1.00000 101 GiB 2.4 GiB 416 MiB 1.2 MiB 1023 MiB 99 GiB 2.38 0.99 12 up osd.2
|
|
3 hdd 0.09859 1.00000 101 GiB 2.5 GiB 484 MiB 983 KiB 1023 MiB 99 GiB 2.45 1.01 14 up osd.3
|
|
-9 0.19717 - 202 GiB 5.0 GiB 988 MiB 2.0 MiB 2.0 GiB 197 GiB 2.46 1.02 - host node3
|
|
4 hdd 0.09859 1.00000 101 GiB 2.5 GiB 540 MiB 1011 KiB 1023 MiB 98 GiB 2.50 1.04 13 up osd.4
|
|
5 hdd 0.09859 1.00000 101 GiB 2.4 GiB 448 MiB 994 KiB 1023 MiB 99 GiB 2.41 1.00 12 up osd.5
|
|
TOTAL 808 GiB 20 GiB 3.5 GiB 7.1 MiB 8.0 GiB 788 GiB 2.41
|
|
MIN/MAX VAR: 0.95/1.04 STDDEV: 0.07
|
|
|
|
[root@ln-ceph-rpm build]# ceph df detail
|
|
--- RAW STORAGE ---
|
|
CLASS SIZE AVAIL USED RAW USED %RAW USED
|
|
hdd 707 GiB 690 GiB 10 GiB 17 GiB 2.43
|
|
TOTAL 808 GiB 788 GiB 12 GiB 20 GiB 2.41
|
|
|
|
--- POOLS ---
|
|
POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR
|
|
device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 227 GiB N/A N/A 0 0 B 0 B
|
|
test 2 32 1.0 GiB 1.0 GiB 566 KiB 261 3.0 GiB 3.0 GiB 1.7 MiB 0.44 227 GiB N/A N/A 261 0 B 0 B
|
|
|
|
|
|
2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.0 is_down=1 is_out=0
|
|
2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.1 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.2 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.3 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.4 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.5 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.6 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:51:32.104+0800 7f54b5669700 0 filter_down_out_osd osd.7 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
|
|
|
|
3. osd.0 out but not backfill
|
|
|
|
[root@ln-ceph-rpm build]# ceph -s
|
|
cluster:
|
|
id: afec64f8-d9ee-4262-9410-fcf907807e2c
|
|
health: HEALTH_WARN
|
|
nobackfill flag(s) set
|
|
Degraded data redundancy: 64/783 objects degraded (8.174%), 7 pgs degraded
|
|
|
|
services:
|
|
mon: 3 daemons, quorum a,b,c (age 6m)
|
|
mgr: x(active, since 6m)
|
|
ioa: 0 daemonsno daemons active
|
|
osd: 8 osds: 7 up (since 2m), 7 in (since 31s); 7 remapped pgs
|
|
flags nobackfill
|
|
|
|
data:
|
|
pools: 2 pools, 33 pgs
|
|
objects: 261 objects, 1.0 GiB
|
|
usage: 17 GiB used, 690 GiB / 707 GiB avail
|
|
pgs: 64/783 objects degraded (8.174%)
|
|
26 active+clean
|
|
4 active+undersized+degraded+remapped+backfill_wait
|
|
3 active+undersized+degraded+remapped+backfilling
|
|
|
|
|
|
[root@ln-ceph-rpm build]# ceph osd df tree
|
|
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME
|
|
-3 0 - 0 B 0 B 0 B 0 B 0 B 0 B 0 0 - root myroot
|
|
-1 0.78870 - 707 GiB 17 GiB 3.2 GiB 6.4 MiB 7.0 GiB 690 GiB 2.43 1.00 - root default
|
|
-11 0.19717 - 202 GiB 4.9 GiB 874 MiB 1.6 MiB 2.0 GiB 197 GiB 2.40 0.99 - host ln-ceph-rpm
|
|
6 hdd 0.09859 1.00000 101 GiB 2.5 GiB 505 MiB 980 KiB 1023 MiB 99 GiB 2.47 1.01 14 up osd.6
|
|
7 hdd 0.09859 1.00000 101 GiB 2.4 GiB 369 MiB 698 KiB 1023 MiB 99 GiB 2.34 0.96 11 up osd.7
|
|
-5 0.19717 - 101 GiB 2.5 GiB 513 MiB 646 KiB 1023 MiB 98 GiB 2.48 1.02 - host node1
|
|
0 0.09859 0 0 B 0 B 0 B 0 B 0 B 0 B 0 0 0 down osd.0
|
|
1 hdd 0.09859 1.00000 101 GiB 2.5 GiB 513 MiB 646 KiB 1023 MiB 98 GiB 2.48 1.02 16 up osd.1
|
|
-6 0.19717 - 202 GiB 4.9 GiB 902 MiB 2.2 MiB 2.0 GiB 197 GiB 2.42 0.99 - host node2
|
|
2 hdd 0.09859 1.00000 101 GiB 2.4 GiB 417 MiB 1.2 MiB 1023 MiB 99 GiB 2.38 0.98 12 up osd.2
|
|
3 hdd 0.09859 1.00000 101 GiB 2.5 GiB 485 MiB 983 KiB 1023 MiB 99 GiB 2.45 1.01 14 up osd.3
|
|
-9 0.19717 - 202 GiB 5.0 GiB 989 MiB 2.0 MiB 2.0 GiB 197 GiB 2.46 1.01 - host node3
|
|
4 hdd 0.09859 1.00000 101 GiB 2.5 GiB 541 MiB 1011 KiB 1023 MiB 98 GiB 2.50 1.03 13 up osd.4
|
|
5 hdd 0.09859 1.00000 101 GiB 2.4 GiB 449 MiB 994 KiB 1023 MiB 99 GiB 2.41 0.99 12 up osd.5
|
|
TOTAL 707 GiB 17 GiB 3.2 GiB 6.4 MiB 7.0 GiB 690 GiB 2.43
|
|
MIN/MAX VAR: 0.96/1.03 STDDEV: 0.05
|
|
|
|
|
|
[root@ln-ceph-rpm build]# ceph df detail
|
|
--- RAW STORAGE ---
|
|
CLASS SIZE AVAIL USED RAW USED %RAW USED
|
|
hdd 707 GiB 690 GiB 10 GiB 17 GiB 2.43
|
|
TOTAL 707 GiB 690 GiB 10 GiB 17 GiB 2.43
|
|
|
|
--- POOLS ---
|
|
POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR
|
|
device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 227 GiB N/A N/A 0 0 B 0 B
|
|
test 2 32 1.0 GiB 1.0 GiB 566 KiB 261 3.0 GiB 3.0 GiB 1.7 MiB 0.44 227 GiB N/A N/A 261 0 B 0 B
|
|
|
|
|
|
2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.0 is_down=1 is_out=1
|
|
2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.1 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.2 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.3 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:53:32.278+0800 7f54b5669700 0 filter_down_out_osd osd.4 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:53:32.279+0800 7f54b5669700 0 filter_down_out_osd osd.5 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:53:32.279+0800 7f54b5669700 0 filter_down_out_osd osd.6 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:53:32.279+0800 7f54b5669700 0 filter_down_out_osd osd.7 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
|
|
|
|
4. osd.0 out and recovery HEALTH_OK
|
|
|
|
[root@ln-ceph-rpm build]# ceph -s
|
|
cluster:
|
|
id: afec64f8-d9ee-4262-9410-fcf907807e2c
|
|
health: HEALTH_OK
|
|
|
|
services:
|
|
mon: 3 daemons, quorum a,b,c (age 9m)
|
|
mgr: x(active, since 9m)
|
|
ioa: 0 daemonsno daemons active
|
|
osd: 8 osds: 7 up (since 5m), 7 in (since 3m)
|
|
|
|
data:
|
|
pools: 2 pools, 33 pgs
|
|
objects: 261 objects, 1.0 GiB
|
|
usage: 17 GiB used, 690 GiB / 707 GiB avail
|
|
pgs: 33 active+clean
|
|
|
|
|
|
[root@ln-ceph-rpm build]# ceph osd df tree
|
|
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME
|
|
-3 0 - 0 B 0 B 0 B 0 B 0 B 0 B 0 0 - root myroot
|
|
-1 0.78870 - 707 GiB 17 GiB 3.4 GiB 6.4 MiB 7.0 GiB 690 GiB 2.47 1.00 - root default
|
|
-11 0.19717 - 202 GiB 4.9 GiB 959 MiB 1.6 MiB 2.0 GiB 197 GiB 2.44 0.99 - host ln-ceph-rpm
|
|
6 hdd 0.09859 1.00000 101 GiB 2.6 GiB 590 MiB 980 KiB 1023 MiB 98 GiB 2.55 1.03 16 up osd.6
|
|
7 hdd 0.09859 1.00000 101 GiB 2.4 GiB 369 MiB 698 KiB 1023 MiB 99 GiB 2.34 0.95 11 up osd.7
|
|
-5 0.19717 - 101 GiB 2.5 GiB 561 MiB 646 KiB 1023 MiB 98 GiB 2.52 1.02 - host node1
|
|
0 0.09859 0 0 B 0 B 0 B 0 B 0 B 0 B 0 0 0 down osd.0
|
|
1 hdd 0.09859 1.00000 101 GiB 2.5 GiB 561 MiB 646 KiB 1023 MiB 98 GiB 2.52 1.02 18 up osd.1
|
|
-6 0.19717 - 202 GiB 5.0 GiB 975 MiB 2.2 MiB 2.0 GiB 197 GiB 2.45 0.99 - host node2
|
|
2 hdd 0.09859 1.00000 101 GiB 2.5 GiB 490 MiB 1.2 MiB 1023 MiB 99 GiB 2.45 0.99 14 up osd.2
|
|
3 hdd 0.09859 1.00000 101 GiB 2.5 GiB 485 MiB 983 KiB 1023 MiB 99 GiB 2.45 0.99 14 up osd.3
|
|
-9 0.19717 - 202 GiB 5.0 GiB 1.0 GiB 2.0 MiB 2.0 GiB 197 GiB 2.48 1.01 - host node3
|
|
4 hdd 0.09859 1.00000 101 GiB 2.5 GiB 541 MiB 1011 KiB 1023 MiB 98 GiB 2.50 1.01 13 up osd.4
|
|
5 hdd 0.09859 1.00000 101 GiB 2.5 GiB 493 MiB 994 KiB 1023 MiB 99 GiB 2.46 1.00 13 up osd.5
|
|
TOTAL 707 GiB 17 GiB 3.4 GiB 6.4 MiB 7.0 GiB 690 GiB 2.47
|
|
MIN/MAX VAR: 0.95/1.03 STDDEV: 0.06
|
|
|
|
|
|
[root@ln-ceph-rpm build]# ceph df detail
|
|
--- RAW STORAGE ---
|
|
CLASS SIZE AVAIL USED RAW USED %RAW USED
|
|
hdd 707 GiB 690 GiB 10 GiB 17 GiB 2.47
|
|
TOTAL 707 GiB 690 GiB 10 GiB 17 GiB 2.47
|
|
|
|
--- POOLS ---
|
|
POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR
|
|
device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 227 GiB N/A N/A 0 0 B 0 B
|
|
test 2 32 1.1 GiB 1.1 GiB 566 KiB 261 3.3 GiB 3.3 GiB 1.7 MiB 0.48 227 GiB N/A N/A 261 0 B 0 B
|
|
|
|
2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.0 is_down=1 is_out=1
|
|
2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.1 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.2 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.3 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.4 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.5 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.6 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
2022-06-01T17:56:02.485+0800 7f54b5669700 0 filter_down_out_osd osd.7 sum=0.875 before_percentage=0.125 after_percentage=0.142857
|
|
|
|
|