Project

General

Profile

Support #20994 » node1状态.log

warrick xu, 08/14/2017 07:07 AM

 
[BEGIN] 2017/8/11 18:13:31
[test@node1 my-cluster]$ ceph -s
cluster f864ebb1-9082-4ee6-8804-e9d795a491a3
health HEALTH_WARN
64 pgs stuck unclean
monmap e1: 1 mons at {node1=192.168.1.11:6789/0}
election epoch 2, quorum 0 node1
osdmap e38: 2 osds: 2 up, 2 in; 64 remapped pgs
flags sortbitwise
pgmap v76: 64 pgs, 1 pools, 0 bytes data, 0 objects
10312 MB used, 29785 GB / 29795 GB avail
64 active+remapped
[test@node1 my-cluster]$ ceph osd treee
no valid command found; 10 closest matches:
osd down <ids> [<ids>...]
osd out <ids> [<ids>...]
osd unset full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise
osd erasure-code-profile ls
osd set full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise
osd erasure-code-profile get <name>
osd erasure-code-profile rm <name>
osd unpause
osd erasure-code-profile set <name> {<profile> [<profile>...]}
osd pool mksnap <poolname> <snap>
Error EINVAL: invalid command
[test@node1 my-cluster]$ ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 29.09760 root default
-2 29.09760 host localhost
0 14.54880 osd.0 up 1.00000 1.00000
1 14.54880 osd.1 up 1.00000 1.00000
[test@node1 my-cluster]$ ceph pg dump
dumped all in format plain
version 76
stamp 2017-08-11 05:19:07.464311
last_osdmap_epoch 38
last_pg_scan 38
full_ratio 0.95
nearfull_ratio 0.85
pg_stat objects mip degr misp unf bytes log disklog state state_stamp v reported up up_primary acting acting_primary last_scrub scrub_stamp last_deep_scrub deep_scrub_stamp
0.3f 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005545 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043454 0'0 2017-08-10 01:26:26.043454
0.3e 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.354849 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043453 0'0 2017-08-10 01:26:26.043453
0.3d 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116656 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043453 0'0 2017-08-10 01:26:26.043453
0.3c 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.354771 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043452 0'0 2017-08-10 01:26:26.043452
0.3b 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116864 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043441 0'0 2017-08-10 01:26:26.043441
0.3a 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116937 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043441 0'0 2017-08-10 01:26:26.043441
0.39 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355059 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043440 0'0 2017-08-10 01:26:26.043440
0.38 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355162 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043440 0'0 2017-08-10 01:26:26.043440
0.37 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.119921 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043440 0'0 2017-08-10 01:26:26.043440
0.36 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005603 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043439 0'0 2017-08-10 01:26:26.043439
0.35 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.120070 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043439 0'0 2017-08-10 01:26:26.043439
0.34 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115932 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043438 0'0 2017-08-10 01:26:26.043438
0.33 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005687 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043438 0'0 2017-08-10 01:26:26.043438
0.32 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116369 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043438 0'0 2017-08-10 01:26:26.043438
0.31 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115857 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043437 0'0 2017-08-10 01:26:26.043437
0.30 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115737 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043437 0'0 2017-08-10 01:26:26.043437
0.2f 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116235 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043436 0'0 2017-08-10 01:26:26.043436
0.2e 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116367 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043436 0'0 2017-08-10 01:26:26.043436
0.15 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116457 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043416 0'0 2017-08-10 01:26:26.043416
0.14 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116584 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043416 0'0 2017-08-10 01:26:26.043416
0.13 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.117025 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043416 0'0 2017-08-10 01:26:26.043416
0.12 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.117561 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043415 0'0 2017-08-10 01:26:26.043415
0.11 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355157 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043415 0'0 2017-08-10 01:26:26.043415
0.10 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355505 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043414 0'0 2017-08-10 01:26:26.043414
0.f 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.118911 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043414 0'0 2017-08-10 01:26:26.043414
0.e 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116704 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043414 0'0 2017-08-10 01:26:26.043414
0.d 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.118969 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043413 0'0 2017-08-10 01:26:26.043413
0.c 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005786 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043413 0'0 2017-08-10 01:26:26.043413
0.b 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.117154 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043413 0'0 2017-08-10 01:26:26.043413
0.a 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355888 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043412 0'0 2017-08-10 01:26:26.043412
0.0 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355682 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043395 0'0 2017-08-10 01:26:26.043395
0.1 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115795 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043396 0'0 2017-08-10 01:26:26.043396
0.2 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005841 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043397 0'0 2017-08-10 01:26:26.043397
0.3 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355628 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043399 0'0 2017-08-10 01:26:26.043399
0.4 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006134 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043399 0'0 2017-08-10 01:26:26.043399
0.5 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355738 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043400 0'0 2017-08-10 01:26:26.043400
0.6 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356237 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043400 0'0 2017-08-10 01:26:26.043400
0.7 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116047 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043401 0'0 2017-08-10 01:26:26.043401
0.8 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.119970 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043401 0'0 2017-08-10 01:26:26.043401
0.9 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.120013 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043402 0'0 2017-08-10 01:26:26.043402
0.16 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.007277 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043417 0'0 2017-08-10 01:26:26.043417
0.17 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356625 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043417 0'0 2017-08-10 01:26:26.043417
0.18 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116155 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043417 0'0 2017-08-10 01:26:26.043417
0.19 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356774 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043418 0'0 2017-08-10 01:26:26.043418
0.1a 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005419 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043418 0'0 2017-08-10 01:26:26.043418
0.1b 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116461 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043419 0'0 2017-08-10 01:26:26.043419
0.1c 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005746 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043419 0'0 2017-08-10 01:26:26.043419
0.1d 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356891 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043419 0'0 2017-08-10 01:26:26.043419
0.1e 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356315 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043420 0'0 2017-08-10 01:26:26.043420
0.1f 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356571 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043420 0'0 2017-08-10 01:26:26.043420
0.20 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.119154 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043421 0'0 2017-08-10 01:26:26.043421
0.21 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006006 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043421 0'0 2017-08-10 01:26:26.043421
0.22 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116293 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043421 0'0 2017-08-10 01:26:26.043421
0.23 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356360 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043432 0'0 2017-08-10 01:26:26.043432
0.24 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116818 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043432 0'0 2017-08-10 01:26:26.043432
0.25 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116191 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043433 0'0 2017-08-10 01:26:26.043433
0.26 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116586 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043433 0'0 2017-08-10 01:26:26.043433
0.27 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116120 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043433 0'0 2017-08-10 01:26:26.043433
0.28 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115983 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043434 0'0 2017-08-10 01:26:26.043434
0.29 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356726 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043434 0'0 2017-08-10 01:26:26.043434
0.2a 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006606 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043434 0'0 2017-08-10 01:26:26.043434
0.2b 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356828 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043435 0'0 2017-08-10 01:26:26.043435
0.2c 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006327 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043435 0'0 2017-08-10 01:26:26.043435
0.2d 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006819 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043436 0'0 2017-08-10 01:26:26.043436
pool 0 0 0 0 0 0 0 0 0
sum 0 0 0 0 0 0 0 0
osdstat kbused kbavail kb hb in hb out
1 5279956 15616400152 15621680108 [0] []
0 5279988 15616400120 15621680108 [1] []
sum 10559944 31232800272 31243360216
[test@node1 my-cluster]$ ls
ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.bootstrap-rgw.keyring ceph.client.admin.keyring ceph.conf ceph-deploy-ceph.log ceph.mon.keyring
[test@node1 my-cluster]$ cat ceph.conf
[global]
fsid = f864ebb1-9082-4ee6-8804-e9d795a491a3
mon_initial_members = node1
mon_host = 192.168.1.11
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
osd pool default size = 2
[test@node1 my-cluster]$ ceph pg dump_stuck clean
clean not valid: clean doesn't represent an int
Invalid command: unused arguments: ['clean']
pg dump_stuck {inactive|unclean|stale|undersized|degraded [inactive|unclean|stale|undersized|degraded...]} {<int>} : show information about stuck pgs
Error EINVAL: invalid command
[test@node1 my-cluster]$ ceph pg dump_stuck unclean
ok
pg_stat state up up_primary acting acting_primary
0.2d active+remapped [0] 0 [0,1] 0
0.2c active+remapped [0] 0 [0,1] 0
0.2b active+remapped [0] 0 [0,1] 0
0.2a active+remapped [0] 0 [0,1] 0
0.29 active+remapped [0] 0 [0,1] 0
0.28 active+remapped [1] 1 [1,0] 1
0.27 active+remapped [1] 1 [1,0] 1
0.26 active+remapped [1] 1 [1,0] 1
0.25 active+remapped [1] 1 [1,0] 1
0.24 active+remapped [1] 1 [1,0] 1
0.23 active+remapped [0] 0 [0,1] 0
0.22 active+remapped [1] 1 [1,0] 1
0.21 active+remapped [0] 0 [0,1] 0
0.20 active+remapped [1] 1 [1,0] 1
0.1f active+remapped [0] 0 [0,1] 0
0.1e active+remapped [0] 0 [0,1] 0
0.1d active+remapped [0] 0 [0,1] 0
0.1c active+remapped [0] 0 [0,1] 0
0.12 active+remapped [1] 1 [1,0] 1
0.13 active+remapped [1] 1 [1,0] 1
0.14 active+remapped [1] 1 [1,0] 1
0.15 active+remapped [1] 1 [1,0] 1
0.2e active+remapped [1] 1 [1,0] 1
0.2f active+remapped [1] 1 [1,0] 1
0.0 active+remapped [0] 0 [0,1] 0
0.30 active+remapped [1] 1 [1,0] 1
0.1 active+remapped [1] 1 [1,0] 1
0.31 active+remapped [1] 1 [1,0] 1
0.2 active+remapped [0] 0 [0,1] 0
0.32 active+remapped [1] 1 [1,0] 1
0.3 active+remapped [0] 0 [0,1] 0
0.33 active+remapped [0] 0 [0,1] 0
0.4 active+remapped [0] 0 [0,1] 0
0.34 active+remapped [1] 1 [1,0] 1
0.5 active+remapped [0] 0 [0,1] 0
0.35 active+remapped [1] 1 [1,0] 1
0.6 active+remapped [0] 0 [0,1] 0
0.3f active+remapped [0] 0 [0,1] 0
0.10 active+remapped [0] 0 [0,1] 0
0.3e active+remapped [0] 0 [0,1] 0
0.f active+remapped [1] 1 [1,0] 1
0.3d active+remapped [1] 1 [1,0] 1
0.e active+remapped [1] 1 [1,0] 1
0.3c active+remapped [0] 0 [0,1] 0
0.d active+remapped [1] 1 [1,0] 1
0.3b active+remapped [1] 1 [1,0] 1
0.c active+remapped [0] 0 [0,1] 0
0.3a active+remapped [1] 1 [1,0] 1
0.b active+remapped [1] 1 [1,0] 1
0.39 active+remapped [0] 0 [0,1] 0
0.a active+remapped [0] 0 [0,1] 0
0.38 active+remapped [0] 0 [0,1] 0
0.9 active+remapped [1] 1 [1,0] 1
0.37 active+remapped [1] 1 [1,0] 1
0.8 active+remapped [1] 1 [1,0] 1
0.36 active+remapped [0] 0 [0,1] 0
0.7 active+remapped [1] 1 [1,0] 1
0.11 active+remapped [0] 0 [0,1] 0
0.16 active+remapped [0] 0 [0,1] 0
0.17 active+remapped [0] 0 [0,1] 0
0.18 active+remapped [1] 1 [1,0] 1
0.19 active+remapped [0] 0 [0,1] 0
0.1a active+remapped [0] 0 [0,1] 0
0.1b active+remapped [1] 1 [1,0] 1
[test@node1 my-cluster]$ ceph health detail
HEALTH_WARN 64 pgs stuck unclean
pg 0.2d is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.2c is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.2b is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.2a is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.29 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.28 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.27 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.26 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.25 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.24 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.23 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.22 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.21 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.20 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.1f is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.1e is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.1d is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.1c is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.12 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.13 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.14 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.15 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.2e is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.2f is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.0 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.30 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.1 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.31 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.2 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.32 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.3 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.33 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.4 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.34 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.5 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.35 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.6 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.3f is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.10 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.3e is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.f is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.3d is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.e is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.3c is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.d is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.3b is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.c is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.3a is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.b is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.39 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.a is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.38 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.9 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.37 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.8 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.36 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.7 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.11 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.16 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.17 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.18 is stuck unclean since forever, current state active+remapped, last acting [1,0]
pg 0.19 is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.1a is stuck unclean since forever, current state active+remapped, last acting [0,1]
pg 0.1b is stuck unclean since forever, current state active+remapped, last acting [1,0]
[test@node1 my-cluster]$ ceph pg 0.1b query
{
"state": "active+remapped",
"snap_trimq": "[]",
"epoch": 38,
"up": [
1
],
"acting": [
1,
0
],
"actingbackfill": [
"0",
"1"
],
"info": {
"pgid": "0.1b",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 1,
"last_epoch_started": 28,
"last_epoch_clean": 28,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 26,
"same_interval_since": 27,
"same_primary_since": 27,
"last_scrub": "0'0",
"last_scrub_stamp": "2017-08-10 01:26:26.043419",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-08-10 01:26:26.043419",
"last_clean_scrub_stamp": "0.000000"
},
"stats": {
"version": "0'0",
"reported_seq": "31",
"reported_epoch": "30",
"state": "active+remapped",
"last_fresh": "2017-08-11 05:14:24.309397",
"last_change": "2017-08-11 04:49:08.116461",
"last_active": "2017-08-11 05:14:24.309397",
"last_peered": "2017-08-11 05:14:24.309397",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2017-08-11 05:14:24.309397",
"last_undegraded": "2017-08-11 05:14:24.309397",
"last_fullsized": "2017-08-11 05:14:24.309397",
"mapping_epoch": 26,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 1,
"last_epoch_clean": 28,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2017-08-10 01:26:26.043419",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-08-10 01:26:26.043419",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0
},
"up": [
1
],
"acting": [
1,
0
],
"blocked_by": [],
"up_primary": 1,
"acting_primary": 1
},
"empty": 1,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 28,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "0",
"pgid": "0.1b",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 1,
"last_epoch_started": 28,
"last_epoch_clean": 28,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 26,
"same_interval_since": 27,
"same_primary_since": 27,
"last_scrub": "0'0",
"last_scrub_stamp": "2017-08-10 01:26:26.043419",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-08-10 01:26:26.043419",
"last_clean_scrub_stamp": "0.000000"
},
"stats": {
"version": "0'0",
"reported_seq": "26",
"reported_epoch": "26",
"state": "remapped+peering",
"last_fresh": "2017-08-11 04:48:55.032262",
"last_change": "2017-08-11 04:48:55.028782",
"last_active": "2017-08-11 04:48:32.971942",
"last_peered": "2017-08-11 04:48:32.971942",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2017-08-11 04:48:55.032262",
"last_undegraded": "2017-08-11 04:48:55.032262",
"last_fullsized": "2017-08-11 04:48:55.032262",
"mapping_epoch": 26,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 1,
"last_epoch_clean": 25,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2017-08-10 01:26:26.043419",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-08-10 01:26:26.043419",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0
},
"up": [
1
],
"acting": [
1,
0
],
"blocked_by": [],
"up_primary": 1,
"acting_primary": 1
},
"empty": 1,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 28,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2017-08-11 04:49:08.112756",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2017-08-11 04:49:07.110269"
}
],
"agent_state": {}
}
[test@node1 my-cluster]$ ceph -s
cluster f864ebb1-9082-4ee6-8804-e9d795a491a3
health HEALTH_WARN
64 pgs stuck unclean
monmap e1: 1 mons at {node1=192.168.1.11:6789/0}
election epoch 2, quorum 0 node1
osdmap e38: 2 osds: 2 up, 2 in; 64 remapped pgs
flags sortbitwise
pgmap v76: 64 pgs, 1 pools, 0 bytes data, 0 objects
10312 MB used, 29785 GB / 29795 GB avail
64 active+remapped

[END] 2017/8/11 18:23:04
    (1-1/1)