[BEGIN] 2017/8/11 18:13:31 [test@node1 my-cluster]$ ceph -s cluster f864ebb1-9082-4ee6-8804-e9d795a491a3 health HEALTH_WARN 64 pgs stuck unclean monmap e1: 1 mons at {node1=192.168.1.11:6789/0} election epoch 2, quorum 0 node1 osdmap e38: 2 osds: 2 up, 2 in; 64 remapped pgs flags sortbitwise pgmap v76: 64 pgs, 1 pools, 0 bytes data, 0 objects 10312 MB used, 29785 GB / 29795 GB avail 64 active+remapped [test@node1 my-cluster]$ ceph osd treee no valid command found; 10 closest matches: osd down [...] osd out [...] osd unset full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise osd erasure-code-profile ls osd set full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise osd erasure-code-profile get osd erasure-code-profile rm osd unpause osd erasure-code-profile set { [...]} osd pool mksnap Error EINVAL: invalid command [test@node1 my-cluster]$ ceph osd tree ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY -1 29.09760 root default -2 29.09760 host localhost 0 14.54880 osd.0 up 1.00000 1.00000 1 14.54880 osd.1 up 1.00000 1.00000 [test@node1 my-cluster]$ ceph pg dump dumped all in format plain version 76 stamp 2017-08-11 05:19:07.464311 last_osdmap_epoch 38 last_pg_scan 38 full_ratio 0.95 nearfull_ratio 0.85 pg_stat objects mip degr misp unf bytes log disklog state state_stamp v reported up up_primary acting acting_primary last_scrub scrub_stamp last_deep_scrub deep_scrub_stamp 0.3f 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005545 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043454 0'0 2017-08-10 01:26:26.043454 0.3e 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.354849 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043453 0'0 2017-08-10 01:26:26.043453 0.3d 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116656 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043453 0'0 2017-08-10 01:26:26.043453 0.3c 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.354771 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043452 0'0 2017-08-10 01:26:26.043452 0.3b 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116864 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043441 0'0 2017-08-10 01:26:26.043441 0.3a 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116937 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043441 0'0 2017-08-10 01:26:26.043441 0.39 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355059 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043440 0'0 2017-08-10 01:26:26.043440 0.38 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355162 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043440 0'0 2017-08-10 01:26:26.043440 0.37 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.119921 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043440 0'0 2017-08-10 01:26:26.043440 0.36 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005603 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043439 0'0 2017-08-10 01:26:26.043439 0.35 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.120070 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043439 0'0 2017-08-10 01:26:26.043439 0.34 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115932 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043438 0'0 2017-08-10 01:26:26.043438 0.33 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005687 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043438 0'0 2017-08-10 01:26:26.043438 0.32 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116369 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043438 0'0 2017-08-10 01:26:26.043438 0.31 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115857 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043437 0'0 2017-08-10 01:26:26.043437 0.30 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115737 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043437 0'0 2017-08-10 01:26:26.043437 0.2f 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116235 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043436 0'0 2017-08-10 01:26:26.043436 0.2e 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116367 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043436 0'0 2017-08-10 01:26:26.043436 0.15 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116457 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043416 0'0 2017-08-10 01:26:26.043416 0.14 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116584 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043416 0'0 2017-08-10 01:26:26.043416 0.13 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.117025 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043416 0'0 2017-08-10 01:26:26.043416 0.12 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.117561 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043415 0'0 2017-08-10 01:26:26.043415 0.11 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355157 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043415 0'0 2017-08-10 01:26:26.043415 0.10 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355505 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043414 0'0 2017-08-10 01:26:26.043414 0.f 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.118911 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043414 0'0 2017-08-10 01:26:26.043414 0.e 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116704 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043414 0'0 2017-08-10 01:26:26.043414 0.d 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.118969 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043413 0'0 2017-08-10 01:26:26.043413 0.c 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005786 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043413 0'0 2017-08-10 01:26:26.043413 0.b 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.117154 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043413 0'0 2017-08-10 01:26:26.043413 0.a 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355888 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043412 0'0 2017-08-10 01:26:26.043412 0.0 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355682 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043395 0'0 2017-08-10 01:26:26.043395 0.1 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115795 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043396 0'0 2017-08-10 01:26:26.043396 0.2 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005841 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043397 0'0 2017-08-10 01:26:26.043397 0.3 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355628 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043399 0'0 2017-08-10 01:26:26.043399 0.4 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006134 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043399 0'0 2017-08-10 01:26:26.043399 0.5 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.355738 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043400 0'0 2017-08-10 01:26:26.043400 0.6 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356237 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043400 0'0 2017-08-10 01:26:26.043400 0.7 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116047 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043401 0'0 2017-08-10 01:26:26.043401 0.8 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.119970 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043401 0'0 2017-08-10 01:26:26.043401 0.9 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.120013 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043402 0'0 2017-08-10 01:26:26.043402 0.16 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.007277 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043417 0'0 2017-08-10 01:26:26.043417 0.17 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356625 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043417 0'0 2017-08-10 01:26:26.043417 0.18 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116155 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043417 0'0 2017-08-10 01:26:26.043417 0.19 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356774 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043418 0'0 2017-08-10 01:26:26.043418 0.1a 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005419 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043418 0'0 2017-08-10 01:26:26.043418 0.1b 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116461 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043419 0'0 2017-08-10 01:26:26.043419 0.1c 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.005746 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043419 0'0 2017-08-10 01:26:26.043419 0.1d 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356891 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043419 0'0 2017-08-10 01:26:26.043419 0.1e 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356315 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043420 0'0 2017-08-10 01:26:26.043420 0.1f 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356571 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043420 0'0 2017-08-10 01:26:26.043420 0.20 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.119154 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043421 0'0 2017-08-10 01:26:26.043421 0.21 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006006 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043421 0'0 2017-08-10 01:26:26.043421 0.22 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116293 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043421 0'0 2017-08-10 01:26:26.043421 0.23 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356360 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043432 0'0 2017-08-10 01:26:26.043432 0.24 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116818 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043432 0'0 2017-08-10 01:26:26.043432 0.25 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116191 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043433 0'0 2017-08-10 01:26:26.043433 0.26 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116586 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043433 0'0 2017-08-10 01:26:26.043433 0.27 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.116120 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043433 0'0 2017-08-10 01:26:26.043433 0.28 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 04:49:08.115983 0'0 30:31 [1] 1 [1,0] 1 0'0 2017-08-10 01:26:26.043434 0'0 2017-08-10 01:26:26.043434 0.29 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356726 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043434 0'0 2017-08-10 01:26:26.043434 0.2a 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006606 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043434 0'0 2017-08-10 01:26:26.043434 0.2b 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:15:17.356828 0'0 35:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043435 0'0 2017-08-10 01:26:26.043435 0.2c 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006327 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043435 0'0 2017-08-10 01:26:26.043435 0.2d 0 0 0 0 0 0 0 0 active+remapped 2017-08-11 05:18:40.006819 0'0 38:26 [0] 0 [0,1] 0 0'0 2017-08-10 01:26:26.043436 0'0 2017-08-10 01:26:26.043436 pool 0 0 0 0 0 0 0 0 0 sum 0 0 0 0 0 0 0 0 osdstat kbused kbavail kb hb in hb out 1 5279956 15616400152 15621680108 [0] [] 0 5279988 15616400120 15621680108 [1] [] sum 10559944 31232800272 31243360216 [test@node1 my-cluster]$ ls ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.bootstrap-rgw.keyring ceph.client.admin.keyring ceph.conf ceph-deploy-ceph.log ceph.mon.keyring [test@node1 my-cluster]$ cat ceph.conf [global] fsid = f864ebb1-9082-4ee6-8804-e9d795a491a3 mon_initial_members = node1 mon_host = 192.168.1.11 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx osd pool default size = 2 [test@node1 my-cluster]$ ceph pg dump_stuck clean clean not valid: clean doesn't represent an int Invalid command: unused arguments: ['clean'] pg dump_stuck {inactive|unclean|stale|undersized|degraded [inactive|unclean|stale|undersized|degraded...]} {} : show information about stuck pgs Error EINVAL: invalid command [test@node1 my-cluster]$ ceph pg dump_stuck unclean ok pg_stat state up up_primary acting acting_primary 0.2d active+remapped [0] 0 [0,1] 0 0.2c active+remapped [0] 0 [0,1] 0 0.2b active+remapped [0] 0 [0,1] 0 0.2a active+remapped [0] 0 [0,1] 0 0.29 active+remapped [0] 0 [0,1] 0 0.28 active+remapped [1] 1 [1,0] 1 0.27 active+remapped [1] 1 [1,0] 1 0.26 active+remapped [1] 1 [1,0] 1 0.25 active+remapped [1] 1 [1,0] 1 0.24 active+remapped [1] 1 [1,0] 1 0.23 active+remapped [0] 0 [0,1] 0 0.22 active+remapped [1] 1 [1,0] 1 0.21 active+remapped [0] 0 [0,1] 0 0.20 active+remapped [1] 1 [1,0] 1 0.1f active+remapped [0] 0 [0,1] 0 0.1e active+remapped [0] 0 [0,1] 0 0.1d active+remapped [0] 0 [0,1] 0 0.1c active+remapped [0] 0 [0,1] 0 0.12 active+remapped [1] 1 [1,0] 1 0.13 active+remapped [1] 1 [1,0] 1 0.14 active+remapped [1] 1 [1,0] 1 0.15 active+remapped [1] 1 [1,0] 1 0.2e active+remapped [1] 1 [1,0] 1 0.2f active+remapped [1] 1 [1,0] 1 0.0 active+remapped [0] 0 [0,1] 0 0.30 active+remapped [1] 1 [1,0] 1 0.1 active+remapped [1] 1 [1,0] 1 0.31 active+remapped [1] 1 [1,0] 1 0.2 active+remapped [0] 0 [0,1] 0 0.32 active+remapped [1] 1 [1,0] 1 0.3 active+remapped [0] 0 [0,1] 0 0.33 active+remapped [0] 0 [0,1] 0 0.4 active+remapped [0] 0 [0,1] 0 0.34 active+remapped [1] 1 [1,0] 1 0.5 active+remapped [0] 0 [0,1] 0 0.35 active+remapped [1] 1 [1,0] 1 0.6 active+remapped [0] 0 [0,1] 0 0.3f active+remapped [0] 0 [0,1] 0 0.10 active+remapped [0] 0 [0,1] 0 0.3e active+remapped [0] 0 [0,1] 0 0.f active+remapped [1] 1 [1,0] 1 0.3d active+remapped [1] 1 [1,0] 1 0.e active+remapped [1] 1 [1,0] 1 0.3c active+remapped [0] 0 [0,1] 0 0.d active+remapped [1] 1 [1,0] 1 0.3b active+remapped [1] 1 [1,0] 1 0.c active+remapped [0] 0 [0,1] 0 0.3a active+remapped [1] 1 [1,0] 1 0.b active+remapped [1] 1 [1,0] 1 0.39 active+remapped [0] 0 [0,1] 0 0.a active+remapped [0] 0 [0,1] 0 0.38 active+remapped [0] 0 [0,1] 0 0.9 active+remapped [1] 1 [1,0] 1 0.37 active+remapped [1] 1 [1,0] 1 0.8 active+remapped [1] 1 [1,0] 1 0.36 active+remapped [0] 0 [0,1] 0 0.7 active+remapped [1] 1 [1,0] 1 0.11 active+remapped [0] 0 [0,1] 0 0.16 active+remapped [0] 0 [0,1] 0 0.17 active+remapped [0] 0 [0,1] 0 0.18 active+remapped [1] 1 [1,0] 1 0.19 active+remapped [0] 0 [0,1] 0 0.1a active+remapped [0] 0 [0,1] 0 0.1b active+remapped [1] 1 [1,0] 1 [test@node1 my-cluster]$ ceph health detail HEALTH_WARN 64 pgs stuck unclean pg 0.2d is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.2c is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.2b is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.2a is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.29 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.28 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.27 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.26 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.25 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.24 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.23 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.22 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.21 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.20 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.1f is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.1e is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.1d is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.1c is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.12 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.13 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.14 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.15 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.2e is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.2f is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.0 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.30 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.1 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.31 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.2 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.32 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.3 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.33 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.4 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.34 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.5 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.35 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.6 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.3f is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.10 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.3e is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.f is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.3d is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.e is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.3c is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.d is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.3b is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.c is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.3a is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.b is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.39 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.a is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.38 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.9 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.37 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.8 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.36 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.7 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.11 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.16 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.17 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.18 is stuck unclean since forever, current state active+remapped, last acting [1,0] pg 0.19 is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.1a is stuck unclean since forever, current state active+remapped, last acting [0,1] pg 0.1b is stuck unclean since forever, current state active+remapped, last acting [1,0] [test@node1 my-cluster]$ ceph pg 0.1b query { "state": "active+remapped", "snap_trimq": "[]", "epoch": 38, "up": [ 1 ], "acting": [ 1, 0 ], "actingbackfill": [ "0", "1" ], "info": { "pgid": "0.1b", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "last_backfill_bitwise": 0, "purged_snaps": "[]", "history": { "epoch_created": 1, "last_epoch_started": 28, "last_epoch_clean": 28, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 26, "same_interval_since": 27, "same_primary_since": 27, "last_scrub": "0'0", "last_scrub_stamp": "2017-08-10 01:26:26.043419", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2017-08-10 01:26:26.043419", "last_clean_scrub_stamp": "0.000000" }, "stats": { "version": "0'0", "reported_seq": "31", "reported_epoch": "30", "state": "active+remapped", "last_fresh": "2017-08-11 05:14:24.309397", "last_change": "2017-08-11 04:49:08.116461", "last_active": "2017-08-11 05:14:24.309397", "last_peered": "2017-08-11 05:14:24.309397", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "2017-08-11 05:14:24.309397", "last_undegraded": "2017-08-11 05:14:24.309397", "last_fullsized": "2017-08-11 05:14:24.309397", "mapping_epoch": 26, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 1, "last_epoch_clean": 28, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "2017-08-10 01:26:26.043419", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2017-08-10 01:26:26.043419", "last_clean_scrub_stamp": "0.000000", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0 }, "up": [ 1 ], "acting": [ 1, 0 ], "blocked_by": [], "up_primary": 1, "acting_primary": 1 }, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 28, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "0", "pgid": "0.1b", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "last_backfill_bitwise": 0, "purged_snaps": "[]", "history": { "epoch_created": 1, "last_epoch_started": 28, "last_epoch_clean": 28, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 26, "same_interval_since": 27, "same_primary_since": 27, "last_scrub": "0'0", "last_scrub_stamp": "2017-08-10 01:26:26.043419", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2017-08-10 01:26:26.043419", "last_clean_scrub_stamp": "0.000000" }, "stats": { "version": "0'0", "reported_seq": "26", "reported_epoch": "26", "state": "remapped+peering", "last_fresh": "2017-08-11 04:48:55.032262", "last_change": "2017-08-11 04:48:55.028782", "last_active": "2017-08-11 04:48:32.971942", "last_peered": "2017-08-11 04:48:32.971942", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "2017-08-11 04:48:55.032262", "last_undegraded": "2017-08-11 04:48:55.032262", "last_fullsized": "2017-08-11 04:48:55.032262", "mapping_epoch": 26, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 1, "last_epoch_clean": 25, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "2017-08-10 01:26:26.043419", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2017-08-10 01:26:26.043419", "last_clean_scrub_stamp": "0.000000", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0 }, "up": [ 1 ], "acting": [ 1, 0 ], "blocked_by": [], "up_primary": 1, "acting_primary": 1 }, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 28, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started\/Primary\/Active", "enter_time": "2017-08-11 04:49:08.112756", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [], "waiting_on_backfill": [], "last_backfill_started": "MIN", "backfill_info": { "begin": "MIN", "end": "MIN", "objects": [] }, "peer_backfill_info": [], "backfills_in_flight": [], "recovering": [], "pg_backend": { "pull_from_peer": [], "pushing": [] } }, "scrub": { "scrubber.epoch_start": "0", "scrubber.active": 0, "scrubber.waiting_on": 0, "scrubber.waiting_on_whom": [] } }, { "name": "Started", "enter_time": "2017-08-11 04:49:07.110269" } ], "agent_state": {} } [test@node1 my-cluster]$ ceph -s cluster f864ebb1-9082-4ee6-8804-e9d795a491a3 health HEALTH_WARN 64 pgs stuck unclean monmap e1: 1 mons at {node1=192.168.1.11:6789/0} election epoch 2, quorum 0 node1 osdmap e38: 2 osds: 2 up, 2 in; 64 remapped pgs flags sortbitwise pgmap v76: 64 pgs, 1 pools, 0 bytes data, 0 objects 10312 MB used, 29785 GB / 29795 GB avail 64 active+remapped [END] 2017/8/11 18:23:04