Bug #57185
Updated by Vikhyat Umrao over 1 year ago
- PG Query
<pre>
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "activating+degraded+remapped",
"epoch": 5861,
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"backfill_targets": [
"14(0)"
],
"acting_recovery_backfill": [
"4(0)",
"8(3)",
"14(0)",
"41(2)",
"51(1)",
"117(4)",
"156(5)"
],
"info": {
"pgid": "9.c78s0",
"last_update": "5210'48054",
"last_complete": "4727'47329",
"log_tail": "4722'45161",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 159430,
"reported_epoch": 5861,
"state": "activating+degraded+remapped",
"last_fresh": "2022-08-18T17:20:04.492434+0000",
"last_change": "2022-08-18T16:34:47.692664+0000",
"last_active": "2022-08-18T16:34:47.612904+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T17:20:04.492434+0000",
"last_undegraded": "2022-08-18T16:34:47.625898+0000",
"last_fullsized": "2022-08-18T17:20:04.492434+0000",
"mapping_epoch": 5215,
"log_start": "4722'45161",
"ondisk_log_start": "4722'45161",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 2893,
"ondisk_log_size": 2893,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 683,
"num_objects_missing": 683,
"num_objects_degraded": 683,
"num_objects_misplaced": 44885,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"2(0)",
"8(3)",
"41(2)",
"51(1)",
"64(0)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "2(0),8(3),41(2),51(1),64(0),117(4),156(5)",
"objects": 72
},
{
"shards": "2(0),8(3),41(2),51(1),117(4),156(5)",
"objects": 32
},
{
"shards": "4(0),8(3),41(2),51(1),117(4),156(5)",
"objects": 44964
},
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 500
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5218,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "2(0)",
"pgid": "9.c78s0",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "9:1e32a2e7:::59178bc5-0d1c-4138-85c4-42b6340da0d1.188855.3_TTcgroas%2f1580272.tl0uQj2FjJApUtlA.rnd:head",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 1,
"reported_epoch": 5138,
"state": "peering",
"last_fresh": "2022-08-18T14:59:15.543131+0000",
"last_change": "2022-08-18T14:59:15.543131+0000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2022-08-18T14:59:15.543131+0000",
"last_undegraded": "2022-08-18T14:59:15.543131+0000",
"last_fullsized": "2022-08-18T14:59:15.543131+0000",
"mapping_epoch": 5215,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2148793070,
"num_objects": 7520,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 7520,
"num_whiteouts": 0,
"num_read": 4,
"num_read_kb": 6,
"num_write": 38,
"num_write_kb": 21,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
8,
41,
51,
117,
156
],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "8(3)",
"pgid": "9.c78s3",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 158780,
"reported_epoch": 5215,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:31:53.399844+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:31:53.399844+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:31:53.399844+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38071,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "14(0)",
"pgid": "9.c78s0",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4722'45161",
"last_user_version": 0,
"last_backfill": "MIN",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"objects_scrubbed": 0,
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "--",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 45568,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [],
"acting": [],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 5218,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "41(2)",
"pgid": "9.c78s2",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 188248,
"reported_epoch": 5210,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:31:53.399844+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:31:53.399844+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:31:53.399844+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38071,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "51(1)",
"pgid": "9.c78s1",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 188294,
"reported_epoch": 5212,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:32:22.161926+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:32:22.161926+0000",
"last_peered": "2022-08-18T16:32:22.161926+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:32:22.161926+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38048,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56144,
"num_bytes_recovered": 2656143919,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "64(0)",
"pgid": "9.c78s0",
"last_update": "4973'48000",
"last_complete": "4973'48000",
"log_tail": "4724'46128",
"last_user_version": 48000,
"last_backfill": "9:1e3213f3:::59178bc5-0d1c-4138-85c4-42b6340da0d1.188831.2_LkZrS(rv%2f1416931.q)Ojo8f06c3YPl46.rnd:head",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 1,
"reported_epoch": 4913,
"state": "peering",
"last_fresh": "2022-08-18T13:48:06.401417+0000",
"last_change": "2022-08-18T13:48:06.401417+0000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2022-08-18T13:48:06.401417+0000",
"last_undegraded": "2022-08-18T13:48:06.401417+0000",
"last_fullsized": "2022-08-18T13:48:06.401417+0000",
"mapping_epoch": 5215,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 286374799,
"num_objects": 5927,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 5927,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 24,
"num_write_kb": 207,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
8,
41,
51,
117,
156
],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 4915,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "117(4)",
"pgid": "9.c78s4",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 188248,
"reported_epoch": 5210,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:31:53.399844+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:31:53.399844+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:31:53.399844+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38071,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "156(5)",
"pgid": "9.c78s5",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 188248,
"reported_epoch": 5210,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:31:53.399844+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:31:53.399844+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:31:53.399844+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38071,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2022-08-18T16:34:47.612959+0000",
"might_have_unfound": [
{
"osd": "2(0)",
"status": "already probed"
},
{
"osd": "8(3)",
"status": "already probed"
},
{
"osd": "14(0)",
"status": "already probed"
},
{
"osd": "41(2)",
"status": "already probed"
},
{
"osd": "51(1)",
"status": "already probed"
},
{
"osd": "64(0)",
"status": "already probed"
},
{
"osd": "117(4)",
"status": "already probed"
},
{
"osd": "156(5)",
"status": "already probed"
}
],
"recovery_progress": {
"backfill_targets": [
"14(0)"
],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"recovery_ops": [],
"read_ops": []
}
}
},
{
"name": "Started",
"enter_time": "2022-08-18T16:34:43.465216+0000"
}
],
"agent_state": {}
}
</pre>
- Pool details
<pre>
root@f28-h28-000-r630:~/tracker57185
# ceph osd pool ls detail
pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 203 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr
pool 2 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 1780 lfor 0/0/403 flags hashpspool stripe_width 0 application rgw
pool 8 'default.rgw.control' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 128 pgp_num 128 autoscale_mode on last_change 1786 lfor 0/0/1779 flags hashpspool stripe_width 0 pg_num_min 128 application rgw
pool 9 'default.rgw.buckets.data' erasure profile myprofile size 6 min_size 4 crush_rule 1 object_hash rjenkins pg_num 4096 pgp_num 4096 autoscale_mode on last_change 4684 lfor 0/0/2046 flags hashpspool stripe_width 16384 pg_num_min 4096 application rgw
pool 10 'default.rgw.buckets.index' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 256 pgp_num 256 autoscale_mode on last_change 1790 lfor 0/0/1788 flags hashpspool stripe_width 0 pg_num_min 256 application rgw
pool 11 'default.rgw.log' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 128 pgp_num 128 autoscale_mode on last_change 1786 lfor 0/0/1781 flags hashpspool stripe_width 0 pg_num_min 128 application rgw
pool 12 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 128 pgp_num 128 autoscale_mode on last_change 1805 lfor 0/0/1801 flags hashpspool stripe_width 0 pg_num_min 128 application rgw
</pre>
<pre>
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "activating+degraded+remapped",
"epoch": 5861,
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"backfill_targets": [
"14(0)"
],
"acting_recovery_backfill": [
"4(0)",
"8(3)",
"14(0)",
"41(2)",
"51(1)",
"117(4)",
"156(5)"
],
"info": {
"pgid": "9.c78s0",
"last_update": "5210'48054",
"last_complete": "4727'47329",
"log_tail": "4722'45161",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 159430,
"reported_epoch": 5861,
"state": "activating+degraded+remapped",
"last_fresh": "2022-08-18T17:20:04.492434+0000",
"last_change": "2022-08-18T16:34:47.692664+0000",
"last_active": "2022-08-18T16:34:47.612904+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T17:20:04.492434+0000",
"last_undegraded": "2022-08-18T16:34:47.625898+0000",
"last_fullsized": "2022-08-18T17:20:04.492434+0000",
"mapping_epoch": 5215,
"log_start": "4722'45161",
"ondisk_log_start": "4722'45161",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 2893,
"ondisk_log_size": 2893,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 683,
"num_objects_missing": 683,
"num_objects_degraded": 683,
"num_objects_misplaced": 44885,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"2(0)",
"8(3)",
"41(2)",
"51(1)",
"64(0)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "2(0),8(3),41(2),51(1),64(0),117(4),156(5)",
"objects": 72
},
{
"shards": "2(0),8(3),41(2),51(1),117(4),156(5)",
"objects": 32
},
{
"shards": "4(0),8(3),41(2),51(1),117(4),156(5)",
"objects": 44964
},
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 500
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5218,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "2(0)",
"pgid": "9.c78s0",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "9:1e32a2e7:::59178bc5-0d1c-4138-85c4-42b6340da0d1.188855.3_TTcgroas%2f1580272.tl0uQj2FjJApUtlA.rnd:head",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 1,
"reported_epoch": 5138,
"state": "peering",
"last_fresh": "2022-08-18T14:59:15.543131+0000",
"last_change": "2022-08-18T14:59:15.543131+0000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2022-08-18T14:59:15.543131+0000",
"last_undegraded": "2022-08-18T14:59:15.543131+0000",
"last_fullsized": "2022-08-18T14:59:15.543131+0000",
"mapping_epoch": 5215,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2148793070,
"num_objects": 7520,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 7520,
"num_whiteouts": 0,
"num_read": 4,
"num_read_kb": 6,
"num_write": 38,
"num_write_kb": 21,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
8,
41,
51,
117,
156
],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "8(3)",
"pgid": "9.c78s3",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 158780,
"reported_epoch": 5215,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:31:53.399844+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:31:53.399844+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:31:53.399844+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38071,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "14(0)",
"pgid": "9.c78s0",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4722'45161",
"last_user_version": 0,
"last_backfill": "MIN",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"objects_scrubbed": 0,
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "--",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 45568,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [],
"acting": [],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 5218,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "41(2)",
"pgid": "9.c78s2",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 188248,
"reported_epoch": 5210,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:31:53.399844+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:31:53.399844+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:31:53.399844+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38071,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "51(1)",
"pgid": "9.c78s1",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 188294,
"reported_epoch": 5212,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:32:22.161926+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:32:22.161926+0000",
"last_peered": "2022-08-18T16:32:22.161926+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:32:22.161926+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38048,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56144,
"num_bytes_recovered": 2656143919,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "64(0)",
"pgid": "9.c78s0",
"last_update": "4973'48000",
"last_complete": "4973'48000",
"log_tail": "4724'46128",
"last_user_version": 48000,
"last_backfill": "9:1e3213f3:::59178bc5-0d1c-4138-85c4-42b6340da0d1.188831.2_LkZrS(rv%2f1416931.q)Ojo8f06c3YPl46.rnd:head",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 1,
"reported_epoch": 4913,
"state": "peering",
"last_fresh": "2022-08-18T13:48:06.401417+0000",
"last_change": "2022-08-18T13:48:06.401417+0000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2022-08-18T13:48:06.401417+0000",
"last_undegraded": "2022-08-18T13:48:06.401417+0000",
"last_fullsized": "2022-08-18T13:48:06.401417+0000",
"mapping_epoch": 5215,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 286374799,
"num_objects": 5927,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 5927,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 24,
"num_write_kb": 207,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
8,
41,
51,
117,
156
],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 4915,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "117(4)",
"pgid": "9.c78s4",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 188248,
"reported_epoch": 5210,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:31:53.399844+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:31:53.399844+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:31:53.399844+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38071,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "156(5)",
"pgid": "9.c78s5",
"last_update": "5210'48054",
"last_complete": "5210'48054",
"log_tail": "4724'46503",
"last_user_version": 48054,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1990,
"epoch_pool_created": 1772,
"last_epoch_started": 5140,
"last_interval_started": 5139,
"last_epoch_clean": 4685,
"last_interval_clean": 4684,
"last_epoch_split": 1990,
"last_epoch_marked_full": 0,
"same_up_since": 5215,
"same_interval_since": 5215,
"same_primary_since": 5213,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "5210'48054",
"reported_seq": 188248,
"reported_epoch": 5210,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2022-08-18T16:31:53.399844+0000",
"last_change": "2022-08-18T14:58:54.061043+0000",
"last_active": "2022-08-18T16:31:53.399844+0000",
"last_peered": "2022-08-18T16:31:53.399844+0000",
"last_clean": "2022-08-18T13:34:41.913797+0000",
"last_became_active": "2022-08-18T14:58:53.809588+0000",
"last_became_peered": "2022-08-18T14:58:53.809588+0000",
"last_unstale": "2022-08-18T16:31:53.399844+0000",
"last_undegraded": "2022-08-18T14:58:53.542830+0000",
"last_fullsized": "2022-08-18T14:58:53.532113+0000",
"mapping_epoch": 5215,
"log_start": "4724'46503",
"ondisk_log_start": "4724'46503",
"created": 1990,
"last_epoch_clean": 4685,
"parent": "0.0",
"parent_split_bits": 12,
"last_scrub": "0'0",
"last_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2022-08-16T17:02:35.249566+0000",
"last_clean_scrub_stamp": "2022-08-16T17:21:58.068345+0000",
"objects_scrubbed": 0,
"log_size": 1551,
"ondisk_log_size": 1551,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 1,
"scrub_schedule": "queued for scrub",
"scrub_duration": 0.0023479350000000002,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 2147351278,
"num_objects": 45568,
"num_object_clones": 0,
"num_object_copies": 273408,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 38071,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 45568,
"num_whiteouts": 0,
"num_read": 86096,
"num_read_kb": 1035336,
"num_write": 562975,
"num_write_kb": 2182704,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 56122,
"num_bytes_recovered": 2654949610,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
14,
51,
41,
8,
117,
156
],
"acting": [
4,
51,
41,
8,
117,
156
],
"avail_no_missing": [
"51(1)",
"8(3)",
"41(2)",
"117(4)",
"156(5)"
],
"object_location_counts": [
{
"shards": "8(3),41(2),51(1),117(4),156(5)",
"objects": 45568
}
],
"blocked_by": [],
"up_primary": 14,
"acting_primary": 4,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 5140,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2022-08-18T16:34:47.612959+0000",
"might_have_unfound": [
{
"osd": "2(0)",
"status": "already probed"
},
{
"osd": "8(3)",
"status": "already probed"
},
{
"osd": "14(0)",
"status": "already probed"
},
{
"osd": "41(2)",
"status": "already probed"
},
{
"osd": "51(1)",
"status": "already probed"
},
{
"osd": "64(0)",
"status": "already probed"
},
{
"osd": "117(4)",
"status": "already probed"
},
{
"osd": "156(5)",
"status": "already probed"
}
],
"recovery_progress": {
"backfill_targets": [
"14(0)"
],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"recovery_ops": [],
"read_ops": []
}
}
},
{
"name": "Started",
"enter_time": "2022-08-18T16:34:43.465216+0000"
}
],
"agent_state": {}
}
</pre>
- Pool details
<pre>
root@f28-h28-000-r630:~/tracker57185
# ceph osd pool ls detail
pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 203 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr
pool 2 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 1780 lfor 0/0/403 flags hashpspool stripe_width 0 application rgw
pool 8 'default.rgw.control' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 128 pgp_num 128 autoscale_mode on last_change 1786 lfor 0/0/1779 flags hashpspool stripe_width 0 pg_num_min 128 application rgw
pool 9 'default.rgw.buckets.data' erasure profile myprofile size 6 min_size 4 crush_rule 1 object_hash rjenkins pg_num 4096 pgp_num 4096 autoscale_mode on last_change 4684 lfor 0/0/2046 flags hashpspool stripe_width 16384 pg_num_min 4096 application rgw
pool 10 'default.rgw.buckets.index' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 256 pgp_num 256 autoscale_mode on last_change 1790 lfor 0/0/1788 flags hashpspool stripe_width 0 pg_num_min 256 application rgw
pool 11 'default.rgw.log' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 128 pgp_num 128 autoscale_mode on last_change 1786 lfor 0/0/1781 flags hashpspool stripe_width 0 pg_num_min 128 application rgw
pool 12 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 128 pgp_num 128 autoscale_mode on last_change 1805 lfor 0/0/1801 flags hashpspool stripe_width 0 pg_num_min 128 application rgw
</pre>