Subset of PGs whose acting primary is osd.34 PG_STAT STATE UP UP_PRIMARY ACTING ACTING_PRIMARY 6.15d active+remapped+backfilling [2,21,24,15] 2 [34,21,24,15] 34 6.15a active+remapped+backfilling [22,35,5,24] 22 [34,35,5,24] 34 6.11d active+remapped+backfilling [22,23,21,32] 22 [34,23,21,32] 34 6.114 active+remapped+backfilling [2,29,23,20] 2 [34,29,23,20] 34 6.d4 active+remapped+backfilling [2,24,5,31] 2 [34,24,5,31] 34 6.c5 active+remapped+backfilling [22,5,24,19] 22 [34,5,24,19] 34 6.90 active+remapped+backfilling [2,8,21,6] 2 [34,8,21,6] 34 6.2c active+remapped+backfilling [2,33,6,28] 2 [34,33,20,19] 34 6.35 active+remapped+backfilling [11,13,20,27] 11 [34,13,20,27] 34 6.f3 active+remapped+backfilling [30,31,12,5] 30 [34,31,12,5] 34 6.1ad active+remapped+backfilling [2,19,28,17] 2 [34,19,28,17] 34 PG query for one of the above PGs 6.15a: { "snap_trimq": "[]", "snap_trimq_len": 0, "state": "active+remapped+backfilling", "epoch": 21841, "up": [ 22, 35, 5, 24 ], "acting": [ 34, 35, 5, 24 ], "backfill_targets": [ "22(0)" ], "acting_recovery_backfill": [ "5(2)", "22(0)", "24(3)", "34(0)", "35(1)" ], "info": { "pgid": "6.15as0", "last_update": "21841'39052", "last_complete": "21841'39052", "log_tail": "18408'35148", "last_user_version": 39052, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 16000, "epoch_pool_created": 247, "last_epoch_started": 21583, "last_interval_started": 21582, "last_epoch_clean": 21568, "last_interval_clean": 21567, "last_epoch_split": 16000, "last_epoch_marked_full": 0, "same_up_since": 21581, "same_interval_since": 21582, "same_primary_since": 21582, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "21841'39052", "reported_seq": 241804, "reported_epoch": 21841, "state": "active+remapped+backfilling", "last_fresh": "2023-09-01T05:46:11.750238+0000", "last_change": "2023-09-01T01:55:36.433405+0000", "last_active": "2023-09-01T05:46:11.750238+0000", "last_peered": "2023-09-01T05:46:11.750238+0000", "last_clean": "2023-09-01T01:46:10.912687+0000", "last_became_active": "2023-09-01T01:55:34.991155+0000", "last_became_peered": "2023-09-01T01:55:34.991155+0000", "last_unstale": "2023-09-01T05:46:11.750238+0000", "last_undegraded": "2023-09-01T05:46:11.750238+0000", "last_fullsized": "2023-09-01T05:46:11.750238+0000", "mapping_epoch": 21582, "log_start": "18408'35148", "ondisk_log_start": "18408'35148", "created": 16000, "last_epoch_clean": 21568, "parent": "0.0", "parent_split_bits": 9, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "objects_scrubbed": 11609, "log_size": 3904, "ondisk_log_size": 3904, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 42, "scrub_schedule": "queued for scrub", "scrub_duration": 41.903327484999998, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 6907494400, "num_objects": 12063, "num_object_clones": 0, "num_object_copies": 48252, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 12063, "num_objects_unfound": 0, "num_objects_dirty": 12063, "num_whiteouts": 0, "num_read": 850, "num_read_kb": 430, "num_write": 55571, "num_write_kb": 726144, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 23678, "num_bytes_recovered": 13785628672, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 22, 35, 5, 24 ], "acting": [ 34, 35, 5, 24 ], "avail_no_missing": [ "34(0)", "5(2)", "24(3)", "35(1)" ], "object_location_counts": [ { "shards": "5(2),22(0),24(3),35(1)", "objects": 0 }, { "shards": "5(2),24(3),34(0),35(1)", "objects": 12063 } ], "blocked_by": [], "up_primary": 22, "acting_primary": 34, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 21583, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "5(2)", "pgid": "6.15as2", "last_update": "21841'39052", "last_complete": "21761'39016", "log_tail": "18408'35148", "last_user_version": 39015, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 16000, "epoch_pool_created": 247, "last_epoch_started": 21583, "last_interval_started": 21582, "last_epoch_clean": 21568, "last_interval_clean": 21567, "last_epoch_split": 16000, "last_epoch_marked_full": 0, "same_up_since": 21581, "same_interval_since": 21582, "same_primary_since": 21582, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "21580'39015", "reported_seq": 241518, "reported_epoch": 21580, "state": "active+clean", "last_fresh": "2023-09-01T01:46:10.912687+0000", "last_change": "2023-08-31T21:03:45.851294+0000", "last_active": "2023-09-01T01:46:10.912687+0000", "last_peered": "2023-09-01T01:46:10.912687+0000", "last_clean": "2023-09-01T01:46:10.912687+0000", "last_became_active": "2023-08-31T21:03:45.851134+0000", "last_became_peered": "2023-08-31T21:03:45.851134+0000", "last_unstale": "2023-09-01T01:46:10.912687+0000", "last_undegraded": "2023-09-01T01:46:10.912687+0000", "last_fullsized": "2023-09-01T01:46:10.912687+0000", "mapping_epoch": 21582, "log_start": "18408'35148", "ondisk_log_start": "18408'35148", "created": 16000, "last_epoch_clean": 21568, "parent": "0.0", "parent_split_bits": 9, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "objects_scrubbed": 11609, "log_size": 3867, "ondisk_log_size": 3867, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 42, "scrub_schedule": "periodic scrub scheduled @ 2023-09-01T05:15:41.307699+0000", "scrub_duration": 41.903327484999998, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 6906642432, "num_objects": 12050, "num_object_clones": 0, "num_object_copies": 48200, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 12050, "num_whiteouts": 0, "num_read": 816, "num_read_kb": 408, "num_write": 55386, "num_write_kb": 724992, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 23678, "num_bytes_recovered": 13785628672, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 22, 35, 5, 24 ], "acting": [ 34, 35, 5, 24 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 22, "acting_primary": 34, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 21583, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "22(0)", "pgid": "6.15as0", "last_update": "21841'39052", "last_complete": "21761'39016", "log_tail": "18408'35148", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 16000, "epoch_pool_created": 247, "last_epoch_started": 21583, "last_interval_started": 21582, "last_epoch_clean": 21568, "last_interval_clean": 21567, "last_epoch_split": 16000, "last_epoch_marked_full": 0, "same_up_since": 21581, "same_interval_since": 21582, "same_primary_since": 21582, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 1, "reported_epoch": 21581, "state": "peering", "last_fresh": "2023-09-01T01:55:31.936751+0000", "last_change": "2023-09-01T01:55:31.936751+0000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "2023-09-01T01:55:31.936751+0000", "last_undegraded": "2023-09-01T01:55:31.936751+0000", "last_fullsized": "2023-09-01T01:55:31.936751+0000", "mapping_epoch": 21582, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 16000, "last_epoch_clean": 21568, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "objects_scrubbed": 0, "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "periodic scrub scheduled @ 2023-09-01T05:16:08.619480+0000", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 12063, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 22, 35, 5, 24 ], "acting": [ 34, 35, 5, 24 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 5, 24, 34, 35 ], "up_primary": 22, "acting_primary": 34, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 21583, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "24(3)", "pgid": "6.15as3", "last_update": "21841'39052", "last_complete": "21761'39016", "log_tail": "18408'35148", "last_user_version": 39015, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 16000, "epoch_pool_created": 247, "last_epoch_started": 21583, "last_interval_started": 21582, "last_epoch_clean": 21568, "last_interval_clean": 21567, "last_epoch_split": 16000, "last_epoch_marked_full": 0, "same_up_since": 21581, "same_interval_since": 21582, "same_primary_since": 21582, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "21580'39015", "reported_seq": 241518, "reported_epoch": 21580, "state": "active+clean", "last_fresh": "2023-09-01T01:46:10.912687+0000", "last_change": "2023-08-31T21:03:45.851294+0000", "last_active": "2023-09-01T01:46:10.912687+0000", "last_peered": "2023-09-01T01:46:10.912687+0000", "last_clean": "2023-09-01T01:46:10.912687+0000", "last_became_active": "2023-08-31T21:03:45.851134+0000", "last_became_peered": "2023-08-31T21:03:45.851134+0000", "last_unstale": "2023-09-01T01:46:10.912687+0000", "last_undegraded": "2023-09-01T01:46:10.912687+0000", "last_fullsized": "2023-09-01T01:46:10.912687+0000", "mapping_epoch": 21582, "log_start": "18408'35148", "ondisk_log_start": "18408'35148", "created": 16000, "last_epoch_clean": 21568, "parent": "0.0", "parent_split_bits": 9, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "objects_scrubbed": 11609, "log_size": 3867, "ondisk_log_size": 3867, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 42, "scrub_schedule": "periodic scrub scheduled @ 2023-09-01T05:15:41.307699+0000", "scrub_duration": 41.903327484999998, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 6906642432, "num_objects": 12050, "num_object_clones": 0, "num_object_copies": 48200, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 12050, "num_whiteouts": 0, "num_read": 816, "num_read_kb": 408, "num_write": 55386, "num_write_kb": 724992, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 23678, "num_bytes_recovered": 13785628672, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 22, 35, 5, 24 ], "acting": [ 34, 35, 5, 24 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 22, "acting_primary": 34, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 21583, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "35(1)", "pgid": "6.15as1", "last_update": "21841'39052", "last_complete": "21761'39016", "log_tail": "18408'35148", "last_user_version": 39015, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 16000, "epoch_pool_created": 247, "last_epoch_started": 21583, "last_interval_started": 21582, "last_epoch_clean": 21568, "last_interval_clean": 21567, "last_epoch_split": 16000, "last_epoch_marked_full": 0, "same_up_since": 21581, "same_interval_since": 21582, "same_primary_since": 21582, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "21580'39015", "reported_seq": 241518, "reported_epoch": 21580, "state": "active+clean", "last_fresh": "2023-09-01T01:46:10.912687+0000", "last_change": "2023-08-31T21:03:45.851294+0000", "last_active": "2023-09-01T01:46:10.912687+0000", "last_peered": "2023-09-01T01:46:10.912687+0000", "last_clean": "2023-09-01T01:46:10.912687+0000", "last_became_active": "2023-08-31T21:03:45.851134+0000", "last_became_peered": "2023-08-31T21:03:45.851134+0000", "last_unstale": "2023-09-01T01:46:10.912687+0000", "last_undegraded": "2023-09-01T01:46:10.912687+0000", "last_fullsized": "2023-09-01T01:46:10.912687+0000", "mapping_epoch": 21582, "log_start": "18408'35148", "ondisk_log_start": "18408'35148", "created": 16000, "last_epoch_clean": 21568, "parent": "0.0", "parent_split_bits": 9, "last_scrub": "20846'38574", "last_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "last_deep_scrub": "19404'38078", "last_deep_scrub_stamp": "2023-08-29T06:19:39.263115+0000", "last_clean_scrub_stamp": "2023-08-30T17:21:38.299628+0000", "objects_scrubbed": 11609, "log_size": 3867, "ondisk_log_size": 3867, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 42, "scrub_schedule": "periodic scrub scheduled @ 2023-09-01T05:15:41.307699+0000", "scrub_duration": 41.903327484999998, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 6906642432, "num_objects": 12050, "num_object_clones": 0, "num_object_copies": 48200, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 12050, "num_whiteouts": 0, "num_read": 816, "num_read_kb": 408, "num_write": 55386, "num_write_kb": 724992, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 23678, "num_bytes_recovered": 13785628672, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 22, 35, 5, 24 ], "acting": [ 34, 35, 5, 24 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 22, "acting_primary": 34, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 21583, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Active", "enter_time": "2023-09-01T01:55:33.861794+0000", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [ "22(0)" ], "waiting_on_backfill": [], "last_backfill_started": "MIN", "backfill_info": { "begin": "MIN", "end": "MIN", "objects": [] }, "peer_backfill_info": [], "backfills_in_flight": [], "recovering": [], "pg_backend": { "recovery_ops": [], "read_ops": [] } } }, { "name": "Started", "enter_time": "2023-09-01T01:55:32.916530+0000" } ], "scrubber": { "active": false, "must_scrub": false, "must_deep_scrub": false, "must_repair": false, "need_auto": false, "scrub_reg_stamp": "2023-09-01T03:17:01.804470+0000", "schedule": "queued for scrub" }, "agent_state": {} }