Project

General

Profile

Download (18.8 KB)

Bug #64966 » pg.2.3.202403181512.json

Victor Kotlyar, 03/18/2024 12:25 PM

 
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "active+undersized+degraded+remapped+backfilling",
"epoch": 36291,
"up": [
191,
146,
213
],
"acting": [
191,
146
],
"backfill_targets": [
"213"
],
"acting_recovery_backfill": [
"146",
"191",
"213"
],
"info": {
"pgid": "2.3",
"last_update": "36291'65319852",
"last_complete": "36291'65319852",
"log_tail": "35197'65309801",
"last_user_version": 65319852,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 318,
"epoch_pool_created": 318,
"last_epoch_started": 36254,
"last_interval_started": 36253,
"last_epoch_clean": 35968,
"last_interval_clean": 35967,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 36252,
"same_interval_since": 36253,
"same_primary_since": 36253,
"last_scrub": "35613'65315243",
"last_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"last_deep_scrub": "34678'65293800",
"last_deep_scrub_stamp": "2024-03-13T09:41:52.962965+0530",
"last_clean_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"prior_readable_until_ub": 0
},
"stats": {
"version": "36291'65319852",
"reported_seq": 71504880,
"reported_epoch": 36291,
"state": "active+undersized+degraded+remapped+backfilling",
"last_fresh": "2024-03-18T17:48:01.461484+0530",
"last_change": "2024-03-18T16:37:01.955588+0530",
"last_active": "2024-03-18T17:48:01.461484+0530",
"last_peered": "2024-03-18T17:48:01.461484+0530",
"last_clean": "2024-03-18T11:37:59.663680+0530",
"last_became_active": "2024-03-18T16:37:01.848482+0530",
"last_became_peered": "2024-03-18T16:37:01.848482+0530",
"last_unstale": "2024-03-18T17:48:01.461484+0530",
"last_undegraded": "2024-03-18T16:37:01.816156+0530",
"last_fullsized": "2024-03-18T16:37:01.816057+0530",
"mapping_epoch": 36253,
"log_start": "35197'65309801",
"ondisk_log_start": "35197'65309801",
"created": 318,
"last_epoch_clean": 35968,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "35613'65315243",
"last_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"last_deep_scrub": "34678'65293800",
"last_deep_scrub_stamp": "2024-03-13T09:41:52.962965+0530",
"last_clean_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"objects_scrubbed": 67716,
"log_size": 10051,
"ondisk_log_size": 10051,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 18,
"scrub_schedule": "queued for scrub",
"scrub_duration": 17.235897828999999,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 12115060243,
"num_objects": 68272,
"num_object_clones": 0,
"num_object_copies": 204816,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 31185,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 68272,
"num_whiteouts": 0,
"num_read": 4913208,
"num_read_kb": 692070685,
"num_write": 65322317,
"num_write_kb": 193605513,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 37015,
"num_bytes_recovered": 6450026676,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
191,
146,
213
],
"acting": [
191,
146
],
"avail_no_missing": [
"191",
"146"
],
"object_location_counts": [
{
"shards": "146,191",
"objects": 68272
}
],
"blocked_by": [],
"up_primary": 191,
"acting_primary": 191,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 36254,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "146",
"pgid": "2.3",
"last_update": "36291'65319852",
"last_complete": "36254'65319533",
"log_tail": "35173'65309501",
"last_user_version": 65319532,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 318,
"epoch_pool_created": 318,
"last_epoch_started": 36254,
"last_interval_started": 36253,
"last_epoch_clean": 35968,
"last_interval_clean": 35967,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 36252,
"same_interval_since": 36253,
"same_primary_since": 36253,
"last_scrub": "35613'65315243",
"last_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"last_deep_scrub": "34678'65293800",
"last_deep_scrub_stamp": "2024-03-13T09:41:52.962965+0530",
"last_clean_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"prior_readable_until_ub": 0
},
"stats": {
"version": "36240'65319532",
"reported_seq": 71503890,
"reported_epoch": 36252,
"state": "remapped+peering",
"last_fresh": "2024-03-18T16:36:59.826326+0530",
"last_change": "2024-03-18T16:36:59.826326+0530",
"last_active": "2024-03-18T16:35:09.403074+0530",
"last_peered": "2024-03-18T16:36:58.801445+0530",
"last_clean": "2024-03-18T11:37:59.663680+0530",
"last_became_active": "2024-03-18T13:27:32.295790+0530",
"last_became_peered": "2024-03-18T16:36:53.736233+0530",
"last_unstale": "2024-03-18T16:36:59.826326+0530",
"last_undegraded": "2024-03-18T16:36:59.826326+0530",
"last_fullsized": "2024-03-18T16:36:59.826326+0530",
"mapping_epoch": 36253,
"log_start": "35173'65309501",
"ondisk_log_start": "35173'65309501",
"created": 318,
"last_epoch_clean": 35968,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "35613'65315243",
"last_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"last_deep_scrub": "34678'65293800",
"last_deep_scrub_stamp": "2024-03-13T09:41:52.962965+0530",
"last_clean_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"objects_scrubbed": 67716,
"log_size": 10031,
"ondisk_log_size": 10031,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 18,
"scrub_schedule": "queued for scrub",
"scrub_duration": 17.235897828999999,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 12119399586,
"num_objects": 68322,
"num_object_clones": 0,
"num_object_copies": 204966,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 68322,
"num_whiteouts": 0,
"num_read": 4912562,
"num_read_kb": 691866455,
"num_write": 65321992,
"num_write_kb": 193485851,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 37015,
"num_bytes_recovered": 6450026676,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
191,
146,
213
],
"acting": [
191,
146
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
191,
213
],
"up_primary": 191,
"acting_primary": 191,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 36254,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "213",
"pgid": "2.3",
"last_update": "36291'65319852",
"last_complete": "36254'65319533",
"log_tail": "35173'65309501",
"last_user_version": 65319532,
"last_backfill": "2:c1172f99:::1000081a9d7.00000000:head",
"purged_snaps": [],
"history": {
"epoch_created": 318,
"epoch_pool_created": 318,
"last_epoch_started": 36254,
"last_interval_started": 36253,
"last_epoch_clean": 35968,
"last_interval_clean": 35967,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 36252,
"same_interval_since": 36253,
"same_primary_since": 36253,
"last_scrub": "35613'65315243",
"last_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"last_deep_scrub": "34678'65293800",
"last_deep_scrub_stamp": "2024-03-13T09:41:52.962965+0530",
"last_clean_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"prior_readable_until_ub": 0
},
"stats": {
"version": "36240'65319532",
"reported_seq": 179,
"reported_epoch": 36247,
"state": "down",
"last_fresh": "2024-03-18T16:36:50.753915+0530",
"last_change": "2024-03-18T16:35:13.473966+0530",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2024-03-18T16:36:50.753915+0530",
"last_undegraded": "2024-03-18T16:36:50.753915+0530",
"last_fullsized": "2024-03-18T16:36:50.753915+0530",
"mapping_epoch": 36253,
"log_start": "35173'65309501",
"ondisk_log_start": "35173'65309501",
"created": 318,
"last_epoch_clean": 35968,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "35613'65315243",
"last_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"last_deep_scrub": "34678'65293800",
"last_deep_scrub_stamp": "2024-03-13T09:41:52.962965+0530",
"last_clean_scrub_stamp": "2024-03-17T09:41:26.694192+0530",
"objects_scrubbed": 0,
"log_size": 10031,
"ondisk_log_size": 10031,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "periodic scrub scheduled @ 2024-03-18T14:42:56.858818+0000",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 6609474304,
"num_objects": 37087,
"num_object_clones": 0,
"num_object_copies": 111351,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 31185,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 37087,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 518,
"num_write_kb": 247545,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
191,
146,
213
],
"acting": [
191,
146
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
27,
146,
191
],
"up_primary": 191,
"acting_primary": 191,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 36254,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2024-03-18T16:37:01.816104+0530",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [
"213"
],
"waiting_on_backfill": [],
"last_backfill_started": "2:c1172f99:::1000081a9d7.00000000:head",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
}
},
{
"name": "Started",
"enter_time": "2024-03-18T16:37:00.795187+0530"
}
],
"scrubber": {
"active": false,
"must_scrub": false,
"must_deep_scrub": false,
"must_repair": false,
"need_auto": false,
"scrub_reg_stamp": "2024-03-18T13:08:05.300273+0530",
"schedule": "queued for scrub"
},
"agent_state": {}
}
(1-1/8)