Project

General

Profile

Bug #16028 ยป pg_query.txt

ceph pg query 10.31 (stuck active+remapped) - Georg Stergiou, 05/25/2016 04:57 PM

 
# ceph pg 10.31 query
{
"state": "active+remapped+backfilling",
"snap_trimq": "[]",
"epoch": 9617,
"up": [
24,
2
],
"acting": [
21,
13
],
"backfill_targets": [
"2",
"24"
],
"actingbackfill": [
"2",
"13",
"21",
"24"
],
"info": {
"pgid": "10.31",
"last_update": "8294'4423149",
"last_complete": "8294'4423149",
"log_tail": "8294'4420100",
"last_user_version": 4423149,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": {
"epoch_created": 4013,
"last_epoch_started": 9617,
"last_epoch_clean": 8851,
"last_epoch_split": 0,
"same_up_since": 9202,
"same_interval_since": 9616,
"same_primary_since": 9615,
"last_scrub": "8294'4423149",
"last_scrub_stamp": "2016-05-23 22:26:07.342255",
"last_deep_scrub": "7667'2175097",
"last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
"last_clean_scrub_stamp": "2016-05-23 22:26:07.342255"
},
"stats": {
"version": "8294'4423149",
"reported_seq": "4424030",
"reported_epoch": "9617",
"state": "active+remapped",
"last_fresh": "2016-05-25 18:18:48.224015",
"last_change": "2016-05-25 18:18:48.224015",
"last_active": "2016-05-25 18:18:48.224015",
"last_peered": "2016-05-25 18:18:48.224015",
"last_clean": "2016-05-25 12:33:45.886778",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-25 18:18:48.224015",
"last_undegraded": "2016-05-25 18:18:48.224015",
"last_fullsized": "2016-05-25 18:18:48.224015",
"mapping_epoch": 9615,
"log_start": "8294'4420100",
"ondisk_log_start": "8294'4420100",
"created": 4013,
"last_epoch_clean": 8851,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "8294'4423149",
"last_scrub_stamp": "2016-05-23 22:26:07.342255",
"last_deep_scrub": "7667'2175097",
"last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
"last_clean_scrub_stamp": "2016-05-23 22:26:07.342255",
"log_size": 3049,
"ondisk_log_size": 3049,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 203105574400,
"num_objects": 2,
"num_object_clones": 0,
"num_object_copies": 8,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 8,
"num_objects_unfound": 0,
"num_objects_dirty": 2,
"num_whiteouts": 0,
"num_read": 14,
"num_read_kb": 0,
"num_write": 4423148,
"num_write_kb": 198365768,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
24,
2
],
"acting": [
21,
13
],
"blocked_by": [],
"up_primary": 24,
"acting_primary": 21
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 9617,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "0"
},
"history": []
}
},
"peer_info": [
{
"peer": "2",
"pgid": "10.31",
"last_update": "8294'4423149",
"last_complete": "8294'4423149",
"log_tail": "8294'4420149",
"last_user_version": 0,
"last_backfill": "-1\/0\/\/0",
"purged_snaps": "[]",
"history": {
"epoch_created": 4013,
"last_epoch_started": 9617,
"last_epoch_clean": 8851,
"last_epoch_split": 0,
"same_up_since": 9202,
"same_interval_since": 9616,
"same_primary_since": 9615,
"last_scrub": "8294'4423149",
"last_scrub_stamp": "2016-05-23 22:26:07.342255",
"last_deep_scrub": "7667'2175097",
"last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
"last_clean_scrub_stamp": "2016-05-23 22:26:07.342255"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 9615,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
24,
2
],
"acting": [
21,
13
],
"blocked_by": [],
"up_primary": 24,
"acting_primary": 21
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 9617,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "0"
},
"history": []
}
},
{
"peer": "13",
"pgid": "10.31",
"last_update": "8294'4423149",
"last_complete": "8294'4423149",
"log_tail": "8294'4420100",
"last_user_version": 4423149,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": {
"epoch_created": 4013,
"last_epoch_started": 9617,
"last_epoch_clean": 8851,
"last_epoch_split": 0,
"same_up_since": 9202,
"same_interval_since": 9616,
"same_primary_since": 9615,
"last_scrub": "8294'4423149",
"last_scrub_stamp": "2016-05-23 22:26:07.342255",
"last_deep_scrub": "7667'2175097",
"last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
"last_clean_scrub_stamp": "2016-05-23 22:26:07.342255"
},
"stats": {
"version": "8294'4423149",
"reported_seq": "4423834",
"reported_epoch": "8853",
"state": "peering",
"last_fresh": "2016-05-25 17:45:35.778744",
"last_change": "2016-05-25 17:45:35.778744",
"last_active": "2016-05-22 22:12:19.771906",
"last_peered": "2016-05-22 22:12:19.771906",
"last_clean": "2016-05-22 22:12:19.771906",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-25 17:45:35.778744",
"last_undegraded": "2016-05-25 17:45:35.778744",
"last_fullsized": "2016-05-25 17:45:35.778744",
"mapping_epoch": 9615,
"log_start": "8294'4420100",
"ondisk_log_start": "8294'4420100",
"created": 4013,
"last_epoch_clean": 8851,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "8294'4423149",
"last_scrub_stamp": "2016-05-23 22:26:07.342255",
"last_deep_scrub": "7667'2175097",
"last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
"last_clean_scrub_stamp": "2016-05-23 22:26:07.342255",
"log_size": 3049,
"ondisk_log_size": 3049,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 203105574400,
"num_objects": 2,
"num_object_clones": 0,
"num_object_copies": 4,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 2,
"num_whiteouts": 0,
"num_read": 14,
"num_read_kb": 0,
"num_write": 4423148,
"num_write_kb": 198365768,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
24,
2
],
"acting": [
21,
13
],
"blocked_by": [],
"up_primary": 24,
"acting_primary": 21
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 9617,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "0"
},
"history": []
}
},
{
"peer": "24",
"pgid": "10.31",
"last_update": "8294'4423149",
"last_complete": "8294'4423149",
"log_tail": "8294'4420149",
"last_user_version": 0,
"last_backfill": "-1\/0\/\/0",
"purged_snaps": "[]",
"history": {
"epoch_created": 4013,
"last_epoch_started": 9617,
"last_epoch_clean": 8851,
"last_epoch_split": 0,
"same_up_since": 9202,
"same_interval_since": 9616,
"same_primary_since": 9615,
"last_scrub": "8294'4423149",
"last_scrub_stamp": "2016-05-23 22:26:07.342255",
"last_deep_scrub": "7667'2175097",
"last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
"last_clean_scrub_stamp": "2016-05-23 22:26:07.342255"
},
"stats": {
"version": "8294'4423149",
"reported_seq": "7",
"reported_epoch": "9614",
"state": "down+peering",
"last_fresh": "2016-05-25 18:18:45.106431",
"last_change": "2016-05-25 18:18:42.383132",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-25 18:18:45.106431",
"last_undegraded": "2016-05-25 18:18:45.106431",
"last_fullsized": "2016-05-25 18:18:45.106431",
"mapping_epoch": 9615,
"log_start": "8294'4420149",
"ondisk_log_start": "8294'4420149",
"created": 4013,
"last_epoch_clean": 8851,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "8294'4423149",
"last_scrub_stamp": "2016-05-23 22:26:07.342255",
"last_deep_scrub": "7667'2175097",
"last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
"last_clean_scrub_stamp": "2016-05-23 22:26:07.342255",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0
},
"up": [
24,
2
],
"acting": [
21,
13
],
"blocked_by": [],
"up_primary": 24,
"acting_primary": 21
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 9617,
"hit_set_history": {
"current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0",
"using_gmt": "0"
},
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Active",
"enter_time": "2016-05-25 18:18:48.111894",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [
"2",
"24"
],
"waiting_on_backfill": [],
"last_backfill_started": "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head",
"backfill_info": {
"begin": "MAX",
"end": "MAX",
"objects": []
},
"peer_backfill_info": [
"2",
{
"begin": "MAX",
"end": "MAX",
"objects": []
},
"24",
{
"begin": "MAX",
"end": "MAX",
"objects": []
}
],
"backfills_in_flight": [
"10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
"10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head"
],
"recovering": [
"10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
"10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head"
],
"pg_backend": {
"pull_from_peer": [],
"pushing": [
{
"pushing": "10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
"pushing_to": [
{
"pushing_to": "2",
"push_info": {
"recovery_progress": {
"first?": 0,
"data_complete?": 0,
"data_recovered_to": 7801405440,
"omap_complete?": 1,
"omap_recovered_to": ""
},
"recovery_info": {
"object": "10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
"at_version": "7667'2175097",
"size": "95731400192",
"object_info": {
"oid": {
"oid": "mysql.it-recht-kanzlei.de.2016-03-26.xva",
"key": "",
"snapid": -2,
"hash": 3148473649,
"max": 0,
"pool": 10,
"namespace": ""
},
"version": "7667'2175097",
"prior_version": "7651'2175096",
"last_reqid": "osd.21.0:2731328",
"user_version": 2175096,
"size": 95731400192,
"mtime": "2016-03-26 19:57:53.420404",
"local_mtime": "2016-03-26 19:58:06.503681",
"lost": 0,
"flags": 52,
"wrlock_by": "unknown.0.0:0",
"snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": 3369082238,
"omap_digest": 4294967295,
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 0,
"snaps": []
},
"head_exists": 0,
"clones": []
},
"copy_subset": "[0~95731400192]",
"clone_subset": "{}"
}
}
},
{
"pushing_to": "24",
"push_info": {
"recovery_progress": {
"first?": 0,
"data_complete?": 0,
"data_recovered_to": 9546235904,
"omap_complete?": 1,
"omap_recovered_to": ""
},
"recovery_info": {
"object": "10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
"at_version": "7667'2175097",
"size": "95731400192",
"object_info": {
"oid": {
"oid": "mysql.it-recht-kanzlei.de.2016-03-26.xva",
"key": "",
"snapid": -2,
"hash": 3148473649,
"max": 0,
"pool": 10,
"namespace": ""
},
"version": "7667'2175097",
"prior_version": "7651'2175096",
"last_reqid": "osd.21.0:2731328",
"user_version": 2175096,
"size": 95731400192,
"mtime": "2016-03-26 19:57:53.420404",
"local_mtime": "2016-03-26 19:58:06.503681",
"lost": 0,
"flags": 52,
"wrlock_by": "unknown.0.0:0",
"snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": 3369082238,
"omap_digest": 4294967295,
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 0,
"snaps": []
},
"head_exists": 0,
"clones": []
},
"copy_subset": "[0~95731400192]",
"clone_subset": "{}"
}
}
}
]
},
{
"pushing": "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head",
"pushing_to": [
{
"pushing_to": "2",
"push_info": {
"recovery_progress": {
"first?": 0,
"data_complete?": 0,
"data_recovered_to": 7793016832,
"omap_complete?": 1,
"omap_recovered_to": ""
},
"recovery_info": {
"object": "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head",
"at_version": "8294'4423149",
"size": "107374174208",
"object_info": {
"oid": {
"oid": "mysql.it-recht-kanzlei.de.2016-05-22.xva",
"key": "",
"snapid": -2,
"hash": 4054523697,
"max": 0,
"pool": 10,
"namespace": ""
},
"version": "8294'4423149",
"prior_version": "8294'4423148",
"last_reqid": "client.174225.0:2248052",
"user_version": 4423149,
"size": 107374174208,
"mtime": "2016-05-22 22:10:37.625987",
"local_mtime": "0.000000",
"lost": 0,
"flags": 4,
"wrlock_by": "unknown.0.0:0",
"snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": 4294967295,
"omap_digest": 4294967295,
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 0,
"snaps": []
},
"head_exists": 0,
"clones": []
},
"copy_subset": "[0~107374174208]",
"clone_subset": "{}"
}
}
},
{
"pushing_to": "24",
"push_info": {
"recovery_progress": {
"first?": 0,
"data_complete?": 0,
"data_recovered_to": 9537847296,
"omap_complete?": 1,
"omap_recovered_to": ""
},
"recovery_info": {
"object": "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head",
"at_version": "8294'4423149",
"size": "107374174208",
"object_info": {
"oid": {
"oid": "mysql.it-recht-kanzlei.de.2016-05-22.xva",
"key": "",
"snapid": -2,
"hash": 4054523697,
"max": 0,
"pool": 10,
"namespace": ""
},
"version": "8294'4423149",
"prior_version": "8294'4423148",
"last_reqid": "client.174225.0:2248052",
"user_version": 4423149,
"size": 107374174208,
"mtime": "2016-05-22 22:10:37.625987",
"local_mtime": "0.000000",
"lost": 0,
"flags": 4,
"wrlock_by": "unknown.0.0:0",
"snaps": [],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": 4294967295,
"omap_digest": 4294967295,
"watchers": {}
},
"snapset": {
"snap_context": {
"seq": 0,
"snaps": []
},
"head_exists": 0,
"clones": []
},
"copy_subset": "[0~107374174208]",
"clone_subset": "{}"
}
}
}
]
}
]
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2016-05-25 18:18:47.103175"
}
],
"agent_state": {}
}
    (1-1/1)