# ceph pg 10.31 query
{
    "state": "active+remapped+backfilling",
    "snap_trimq": "[]",
    "epoch": 9617,
    "up": [
        24,
        2
    ],
    "acting": [
        21,
        13
    ],
    "backfill_targets": [
        "2",
        "24"
    ],
    "actingbackfill": [
        "2",
        "13",
        "21",
        "24"
    ],
    "info": {
        "pgid": "10.31",
        "last_update": "8294'4423149",
        "last_complete": "8294'4423149",
        "log_tail": "8294'4420100",
        "last_user_version": 4423149,
        "last_backfill": "MAX",
        "purged_snaps": "[]",
        "history": {
            "epoch_created": 4013,
            "last_epoch_started": 9617,
            "last_epoch_clean": 8851,
            "last_epoch_split": 0,
            "same_up_since": 9202,
            "same_interval_since": 9616,
            "same_primary_since": 9615,
            "last_scrub": "8294'4423149",
            "last_scrub_stamp": "2016-05-23 22:26:07.342255",
            "last_deep_scrub": "7667'2175097",
            "last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
            "last_clean_scrub_stamp": "2016-05-23 22:26:07.342255"
        },
        "stats": {
            "version": "8294'4423149",
            "reported_seq": "4424030",
            "reported_epoch": "9617",
            "state": "active+remapped",
            "last_fresh": "2016-05-25 18:18:48.224015",
            "last_change": "2016-05-25 18:18:48.224015",
            "last_active": "2016-05-25 18:18:48.224015",
            "last_peered": "2016-05-25 18:18:48.224015",
            "last_clean": "2016-05-25 12:33:45.886778",
            "last_became_active": "0.000000",
            "last_became_peered": "0.000000",
            "last_unstale": "2016-05-25 18:18:48.224015",
            "last_undegraded": "2016-05-25 18:18:48.224015",
            "last_fullsized": "2016-05-25 18:18:48.224015",
            "mapping_epoch": 9615,
            "log_start": "8294'4420100",
            "ondisk_log_start": "8294'4420100",
            "created": 4013,
            "last_epoch_clean": 8851,
            "parent": "0.0",
            "parent_split_bits": 0,
            "last_scrub": "8294'4423149",
            "last_scrub_stamp": "2016-05-23 22:26:07.342255",
            "last_deep_scrub": "7667'2175097",
            "last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
            "last_clean_scrub_stamp": "2016-05-23 22:26:07.342255",
            "log_size": 3049,
            "ondisk_log_size": 3049,
            "stats_invalid": "0",
            "stat_sum": {
                "num_bytes": 203105574400,
                "num_objects": 2,
                "num_object_clones": 0,
                "num_object_copies": 8,
                "num_objects_missing_on_primary": 0,
                "num_objects_degraded": 0,
                "num_objects_misplaced": 8,
                "num_objects_unfound": 0,
                "num_objects_dirty": 2,
                "num_whiteouts": 0,
                "num_read": 14,
                "num_read_kb": 0,
                "num_write": 4423148,
                "num_write_kb": 198365768,
                "num_scrub_errors": 0,
                "num_shallow_scrub_errors": 0,
                "num_deep_scrub_errors": 0,
                "num_objects_recovered": 0,
                "num_bytes_recovered": 0,
                "num_keys_recovered": 0,
                "num_objects_omap": 0,
                "num_objects_hit_set_archive": 0,
                "num_bytes_hit_set_archive": 0
            },
            "up": [
                24,
                2
            ],
            "acting": [
                21,
                13
            ],
            "blocked_by": [],
            "up_primary": 24,
            "acting_primary": 21
        },
        "empty": 0,
        "dne": 0,
        "incomplete": 0,
        "last_epoch_started": 9617,
        "hit_set_history": {
            "current_last_update": "0'0",
            "current_last_stamp": "0.000000",
            "current_info": {
                "begin": "0.000000",
                "end": "0.000000",
                "version": "0'0",
                "using_gmt": "0"
            },
            "history": []
        }
    },
    "peer_info": [
        {
            "peer": "2",
            "pgid": "10.31",
            "last_update": "8294'4423149",
            "last_complete": "8294'4423149",
            "log_tail": "8294'4420149",
            "last_user_version": 0,
            "last_backfill": "-1\/0\/\/0",
            "purged_snaps": "[]",
            "history": {
                "epoch_created": 4013,
                "last_epoch_started": 9617,
                "last_epoch_clean": 8851,
                "last_epoch_split": 0,
                "same_up_since": 9202,
                "same_interval_since": 9616,
                "same_primary_since": 9615,
                "last_scrub": "8294'4423149",
                "last_scrub_stamp": "2016-05-23 22:26:07.342255",
                "last_deep_scrub": "7667'2175097",
                "last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
                "last_clean_scrub_stamp": "2016-05-23 22:26:07.342255"
            },
            "stats": {
                "version": "0'0",
                "reported_seq": "0",
                "reported_epoch": "0",
                "state": "inactive",
                "last_fresh": "0.000000",
                "last_change": "0.000000",
                "last_active": "0.000000",
                "last_peered": "0.000000",
                "last_clean": "0.000000",
                "last_became_active": "0.000000",
                "last_became_peered": "0.000000",
                "last_unstale": "0.000000",
                "last_undegraded": "0.000000",
                "last_fullsized": "0.000000",
                "mapping_epoch": 9615,
                "log_start": "0'0",
                "ondisk_log_start": "0'0",
                "created": 0,
                "last_epoch_clean": 0,
                "parent": "0.0",
                "parent_split_bits": 0,
                "last_scrub": "0'0",
                "last_scrub_stamp": "0.000000",
                "last_deep_scrub": "0'0",
                "last_deep_scrub_stamp": "0.000000",
                "last_clean_scrub_stamp": "0.000000",
                "log_size": 0,
                "ondisk_log_size": 0,
                "stats_invalid": "0",
                "stat_sum": {
                    "num_bytes": 0,
                    "num_objects": 0,
                    "num_object_clones": 0,
                    "num_object_copies": 0,
                    "num_objects_missing_on_primary": 0,
                    "num_objects_degraded": 0,
                    "num_objects_misplaced": 0,
                    "num_objects_unfound": 0,
                    "num_objects_dirty": 0,
                    "num_whiteouts": 0,
                    "num_read": 0,
                    "num_read_kb": 0,
                    "num_write": 0,
                    "num_write_kb": 0,
                    "num_scrub_errors": 0,
                    "num_shallow_scrub_errors": 0,
                    "num_deep_scrub_errors": 0,
                    "num_objects_recovered": 0,
                    "num_bytes_recovered": 0,
                    "num_keys_recovered": 0,
                    "num_objects_omap": 0,
                    "num_objects_hit_set_archive": 0,
                    "num_bytes_hit_set_archive": 0
                },
                "up": [
                    24,
                    2
                ],
                "acting": [
                    21,
                    13
                ],
                "blocked_by": [],
                "up_primary": 24,
                "acting_primary": 21
            },
            "empty": 0,
            "dne": 0,
            "incomplete": 1,
            "last_epoch_started": 9617,
            "hit_set_history": {
                "current_last_update": "0'0",
                "current_last_stamp": "0.000000",
                "current_info": {
                    "begin": "0.000000",
                    "end": "0.000000",
                    "version": "0'0",
                    "using_gmt": "0"
                },
                "history": []
            }
        },
        {
            "peer": "13",
            "pgid": "10.31",
            "last_update": "8294'4423149",
            "last_complete": "8294'4423149",
            "log_tail": "8294'4420100",
            "last_user_version": 4423149,
            "last_backfill": "MAX",
            "purged_snaps": "[]",
            "history": {
                "epoch_created": 4013,
                "last_epoch_started": 9617,
                "last_epoch_clean": 8851,
                "last_epoch_split": 0,
                "same_up_since": 9202,
                "same_interval_since": 9616,
                "same_primary_since": 9615,
                "last_scrub": "8294'4423149",
                "last_scrub_stamp": "2016-05-23 22:26:07.342255",
                "last_deep_scrub": "7667'2175097",
                "last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
                "last_clean_scrub_stamp": "2016-05-23 22:26:07.342255"
            },
            "stats": {
                "version": "8294'4423149",
                "reported_seq": "4423834",
                "reported_epoch": "8853",
                "state": "peering",
                "last_fresh": "2016-05-25 17:45:35.778744",
                "last_change": "2016-05-25 17:45:35.778744",
                "last_active": "2016-05-22 22:12:19.771906",
                "last_peered": "2016-05-22 22:12:19.771906",
                "last_clean": "2016-05-22 22:12:19.771906",
                "last_became_active": "0.000000",
                "last_became_peered": "0.000000",
                "last_unstale": "2016-05-25 17:45:35.778744",
                "last_undegraded": "2016-05-25 17:45:35.778744",
                "last_fullsized": "2016-05-25 17:45:35.778744",
                "mapping_epoch": 9615,
                "log_start": "8294'4420100",
                "ondisk_log_start": "8294'4420100",
                "created": 4013,
                "last_epoch_clean": 8851,
                "parent": "0.0",
                "parent_split_bits": 0,
                "last_scrub": "8294'4423149",
                "last_scrub_stamp": "2016-05-23 22:26:07.342255",
                "last_deep_scrub": "7667'2175097",
                "last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
                "last_clean_scrub_stamp": "2016-05-23 22:26:07.342255",
                "log_size": 3049,
                "ondisk_log_size": 3049,
                "stats_invalid": "0",
                "stat_sum": {
                    "num_bytes": 203105574400,
                    "num_objects": 2,
                    "num_object_clones": 0,
                    "num_object_copies": 4,
                    "num_objects_missing_on_primary": 0,
                    "num_objects_degraded": 0,
                    "num_objects_misplaced": 0,
                    "num_objects_unfound": 0,
                    "num_objects_dirty": 2,
                    "num_whiteouts": 0,
                    "num_read": 14,
                    "num_read_kb": 0,
                    "num_write": 4423148,
                    "num_write_kb": 198365768,
                    "num_scrub_errors": 0,
                    "num_shallow_scrub_errors": 0,
                    "num_deep_scrub_errors": 0,
                    "num_objects_recovered": 0,
                    "num_bytes_recovered": 0,
                    "num_keys_recovered": 0,
                    "num_objects_omap": 0,
                    "num_objects_hit_set_archive": 0,
                    "num_bytes_hit_set_archive": 0
                },
                "up": [
                    24,
                    2
                ],
                "acting": [
                    21,
                    13
                ],
                "blocked_by": [],
                "up_primary": 24,
                "acting_primary": 21
            },
            "empty": 0,
            "dne": 0,
            "incomplete": 0,
            "last_epoch_started": 9617,
            "hit_set_history": {
                "current_last_update": "0'0",
                "current_last_stamp": "0.000000",
                "current_info": {
                    "begin": "0.000000",
                    "end": "0.000000",
                    "version": "0'0",
                    "using_gmt": "0"
                },
                "history": []
            }
        },
        {
            "peer": "24",
            "pgid": "10.31",
            "last_update": "8294'4423149",
            "last_complete": "8294'4423149",
            "log_tail": "8294'4420149",
            "last_user_version": 0,
            "last_backfill": "-1\/0\/\/0",
            "purged_snaps": "[]",
            "history": {
                "epoch_created": 4013,
                "last_epoch_started": 9617,
                "last_epoch_clean": 8851,
                "last_epoch_split": 0,
                "same_up_since": 9202,
                "same_interval_since": 9616,
                "same_primary_since": 9615,
                "last_scrub": "8294'4423149",
                "last_scrub_stamp": "2016-05-23 22:26:07.342255",
                "last_deep_scrub": "7667'2175097",
                "last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
                "last_clean_scrub_stamp": "2016-05-23 22:26:07.342255"
            },
            "stats": {
                "version": "8294'4423149",
                "reported_seq": "7",
                "reported_epoch": "9614",
                "state": "down+peering",
                "last_fresh": "2016-05-25 18:18:45.106431",
                "last_change": "2016-05-25 18:18:42.383132",
                "last_active": "0.000000",
                "last_peered": "0.000000",
                "last_clean": "0.000000",
                "last_became_active": "0.000000",
                "last_became_peered": "0.000000",
                "last_unstale": "2016-05-25 18:18:45.106431",
                "last_undegraded": "2016-05-25 18:18:45.106431",
                "last_fullsized": "2016-05-25 18:18:45.106431",
                "mapping_epoch": 9615,
                "log_start": "8294'4420149",
                "ondisk_log_start": "8294'4420149",
                "created": 4013,
                "last_epoch_clean": 8851,
                "parent": "0.0",
                "parent_split_bits": 0,
                "last_scrub": "8294'4423149",
                "last_scrub_stamp": "2016-05-23 22:26:07.342255",
                "last_deep_scrub": "7667'2175097",
                "last_deep_scrub_stamp": "2016-05-17 17:17:50.979695",
                "last_clean_scrub_stamp": "2016-05-23 22:26:07.342255",
                "log_size": 3000,
                "ondisk_log_size": 3000,
                "stats_invalid": "0",
                "stat_sum": {
                    "num_bytes": 0,
                    "num_objects": 0,
                    "num_object_clones": 0,
                    "num_object_copies": 0,
                    "num_objects_missing_on_primary": 0,
                    "num_objects_degraded": 0,
                    "num_objects_misplaced": 0,
                    "num_objects_unfound": 0,
                    "num_objects_dirty": 0,
                    "num_whiteouts": 0,
                    "num_read": 0,
                    "num_read_kb": 0,
                    "num_write": 0,
                    "num_write_kb": 0,
                    "num_scrub_errors": 0,
                    "num_shallow_scrub_errors": 0,
                    "num_deep_scrub_errors": 0,
                    "num_objects_recovered": 0,
                    "num_bytes_recovered": 0,
                    "num_keys_recovered": 0,
                    "num_objects_omap": 0,
                    "num_objects_hit_set_archive": 0,
                    "num_bytes_hit_set_archive": 0
                },
                "up": [
                    24,
                    2
                ],
                "acting": [
                    21,
                    13
                ],
                "blocked_by": [],
                "up_primary": 24,
                "acting_primary": 21
            },
            "empty": 0,
            "dne": 0,
            "incomplete": 1,
            "last_epoch_started": 9617,
            "hit_set_history": {
                "current_last_update": "0'0",
                "current_last_stamp": "0.000000",
                "current_info": {
                    "begin": "0.000000",
                    "end": "0.000000",
                    "version": "0'0",
                    "using_gmt": "0"
                },
                "history": []
            }
        }
    ],
    "recovery_state": [
        {
            "name": "Started\/Primary\/Active",
            "enter_time": "2016-05-25 18:18:48.111894",
            "might_have_unfound": [],
            "recovery_progress": {
                "backfill_targets": [
                    "2",
                    "24"
                ],
                "waiting_on_backfill": [],
                "last_backfill_started": "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head",
                "backfill_info": {
                    "begin": "MAX",
                    "end": "MAX",
                    "objects": []
                },
                "peer_backfill_info": [
                    "2",
                    {
                        "begin": "MAX",
                        "end": "MAX",
                        "objects": []
                    },
                    "24",
                    {
                        "begin": "MAX",
                        "end": "MAX",
                        "objects": []
                    }
                ],
                "backfills_in_flight": [
                    "10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
                    "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head"
                ],
                "recovering": [
                    "10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
                    "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head"
                ],
                "pg_backend": {
                    "pull_from_peer": [],
                    "pushing": [
                        {
                            "pushing": "10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
                            "pushing_to": [
                                {
                                    "pushing_to": "2",
                                    "push_info": {
                                        "recovery_progress": {
                                            "first?": 0,
                                            "data_complete?": 0,
                                            "data_recovered_to": 7801405440,
                                            "omap_complete?": 1,
                                            "omap_recovered_to": ""
                                        },
                                        "recovery_info": {
                                            "object": "10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
                                            "at_version": "7667'2175097",
                                            "size": "95731400192",
                                            "object_info": {
                                                "oid": {
                                                    "oid": "mysql.it-recht-kanzlei.de.2016-03-26.xva",
                                                    "key": "",
                                                    "snapid": -2,
                                                    "hash": 3148473649,
                                                    "max": 0,
                                                    "pool": 10,
                                                    "namespace": ""
                                                },
                                                "version": "7667'2175097",
                                                "prior_version": "7651'2175096",
                                                "last_reqid": "osd.21.0:2731328",
                                                "user_version": 2175096,
                                                "size": 95731400192,
                                                "mtime": "2016-03-26 19:57:53.420404",
                                                "local_mtime": "2016-03-26 19:58:06.503681",
                                                "lost": 0,
                                                "flags": 52,
                                                "wrlock_by": "unknown.0.0:0",
                                                "snaps": [],
                                                "truncate_seq": 0,
                                                "truncate_size": 0,
                                                "data_digest": 3369082238,
                                                "omap_digest": 4294967295,
                                                "watchers": {}
                                            },
                                            "snapset": {
                                                "snap_context": {
                                                    "seq": 0,
                                                    "snaps": []
                                                },
                                                "head_exists": 0,
                                                "clones": []
                                            },
                                            "copy_subset": "[0~95731400192]",
                                            "clone_subset": "{}"
                                        }
                                    }
                                },
                                {
                                    "pushing_to": "24",
                                    "push_info": {
                                        "recovery_progress": {
                                            "first?": 0,
                                            "data_complete?": 0,
                                            "data_recovered_to": 9546235904,
                                            "omap_complete?": 1,
                                            "omap_recovered_to": ""
                                        },
                                        "recovery_info": {
                                            "object": "10\/bba9e531\/mysql.it-recht-kanzlei.de.2016-03-26.xva\/head",
                                            "at_version": "7667'2175097",
                                            "size": "95731400192",
                                            "object_info": {
                                                "oid": {
                                                    "oid": "mysql.it-recht-kanzlei.de.2016-03-26.xva",
                                                    "key": "",
                                                    "snapid": -2,
                                                    "hash": 3148473649,
                                                    "max": 0,
                                                    "pool": 10,
                                                    "namespace": ""
                                                },
                                                "version": "7667'2175097",
                                                "prior_version": "7651'2175096",
                                                "last_reqid": "osd.21.0:2731328",
                                                "user_version": 2175096,
                                                "size": 95731400192,
                                                "mtime": "2016-03-26 19:57:53.420404",
                                                "local_mtime": "2016-03-26 19:58:06.503681",
                                                "lost": 0,
                                                "flags": 52,
                                                "wrlock_by": "unknown.0.0:0",
                                                "snaps": [],
                                                "truncate_seq": 0,
                                                "truncate_size": 0,
                                                "data_digest": 3369082238,
                                                "omap_digest": 4294967295,
                                                "watchers": {}
                                            },
                                            "snapset": {
                                                "snap_context": {
                                                    "seq": 0,
                                                    "snaps": []
                                                },
                                                "head_exists": 0,
                                                "clones": []
                                            },
                                            "copy_subset": "[0~95731400192]",
                                            "clone_subset": "{}"
                                        }
                                    }
                                }
                            ]
                        },
                        {
                            "pushing": "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head",
                            "pushing_to": [
                                {
                                    "pushing_to": "2",
                                    "push_info": {
                                        "recovery_progress": {
                                            "first?": 0,
                                            "data_complete?": 0,
                                            "data_recovered_to": 7793016832,
                                            "omap_complete?": 1,
                                            "omap_recovered_to": ""
                                        },
                                        "recovery_info": {
                                            "object": "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head",
                                            "at_version": "8294'4423149",
                                            "size": "107374174208",
                                            "object_info": {
                                                "oid": {
                                                    "oid": "mysql.it-recht-kanzlei.de.2016-05-22.xva",
                                                    "key": "",
                                                    "snapid": -2,
                                                    "hash": 4054523697,
                                                    "max": 0,
                                                    "pool": 10,
                                                    "namespace": ""
                                                },
                                                "version": "8294'4423149",
                                                "prior_version": "8294'4423148",
                                                "last_reqid": "client.174225.0:2248052",
                                                "user_version": 4423149,
                                                "size": 107374174208,
                                                "mtime": "2016-05-22 22:10:37.625987",
                                                "local_mtime": "0.000000",
                                                "lost": 0,
                                                "flags": 4,
                                                "wrlock_by": "unknown.0.0:0",
                                                "snaps": [],
                                                "truncate_seq": 0,
                                                "truncate_size": 0,
                                                "data_digest": 4294967295,
                                                "omap_digest": 4294967295,
                                                "watchers": {}
                                            },
                                            "snapset": {
                                                "snap_context": {
                                                    "seq": 0,
                                                    "snaps": []
                                                },
                                                "head_exists": 0,
                                                "clones": []
                                            },
                                            "copy_subset": "[0~107374174208]",
                                            "clone_subset": "{}"
                                        }
                                    }
                                },
                                {
                                    "pushing_to": "24",
                                    "push_info": {
                                        "recovery_progress": {
                                            "first?": 0,
                                            "data_complete?": 0,
                                            "data_recovered_to": 9537847296,
                                            "omap_complete?": 1,
                                            "omap_recovered_to": ""
                                        },
                                        "recovery_info": {
                                            "object": "10\/f1ab1f31\/mysql.it-recht-kanzlei.de.2016-05-22.xva\/head",
                                            "at_version": "8294'4423149",
                                            "size": "107374174208",
                                            "object_info": {
                                                "oid": {
                                                    "oid": "mysql.it-recht-kanzlei.de.2016-05-22.xva",
                                                    "key": "",
                                                    "snapid": -2,
                                                    "hash": 4054523697,
                                                    "max": 0,
                                                    "pool": 10,
                                                    "namespace": ""
                                                },
                                                "version": "8294'4423149",
                                                "prior_version": "8294'4423148",
                                                "last_reqid": "client.174225.0:2248052",
                                                "user_version": 4423149,
                                                "size": 107374174208,
                                                "mtime": "2016-05-22 22:10:37.625987",
                                                "local_mtime": "0.000000",
                                                "lost": 0,
                                                "flags": 4,
                                                "wrlock_by": "unknown.0.0:0",
                                                "snaps": [],
                                                "truncate_seq": 0,
                                                "truncate_size": 0,
                                                "data_digest": 4294967295,
                                                "omap_digest": 4294967295,
                                                "watchers": {}
                                            },
                                            "snapset": {
                                                "snap_context": {
                                                    "seq": 0,
                                                    "snaps": []
                                                },
                                                "head_exists": 0,
                                                "clones": []
                                            },
                                            "copy_subset": "[0~107374174208]",
                                            "clone_subset": "{}"
                                        }
                                    }
                                }
                            ]
                        }
                    ]
                }
            },
            "scrub": {
                "scrubber.epoch_start": "0",
                "scrubber.active": 0,
                "scrubber.waiting_on": 0,
                "scrubber.waiting_on_whom": []
            }
        },
        {
            "name": "Started",
            "enter_time": "2016-05-25 18:18:47.103175"
        }
    ],
    "agent_state": {}
}