Same root cause as https://tracker.ceph.com/issues/45619.
http://pulpito.ceph.com/teuthology-2020-05-30_03:05:02-rgw-master-distro-basic-smithi/5104753/
2020-05-31T02:35:12.030+0000 7fa8991bd700 0 [balancer INFO root] ceph osd pg-upmap-items 10.1d mappings [{'from': 2, 'to': 0}]
2020-05-31T02:35:12.030+0000 7f9185c41700 1 -- [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] <== mgr.4099 172.21.15.181:0/13370 272 ==== mon_command({"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.1d", "id": [2, 0]} v 0) v1 ==== 123+0+0 (secure 0 0 0) 0x55aad79d8a80 con 0x55aad7965000
2020-05-31T02:35:13.030+0000 7f25b939e700 10 osd.0 52 _make_pg 10.1d
2020-05-31T02:35:13.714+0000 7f9185c41700 20 mon.a@0(leader).mgrstat health checks:
{
"PG_AVAILABILITY": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Reduced data availability: 1 pg inactive, 1 pg peering",
"count": 2
},
"detail": [
{
"message": "pg 10.1d is stuck peering since forever, current state peering, last acting [0,1]"
}
]
}
}
2020-05-31T02:35:14.058+0000 7f25b939e700 10 osd.0 pg_epoch: 53 pg[10.1d( v 51'26 (0'0,51'26] local-lis/les=52/53 n=5 ec=34/34 lis/c=52/52 les/c/f=53/53/0 sis=52) [0,1] r=0 lpr=52 crt=51'26 mlcod 0'0 active+clean] share_pg_info
2020-05-31T02:35:19.718+0000 7f9185c41700 20 mon.a@0(leader).mgrstat pending_digest:
"num_pg_by_state": [
{
"state": "active+clean",
"num": 585
}
],
http://pulpito.ceph.com/teuthology-2020-05-30_03:05:02-rgw-master-distro-basic-smithi/5104752/
2020-05-31T02:35:12.030+0000 7fa8991bd700 0 [balancer INFO root] ceph osd pg-upmap-items 10.1d mappings [{'from': 2, 'to': 0}]
2020-05-31T02:35:12.030+0000 7f9185c41700 1 -- [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] <== mgr.4099 172.21.15.181:0/13370 272 ==== mon_command({"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.1d", "id": [2, 0]} v 0) v1 ==== 123+0+0 (secure 0 0 0) 0x55aad79d8a80 con 0x55aad7965000
2020-05-31T02:35:13.030+0000 7f25b939e700 10 osd.0 52 _make_pg 10.1d
2020-05-31T02:35:13.714+0000 7f9185c41700 20 mon.a@0(leader).mgrstat health checks:
{
"PG_AVAILABILITY": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Reduced data availability: 1 pg inactive, 1 pg peering",
"count": 2
},
"detail": [
{
"message": "pg 10.1d is stuck peering since forever, current state peering, last acting [0,1]"
}
]
}
}
2020-05-31T02:35:14.058+0000 7f25b939e700 10 osd.0 pg_epoch: 53 pg[10.1d( v 51'26 (0'0,51'26] local-lis/les=52/53 n=5 ec=34/34 lis/c=52/52 les/c/f=53/53/0 sis=52) [0,1] r=0 lpr=52 crt=51'26 mlcod 0'0 active+clean] share_pg_info
2020-05-31T02:35:19.718+0000 7f9185c41700 20 mon.a@0(leader).mgrstat pending_digest:
"num_pg_by_state": [
{
"state": "active+clean",
"num": 585
}
],
2020-05-31T02:26:03.165+0000 7fafe17de700 10 osd.0 pg_epoch: 38 pg[10.10( empty local-lis/les=37/38 n=0 ec=34/34 lis/c=37/37 les/c/f=38/38/0 sis=37) [0,2] r=0 lpr=37 crt=0'0 mlcod 0'0 active+clean] share_pg_info
2020-05-31T02:26:08.957+0000 7fcc2b07c700 20 mon.a@0(leader).mgrstat pending_digest:
"num_pg_by_state": [
{
"state": "active+clean",
"num": 329
}
],