-273> 2020-07-06 18:40:14.306835 7f83a92be700 10 osd.24 45014 do_waiters -- start -272> 2020-07-06 18:40:14.306840 7f83a92be700 10 osd.24 45014 do_waiters -- finish -271> 2020-07-06 18:40:14.306842 7f83a92be700 20 osd.24 45014 _dispatch 0x7f83cef418c0 osd_map(45014..45014 src has 38101..45014) v3 -270> 2020-07-06 18:40:14.306852 7f83a92be700 3 osd.24.objecter handle_osd_map ignoring epochs [45014,45014] <= 45014 -269> 2020-07-06 18:40:14.306856 7f83a92be700 20 osd.24.objecter dump_active .. 0 homeless -268> 2020-07-06 18:40:14.306862 7f83a92be700 3 osd.24 45014 handle_osd_map epochs [45014,45014], i have 45014, src has [38101,45014] -267> 2020-07-06 18:40:14.306865 7f83a92be700 10 osd.24 45014 no new maps here, dropping -266> 2020-07-06 18:40:14.306869 7f83a92be700 10 osd.24 45014 do_waiters -- start -265> 2020-07-06 18:40:14.306871 7f83a92be700 10 osd.24 45014 do_waiters -- finish -264> 2020-07-06 18:40:14.306874 7f83a92be700 10 osd.24 45014 do_waiters -- start -263> 2020-07-06 18:40:14.306876 7f83a92be700 10 osd.24 45014 do_waiters -- finish -262> 2020-07-06 18:40:14.306878 7f83a92be700 20 osd.24 45014 _dispatch 0x7f83cf0d8760 pg_notify(2.29(8) epoch 45014) v5 -261> 2020-07-06 18:40:14.306894 7f83a92be700 7 osd.24 45014 handle_pg_notify from osd.135 -260> 2020-07-06 18:40:14.306898 7f83a92be700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83cf0d8760 -259> 2020-07-06 18:40:14.306902 7f8389c90700 10 log is not dirty -258> 2020-07-06 18:40:14.306942 7f83a92be700 10 osd.24 45014 do_waiters -- start -257> 2020-07-06 18:40:14.306947 7f83a92be700 10 osd.24 45014 do_waiters -- finish -256> 2020-07-06 18:40:14.306938 7f838ac92700 10 osd.24 pg_epoch: 45014 pg[2.29( v 24074'367 (0'0,24074'367] local-les=45014 n=78 ec=224 les/c/f 44999/44646/0 45012/45013/45013) [24,144,11] r=0 lpr=45013 pi=44298-45012/25 crt=24074'367 lcod 0'0 mlcod 0'0 activating+degraded] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MNotifyRec from 135 notify: (query_epoch:45014, epoch_sent:45014, info:2.29( v 24074'367 (0'0,24074'367] local-les=44646 n=78 ec=224 les/c/f 44646/44646/0 45012/45013/45013)) features: 0x7ffffffefdfbfff -255> 2020-07-06 18:40:14.306960 7f838ac92700 10 osd.24 pg_epoch: 45014 pg[2.29( v 24074'367 (0'0,24074'367] local-les=45014 n=78 ec=224 les/c/f 44999/44646/0 45012/45013/45013) [24,144,11] r=0 lpr=45013 pi=44298-45012/25 crt=24074'367 lcod 0'0 mlcod 0'0 activating+degraded] state: Active: got notify from 135, already have info from that osd, ignoring -254> 2020-07-06 18:40:14.306970 7f838ac92700 10 log is not dirty -253> 2020-07-06 18:40:14.307580 7f83a8abd700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.197:6818/3452362 conn(0x7f83d0d9d000 sd=1494 :6805 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=96695 cs=1 l=0). == rx == osd.28 seq 5 0x7f83cf9b9200 osd_map(45014..45014 src has 38101..45014) v3 -252> 2020-07-06 18:40:14.307618 7f83a8abd700 10 osd.24 45014 do_waiters -- start -251> 2020-07-06 18:40:14.307620 7f83a8abd700 10 osd.24 45014 do_waiters -- finish -250> 2020-07-06 18:40:14.307622 7f83a8abd700 20 osd.24 45014 _dispatch 0x7f83cf9b9200 osd_map(45014..45014 src has 38101..45014) v3 -249> 2020-07-06 18:40:14.307628 7f83a8abd700 3 osd.24.objecter handle_osd_map ignoring epochs [45014,45014] <= 45014 -248> 2020-07-06 18:40:14.307631 7f83a8abd700 20 osd.24.objecter dump_active .. 0 homeless -247> 2020-07-06 18:40:14.307634 7f83a8abd700 3 osd.24 45014 handle_osd_map epochs [45014,45014], i have 45014, src has [38101,45014] -246> 2020-07-06 18:40:14.307637 7f83a8abd700 10 osd.24 45014 no new maps here, dropping -245> 2020-07-06 18:40:14.307639 7f83a8abd700 10 osd.24 45014 do_waiters -- start -244> 2020-07-06 18:40:14.307641 7f83a8abd700 10 osd.24 45014 do_waiters -- finish -243> 2020-07-06 18:40:14.307698 7f83a8abd700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.197:6818/3452362 conn(0x7f83d0d9d000 sd=1494 :6805 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=96695 cs=1 l=0). == rx == osd.28 seq 6 0x7f83d1110760 pg_notify(1.42c(8) epoch 45014) v5 -242> 2020-07-06 18:40:14.307722 7f83a8abd700 10 osd.24 45014 do_waiters -- start -241> 2020-07-06 18:40:14.307726 7f83a8abd700 10 osd.24 45014 do_waiters -- finish -240> 2020-07-06 18:40:14.307727 7f83a8abd700 20 osd.24 45014 _dispatch 0x7f83d1110760 pg_notify(1.42c(8) epoch 45014) v5 -239> 2020-07-06 18:40:14.307732 7f83a8abd700 7 osd.24 45014 handle_pg_notify from osd.28 -238> 2020-07-06 18:40:14.307734 7f83a8abd700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83d1110760 -237> 2020-07-06 18:40:14.307785 7f83a8abd700 10 osd.24 45014 do_waiters -- start -236> 2020-07-06 18:40:14.307788 7f83a8abd700 10 osd.24 45014 do_waiters -- finish -235> 2020-07-06 18:40:14.307785 7f838cc96700 10 osd.24 pg_epoch: 45014 pg[1.42c( v 44756'19846432 lc 44133'19846430 (33550'19843351,44756'19846432] local-les=44999 n=5552 ec=30458 les/c/f 44999/44790/0 45012/45012/45012) [24,150,4]/[24,4,12] r=0 lpr=45012 pi=44281-45011/21 crt=44756'19846432 lcod 0'0 mlcod 0'0 remapped+peering m=2] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MNotifyRec from 28 notify: (query_epoch:45014, epoch_sent:45014, info:1.42c( v 44756'19846432 (33550'19843431,44756'19846432] local-les=44790 n=5552 ec=30458 les/c/f 44790/44790/0 45012/45012/45012)) features: 0x7ffffffefdfbfff -234> 2020-07-06 18:40:14.307832 7f838cc96700 10 osd.24 pg_epoch: 45014 pg[1.42c( v 44756'19846432 lc 44133'19846430 (33550'19843351,44756'19846432] local-les=44999 n=5552 ec=30458 les/c/f 44999/44790/0 45012/45012/45012) [24,150,4]/[24,4,12] r=0 lpr=45012 pi=44281-45011/21 crt=44756'19846432 lcod 0'0 mlcod 0'0 remapped+peering m=2] got dup osd.28 info 1.42c( v 44756'19846432 (33550'19843431,44756'19846432] local-les=44790 n=5552 ec=30458 les/c/f 44790/44790/0 45012/45012/45012), identical to ours -233> 2020-07-06 18:40:14.307895 7f838cc96700 10 log is not dirty -232> 2020-07-06 18:40:14.310047 7f83ad2c6700 1 -- 172.26.224.197:6806/2876125 >> 172.26.224.166:0/629113 conn(0x7f83d083f800 sd=1424 :6806 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12138 cs=1 l=1). == rx == osd.23 seq 1 0x7f83c38fea00 osd_ping(ping e45014 stamp 2020-07-06 18:40:14.307073) v2 -231> 2020-07-06 18:40:14.310054 7f83adac7700 1 -- 172.26.216.197:6815/2876125 >> 172.26.224.166:0/629113 conn(0x7f83d0eae000 sd=1531 :6815 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12139 cs=1 l=1). == rx == osd.23 seq 1 0x7f83cf97a400 osd_ping(ping e45014 stamp 2020-07-06 18:40:14.307073) v2 -230> 2020-07-06 18:40:14.310105 7f83ad2c6700 1 -- 172.26.224.197:6806/2876125 >> 172.26.224.166:0/629113 conn(0x7f83d083f800 sd=1424 :6806 s=STATE_OPEN pgs=12138 cs=1 l=1). == tx == 0x7f83c38fe800 osd_ping(ping_reply e45014 stamp 2020-07-06 18:40:14.307073) v2 -229> 2020-07-06 18:40:14.310142 7f83ad2c6700 20 osd.24 45014 share_map_peer 0x7f83d07be800 already has epoch 45014 -228> 2020-07-06 18:40:14.310165 7f83adac7700 1 -- 172.26.216.197:6815/2876125 >> 172.26.224.166:0/629113 conn(0x7f83d0eae000 sd=1531 :6815 s=STATE_OPEN pgs=12139 cs=1 l=1). == tx == 0x7f83cfb85e00 osd_ping(ping_reply e45014 stamp 2020-07-06 18:40:14.307073) v2 -227> 2020-07-06 18:40:14.310205 7f83adac7700 20 osd.24 45014 share_map_peer 0x7f83d07be800 already has epoch 45014 -226> 2020-07-06 18:40:14.335432 7f83a72ba700 1 -- 172.26.224.197:6806/2876125 >> 172.26.216.143:0/32319 conn(0x7f83d079b000 sd=1404 :6806 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=9638 cs=1 l=1). == rx == osd.139 seq 1 0x7f83d098d600 osd_ping(ping e45014 stamp 2020-07-06 18:40:14.331379) v2 -225> 2020-07-06 18:40:14.335478 7f83a72ba700 1 -- 172.26.224.197:6806/2876125 >> 172.26.216.143:0/32319 conn(0x7f83d079b000 sd=1404 :6806 s=STATE_OPEN pgs=9638 cs=1 l=1). == tx == 0x7f83d02ed600 osd_ping(ping_reply e45014 stamp 2020-07-06 18:40:14.331379) v2 -224> 2020-07-06 18:40:14.335504 7f83a72ba700 20 osd.24 45014 share_map_peer 0x7f83d0719000 already has epoch 45014 -223> 2020-07-06 18:40:14.335489 7f83aaac1700 1 -- 172.26.216.197:6815/2876125 >> 172.26.216.143:0/32319 conn(0x7f83d0e8c000 sd=1521 :6815 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=9639 cs=1 l=1). == rx == osd.139 seq 1 0x7f83cf7a9000 osd_ping(ping e45014 stamp 2020-07-06 18:40:14.331379) v2 -222> 2020-07-06 18:40:14.335547 7f83aaac1700 1 -- 172.26.216.197:6815/2876125 >> 172.26.216.143:0/32319 conn(0x7f83d0e8c000 sd=1521 :6815 s=STATE_OPEN pgs=9639 cs=1 l=1). == tx == 0x7f83cfea4200 osd_ping(ping_reply e45014 stamp 2020-07-06 18:40:14.331379) v2 -221> 2020-07-06 18:40:14.335592 7f83aaac1700 20 osd.24 45014 share_map_peer 0x7f83d0719000 already has epoch 45014 -220> 2020-07-06 18:40:14.343119 7f83a82bc700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.176:6810/536802 conn(0x7f83d09af800 sd=1460 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=621 cs=1 l=0). == rx == osd.153 seq 32 0x7f83d1719b00 osd_map(45014..45014 src has 38101..45014) v3 -219> 2020-07-06 18:40:14.343221 7f83a82bc700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.176:6810/536802 conn(0x7f83d09af800 sd=1460 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=621 cs=1 l=0). == rx == osd.153 seq 33 0x7f83d1a090e0 pg_notify(1.23e(25) epoch 45014) v5 -218> 2020-07-06 18:40:14.343247 7f83a82bc700 10 osd.24 45014 do_waiters -- start -217> 2020-07-06 18:40:14.343254 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -216> 2020-07-06 18:40:14.343256 7f83a82bc700 20 osd.24 45014 _dispatch 0x7f83d1719b00 osd_map(45014..45014 src has 38101..45014) v3 -215> 2020-07-06 18:40:14.343264 7f83a82bc700 3 osd.24.objecter handle_osd_map ignoring epochs [45014,45014] <= 45014 -214> 2020-07-06 18:40:14.343267 7f83a82bc700 20 osd.24.objecter dump_active .. 0 homeless -213> 2020-07-06 18:40:14.343272 7f83a82bc700 3 osd.24 45014 handle_osd_map epochs [45014,45014], i have 45014, src has [38101,45014] -212> 2020-07-06 18:40:14.343274 7f83a82bc700 10 osd.24 45014 no new maps here, dropping -211> 2020-07-06 18:40:14.343278 7f83a82bc700 10 osd.24 45014 do_waiters -- start -210> 2020-07-06 18:40:14.343279 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -209> 2020-07-06 18:40:14.343281 7f83a82bc700 10 osd.24 45014 do_waiters -- start -208> 2020-07-06 18:40:14.343282 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -207> 2020-07-06 18:40:14.343284 7f83a82bc700 20 osd.24 45014 _dispatch 0x7f83d1a090e0 pg_notify(1.23e(25) epoch 45014) v5 -206> 2020-07-06 18:40:14.343290 7f83a82bc700 7 osd.24 45014 handle_pg_notify from osd.153 -205> 2020-07-06 18:40:14.343292 7f83a82bc700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83d1a090e0 -204> 2020-07-06 18:40:14.343358 7f83a82bc700 10 osd.24 45014 do_waiters -- start -203> 2020-07-06 18:40:14.343361 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -202> 2020-07-06 18:40:14.343374 7f839049d700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 44999/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 activating+degraded+remapped m=2] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MNotifyRec from 153 notify: (query_epoch:45014, epoch_sent:45014, info:1.23e( v 44314'17115402 (33550'17112402,44314'17115402] lb MIN (bitwise) local-les=44984 n=0 ec=30458 les/c/f 44999/44645/0 45012/45013/45013)) features: 0x7ffffffefdfbfff -201> 2020-07-06 18:40:14.343416 7f839049d700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 44999/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 activating+degraded+remapped m=2] state: Active: got notify from 153, already have info from that osd, ignoring -200> 2020-07-06 18:40:14.343468 7f839049d700 10 log is not dirty -199> 2020-07-06 18:40:14.344584 7f83a82bc700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.176:6810/536802 conn(0x7f83d09af800 sd=1460 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=621 cs=1 l=0). == rx == osd.153 seq 34 0x7f83d1a090e0 pg_info(1 pgs e45014:2.26a(8)) v4 -198> 2020-07-06 18:40:14.344613 7f83a82bc700 10 osd.24 45014 do_waiters -- start -197> 2020-07-06 18:40:14.344615 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -196> 2020-07-06 18:40:14.344619 7f83a82bc700 20 osd.24 45014 _dispatch 0x7f83d1a090e0 pg_info(1 pgs e45014:2.26a(8)) v4 -195> 2020-07-06 18:40:14.344626 7f83a82bc700 7 osd.24 45014 handle_pg_info pg_info(1 pgs e45014:2.26a(8)) v4 from osd.153 -194> 2020-07-06 18:40:14.344629 7f83a82bc700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83d1a090e0 -193> 2020-07-06 18:40:14.344659 7f83a82bc700 10 osd.24 45014 do_waiters -- start -192> 2020-07-06 18:40:14.344663 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -191> 2020-07-06 18:40:14.344662 7f838d497700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=44999 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MInfoRec from 153 info: 2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) -190> 2020-07-06 18:40:14.344686 7f838d497700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=44999 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive NOTIFY] state: got info from osd.153 2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) -189> 2020-07-06 18:40:14.344700 7f838d497700 5 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=44999 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive NOTIFY] exit Started/Stray 1.133501 7 0.000605 -188> 2020-07-06 18:40:14.344713 7f838d497700 5 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=44999 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive -187> 2020-07-06 18:40:14.344724 7f838d497700 5 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=44999 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive/RepNotRecovering -186> 2020-07-06 18:40:14.344735 7f838d497700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=44999 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate -185> 2020-07-06 18:40:14.344746 7f838d497700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive] activate - no missing, moving last_complete 24074'356 -> 24074'356 -184> 2020-07-06 18:40:14.344778 7f838d497700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive] state: Activate Finished -183> 2020-07-06 18:40:14.344853 7f838d497700 10 log is not dirty -182> 2020-07-06 18:40:14.345087 7f8394bc9700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 crt=24074'356 lcod 0'0 inactive] _activate_committed 45014 telling primary -181> 2020-07-06 18:40:14.345148 7f8394bc9700 20 osd.24 45014 share_map_peer 0x7f83d09af800 already has epoch 45014 -180> 2020-07-06 18:40:14.345154 7f8394bc9700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.176:6810/536802 conn(0x7f83d09af800 sd=1460 :-1 s=STATE_OPEN pgs=621 cs=1 l=0). == tx == 0x7f83d04d7a40 pg_info(1 pgs e45014:2.26a) v4 -179> 2020-07-06 18:40:14.345241 7f8394bc9700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 luod=0'0 crt=24074'356 lcod 0'0 active] flushed -178> 2020-07-06 18:40:14.345300 7f838bc94700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 luod=0'0 crt=24074'356 lcod 0'0 active] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 FlushedEvt -177> 2020-07-06 18:40:14.345335 7f838bc94700 15 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 luod=0'0 crt=24074'356 lcod 0'0 active] requeue_ops -176> 2020-07-06 18:40:14.345352 7f838bc94700 10 log is not dirty -175> 2020-07-06 18:40:14.345577 7f83a82bc700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.176:6810/536802 conn(0x7f83d09af800 sd=1460 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=621 cs=1 l=0). == rx == osd.153 seq 35 0x7f83d04d7a40 pg_info(1 pgs e45014:2.26a) v4 -174> 2020-07-06 18:40:14.345605 7f83a82bc700 10 osd.24 45014 do_waiters -- start -173> 2020-07-06 18:40:14.345612 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -172> 2020-07-06 18:40:14.345614 7f83a82bc700 20 osd.24 45014 _dispatch 0x7f83d04d7a40 pg_info(1 pgs e45014:2.26a) v4 -171> 2020-07-06 18:40:14.345620 7f83a82bc700 7 osd.24 45014 handle_pg_info pg_info(1 pgs e45014:2.26a) v4 from osd.153 -170> 2020-07-06 18:40:14.345626 7f83a82bc700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83d04d7a40 -169> 2020-07-06 18:40:14.345652 7f83a82bc700 10 osd.24 45014 do_waiters -- start -168> 2020-07-06 18:40:14.345655 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -167> 2020-07-06 18:40:14.345700 7f838a491700 10 osd.24 pg_epoch: 45014 pg[2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 44999/44660/0 45012/45013/43970) [153,24,7] r=1 lpr=45013 pi=43869-45012/63 luod=0'0 crt=24074'356 lcod 0'0 active] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MInfoRec from 153 info: 2.26a( v 24074'356 (0'0,24074'356] local-les=45014 n=76 ec=24411 les/c/f 45014/44660/0 45012/45013/43970) -166> 2020-07-06 18:40:14.345741 7f83a82bc700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.176:6810/536802 conn(0x7f83d09af800 sd=1460 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=621 cs=1 l=0). == rx == osd.153 seq 36 0x7f83d04d7a40 pg_info(1 pgs e45014:1.23e) v4 -165> 2020-07-06 18:40:14.345765 7f838a491700 10 log is not dirty -164> 2020-07-06 18:40:14.345767 7f83a82bc700 10 osd.24 45014 do_waiters -- start -163> 2020-07-06 18:40:14.345775 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -162> 2020-07-06 18:40:14.345776 7f83a82bc700 20 osd.24 45014 _dispatch 0x7f83d04d7a40 pg_info(1 pgs e45014:1.23e) v4 -161> 2020-07-06 18:40:14.345782 7f83a82bc700 7 osd.24 45014 handle_pg_info pg_info(1 pgs e45014:1.23e) v4 from osd.153 -160> 2020-07-06 18:40:14.345785 7f83a82bc700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83d04d7a40 -159> 2020-07-06 18:40:14.345860 7f83a82bc700 10 osd.24 45014 do_waiters -- start -158> 2020-07-06 18:40:14.345864 7f83a82bc700 10 osd.24 45014 do_waiters -- finish -157> 2020-07-06 18:40:14.345877 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 44999/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 activating+degraded+remapped m=2] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MInfoRec from 153 info: 1.23e( v 44314'17115402 (33550'17112402,44314'17115402] lb MIN (bitwise) local-les=44984 n=0 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) -156> 2020-07-06 18:40:14.345925 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 44999/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 activating+degraded+remapped m=2] state: peer osd.153 activated and committed -155> 2020-07-06 18:40:14.345944 7f838fc9c700 15 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 44999/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 activating+degraded+remapped m=2] publish_stats_to_osd 45014: no change since 2020-07-06 18:40:14.211834 -154> 2020-07-06 18:40:14.345966 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 44999/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 activating+degraded+remapped m=2] all_activated_and_committed -153> 2020-07-06 18:40:14.346006 7f838fc9c700 10 log is not dirty -152> 2020-07-06 18:40:14.346017 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 44999/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 activating+degraded+remapped m=2] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 AllReplicasActivated -151> 2020-07-06 18:40:14.346034 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] share_pg_info -150> 2020-07-06 18:40:14.346065 7f838fc9c700 20 osd.24 45014 share_map_peer 0x7f83d0807800 already has epoch 45014 -149> 2020-07-06 18:40:14.346070 7f838fc9c700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.133:6800/201624 conn(0x7f83d0807800 sd=1417 :6805 s=STATE_OPEN pgs=5113 cs=1 l=0). == tx == 0x7f83d19ffde0 pg_info(1 pgs e45014:1.23e) v4 -148> 2020-07-06 18:40:14.346187 7f838fc9c700 20 osd.24 45014 share_map_peer 0x7f83d09e0800 already has epoch 45014 -147> 2020-07-06 18:40:14.346196 7f838fc9c700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.133:6816/2554898 conn(0x7f83d09e0800 sd=1485 :-1 s=STATE_OPEN pgs=3957 cs=1 l=0). == tx == 0x7f83d00fcd00 pg_info(1 pgs e45014:1.23e) v4 -146> 2020-07-06 18:40:14.346265 7f838fc9c700 20 osd.24 45014 share_map_peer 0x7f83d07e8800 already has epoch 45014 -145> 2020-07-06 18:40:14.346272 7f838fc9c700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.165:6806/1500877 conn(0x7f83d07e8800 sd=1415 :6805 s=STATE_OPEN pgs=81733 cs=1 l=0). == tx == 0x7f83d00fad20 pg_info(1 pgs e45014:1.23e) v4 -144> 2020-07-06 18:40:14.346345 7f838fc9c700 20 osd.24 45014 share_map_peer 0x7f83d09af800 already has epoch 45014 -143> 2020-07-06 18:40:14.346351 7f838fc9c700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.176:6810/536802 conn(0x7f83d09af800 sd=1460 :-1 s=STATE_OPEN pgs=621 cs=1 l=0). == tx == 0x7f83d00faf00 pg_info(1 pgs e45014:1.23e) v4 -142> 2020-07-06 18:40:14.346407 7f838fc9c700 20 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] _update_calc_stats osd.153 misplaced 5618 but partially backfilled 0 -141> 2020-07-06 18:40:14.346431 7f838fc9c700 20 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] _update_calc_stats osd.7 misplaced 5618 but partially backfilled 0 -140> 2020-07-06 18:40:14.346449 7f838fc9c700 15 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] publish_stats_to_osd 45014:12255689 -139> 2020-07-06 18:40:14.346465 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] check_local -138> 2020-07-06 18:40:14.346480 7f838fc9c700 15 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] requeue_ops -137> 2020-07-06 18:40:14.346494 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] needs_recovery primary has 2 missing -136> 2020-07-06 18:40:14.346509 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] activate not all replicas are up-to-date, queueing recovery -135> 2020-07-06 18:40:14.346532 7f838fc9c700 20 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] _update_calc_stats osd.153 misplaced 5618 but partially backfilled 0 -134> 2020-07-06 18:40:14.346548 7f838fc9c700 20 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] _update_calc_stats osd.7 misplaced 5618 but partially backfilled 0 -133> 2020-07-06 18:40:14.346565 7f838fc9c700 15 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] publish_stats_to_osd 45014: no change since 2020-07-06 18:40:14.346401 -132> 2020-07-06 18:40:14.346589 7f838fc9c700 5 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] on activate: bft=7,153 from MIN -131> 2020-07-06 18:40:14.346610 7f838fc9c700 5 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] target shard 7 from MIN -130> 2020-07-06 18:40:14.346624 7f838fc9c700 5 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] target shard 153 from MIN -129> 2020-07-06 18:40:14.346638 7f838fc9c700 20 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] hit_set_clear -128> 2020-07-06 18:40:14.346654 7f838fc9c700 20 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] agent_stop -127> 2020-07-06 18:40:14.346681 7f838fc9c700 10 log is not dirty -126> 2020-07-06 18:40:14.346732 7f838fc9c700 10 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 DoRecovery -125> 2020-07-06 18:40:14.346752 7f838fc9c700 5 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] exit Started/Primary/Active/Activating 0.134821 13 0.004127 -124> 2020-07-06 18:40:14.346772 7f838fc9c700 5 osd.24 pg_epoch: 45014 pg[1.23e( v 44314'17115402 lc 44024'17115400 (33550'17112378,44314'17115402] local-les=45014 n=5618 ec=30458 les/c/f 45014/44645/0 45012/45013/45013) [24,153,7]/[24,10,4] r=0 lpr=45013 pi=44281-45012/25 bft=7,153 crt=44314'17115402 lcod 0'0 mlcod 0'0 active+degraded+remapped m=2] enter Started/Primary/Active/WaitLocalRecoveryReserved -123> 2020-07-06 18:40:14.346792 7f838fc9c700 10 log is not dirty -122> 2020-07-06 18:40:14.361510 7f83a72ba700 1 -- 172.26.216.197:6815/2876125 >> 172.26.216.176:0/53417 conn(0x7f83d12e5000 sd=1594 :6815 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=10590 cs=1 l=1). == rx == osd.155 seq 1 0x7f83d098d400 osd_ping(ping e45014 stamp 2020-07-06 18:40:14.357832) v2 -121> 2020-07-06 18:40:14.361585 7f83a72ba700 1 -- 172.26.216.197:6815/2876125 >> 172.26.216.176:0/53417 conn(0x7f83d12e5000 sd=1594 :6815 s=STATE_OPEN pgs=10590 cs=1 l=1). == tx == 0x7f83d02ed600 osd_ping(ping_reply e45014 stamp 2020-07-06 18:40:14.357832) v2 -120> 2020-07-06 18:40:14.361542 7f83a6ab9700 1 -- 172.26.224.197:6806/2876125 >> 172.26.216.176:0/53417 conn(0x7f83d0d55000 sd=1595 :6806 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=10589 cs=1 l=1). == rx == osd.155 seq 1 0x7f83d061ba00 osd_ping(ping e45014 stamp 2020-07-06 18:40:14.357832) v2 -119> 2020-07-06 18:40:14.361638 7f83a72ba700 20 osd.24 45014 share_map_peer 0x7f83d079f800 already has epoch 45014 -118> 2020-07-06 18:40:14.361655 7f83a6ab9700 1 -- 172.26.224.197:6806/2876125 >> 172.26.216.176:0/53417 conn(0x7f83d0d55000 sd=1595 :6806 s=STATE_OPEN pgs=10589 cs=1 l=1). == tx == 0x7f83d061ce00 osd_ping(ping_reply e45014 stamp 2020-07-06 18:40:14.357832) v2 -117> 2020-07-06 18:40:14.361749 7f83a6ab9700 20 osd.24 45014 share_map_peer 0x7f83d079f800 already has epoch 45014 -116> 2020-07-06 18:40:14.418434 7f83a6ab9700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.145:6800/60236 conn(0x7f83d109c000 sd=1581 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=77152 cs=1 l=0). == rx == osd.140 seq 17 0x7f83d06b41c0 osd_map(45014..45014 src has 38101..45014) v3 -115> 2020-07-06 18:40:14.418523 7f83a6ab9700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.145:6800/60236 conn(0x7f83d109c000 sd=1581 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=77152 cs=1 l=0). == rx == osd.140 seq 18 0x7f83cfe723c0 pg_notify(2.157(23) epoch 45014) v5 -114> 2020-07-06 18:40:14.418555 7f83a6ab9700 10 osd.24 45014 do_waiters -- start -113> 2020-07-06 18:40:14.418564 7f83a6ab9700 10 osd.24 45014 do_waiters -- finish -112> 2020-07-06 18:40:14.418566 7f83a6ab9700 20 osd.24 45014 _dispatch 0x7f83d06b41c0 osd_map(45014..45014 src has 38101..45014) v3 -111> 2020-07-06 18:40:14.418577 7f83a6ab9700 3 osd.24.objecter handle_osd_map ignoring epochs [45014,45014] <= 45014 -110> 2020-07-06 18:40:14.418582 7f83a6ab9700 20 osd.24.objecter dump_active .. 0 homeless -109> 2020-07-06 18:40:14.418587 7f83a6ab9700 3 osd.24 45014 handle_osd_map epochs [45014,45014], i have 45014, src has [38101,45014] -108> 2020-07-06 18:40:14.418590 7f83a6ab9700 10 osd.24 45014 no new maps here, dropping -107> 2020-07-06 18:40:14.418594 7f83a6ab9700 10 osd.24 45014 do_waiters -- start -106> 2020-07-06 18:40:14.418595 7f83a6ab9700 10 osd.24 45014 do_waiters -- finish -105> 2020-07-06 18:40:14.418598 7f83a6ab9700 10 osd.24 45014 do_waiters -- start -104> 2020-07-06 18:40:14.418599 7f83a6ab9700 10 osd.24 45014 do_waiters -- finish -103> 2020-07-06 18:40:14.418601 7f83a6ab9700 20 osd.24 45014 _dispatch 0x7f83cfe723c0 pg_notify(2.157(23) epoch 45014) v5 -102> 2020-07-06 18:40:14.418607 7f83a6ab9700 7 osd.24 45014 handle_pg_notify from osd.140 -101> 2020-07-06 18:40:14.418610 7f83a6ab9700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83cfe723c0 -100> 2020-07-06 18:40:14.418656 7f83a6ab9700 10 osd.24 45014 do_waiters -- start -99> 2020-07-06 18:40:14.418660 7f83a6ab9700 10 osd.24 45014 do_waiters -- finish -98> 2020-07-06 18:40:14.418680 7f838948f700 10 osd.24 pg_epoch: 45014 pg[2.157( v 29677'2317 (0'0,29677'2317] local-les=45013 n=87 ec=224 les/c/f 45013/44734/0 45012/45012/45012) [24,23,143] r=0 lpr=45012 pi=44298-45011/23 crt=29677'2317 lcod 0'0 mlcod 0'0 active+recovery_wait+degraded] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MNotifyRec from 140 notify: (query_epoch:45014, epoch_sent:45014, info:2.157( v 29677'2317 (0'0,29677'2317] local-les=44646 n=87 ec=224 les/c/f 44646/44734/0 45012/45012/45012)) features: 0x7ffffffefdfbfff -97> 2020-07-06 18:40:14.418715 7f838948f700 10 osd.24 pg_epoch: 45014 pg[2.157( v 29677'2317 (0'0,29677'2317] local-les=45013 n=87 ec=224 les/c/f 45013/44734/0 45012/45012/45012) [24,23,143] r=0 lpr=45012 pi=44298-45011/23 crt=29677'2317 lcod 0'0 mlcod 0'0 active+recovery_wait+degraded] state: Active: got notify from 140, already have info from that osd, ignoring -96> 2020-07-06 18:40:14.418736 7f838948f700 10 log is not dirty -95> 2020-07-06 18:40:14.419270 7f83a6ab9700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.145:6800/60236 conn(0x7f83d109c000 sd=1581 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=77152 cs=1 l=0). == rx == osd.140 seq 19 0x7f83cfe723c0 pg_notify(1.367(137) epoch 45014) v5 -94> 2020-07-06 18:40:14.419312 7f83a6ab9700 10 osd.24 45014 do_waiters -- start -93> 2020-07-06 18:40:14.419319 7f83a6ab9700 10 osd.24 45014 do_waiters -- finish -92> 2020-07-06 18:40:14.419321 7f83a6ab9700 20 osd.24 45014 _dispatch 0x7f83cfe723c0 pg_notify(1.367(137) epoch 45014) v5 -91> 2020-07-06 18:40:14.419329 7f83a6ab9700 7 osd.24 45014 handle_pg_notify from osd.140 -90> 2020-07-06 18:40:14.419331 7f83a6ab9700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83cfe723c0 -89> 2020-07-06 18:40:14.419406 7f83a6ab9700 10 osd.24 45014 do_waiters -- start -88> 2020-07-06 18:40:14.419411 7f83a6ab9700 10 osd.24 45014 do_waiters -- finish -87> 2020-07-06 18:40:14.419409 7f838ec9a700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45009/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 activating+undersized+degraded+remapped] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MNotifyRec from 140 notify: (query_epoch:45014, epoch_sent:45014, info:1.367( v 43961'42232566 (34902'42229566,43961'42232566] lb MIN (bitwise) local-les=44012 n=0 ec=30458 les/c/f 45009/44607/0 45012/45013/45013)) features: 0x7ffffffefdfbfff -86> 2020-07-06 18:40:14.419444 7f838ec9a700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45009/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 activating+undersized+degraded+remapped] state: Active: got notify from 140, already have info from that osd, ignoring -85> 2020-07-06 18:40:14.419470 7f838ec9a700 10 log is not dirty -84> 2020-07-06 18:40:14.420684 7f83a6ab9700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.145:6800/60236 conn(0x7f83d109c000 sd=1581 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=77152 cs=1 l=0). == rx == osd.140 seq 20 0x7f83cfe723c0 pg_info(1 pgs e45014:1.367) v4 -83> 2020-07-06 18:40:14.420728 7f83a6ab9700 10 osd.24 45014 do_waiters -- start -82> 2020-07-06 18:40:14.420736 7f83a6ab9700 10 osd.24 45014 do_waiters -- finish -81> 2020-07-06 18:40:14.420738 7f83a6ab9700 20 osd.24 45014 _dispatch 0x7f83cfe723c0 pg_info(1 pgs e45014:1.367) v4 -80> 2020-07-06 18:40:14.420746 7f83a6ab9700 7 osd.24 45014 handle_pg_info pg_info(1 pgs e45014:1.367) v4 from osd.140 -79> 2020-07-06 18:40:14.420749 7f83a6ab9700 15 osd.24 45014 require_same_or_newer_map 45014 (i am 45014) 0x7f83cfe723c0 -78> 2020-07-06 18:40:14.420818 7f83a6ab9700 10 osd.24 45014 do_waiters -- start -77> 2020-07-06 18:40:14.420823 7f83a6ab9700 10 osd.24 45014 do_waiters -- finish -76> 2020-07-06 18:40:14.420831 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45009/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 activating+undersized+degraded+remapped] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 MInfoRec from 140 info: 1.367( v 43961'42232566 (34902'42229566,43961'42232566] lb MIN (bitwise) local-les=44012 n=0 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) -75> 2020-07-06 18:40:14.420864 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45009/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 activating+undersized+degraded+remapped] state: peer osd.140 activated and committed -74> 2020-07-06 18:40:14.420881 7f838b493700 15 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45009/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 activating+undersized+degraded+remapped] publish_stats_to_osd 45014: no change since 2020-07-06 18:40:14.212296 -73> 2020-07-06 18:40:14.420917 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45009/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 activating+undersized+degraded+remapped] all_activated_and_committed -72> 2020-07-06 18:40:14.420944 7f838b493700 10 log is not dirty -71> 2020-07-06 18:40:14.420954 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45009/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 activating+undersized+degraded+remapped] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 AllReplicasActivated -70> 2020-07-06 18:40:14.420968 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] share_pg_info -69> 2020-07-06 18:40:14.420994 7f838b493700 20 osd.24 45014 share_map_peer 0x7f83d109c000 already has epoch 45014 -68> 2020-07-06 18:40:14.420999 7f838b493700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.145:6800/60236 conn(0x7f83d109c000 sd=1581 :-1 s=STATE_OPEN pgs=77152 cs=1 l=0). == tx == 0x7f83c31ded20 pg_info(1 pgs e45014:1.367) v4 -67> 2020-07-06 18:40:14.421078 7f838b493700 20 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] _update_calc_stats osd.140 misplaced 6476 but partially backfilled 0 -66> 2020-07-06 18:40:14.421099 7f838b493700 15 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] publish_stats_to_osd 45014:36179363 -65> 2020-07-06 18:40:14.421111 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] check_local -64> 2020-07-06 18:40:14.421123 7f838b493700 15 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] requeue_ops 0x7f83d03a4300,0x7f83d07e3800,0x7f83d0f5e200,0x7f83d0f5e300,0x7f83d0f5d900,0x7f83d07e3000 -63> 2020-07-06 18:40:14.421165 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] needs_recovery is recovered -62> 2020-07-06 18:40:14.421176 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] needs_backfill osd.140 has last_backfill MIN -61> 2020-07-06 18:40:14.421187 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] activate queueing backfill -60> 2020-07-06 18:40:14.421204 7f838b493700 20 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] _update_calc_stats osd.140 misplaced 6476 but partially backfilled 0 -59> 2020-07-06 18:40:14.421217 7f838b493700 15 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] publish_stats_to_osd 45014: no change since 2020-07-06 18:40:14.421076 -58> 2020-07-06 18:40:14.421238 7f838b493700 5 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] on activate: bft=140 from MIN -57> 2020-07-06 18:40:14.421250 7f838b493700 5 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] target shard 140 from MIN -56> 2020-07-06 18:40:14.421260 7f838b493700 20 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] hit_set_clear -55> 2020-07-06 18:40:14.421269 7f838b493700 20 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] agent_stop -54> 2020-07-06 18:40:14.421290 7f838b493700 10 log is not dirty -53> 2020-07-06 18:40:14.421330 7f838b493700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] handle_peering_event: epoch_sent: 45014 epoch_requested: 45014 RequestBackfill -52> 2020-07-06 18:40:14.421345 7f838b493700 5 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] exit Started/Primary/Active/Activating 0.208962 7 0.002408 -51> 2020-07-06 18:40:14.421359 7f838b493700 5 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped] enter Started/Primary/Active/WaitLocalBackfillReserved -50> 2020-07-06 18:40:14.421373 7f838b493700 10 log is not dirty -49> 2020-07-06 18:40:14.421387 7f8385487700 10 osd.24 45014 dequeue_op 0x7f83d03a4300 prio 63 cost 81 latency 1.064879 osd_op(osd.60.43950:1579 1.aad8e367 (undecoded) ondisk+retry+write+ignore_overlay+enforce_snapc+known_if_redirected e45013) v7 pg pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] -48> 2020-07-06 18:40:14.421430 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] handle_message: 0x7f83d03a4300 -47> 2020-07-06 18:40:14.421469 7f8385487700 20 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] op_has_sufficient_caps pool=1 (cinder ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes -46> 2020-07-06 18:40:14.421529 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] do_op osd_op(osd.60.43950:1579 1.aad8e367 rbd_data.17fdbcbcf6f7e5be5.00000000000005c4 [copy-from ver 2117975801] snapc 0=[] RETRY=136 ondisk+retry+write+ignore_overlay+enforce_snapc+known_if_redirected e45013) v7 may_write -> write-ordered flags ondisk+retry+write+ignore_overlay+enforce_snapc+known_if_redirected -45> 2020-07-06 18:40:14.422731 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] get_object_context: obc NOT found in cache: 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head -44> 2020-07-06 18:40:14.422910 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] populate_obc_watchers 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head -43> 2020-07-06 18:40:14.422935 7f8385487700 20 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] ReplicatedPG::check_blacklisted_obc_watchers for obc 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head -42> 2020-07-06 18:40:14.422953 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] get_object_context: creating obc from disk: 0x7f83d0e73980 -41> 2020-07-06 18:40:14.422968 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] get_object_context: 0x7f83d0e73980 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head rwstate(none n=0 w=0) oi: 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head(33550'42224045 client.882753258.0:622105207 dirty|omap_digest s 4194304 uv 2089023391 od ffffffff) ssc: 0x7f83d0caee00 snapset: 0=[]:[]+head -40> 2020-07-06 18:40:14.422999 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] find_object_context 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head @head oi=1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head(33550'42224045 client.882753258.0:622105207 dirty|omap_digest s 4194304 uv 2089023391 od ffffffff) -39> 2020-07-06 18:40:14.423041 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] execute_ctx 0x7f83d11c7800 -38> 2020-07-06 18:40:14.423057 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] do_op 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head [copy-from ver 2117975801] ov 33550'42224045 av 45014'42232567 snapc 0=[] snapset 0=[]:[]+head -37> 2020-07-06 18:40:14.423083 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] do_osd_op 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head [copy-from ver 2117975801] -36> 2020-07-06 18:40:14.423099 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] do_osd_op copy-from ver 2117975801 -35> 2020-07-06 18:40:14.423127 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] start_copy 1:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head from 11:e6c71b55:::rbd_data.17fdbcbcf6f7e5be5.00000000000005c4:head @11 v2117975801 flags 15 -34> 2020-07-06 18:40:14.423150 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] _copy_some 0x7f83d0e73980 0x7f83c21ce9d8 -33> 2020-07-06 18:40:14.423190 7f8385487700 10 osd.24.objecter _op_submit op 0x7f83d14b3600 -32> 2020-07-06 18:40:14.423215 7f8385487700 10 osd.24.objecter _calc_target pgid 11.aad8e367 acting [60,160,166] -31> 2020-07-06 18:40:14.423234 7f8385487700 1 Event(0x7f83c01707c8 owner=140203506026240 nevent=5000 time_id=1).wakeup -30> 2020-07-06 18:40:14.423249 7f8385487700 20 osd.24.objecter _get_session s=0x7f83d06cf000 osd=60 2 -29> 2020-07-06 18:40:14.423258 7f8385487700 20 osd.24.objecter note: not requesting commit -28> 2020-07-06 18:40:14.423267 7f8385487700 10 osd.24.objecter _op_submit oid rbd_data.17fdbcbcf6f7e5be5.00000000000005c4 '@11' '@11' [assert-version v2117975801,copy-get max 8388608] tid 1 osd.60 -27> 2020-07-06 18:40:14.423278 7f8385487700 20 osd.24.objecter get_session s=0x7f83d06cf000 osd=60 2 -26> 2020-07-06 18:40:14.423281 7f8385487700 15 osd.24.objecter _session_op_assign 60 1 -25> 2020-07-06 18:40:14.423284 7f8385487700 15 osd.24.objecter _send_op 1 to osd.60 -24> 2020-07-06 18:40:14.423287 7f8385487700 1 -- :/2876125 >> 172.26.216.232:6800/2612269 conn(0x7f83d1abb000 sd=-1 :-1 s=STATE_CONNECTING pgs=0 cs=0 l=1). == tx == 0x7f83d15ade40 osd_op(unknown.0.45012:1 11.aad8e367 rbd_data.17fdbcbcf6f7e5be5.00000000000005c4 [assert-version v2117975801,copy-get max 8388608] snapc 0=[] ack+read+ignore_cache+ignore_overlay+flush+map_snap_clone+known_if_redirected e45014) v7 -23> 2020-07-06 18:40:14.423315 7f8385487700 1 Event(0x7f83c01707c8 owner=140203506026240 nevent=5000 time_id=1).wakeup -22> 2020-07-06 18:40:14.423326 7f8385487700 20 osd.24.objecter put_session s=0x7f83d06cf000 osd=60 3 -21> 2020-07-06 18:40:14.423330 7f8385487700 5 osd.24.objecter 1 unacked, 0 uncommitted -20> 2020-07-06 18:40:14.423337 7f8385487700 10 osd.24 45014 dequeue_op 0x7f83d03a4300 finish -19> 2020-07-06 18:40:14.423348 7f8385487700 10 osd.24 45014 dequeue_op 0x7f83d07e3800 prio 63 cost 78 latency 1.066773 osd_op(osd.60.43950:1580 1.4cb91367 (undecoded) ondisk+retry+write+ignore_overlay+enforce_snapc+known_if_redirected e45013) v7 pg pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] -18> 2020-07-06 18:40:14.423373 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] handle_message: 0x7f83d07e3800 -17> 2020-07-06 18:40:14.423393 7f8385487700 20 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] op_has_sufficient_caps pool=1 (cinder ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes -16> 2020-07-06 18:40:14.423418 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] do_op osd_op(osd.60.43950:1580 1.4cb91367 rbd_data.4670a6238e1f29.0000000000000e29 [copy-from ver 2213266406] snapc 0=[] RETRY=136 ondisk+retry+write+ignore_overlay+enforce_snapc+known_if_redirected e45013) v7 may_write -> write-ordered flags ondisk+retry+write+ignore_overlay+enforce_snapc+known_if_redirected -15> 2020-07-06 18:40:14.423445 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] get_object_context: obc NOT found in cache: 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head -14> 2020-07-06 18:40:14.423527 7f83ac2c4700 1 -- 172.26.216.197:0/2876125 learned_addr learned my addr 172.26.216.197:0/2876125 -13> 2020-07-06 18:40:14.423540 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] populate_obc_watchers 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head -12> 2020-07-06 18:40:14.423558 7f8385487700 20 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] ReplicatedPG::check_blacklisted_obc_watchers for obc 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head -11> 2020-07-06 18:40:14.423578 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] get_object_context: creating obc from disk: 0x7f83d1940280 -10> 2020-07-06 18:40:14.423593 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] get_object_context: 0x7f83d1940280 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head rwstate(none n=0 w=0) oi: 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head(40787'42230661 client.4337709373.0:890373 dirty|omap_digest s 4194304 uv 2145699725 od ffffffff) ssc: 0x7f83d1113dc0 snapset: 0=[]:[]+head -9> 2020-07-06 18:40:14.423615 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] find_object_context 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head @head oi=1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head(40787'42230661 client.4337709373.0:890373 dirty|omap_digest s 4194304 uv 2145699725 od ffffffff) -8> 2020-07-06 18:40:14.423642 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] execute_ctx 0x7f83d1532700 -7> 2020-07-06 18:40:14.423657 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] do_op 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head [copy-from ver 2213266406] ov 40787'42230661 av 45014'42232567 snapc 0=[] snapset 0=[]:[]+head -6> 2020-07-06 18:40:14.423674 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] do_osd_op 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head [copy-from ver 2213266406] -5> 2020-07-06 18:40:14.423690 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] do_osd_op copy-from ver 2213266406 -4> 2020-07-06 18:40:14.423706 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] start_copy 1:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head from 11:e6c89d32:::rbd_data.4670a6238e1f29.0000000000000e29:head @11 v2213266406 flags 15 -3> 2020-07-06 18:40:14.423727 7f8385487700 10 osd.24 pg_epoch: 45014 pg[1.367( v 43961'42232566 (34902'42229517,43961'42232566] local-les=45014 n=6476 ec=30458 les/c/f 45014/44607/0 45012/45013/45013) [140,24]/[24] r=0 lpr=45013 pi=44605-45012/21 bft=140 crt=43961'42232566 lcod 0'0 mlcod 0'0 active+undersized+degraded+remapped+wait_backfill] _copy_some 0x7f83d1940280 0x7f83c21d1418 -2> 2020-07-06 18:40:14.423797 7f83ac2c4700 2 -- 172.26.216.197:0/2876125 >> 172.26.216.232:6800/2612269 conn(0x7f83d1abb000 sd=1637 :-1 s=STATE_CONNECTING_WAIT_ACK_SEQ pgs=0 cs=0 l=1)._process_connection got newly_acked_seq 0 vs out_seq 0 -1> 2020-07-06 18:40:14.423877 7f83ac2c4700 10 osd.24.objecter ms_handle_connect 0x7f83d1abb000 0> 2020-07-06 18:40:14.432978 7f8385487700 -1 common/Throttle.cc: In function 'int64_t Throttle::take(int64_t)' thread 7f8385487700 time 2020-07-06 18:40:14.423756 common/Throttle.cc: 148: FAILED assert(c >= 0) ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x85) [0x7f83b5f8b5b5] 2: (Throttle::take(long)+0x2fc) [0x7f83b5f6f73c] 3: (Objecter::_op_submit_with_budget(Objecter::Op*, ceph::shunique_lock&, unsigned long*, int*)+0x2d2) [0x7f83b5b48652] 4: (Objecter::op_submit(Objecter::Op*, unsigned long*, int*)+0xed) [0x7f83b5b488dd] 5: (ReplicatedPG::_copy_some(std::shared_ptr, std::shared_ptr)+0xfb0) [0x7f83b5a42420] 6: (ReplicatedPG::start_copy(ReplicatedPG::CopyCallback*, std::shared_ptr, hobject_t, object_locator_t, unsigned long, unsigned int, bool, unsigned int, unsigned int)+0x682) [0x7f83b5a433c2] 7: (ReplicatedPG::do_osd_ops(ReplicatedPG::OpContext*, std::vector >&)+0xa34a) [0x7f83b5a7ccca] 8: (ReplicatedPG::prepare_transaction(ReplicatedPG::OpContext*)+0xbf) [0x7f83b5a8a83f] 9: (ReplicatedPG::execute_ctx(ReplicatedPG::OpContext*)+0x8f0) [0x7f83b5a8b6f0] 10: (ReplicatedPG::do_op(std::shared_ptr&)+0x3740) [0x7f83b5a90560] 11: (ReplicatedPG::do_request(std::shared_ptr&, ThreadPool::TPHandle&)+0x747) [0x7f83b5a4be57] 12: (OSD::dequeue_op(boost::intrusive_ptr, std::shared_ptr, ThreadPool::TPHandle&)+0x41d) [0x7f83b5900a8d] 13: (PGQueueable::RunVis::operator()(std::shared_ptr&)+0x6d) [0x7f83b5900cdd] 14: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x869) [0x7f83b5905809] 15: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x887) [0x7f83b5f7b557] 16: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x7f83b5f7d4c0] 17: (()+0x7dc5) [0x7f83b3eb9dc5] 18: (clone()+0x6d) [0x7f83b2544ced] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 0 none 0/ 0 lockdep 0/ 0 context 0/ 0 crush 0/ 0 mds 0/ 0 mds_balancer 0/ 0 mds_locker 0/ 0 mds_log 0/ 0 mds_log_expire 0/ 0 mds_migrator 0/ 0 buffer 0/ 0 timer 0/ 0 filer 0/ 0 striper 20/20 objecter 0/ 0 rados 0/ 0 rbd 0/ 5 rbd_mirror 0/ 0 rbd_replay 0/ 0 journaler 0/ 0 objectcacher 0/ 0 client 20/20 osd 0/ 0 optracker 0/ 0 objclass 0/ 0 filestore 0/ 0 journal 5/ 5 ms 0/ 0 mon 0/ 0 monc 0/ 0 paxos 0/ 0 tp 0/ 0 auth 0/ 0 crypto 0/ 0 finisher 0/ 0 heartbeatmap 0/ 0 perfcounter 0/ 0 rgw 0/ 0 civetweb 0/ 0 javaclient 0/ 0 asok 1/ 1 throttle 0/ 0 refs 0/ 0 xio 1/ 5 compressor 1/ 5 newstore 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 1/ 5 kinetic 1/ 5 fuse -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-osd.24.log --- end dump of recent events --- 2020-07-06 18:40:14.439056 7f83ab2c2700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.211:6813/2669148 conn(0x7f83d0df3800 sd=1587 :6805 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=479 cs=1 l=0). == rx == osd.96 seq 5 0x7f83d151cac0 osd_map(45014..45014 src has 38101..45014) v3 2020-07-06 18:40:14.439199 7f83ab2c2700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.211:6813/2669148 conn(0x7f83d0df3800 sd=1587 :6805 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=479 cs=1 l=0). == rx == osd.96 seq 6 0x7f83d1a6b4a0 pg_notify(1.54c(15) epoch 45014) v5 2020-07-06 18:40:14.439233 7f83ab2c2700 10 osd.24 45014 do_waiters -- start 2020-07-06 18:40:14.439239 7f83ab2c2700 10 osd.24 45014 do_waiters -- finish 2020-07-06 18:40:14.439240 7f83ab2c2700 20 osd.24 45014 _dispatch 0x7f83d151cac0 osd_map(45014..45014 src has 38101..45014) v3 2020-07-06 18:40:14.439975 7f83ac2c4700 1 -- 172.26.216.197:0/2876125 >> 172.26.216.232:6800/2612269 conn(0x7f83d1abb000 sd=1637 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=24277 cs=1 l=1). == rx == osd.60 seq 1 0x7f83d15ade40 osd_op_reply(1 rbd_data.17fdbcbcf6f7e5be5.00000000000005c4 [assert-version v2117975801,copy-get max 8388608] v0'0 uv2117975801 ondisk = 0) v7 2020-07-06 18:40:14.440004 7f83ac2c4700 10 osd.24.objecter ms_dispatch 0x7f83c009e140 osd_op_reply(1 rbd_data.17fdbcbcf6f7e5be5.00000000000005c4 [assert-version v2117975801,copy-get max 8388608] v0'0 uv2117975801 ondisk = 0) v7 2020-07-06 18:40:14.440012 7f83ac2c4700 10 osd.24.objecter in handle_osd_op_reply 2020-07-06 18:40:14.446397 7f83acac5700 1 -- 172.26.224.197:6806/2876125 >> 172.26.224.165:0/500874 conn(0x7f83d0798000 sd=1393 :6806 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12559 cs=1 l=1). == rx == osd.14 seq 1 0x7f83cf980a00 osd_ping(ping e45014 stamp 2020-07-06 18:40:14.448379) v2 2020-07-06 18:40:14.446452 7f83acac5700 1 -- 172.26.224.197:6806/2876125 >> 172.26.224.165:0/500874 conn(0x7f83d0798000 sd=1393 :6806 s=STATE_OPEN pgs=12559 cs=1 l=1). == tx == 0x7f83cf667c00 osd_ping(ping_reply e45014 stamp 2020-07-06 18:40:14.448379) v2 2020-07-06 18:40:14.446495 7f83acac5700 10 osd.24 45014 note_peer_epoch osd.14 has 45014 2020-07-06 18:40:14.446504 7f83acac5700 20 osd.24 45014 share_map_peer 0x7f83d07b0800 already has epoch 45014 2020-07-06 18:40:14.483186 7f8385487700 -1 *** Caught signal (Aborted) ** in thread 7f8385487700 thread_name:tp_osd_tp ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374) 1: (()+0x91341a) [0x7f83b5e8b41a] 2: (()+0xf100) [0x7f83b3ec1100] 3: (gsignal()+0x37) [0x7f83b24835f7] 4: (abort()+0x148) [0x7f83b2484ce8] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x267) [0x7f83b5f8b797] 6: (Throttle::take(long)+0x2fc) [0x7f83b5f6f73c] 7: (Objecter::_op_submit_with_budget(Objecter::Op*, ceph::shunique_lock&, unsigned long*, int*)+0x2d2) [0x7f83b5b48652] 8: (Objecter::op_submit(Objecter::Op*, unsigned long*, int*)+0xed) [0x7f83b5b488dd] 9: (ReplicatedPG::_copy_some(std::shared_ptr, std::shared_ptr)+0xfb0) [0x7f83b5a42420] 10: (ReplicatedPG::start_copy(ReplicatedPG::CopyCallback*, std::shared_ptr, hobject_t, object_locator_t, unsigned long, unsigned int, bool, unsigned int, unsigned int)+0x682) [0x7f83b5a433c2] 11: (ReplicatedPG::do_osd_ops(ReplicatedPG::OpContext*, std::vector >&)+0xa34a) [0x7f83b5a7ccca] 12: (ReplicatedPG::prepare_transaction(ReplicatedPG::OpContext*)+0xbf) [0x7f83b5a8a83f] 13: (ReplicatedPG::execute_ctx(ReplicatedPG::OpContext*)+0x8f0) [0x7f83b5a8b6f0] 14: (ReplicatedPG::do_op(std::shared_ptr&)+0x3740) [0x7f83b5a90560] 15: (ReplicatedPG::do_request(std::shared_ptr&, ThreadPool::TPHandle&)+0x747) [0x7f83b5a4be57] 16: (OSD::dequeue_op(boost::intrusive_ptr, std::shared_ptr, ThreadPool::TPHandle&)+0x41d) [0x7f83b5900a8d] 17: (PGQueueable::RunVis::operator()(std::shared_ptr&)+0x6d) [0x7f83b5900cdd] 18: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x869) [0x7f83b5905809] 19: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x887) [0x7f83b5f7b557] 20: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x7f83b5f7d4c0] 21: (()+0x7dc5) [0x7f83b3eb9dc5] 22: (clone()+0x6d) [0x7f83b2544ced] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- -12> 2020-07-06 18:40:14.439056 7f83ab2c2700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.211:6813/2669148 conn(0x7f83d0df3800 sd=1587 :6805 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=479 cs=1 l=0). == rx == osd.96 seq 5 0x7f83d151cac0 osd_map(45014..45014 src has 38101..45014) v3 -11> 2020-07-06 18:40:14.439199 7f83ab2c2700 1 -- 172.26.224.197:6805/2876125 >> 172.26.224.211:6813/2669148 conn(0x7f83d0df3800 sd=1587 :6805 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=479 cs=1 l=0). == rx == osd.96 seq 6 0x7f83d1a6b4a0 pg_notify(1.54c(15) epoch 45014) v5 -10> 2020-07-06 18:40:14.439233 7f83ab2c2700 10 osd.24 45014 do_waiters -- start -9> 2020-07-06 18:40:14.439239 7f83ab2c2700 10 osd.24 45014 do_waiters -- finish -8> 2020-07-06 18:40:14.439240 7f83ab2c2700 20 osd.24 45014 _dispatch 0x7f83d151cac0 osd_map(45014..45014 src has 38101..45014) v3 -7> 2020-07-06 18:40:14.439975 7f83ac2c4700 1 -- 172.26.216.197:0/2876125 >> 172.26.216.232:6800/2612269 conn(0x7f83d1abb000 sd=1637 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=24277 cs=1 l=1). == rx == osd.60 seq 1 0x7f83d15ade40 osd_op_reply(1 rbd_data.17fdbcbcf6f7e5be5.00000000000005c4 [assert-version v2117975801,copy-get max 8388608] v0'0 uv2117975801 ondisk = 0) v7 -6> 2020-07-06 18:40:14.440004 7f83ac2c4700 10 osd.24.objecter ms_dispatch 0x7f83c009e140 osd_op_reply(1 rbd_data.17fdbcbcf6f7e5be5.00000000000005c4 [assert-version v2117975801,copy-get max 8388608] v0'0 uv2117975801 ondisk = 0) v7 -5> 2020-07-06 18:40:14.440012 7f83ac2c4700 10 osd.24.objecter in handle_osd_op_reply -4> 2020-07-06 18:40:14.446397 7f83acac5700 1 -- 172.26.224.197:6806/2876125 >> 172.26.224.165:0/500874 conn(0x7f83d0798000 sd=1393 :6806 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12559 cs=1 l=1). == rx == osd.14 seq 1 0x7f83cf980a00 osd_ping(ping e45014 stamp 2020-07-06 18:40:14.448379) v2 -3> 2020-07-06 18:40:14.446452 7f83acac5700 1 -- 172.26.224.197:6806/2876125 >> 172.26.224.165:0/500874 conn(0x7f83d0798000 sd=1393 :6806 s=STATE_OPEN pgs=12559 cs=1 l=1). == tx == 0x7f83cf667c00 osd_ping(ping_reply e45014 stamp 2020-07-06 18:40:14.448379) v2 -2> 2020-07-06 18:40:14.446495 7f83acac5700 10 osd.24 45014 note_peer_epoch osd.14 has 45014 -1> 2020-07-06 18:40:14.446504 7f83acac5700 20 osd.24 45014 share_map_peer 0x7f83d07b0800 already has epoch 45014 0> 2020-07-06 18:40:14.483186 7f8385487700 -1 *** Caught signal (Aborted) ** in thread 7f8385487700 thread_name:tp_osd_tp ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374) 1: (()+0x91341a) [0x7f83b5e8b41a] 2: (()+0xf100) [0x7f83b3ec1100] 3: (gsignal()+0x37) [0x7f83b24835f7] 4: (abort()+0x148) [0x7f83b2484ce8] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x267) [0x7f83b5f8b797] 6: (Throttle::take(long)+0x2fc) [0x7f83b5f6f73c] 7: (Objecter::_op_submit_with_budget(Objecter::Op*, ceph::shunique_lock&, unsigned long*, int*)+0x2d2) [0x7f83b5b48652] 8: (Objecter::op_submit(Objecter::Op*, unsigned long*, int*)+0xed) [0x7f83b5b488dd] 9: (ReplicatedPG::_copy_some(std::shared_ptr, std::shared_ptr)+0xfb0) [0x7f83b5a42420] 10: (ReplicatedPG::start_copy(ReplicatedPG::CopyCallback*, std::shared_ptr, hobject_t, object_locator_t, unsigned long, unsigned int, bool, unsigned int, unsigned int)+0x682) [0x7f83b5a433c2] 11: (ReplicatedPG::do_osd_ops(ReplicatedPG::OpContext*, std::vector >&)+0xa34a) [0x7f83b5a7ccca] 12: (ReplicatedPG::prepare_transaction(ReplicatedPG::OpContext*)+0xbf) [0x7f83b5a8a83f] 13: (ReplicatedPG::execute_ctx(ReplicatedPG::OpContext*)+0x8f0) [0x7f83b5a8b6f0] 14: (ReplicatedPG::do_op(std::shared_ptr&)+0x3740) [0x7f83b5a90560] 15: (ReplicatedPG::do_request(std::shared_ptr&, ThreadPool::TPHandle&)+0x747) [0x7f83b5a4be57] 16: (OSD::dequeue_op(boost::intrusive_ptr, std::shared_ptr, ThreadPool::TPHandle&)+0x41d) [0x7f83b5900a8d] 17: (PGQueueable::RunVis::operator()(std::shared_ptr&)+0x6d) [0x7f83b5900cdd] 18: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x869) [0x7f83b5905809] 19: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x887) [0x7f83b5f7b557] 20: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x7f83b5f7d4c0] 21: (()+0x7dc5) [0x7f83b3eb9dc5] 22: (clone()+0x6d) [0x7f83b2544ced] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 0 none 0/ 0 lockdep 0/ 0 context 0/ 0 crush 0/ 0 mds 0/ 0 mds_balancer 0/ 0 mds_locker 0/ 0 mds_log 0/ 0 mds_log_expire 0/ 0 mds_migrator 0/ 0 buffer 0/ 0 timer 0/ 0 filer 0/ 0 striper 20/20 objecter 0/ 0 rados 0/ 0 rbd 0/ 5 rbd_mirror 0/ 0 rbd_replay 0/ 0 journaler 0/ 0 objectcacher 0/ 0 client 20/20 osd 0/ 0 optracker 0/ 0 objclass 0/ 0 filestore 0/ 0 journal 5/ 5 ms 0/ 0 mon 0/ 0 monc 0/ 0 paxos 0/ 0 tp 0/ 0 auth 0/ 0 crypto 0/ 0 finisher 0/ 0 heartbeatmap 0/ 0 perfcounter 0/ 0 rgw 0/ 0 civetweb 0/ 0 javaclient 0/ 0 asok 1/ 1 throttle 0/ 0 refs 0/ 0 xio 1/ 5 compressor 1/ 5 newstore 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 1/ 5 kinetic 1/ 5 fuse -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-osd.24.log --- end dump of recent events ---