-405> 2021-04-29 14:08:01.092 7fc30fab9700 20 osd.76 221098 Recovery event scheduled at 2021-04-29 14:08:01.118614 -404> 2021-04-29 14:08:01.092 7fc30fab9700 20 osd.76 op_wq(2) _process empty q, waiting -403> 2021-04-29 14:08:01.099 7fc32d2f4700 15 osd.76 221098 enqueue_op 0x560235109e40 prio 127 cost 752 latency 0.000011 epoch 221098 osd_repop(client.421625607.0:227253 6.2 e221098/220775) v3 -402> 2021-04-29 14:08:01.099 7fc32d2f4700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.2 PGOpItem(op=osd_repop(client.421625607.0:227253 6.2 e221098/220775) v3) prio 127 cost 752 e221098) -401> 2021-04-29 14:08:01.099 7fc30fab9700 20 osd.76 op_wq(2) _process 6.2 to_process <> waiting <> waiting_peering {} -400> 2021-04-29 14:08:01.099 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.2 PGOpItem(op=osd_repop(client.421625607.0:227253 6.2 e221098/220775) v3) prio 127 cost 752 e221098) queued -399> 2021-04-29 14:08:01.099 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089079 (221026'538086498,221098'538089079] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089079 lcod 221098'538089078 active mbc={}] lock -398> 2021-04-29 14:08:01.099 7fc30fab9700 20 osd.76 op_wq(2) _process 6.2 to_process waiting <> waiting_peering {} -397> 2021-04-29 14:08:01.099 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.2 PGOpItem(op=osd_repop(client.421625607.0:227253 6.2 e221098/220775) v3) prio 127 cost 752 e221098) pg 0x5601e563c000 -396> 2021-04-29 14:08:01.099 7fc30fab9700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -395> 2021-04-29 14:08:01.099 7fc30fab9700 10 osd.76 221098 dequeue_op 0x560235109e40 prio 127 cost 752 latency 0.000147 osd_repop(client.421625607.0:227253 6.2 e221098/220775) v3 pg pg[6.2( v 221098'538089079 (221026'538086498,221098'538089079] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089079 lcod 221098'538089078 active mbc={}] -394> 2021-04-29 14:08:01.099 7fc30fab9700 20 osd.76 221098 share_map osd.40 v2:192.168.148.51:6813/589296 221098 -393> 2021-04-29 14:08:01.099 7fc30fab9700 20 osd.76 221098 should_share_map osd.40 v2:192.168.148.51:6813/589296 221098 -392> 2021-04-29 14:08:01.099 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089079 (221026'538086498,221098'538089079] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089079 lcod 221098'538089078 active mbc={}] _handle_message: 0x560235109e40 -391> 2021-04-29 14:08:01.099 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089079 (221026'538086498,221098'538089079] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089079 lcod 221098'538089078 active mbc={}] do_repop 6:40a70a2f:::incoming128-9:head v 221098'538089080 (transaction) 189 -390> 2021-04-29 14:08:01.099 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089079 (221026'538086498,221098'538089079] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089079 lcod 221098'538089078 active mbc={}] do_repop missing before {} -389> 2021-04-29 14:08:01.099 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089079 (221026'538086498,221098'538089079] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089079 lcod 221098'538089078 active mbc={}] append_log log((221026'538086498,221098'538089079], crt=221098'538089079) [221098'538089080 (221098'538089079) modify 6:40a70a2f:::incoming128-9:head by client.421625607.0:227253 2021-04-29 14:08:01.095878 0] -388> 2021-04-29 14:08:01.099 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089079 lcod 221098'538089078 active mbc={}] add_log_entry 221098'538089080 (221098'538089079) modify 6:40a70a2f:::incoming128-9:head by client.421625607.0:227253 2021-04-29 14:08:01.095878 0 -387> 2021-04-29 14:08:01.099 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089078 active mbc={}] rollforward: entry=221098'538089080 (221098'538089079) modify 6:40a70a2f:::incoming128-9:head by client.421625607.0:227253 2021-04-29 14:08:01.095878 0 -386> 2021-04-29 14:08:01.099 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089078 active mbc={}] append_log approx pg log length = 2582 -385> 2021-04-29 14:08:01.099 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089078 active mbc={}] append_log transaction_applied = 1 -384> 2021-04-29 14:08:01.099 7fc30fab9700 10 trim proposed trim_to = 221026'538086498 -383> 2021-04-29 14:08:01.099 7fc30fab9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 221098'538089080, trimmed: , trimmed_dups: , clear_divergent_priors: 0 -382> 2021-04-29 14:08:01.100 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089078 active mbc={}] do_repop missing after{} -381> 2021-04-29 14:08:01.100 7fc30fab9700 10 osd.76 221098 dequeue_op 0x560235109e40 finish -380> 2021-04-29 14:08:01.100 7fc30fab9700 20 osd.76 op_wq(2) _process empty q, waiting -379> 2021-04-29 14:08:01.100 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089078 active mbc={}] lock -378> 2021-04-29 14:08:01.100 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089078 active mbc={}] repop_commit on op osd_repop(client.421625607.0:227253 6.2 e221098/220775 6:40a70a2f:::incoming128-9:head v 221098'538089080) v3, sending commit to osd.40 -377> 2021-04-29 14:08:01.100 7fc30fab9700 20 osd.76 221098 share_map_peer 0x560228ba4c00 already has epoch 221098 -376> 2021-04-29 14:08:01.100 7fc30fab9700 20 osd.76 op_wq(2) _process empty q, waiting -375> 2021-04-29 14:08:01.117 7fc3272e8700 20 osd.76 221098 do_recovery wake up at 2021-04-29 14:08:01.118713, re-queuing recovery -374> 2021-04-29 14:08:01.117 7fc3272e8700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.cf PGRecovery(pgid=6.cfepoch_queued=221098reserved_pushes=1) prio 5 cost 20971520 e221098 reserved_pushes 1) -373> 2021-04-29 14:08:01.117 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process <> waiting <> waiting_peering {} -372> 2021-04-29 14:08:01.117 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGRecovery(pgid=6.cfepoch_queued=221098reserved_pushes=1) prio 5 cost 20971520 e221098 reserved_pushes 1) queued -371> 2021-04-29 14:08:01.117 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] lock -370> 2021-04-29 14:08:01.117 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process waiting <> waiting_peering {} -369> 2021-04-29 14:08:01.117 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGRecovery(pgid=6.cfepoch_queued=221098reserved_pushes=1) prio 5 cost 20971520 e221098 reserved_pushes 1) pg 0x5601da35b000 -368> 2021-04-29 14:08:01.117 7fc30fab9700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -367> 2021-04-29 14:08:01.117 7fc30fab9700 10 osd.76 221098 do_recovery starting 1 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] -366> 2021-04-29 14:08:01.117 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] recover_replicas(1) -365> 2021-04-29 14:08:01.117 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] peer osd.58 missing 2 objects. -364> 2021-04-29 14:08:01.117 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] peer osd.58 missing {6:f33f1367:::incoming128-44:head=221098'634239124(221082'634238369) flags = none,6:f382cb1f:::incoming128-85:head=221098'634239123(221082'634238371) flags = none} -363> 2021-04-29 14:08:01.117 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] recover_replicas: recover_object_replicas(6:f382cb1f:::incoming128-85:head) -362> 2021-04-29 14:08:01.117 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] prep_object_replica_pushes: on 6:f382cb1f:::incoming128-85:head -361> 2021-04-29 14:08:01.118 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] get_object_context: found obc in cache: 0x56023936b440 -360> 2021-04-29 14:08:01.118 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] get_object_context: 0x56023936b440 6:f382cb1f:::incoming128-85:head rwstate(none n=0 w=0) oi: 6:f382cb1f:::incoming128-85:head(221098'634239123 client.421491796.0:232029 dirty|omap|data_digest s 0 uv 634239123 dd ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x5601e7453560 snapset: 0=[]:{} -359> 2021-04-29 14:08:01.118 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] recovery got recovery read lock on 6:f382cb1f:::incoming128-85:head -358> 2021-04-29 14:08:01.118 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] start_recovery_op 6:f382cb1f:::incoming128-85:head -357> 2021-04-29 14:08:01.118 7fc30fab9700 10 osd.76 221098 start_recovery_op pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] 6:f382cb1f:::incoming128-85:head (0/1 rops) -356> 2021-04-29 14:08:01.118 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] recover_object: 6:f382cb1f:::incoming128-85:head -355> 2021-04-29 14:08:01.118 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] start_pushes soid 6:f382cb1f:::incoming128-85:head -354> 2021-04-29 14:08:01.118 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] prep_push_to_replica: 6:f382cb1f:::incoming128-85:head v221098'634239123 size 0 to osd.58 -353> 2021-04-29 14:08:01.118 7fc30fab9700 15 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] push_to_replica snapset is 0=[]:{} -352> 2021-04-29 14:08:01.118 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] calc_head_subsets 6:f382cb1f:::incoming128-85:head clone_overlap {} -351> 2021-04-29 14:08:01.118 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] calc_head_subsets 6:f382cb1f:::incoming128-85:head data_subset [] clone_subsets {} -350> 2021-04-29 14:08:01.118 7fc30fab9700 7 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] build_push_op 6:f382cb1f:::incoming128-85:head v 221098'634239123 size 0 recovery_info: ObjectRecoveryInfo(6:f382cb1f:::incoming128-85:head@221098'634239123, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}) -349> 2021-04-29 14:08:01.127 7fc32daf5700 15 osd.76 221098 enqueue_op 0x560248bc6f20 prio 63 cost 127 latency 0.000015 epoch 221097 osd_op(client.421492180.0:227642 6.cf 6.f8d341cf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221097) v8 -348> 2021-04-29 14:08:01.127 7fc32daf5700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.cf PGOpItem(op=osd_op(client.421492180.0:227642 6.cf 6.f8d341cf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221097) v8) prio 63 cost 127 e221097) -347> 2021-04-29 14:08:01.128 7fc32daf5700 15 osd.76 221098 enqueue_op 0x56026dbe0420 prio 63 cost 127 latency 0.000012 epoch 221098 osd_op(client.421492180.0:227661 6.4f 6.57bcf64f (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8 -346> 2021-04-29 14:08:01.128 7fc32daf5700 20 osd.76 op_wq(4) _enqueue OpQueueItem(6.4f PGOpItem(op=osd_op(client.421492180.0:227661 6.4f 6.57bcf64f (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8) prio 63 cost 127 e221098) -345> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 op_wq(4) _process 6.4f to_process <> waiting <> waiting_peering {} -344> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 op_wq(4) _process OpQueueItem(6.4f PGOpItem(op=osd_op(client.421492180.0:227661 6.4f 6.57bcf64f (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8) prio 63 cost 127 e221098) queued -343> 2021-04-29 14:08:01.128 7fc30eab7700 30 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] lock -342> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 op_wq(4) _process 6.4f to_process waiting <> waiting_peering {} -341> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 op_wq(4) _process OpQueueItem(6.4f PGOpItem(op=osd_op(client.421492180.0:227661 6.4f 6.57bcf64f (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8) prio 63 cost 127 e221098) pg 0x5601dc8e6000 -340> 2021-04-29 14:08:01.128 7fc30eab7700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[63,52428800,{"priority":63,"num_keys":1,"first_item_cost":127}]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -339> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 221098 dequeue_op 0x56026dbe0420 prio 63 cost 127 latency 0.000136 osd_op(client.421492180.0:227661 6.4f 6.57bcf64f (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8 pg pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] -338> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 221098 share_map client.421492180 192.168.146.83:0/4103370583 221098 -337> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 221098 should_share_map client.421492180 192.168.146.83:0/4103370583 221098 -336> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] _handle_message: 0x56026dbe0420 -335> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_op: op osd_op(client.421492180.0:227661 6.4f 6:f26f3dea:::incoming128-91:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221098) v8 -334> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] op_has_sufficient_caps session=0x56021fee1200 pool=6 (gnocchi ) pool_app_metadata={rbd={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes -333> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_op osd_op(client.421492180.0:227661 6.4f 6:f26f3dea:::incoming128-91:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221098) v8 may_write -> write-ordered flags ondisk+write+skiprwlocks+known_if_redirected -332> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] get_object_context: found obc in cache: 0x5602ba7e0900 -331> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] get_object_context: 0x5602ba7e0900 6:f26f3dea:::incoming128-91:head rwstate(none n=0 w=0) oi: 6:f26f3dea:::incoming128-91:head(221098'612784294 client.421493281.0:227823 dirty|omap|data_digest s 0 uv 612784294 dd ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x56027fad1e60 snapset: 0=[]:{} -330> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] find_object_context 6:f26f3dea:::incoming128-91:head @head oi=6:f26f3dea:::incoming128-91:head(221098'612784294 client.421493281.0:227823 dirty|omap|data_digest s 0 uv 612784294 dd ffffffff alloc_hint [0 0 0]) -329> 2021-04-29 14:08:01.128 7fc30eab7700 25 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_op oi 6:f26f3dea:::incoming128-91:head(221098'612784294 client.421493281.0:227823 dirty|omap|data_digest s 0 uv 612784294 dd ffffffff alloc_hint [0 0 0]) -328> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_op: skipping rw locks -327> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_op obc obc(6:f26f3dea:::incoming128-91:head rwstate(none n=0 w=0)) -326> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] execute_ctx 0x5601db21c700 -325> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] execute_ctx 6:f26f3dea:::incoming128-91:head [omap-set-vals] ov 221098'612784294 av 221098'612784295 snapc 0=[] snapset 0=[]:{} -324> 2021-04-29 14:08:01.128 7fc30eab7700 30 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] execute_ctx user_at_version 612784294 -323> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_osd_op 6:f26f3dea:::incoming128-91:head [omap-set-vals] -322> 2021-04-29 14:08:01.128 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_osd_op omap-set-vals -321> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] setting vals: -320> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] measure_72635cbc-fb7f-40ac-ac19-da3e707b68db_a5d63f1e-fbb9-46ce-8066-c390c4796ecb_20210429_12:08:01 -319> 2021-04-29 14:08:01.128 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] make_writeable 6:f26f3dea:::incoming128-91:head snapset=0=[]:{} snapc=0=[] -318> 2021-04-29 14:08:01.129 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] make_writeable 6:f26f3dea:::incoming128-91:head done, snapset=0=[]:{} -317> 2021-04-29 14:08:01.129 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] finish_ctx 6:f26f3dea:::incoming128-91:head 0x5601db21c700 op modify -316> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] set mtime to 2021-04-29 14:08:01.129470 -315> 2021-04-29 14:08:01.129 7fc32daf5700 15 osd.76 221098 enqueue_op 0x5602b6c97a20 prio 63 cost 127 latency 0.000005 epoch 221098 osd_op(client.421492180.0:227672 6.cf 6.e6c8fccf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8 -314> 2021-04-29 14:08:01.129 7fc32daf5700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.cf PGOpItem(op=osd_op(client.421492180.0:227672 6.cf 6.e6c8fccf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8) prio 63 cost 127 e221098) -313> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] final snapset 0=[]:{} in 6:f26f3dea:::incoming128-91:head -312> 2021-04-29 14:08:01.129 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] zeroing write result code 0 -311> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] calc_trim_to_aggressive limit = 221098'612784294 -310> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] calc_trim_to_aggressive approx pg log length = 3056 -309> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] calc_trim_to_aggressive num_to_trim = 56 -308> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] new_repop rep_tid 114279196 on osd_op(client.421492180.0:227661 6.4f 6:f26f3dea:::incoming128-91:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221098) v8 -307> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] new_repop: repgather(0x560222e1c360 0'0 rep_tid=114279196 committed?=0 r=0) -306> 2021-04-29 14:08:01.129 7fc30eab7700 7 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] issue_repop rep_tid 114279196 o 6:f26f3dea:::incoming128-91:head -305> 2021-04-29 14:08:01.129 7fc30eab7700 20 osd.76 221098 share_map_peer 0x5601f4fd4c00 already has epoch 221098 -304> 2021-04-29 14:08:01.129 7fc30eab7700 20 osd.76 221098 share_map_peer 0x5601f51b8400 already has epoch 221098 -303> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784294 (221006'612781238,221098'612784294] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] append_log log((221006'612781238,221098'612784294], crt=221098'612784294) [221098'612784295 (221098'612784294) modify 6:f26f3dea:::incoming128-91:head by client.421492180.0:227661 2021-04-29 14:08:01.129470 0] -302> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 lua=221098'612784294 crt=221098'612784294 lcod 221098'612784293 mlcod 221098'612784293 active+clean] add_log_entry 221098'612784295 (221098'612784294) modify 6:f26f3dea:::incoming128-91:head by client.421492180.0:227661 2021-04-29 14:08:01.129470 0 -301> 2021-04-29 14:08:01.129 7fc30eab7700 20 earliest_dup_version = 612781296 -300> 2021-04-29 14:08:01.129 7fc30eab7700 20 trim 221098'612784295 (221098'612784294) modify 6:f26f3dea:::incoming128-91:head by client.421492180.0:227661 2021-04-29 14:08:01.129470 0 -299> 2021-04-29 14:08:01.129 7fc30eab7700 20 trim dup log_dup(reqid=client.413444013.0:25685385 v=221007'612781295 uv=612781295 rc=0) -298> 2021-04-29 14:08:01.129 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 lua=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] rollforward: entry=221098'612784295 (221098'612784294) modify 6:f26f3dea:::incoming128-91:head by client.421492180.0:227661 2021-04-29 14:08:01.129470 0 -297> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 lua=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] append_log approx pg log length = 3057 -296> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 lua=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] append_log transaction_applied = 1 -295> 2021-04-29 14:08:01.129 7fc30eab7700 10 trim proposed trim_to = 221006'612781238 -294> 2021-04-29 14:08:01.129 7fc30eab7700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 221098'612784295, trimmed: , trimmed_dups: , clear_divergent_priors: 0 -293> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 lua=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] op_applied version 221098'612784295 -292> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] eval_repop repgather(0x560222e1c360 221098'612784295 rep_tid=114279196 committed?=0 r=0) -291> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 221098 dequeue_op 0x56026dbe0420 finish -290> 2021-04-29 14:08:01.129 7fc30eab7700 20 osd.76 op_wq(4) _process empty q, waiting -289> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] send_pushes: sending push PushOp(6:f382cb1f:::incoming128-85:head, version: 221098'634239123, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 26, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:f382cb1f:::incoming128-85:head@221098'634239123, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false)) to osd.58 -288> 2021-04-29 14:08:01.129 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] started 1 -287> 2021-04-29 14:08:01.129 7fc30fab9700 10 osd.76 221098 do_recovery started 1/1 on pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] -286> 2021-04-29 14:08:01.129 7fc30fab9700 10 osd.76 221098 release_reserved_pushes(1), recovery_ops_reserved 1 -> 0 -285> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process <> waiting <> waiting_peering {} -284> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=osd_op(client.421492180.0:227642 6.cf 6.f8d341cf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221097) v8) prio 63 cost 127 e221097) queued -283> 2021-04-29 14:08:01.129 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] lock -282> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process waiting <> waiting_peering {} -281> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=osd_op(client.421492180.0:227642 6.cf 6.f8d341cf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221097) v8) prio 63 cost 127 e221097) pg 0x5601da35b000 -280> 2021-04-29 14:08:01.129 7fc30eab7700 30 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] lock -279> 2021-04-29 14:08:01.129 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] op_commit: 114279196 -278> 2021-04-29 14:08:01.129 7fc30fab9700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[63,52428800,{"priority":63,"num_keys":1,"first_item_cost":127}]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -277> 2021-04-29 14:08:01.129 7fc30eab7700 20 osd.76 op_wq(4) _process empty q, waiting -276> 2021-04-29 14:08:01.129 7fc30fab9700 10 osd.76 221098 dequeue_op 0x560248bc6f20 prio 63 cost 127 latency 0.002270 osd_op(client.421492180.0:227642 6.cf 6.f8d341cf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221097) v8 pg pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] -275> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 221098 share_map client.421492180 192.168.146.83:0/4103370583 221097 -274> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 221098 should_share_map client.421492180 192.168.146.83:0/4103370583 221097 -273> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 221098 client session last_sent_epoch: 220998 versus osdmap epoch 221098 -272> 2021-04-29 14:08:01.129 7fc30fab9700 10 osd.76 221098 client.421492180 has old map 221097 < 221098 -271> 2021-04-29 14:08:01.129 7fc30fab9700 10 osd.76 221098 send_incremental_map 221097 -> 221098 to 0x5602d7143000 192.168.146.83:0/4103370583 -270> 2021-04-29 14:08:01.129 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _handle_message: 0x560248bc6f20 -269> 2021-04-29 14:08:01.129 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op: op osd_op(client.421492180.0:227642 6.cf 6:f382cb1f:::incoming128-85:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221097) v8 -268> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] op_has_sufficient_caps session=0x56021fee1200 pool=6 (gnocchi ) pool_app_metadata={rbd={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes -267> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op osd_op(client.421492180.0:227642 6.cf 6:f382cb1f:::incoming128-85:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221097) v8 may_write -> write-ordered flags ondisk+write+skiprwlocks+known_if_redirected -266> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] readable_with_acting: locs:76,110 -265> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] get_object_context: found obc in cache: 0x56023936b440 -264> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] get_object_context: 0x56023936b440 6:f382cb1f:::incoming128-85:head rwstate(read n=1 w=0) oi: 6:f382cb1f:::incoming128-85:head(221098'634239123 client.421491796.0:232029 dirty|omap|data_digest s 0 uv 634239123 dd ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x5601e7453560 snapset: 0=[]:{} -263> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] find_object_context 6:f382cb1f:::incoming128-85:head @head oi=6:f382cb1f:::incoming128-85:head(221098'634239123 client.421491796.0:232029 dirty|omap|data_digest s 0 uv 634239123 dd ffffffff alloc_hint [0 0 0]) -262> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] readable_with_acting: locs:76,110 -261> 2021-04-29 14:08:01.130 7fc30fab9700 25 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op oi 6:f382cb1f:::incoming128-85:head(221098'634239123 client.421491796.0:232029 dirty|omap|data_digest s 0 uv 634239123 dd ffffffff alloc_hint [0 0 0]) -260> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op: skipping rw locks -259> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op obc obc(6:f382cb1f:::incoming128-85:head rwstate(read n=1 w=0)) -258> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] execute_ctx 0x56028dfcb500 -257> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] execute_ctx 6:f382cb1f:::incoming128-85:head [omap-set-vals] ov 221098'634239123 av 221098'634239125 snapc 0=[] snapset 0=[]:{} -256> 2021-04-29 14:08:01.130 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] execute_ctx user_at_version 634239123 -255> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_osd_op 6:f382cb1f:::incoming128-85:head [omap-set-vals] -254> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_osd_op omap-set-vals -253> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] setting vals: -252> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] measure_d9b1eaa1-19d8-4084-a014-a688900f35d5_337a1f8b-1f29-4616-9f3b-e5d9ec994197_20210429_12:08:01 -251> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] make_writeable 6:f382cb1f:::incoming128-85:head snapset=0=[]:{} snapc=0=[] -250> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] make_writeable 6:f382cb1f:::incoming128-85:head done, snapset=0=[]:{} -249> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] finish_ctx 6:f382cb1f:::incoming128-85:head 0x56028dfcb500 op modify -248> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] set mtime to 2021-04-29 14:08:01.128557 -247> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] final snapset 0=[]:{} in 6:f382cb1f:::incoming128-85:head -246> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] zeroing write result code 0 -245> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] calc_trim_to_aggressive limit = 221098'634239124 -244> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] calc_trim_to_aggressive approx pg log length = 3075 -243> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] calc_trim_to_aggressive num_to_trim = 75 -242> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] new_repop rep_tid 114279197 on osd_op(client.421492180.0:227642 6.cf 6:f382cb1f:::incoming128-85:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221097) v8 -241> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] new_repop: repgather(0x5602f1491680 0'0 rep_tid=114279197 committed?=0 r=0) -240> 2021-04-29 14:08:01.130 7fc30fab9700 7 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] issue_repop rep_tid 114279197 o 6:f382cb1f:::incoming128-85:head -239> 2021-04-29 14:08:01.130 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] issue_repop missing_loc before: 76,110 -238> 2021-04-29 14:08:01.130 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] issue_repop missing_loc after: 76,110 -237> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] should_send_op issue_repop shipping empty opt to osd.58, object 6:f382cb1f:::incoming128-85:head which is pending recovery in async_recovery_targets -236> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 221098 share_map_peer 0x560238a77800 already has epoch 221098 -235> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 221098 share_map_peer 0x5601ff8e8400 already has epoch 221098 -234> 2021-04-29 14:08:01.130 7fc32d2f4700 15 osd.76 221098 enqueue_op 0x56020277c160 prio 127 cost 754 latency 0.000012 epoch 221098 osd_repop(client.421492180.0:227663 6.3d e221098/220465) v3 -233> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239124 (221020'634236049,221098'634239124] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] append_log log((221020'634236049,221098'634239124], crt=221098'634239124) [221098'634239125 (221098'634239123) modify 6:f382cb1f:::incoming128-85:head by client.421492180.0:227642 2021-04-29 14:08:01.128557 0] -232> 2021-04-29 14:08:01.130 7fc32d2f4700 20 osd.76 op_wq(1) _enqueue OpQueueItem(6.3d PGOpItem(op=osd_repop(client.421492180.0:227663 6.3d e221098/220465) v3) prio 127 cost 754 e221098) -231> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239124 rops=1 crt=221098'634239124 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] add_log_entry 221098'634239125 (221098'634239123) modify 6:f382cb1f:::incoming128-85:head by client.421492180.0:227642 2021-04-29 14:08:01.128557 0 -230> 2021-04-29 14:08:01.130 7fc30fab9700 20 earliest_dup_version = 634236126 -229> 2021-04-29 14:08:01.130 7fc30fab9700 20 trim 221098'634239125 (221098'634239123) modify 6:f382cb1f:::incoming128-85:head by client.421492180.0:227642 2021-04-29 14:08:01.128557 0 -228> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] rollforward: entry=221098'634239125 (221098'634239123) modify 6:f382cb1f:::incoming128-85:head by client.421492180.0:227642 2021-04-29 14:08:01.128557 0 -227> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] append_log approx pg log length = 3076 -226> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] append_log transaction_applied = 1 -225> 2021-04-29 14:08:01.130 7fc32daf5700 15 osd.76 221098 enqueue_op 0x56023e8f4000 prio 196 cost 0 latency 0.000007 epoch 221098 osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2 -224> 2021-04-29 14:08:01.130 7fc30fab9700 10 trim proposed trim_to = 221020'634236049 -223> 2021-04-29 14:08:01.130 7fc32daf5700 20 osd.76 op_wq(4) _enqueue OpQueueItem(6.4f PGOpItem(op=osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2) prio 196 cost 0 e221098) -222> 2021-04-29 14:08:01.130 7fc30fab9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 221098'634239125, trimmed: , trimmed_dups: , clear_divergent_priors: 0 -221> 2021-04-29 14:08:01.130 7fc3102ba700 20 osd.76 op_wq(1) _process 6.3d to_process <> waiting <> waiting_peering {} -220> 2021-04-29 14:08:01.130 7fc30eab7700 20 osd.76 op_wq(4) _process 6.4f to_process <> waiting <> waiting_peering {} -219> 2021-04-29 14:08:01.130 7fc30eab7700 20 osd.76 op_wq(4) _process OpQueueItem(6.4f PGOpItem(op=osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2) prio 196 cost 0 e221098) queued -218> 2021-04-29 14:08:01.130 7fc3102ba700 20 osd.76 op_wq(1) _process OpQueueItem(6.3d PGOpItem(op=osd_repop(client.421492180.0:227663 6.3d e221098/220465) v3) prio 127 cost 754 e221098) queued -217> 2021-04-29 14:08:01.130 7fc30eab7700 30 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] lock -216> 2021-04-29 14:08:01.130 7fc30eab7700 20 osd.76 op_wq(4) _process 6.4f to_process waiting <> waiting_peering {} -215> 2021-04-29 14:08:01.130 7fc30eab7700 20 osd.76 op_wq(4) _process OpQueueItem(6.4f PGOpItem(op=osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2) prio 196 cost 0 e221098) pg 0x5601dc8e6000 -214> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] op_applied version 221098'634239125 -213> 2021-04-29 14:08:01.130 7fc3102ba700 30 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606631 (221046'639604024,221098'639606631] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606631 lcod 221098'639606630 active mbc={}] lock -212> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] eval_repop repgather(0x5602f1491680 221098'634239125 rep_tid=114279197 committed?=0 r=0) -211> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 221098 dequeue_op 0x560248bc6f20 finish -210> 2021-04-29 14:08:01.130 7fc3102ba700 20 osd.76 op_wq(1) _process 6.3d to_process waiting <> waiting_peering {} -209> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process <> waiting <> waiting_peering {} -208> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=osd_op(client.421492180.0:227672 6.cf 6.e6c8fccf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8) prio 63 cost 127 e221098) queued -207> 2021-04-29 14:08:01.130 7fc3102ba700 20 osd.76 op_wq(1) _process OpQueueItem(6.3d PGOpItem(op=osd_repop(client.421492180.0:227663 6.3d e221098/220465) v3) prio 127 cost 754 e221098) pg 0x5601ee25e000 -206> 2021-04-29 14:08:01.130 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] lock -205> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process waiting <> waiting_peering {} -204> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=osd_op(client.421492180.0:227672 6.cf 6.e6c8fccf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8) prio 63 cost 127 e221098) pg 0x5601da35b000 -203> 2021-04-29 14:08:01.130 7fc30eab7700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -202> 2021-04-29 14:08:01.130 7fc3102ba700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -201> 2021-04-29 14:08:01.130 7fc30eab7700 10 osd.76 221098 dequeue_op 0x56023e8f4000 prio 196 cost 0 latency 0.000186 osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2 pg pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] -200> 2021-04-29 14:08:01.130 7fc30fab9700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -199> 2021-04-29 14:08:01.130 7fc30eab7700 20 osd.76 221098 share_map osd.62 v2:192.168.148.54:6826/796801 221098 -198> 2021-04-29 14:08:01.130 7fc30eab7700 20 osd.76 221098 should_share_map osd.62 v2:192.168.148.54:6826/796801 221098 -197> 2021-04-29 14:08:01.130 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] _handle_message: 0x56023e8f4000 -196> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 221098 dequeue_op 0x5602b6c97a20 prio 63 cost 127 latency 0.001630 osd_op(client.421492180.0:227672 6.cf 6.e6c8fccf (undecoded) ondisk+write+skiprwlocks+known_if_redirected e221098) v8 pg pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] -195> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 221098 share_map client.421492180 192.168.146.83:0/4103370583 221098 -194> 2021-04-29 14:08:01.130 7fc3102ba700 10 osd.76 221098 dequeue_op 0x56020277c160 prio 127 cost 754 latency 0.000304 osd_repop(client.421492180.0:227663 6.3d e221098/220465) v3 pg pg[6.3d( v 221098'639606631 (221046'639604024,221098'639606631] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606631 lcod 221098'639606630 active mbc={}] -193> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 221098 should_share_map client.421492180 192.168.146.83:0/4103370583 221098 -192> 2021-04-29 14:08:01.130 7fc30eab7700 7 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_repop_reply: tid 114279196 op ack_type 4 from 62 -191> 2021-04-29 14:08:01.130 7fc30eab7700 10 osd.76 221098 dequeue_op 0x56023e8f4000 finish -190> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _handle_message: 0x5602b6c97a20 -189> 2021-04-29 14:08:01.130 7fc3102ba700 20 osd.76 221098 share_map osd.112 v2:192.168.148.51:6808/588120 221098 -188> 2021-04-29 14:08:01.130 7fc30eab7700 20 osd.76 op_wq(4) _process empty q, waiting -187> 2021-04-29 14:08:01.130 7fc3102ba700 20 osd.76 221098 should_share_map osd.112 v2:192.168.148.51:6808/588120 221098 -186> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op: op osd_op(client.421492180.0:227672 6.cf 6:f33f1367:::incoming128-44:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221098) v8 -185> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] op_has_sufficient_caps session=0x56021fee1200 pool=6 (gnocchi ) pool_app_metadata={rbd={}} need_read_cap=0 need_write_cap=1 classes=[] -> yes -184> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op osd_op(client.421492180.0:227672 6.cf 6:f33f1367:::incoming128-44:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221098) v8 may_write -> write-ordered flags ondisk+write+skiprwlocks+known_if_redirected -183> 2021-04-29 14:08:01.130 7fc3102ba700 10 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606631 (221046'639604024,221098'639606631] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606631 lcod 221098'639606630 active mbc={}] _handle_message: 0x56020277c160 -182> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] readable_with_acting: locs:76,110 -181> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] get_object_context: found obc in cache: 0x5602700b5440 -180> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] get_object_context: 0x5602700b5440 6:f33f1367:::incoming128-44:head rwstate(none n=0 w=0) oi: 6:f33f1367:::incoming128-44:head(221098'634239124 client.421553675.0:230960 dirty|omap|data_digest s 0 uv 634239124 dd ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x560236f2d9e0 snapset: 0=[]:{} -179> 2021-04-29 14:08:01.130 7fc3102ba700 10 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606631 (221046'639604024,221098'639606631] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606631 lcod 221098'639606630 active mbc={}] do_repop 6:bc3bb5bd:::incoming128-55:head v 221098'639606632 (transaction) 190 -178> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] find_object_context 6:f33f1367:::incoming128-44:head @head oi=6:f33f1367:::incoming128-44:head(221098'634239124 client.421553675.0:230960 dirty|omap|data_digest s 0 uv 634239124 dd ffffffff alloc_hint [0 0 0]) -177> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] readable_with_acting: locs:76,110 -176> 2021-04-29 14:08:01.130 7fc3102ba700 30 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606631 (221046'639604024,221098'639606631] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606631 lcod 221098'639606630 active mbc={}] do_repop missing before {} -175> 2021-04-29 14:08:01.130 7fc30fab9700 25 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op oi 6:f33f1367:::incoming128-44:head(221098'634239124 client.421553675.0:230960 dirty|omap|data_digest s 0 uv 634239124 dd ffffffff alloc_hint [0 0 0]) -174> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op: skipping rw locks -173> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_op obc obc(6:f33f1367:::incoming128-44:head rwstate(none n=0 w=0)) -172> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] execute_ctx 0x5601ff653100 -171> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] execute_ctx 6:f33f1367:::incoming128-44:head [omap-set-vals] ov 221098'634239124 av 221098'634239126 snapc 0=[] snapset 0=[]:{} -170> 2021-04-29 14:08:01.130 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] execute_ctx user_at_version 634239124 -169> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_osd_op 6:f33f1367:::incoming128-44:head [omap-set-vals] -168> 2021-04-29 14:08:01.130 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_osd_op omap-set-vals -167> 2021-04-29 14:08:01.130 7fc3102ba700 10 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606631 (221046'639604024,221098'639606631] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606631 lcod 221098'639606630 active mbc={}] append_log log((221046'639604024,221098'639606631], crt=221098'639606631) [221098'639606632 (221098'639606629) modify 6:bc3bb5bd:::incoming128-55:head by client.421492180.0:227663 2021-04-29 14:08:01.129596 0] -166> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] setting vals: -165> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] measure_4344dc93-fbc6-4676-81a9-afc2228a71ac_5adfe784-3503-43c5-859d-4d08f8f15819_20210429_12:08:01 -164> 2021-04-29 14:08:01.130 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] make_writeable 6:f33f1367:::incoming128-44:head snapset=0=[]:{} snapc=0=[] -163> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] make_writeable 6:f33f1367:::incoming128-44:head done, snapset=0=[]:{} -162> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] finish_ctx 6:f33f1367:::incoming128-44:head 0x5601ff653100 op modify -161> 2021-04-29 14:08:01.130 7fc3102ba700 10 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606632 (221046'639604024,221098'639606632] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606631 lcod 221098'639606630 active mbc={}] add_log_entry 221098'639606632 (221098'639606629) modify 6:bc3bb5bd:::incoming128-55:head by client.421492180.0:227663 2021-04-29 14:08:01.129596 0 -160> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] set mtime to 2021-04-29 14:08:01.129961 -159> 2021-04-29 14:08:01.131 7fc3102ba700 20 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606632 (221046'639604024,221098'639606632] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606632 lcod 221098'639606630 active mbc={}] rollforward: entry=221098'639606632 (221098'639606629) modify 6:bc3bb5bd:::incoming128-55:head by client.421492180.0:227663 2021-04-29 14:08:01.129596 0 -158> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] final snapset 0=[]:{} in 6:f33f1367:::incoming128-44:head -157> 2021-04-29 14:08:01.131 7fc3102ba700 10 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606632 (221046'639604024,221098'639606632] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606632 lcod 221098'639606630 active mbc={}] append_log approx pg log length = 2608 -156> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] zeroing write result code 0 -155> 2021-04-29 14:08:01.131 7fc3102ba700 10 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606632 (221046'639604024,221098'639606632] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606632 lcod 221098'639606630 active mbc={}] append_log transaction_applied = 1 -154> 2021-04-29 14:08:01.131 7fc3102ba700 10 trim proposed trim_to = 221046'639604024 -153> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] calc_trim_to_aggressive limit = 221098'634239124 -152> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] calc_trim_to_aggressive approx pg log length = 3076 -151> 2021-04-29 14:08:01.131 7fc3102ba700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 221098'639606632, trimmed: , trimmed_dups: , clear_divergent_priors: 0 -150> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] calc_trim_to_aggressive num_to_trim = 76 -149> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] new_repop rep_tid 114279198 on osd_op(client.421492180.0:227672 6.cf 6:f33f1367:::incoming128-44:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221098) v8 -148> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] new_repop: repgather(0x5602f1490fc0 0'0 rep_tid=114279198 committed?=0 r=0) -147> 2021-04-29 14:08:01.131 7fc30fab9700 7 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] issue_repop rep_tid 114279198 o 6:f33f1367:::incoming128-44:head -146> 2021-04-29 14:08:01.131 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] issue_repop missing_loc before: 76,110 -145> 2021-04-29 14:08:01.131 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] issue_repop missing_loc after: 76,110 -144> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] should_send_op issue_repop shipping empty opt to osd.58, object 6:f33f1367:::incoming128-44:head which is pending recovery in async_recovery_targets -143> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 221098 share_map_peer 0x560238a77800 already has epoch 221098 -142> 2021-04-29 14:08:01.131 7fc32d2f4700 15 osd.76 221098 enqueue_op 0x5602ba38f8c0 prio 127 cost 752 latency 0.000012 epoch 221098 osd_repop(client.421626636.0:228463 6.2 e221098/220775) v3 -141> 2021-04-29 14:08:01.131 7fc32d2f4700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.2 PGOpItem(op=osd_repop(client.421626636.0:228463 6.2 e221098/220775) v3) prio 127 cost 752 e221098) -140> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 221098 share_map_peer 0x5601ff8e8400 already has epoch 221098 -139> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239125 (221020'634236049,221098'634239125] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] append_log log((221020'634236049,221098'634239125], crt=221098'634239125) [221098'634239126 (221098'634239124) modify 6:f33f1367:::incoming128-44:head by client.421492180.0:227672 2021-04-29 14:08:01.129961 0] -138> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239125 rops=1 crt=221098'634239125 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] add_log_entry 221098'634239126 (221098'634239124) modify 6:f33f1367:::incoming128-44:head by client.421492180.0:227672 2021-04-29 14:08:01.129961 0 -137> 2021-04-29 14:08:01.131 7fc30fab9700 20 earliest_dup_version = 634236127 -136> 2021-04-29 14:08:01.131 7fc30fab9700 20 trim 221098'634239126 (221098'634239124) modify 6:f33f1367:::incoming128-44:head by client.421492180.0:227672 2021-04-29 14:08:01.129961 0 -135> 2021-04-29 14:08:01.131 7fc3102ba700 30 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606632 (221046'639604024,221098'639606632] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606632 lcod 221098'639606630 active mbc={}] do_repop missing after{} -134> 2021-04-29 14:08:01.131 7fc3102ba700 10 osd.76 221098 dequeue_op 0x56020277c160 finish -133> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239125 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] rollforward: entry=221098'634239126 (221098'634239124) modify 6:f33f1367:::incoming128-44:head by client.421492180.0:227672 2021-04-29 14:08:01.129961 0 -132> 2021-04-29 14:08:01.131 7fc3102ba700 20 osd.76 op_wq(1) _process empty q, waiting -131> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239125 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] append_log approx pg log length = 3077 -130> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239125 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] append_log transaction_applied = 1 -129> 2021-04-29 14:08:01.131 7fc30fab9700 10 trim proposed trim_to = 221020'634236049 -128> 2021-04-29 14:08:01.131 7fc30fab9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 221098'634239126, trimmed: , trimmed_dups: , clear_divergent_priors: 0 -127> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 lua=221098'634239125 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] op_applied version 221098'634239126 -126> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] eval_repop repgather(0x5602f1490fc0 221098'634239126 rep_tid=114279198 committed?=0 r=0) -125> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 221098 dequeue_op 0x5602b6c97a20 finish -124> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 op_wq(2) _process 6.2 to_process <> waiting <> waiting_peering {} -123> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.2 PGOpItem(op=osd_repop(client.421626636.0:228463 6.2 e221098/220775) v3) prio 127 cost 752 e221098) queued -122> 2021-04-29 14:08:01.131 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089079 active mbc={}] lock -121> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 op_wq(2) _process 6.2 to_process waiting <> waiting_peering {} -120> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.2 PGOpItem(op=osd_repop(client.421626636.0:228463 6.2 e221098/220775) v3) prio 127 cost 752 e221098) pg 0x5601e563c000 -119> 2021-04-29 14:08:01.131 7fc30fab9700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -118> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 221098 dequeue_op 0x5602ba38f8c0 prio 127 cost 752 latency 0.000252 osd_repop(client.421626636.0:228463 6.2 e221098/220775) v3 pg pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089079 active mbc={}] -117> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 221098 share_map osd.40 v2:192.168.148.51:6813/589296 221098 -116> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 221098 should_share_map osd.40 v2:192.168.148.51:6813/589296 221098 -115> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089079 active mbc={}] _handle_message: 0x5602ba38f8c0 -114> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089079 active mbc={}] do_repop 6:40a70a2f:::incoming128-9:head v 221098'538089081 (transaction) 189 -113> 2021-04-29 14:08:01.131 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089079 active mbc={}] do_repop missing before {} -112> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089080 (221026'538086498,221098'538089080] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089079 active mbc={}] append_log log((221026'538086498,221098'538089080], crt=221098'538089080) [221098'538089081 (221098'538089080) modify 6:40a70a2f:::incoming128-9:head by client.421626636.0:228463 2021-04-29 14:08:01.130522 0] -111> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089081 (221026'538086498,221098'538089081] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089080 lcod 221098'538089079 active mbc={}] add_log_entry 221098'538089081 (221098'538089080) modify 6:40a70a2f:::incoming128-9:head by client.421626636.0:228463 2021-04-29 14:08:01.130522 0 -110> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089081 (221026'538086498,221098'538089081] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089081 lcod 221098'538089079 active mbc={}] rollforward: entry=221098'538089081 (221098'538089080) modify 6:40a70a2f:::incoming128-9:head by client.421626636.0:228463 2021-04-29 14:08:01.130522 0 -109> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089081 (221026'538086498,221098'538089081] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089081 lcod 221098'538089079 active mbc={}] append_log approx pg log length = 2583 -108> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089081 (221026'538086498,221098'538089081] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089081 lcod 221098'538089079 active mbc={}] append_log transaction_applied = 1 -107> 2021-04-29 14:08:01.131 7fc30fab9700 10 trim proposed trim_to = 221026'538086498 -106> 2021-04-29 14:08:01.131 7fc30fab9700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 221098'538089081, trimmed: , trimmed_dups: , clear_divergent_priors: 0 -105> 2021-04-29 14:08:01.131 7fc3102ba700 30 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606632 (221046'639604024,221098'639606632] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606632 lcod 221098'639606630 active mbc={}] lock -104> 2021-04-29 14:08:01.131 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089081 (221026'538086498,221098'538089081] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089081 lcod 221098'538089079 active mbc={}] do_repop missing after{} -103> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 221098 dequeue_op 0x5602ba38f8c0 finish -102> 2021-04-29 14:08:01.131 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] lock -101> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] op_commit: 114279197 -100> 2021-04-29 14:08:01.131 7fc3102ba700 10 osd.76 pg_epoch: 221098 pg[6.3d( v 221098'639606632 (221046'639604024,221098'639606632] local-lis/les=220465/220466 n=21564 ec=269/269 lis/c 220465/220465 les/c/f 220466/220466/0 220456/220465/220465) [112,76,93] r=1 lpr=220465 luod=0'0 lua=220464'639579710 crt=221098'639606632 lcod 221098'639606630 active mbc={}] repop_commit on op osd_repop(client.421492180.0:227663 6.3d e221098/220465 6:bc3bb5bd:::incoming128-55:head v 221098'639606632) v3, sending commit to osd.112 -99> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 op_wq(2) _process empty q, waiting -98> 2021-04-29 14:08:01.131 7fc3102ba700 20 osd.76 221098 share_map_peer 0x56029d227c00 already has epoch 221098 -97> 2021-04-29 14:08:01.131 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] lock -96> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] op_commit: 114279198 -95> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 op_wq(2) _process empty q, waiting -94> 2021-04-29 14:08:01.131 7fc3102ba700 20 osd.76 op_wq(1) _process empty q, waiting -93> 2021-04-29 14:08:01.131 7fc32daf5700 15 osd.76 221098 enqueue_op 0x5601f793c420 prio 196 cost 0 latency 0.000007 epoch 221098 osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2 -92> 2021-04-29 14:08:01.131 7fc32daf5700 20 osd.76 op_wq(4) _enqueue OpQueueItem(6.4f PGOpItem(op=osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2) prio 196 cost 0 e221098) -91> 2021-04-29 14:08:01.131 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089081 (221026'538086498,221098'538089081] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089081 lcod 221098'538089079 active mbc={}] lock -90> 2021-04-29 14:08:01.131 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.2( v 221098'538089081 (221026'538086498,221098'538089081] local-lis/les=220775/220776 n=21533 ec=269/269 lis/c 220775/220775 les/c/f 220776/220776/0 220723/220775/220514) [40,50,76] r=2 lpr=220775 luod=0'0 lua=220774'538073520 crt=221098'538089081 lcod 221098'538089079 active mbc={}] repop_commit on op osd_repop(client.421626636.0:228463 6.2 e221098/220775 6:40a70a2f:::incoming128-9:head v 221098'538089081) v3, sending commit to osd.40 -89> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 221098 share_map_peer 0x560228ba4c00 already has epoch 221098 -88> 2021-04-29 14:08:01.131 7fc30eab7700 20 osd.76 op_wq(4) _process 6.4f to_process <> waiting <> waiting_peering {} -87> 2021-04-29 14:08:01.131 7fc30eab7700 20 osd.76 op_wq(4) _process OpQueueItem(6.4f PGOpItem(op=osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2) prio 196 cost 0 e221098) queued -86> 2021-04-29 14:08:01.131 7fc30fab9700 20 osd.76 op_wq(2) _process empty q, waiting -85> 2021-04-29 14:08:01.131 7fc30eab7700 30 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] lock -84> 2021-04-29 14:08:01.131 7fc30eab7700 20 osd.76 op_wq(4) _process 6.4f to_process waiting <> waiting_peering {} -83> 2021-04-29 14:08:01.131 7fc30eab7700 20 osd.76 op_wq(4) _process OpQueueItem(6.4f PGOpItem(op=osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2) prio 196 cost 0 e221098) pg 0x5601dc8e6000 -82> 2021-04-29 14:08:01.131 7fc30eab7700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -81> 2021-04-29 14:08:01.131 7fc30eab7700 10 osd.76 221098 dequeue_op 0x5601f793c420 prio 196 cost 0 latency 0.000111 osd_repop_reply(client.421492180.0:227661 6.4f e221098/214501) v2 pg pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] -80> 2021-04-29 14:08:01.131 7fc30eab7700 20 osd.76 221098 share_map osd.100 v2:192.168.148.58:6828/1038564 221098 -79> 2021-04-29 14:08:01.131 7fc30eab7700 20 osd.76 221098 should_share_map osd.100 v2:192.168.148.58:6828/1038564 221098 -78> 2021-04-29 14:08:01.131 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] _handle_message: 0x5601f793c420 -77> 2021-04-29 14:08:01.131 7fc30eab7700 7 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] do_repop_reply: tid 114279196 op ack_type 4 from 100 -76> 2021-04-29 14:08:01.131 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 luod=221098'612784294 crt=221098'612784295 lcod 221098'612784293 mlcod 221098'612784293 active+clean] repop_all_committed: repop tid 114279196 all committed -75> 2021-04-29 14:08:01.131 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784293 active+clean] eval_repop repgather(0x560222e1c360 221098'612784295 rep_tid=114279196 committed?=1 r=0) -74> 2021-04-29 14:08:01.131 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784293 active+clean] commit: repgather(0x560222e1c360 221098'612784295 rep_tid=114279196 committed?=1 r=0) -73> 2021-04-29 14:08:01.131 7fc30eab7700 15 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784293 active+clean] log_op_stats osd_op(client.421492180.0:227661 6.4f 6:f26f3dea:::incoming128-91:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221098) v8 inb 0 outb 0 lat 0.003421 -72> 2021-04-29 14:08:01.131 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784293 active+clean] sending reply on osd_op(client.421492180.0:227661 6.4f 6:f26f3dea:::incoming128-91:head [omap-set-vals] snapc 0=[] ondisk+write+skiprwlocks+known_if_redirected e221098) v8 0x560214417b80 -71> 2021-04-29 14:08:01.132 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784293 active+clean] publish_stats_to_osd reporting purged_snaps [] -70> 2021-04-29 14:08:01.132 7fc30eab7700 15 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784293 active+clean] publish_stats_to_osd 221098:789097831 -69> 2021-04-29 14:08:01.132 7fc30eab7700 10 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784294 active+clean] removing repgather(0x560222e1c360 221098'612784295 rep_tid=114279196 committed?=1 r=0) -68> 2021-04-29 14:08:01.132 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784294 active+clean] q front is repgather(0x560222e1c360 221098'612784295 rep_tid=114279196 committed?=1 r=0) -67> 2021-04-29 14:08:01.132 7fc30eab7700 15 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784294 active+clean] do_osd_op_effects client.421492180 con 0x5602d7143000 -66> 2021-04-29 14:08:01.132 7fc30eab7700 20 osd.76 pg_epoch: 221098 pg[6.4f( v 221098'612784295 (221006'612781238,221098'612784295] local-lis/les=214501/214502 n=21861 ec=269/269 lis/c 214501/214501 les/c/f 214502/214502/0 214448/214501/214219) [76,62,100] r=0 lpr=214501 crt=221098'612784295 lcod 221098'612784294 mlcod 221098'612784294 active+clean] remove_repop repgather(0x560222e1c360 221098'612784295 rep_tid=114279196 committed?=1 r=0) -65> 2021-04-29 14:08:01.132 7fc30eab7700 10 osd.76 221098 dequeue_op 0x5601f793c420 finish -64> 2021-04-29 14:08:01.132 7fc32daf5700 15 osd.76 221098 enqueue_op 0x56027842a2c0 prio 196 cost 0 latency 0.000007 epoch 221098 osd_repop_reply(client.421492180.0:227642 6.cf e221098/221088) v2 -63> 2021-04-29 14:08:01.132 7fc30eab7700 20 osd.76 op_wq(4) _process empty q, waiting -62> 2021-04-29 14:08:01.132 7fc32daf5700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.cf PGOpItem(op=osd_repop_reply(client.421492180.0:227642 6.cf e221098/221088) v2) prio 196 cost 0 e221098) -61> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process <> waiting <> waiting_peering {} -60> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=osd_repop_reply(client.421492180.0:227642 6.cf e221098/221088) v2) prio 196 cost 0 e221098) queued -59> 2021-04-29 14:08:01.132 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] lock -58> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process waiting <> waiting_peering {} -57> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=osd_repop_reply(client.421492180.0:227642 6.cf e221098/221088) v2) prio 196 cost 0 e221098) pg 0x5601da35b000 -56> 2021-04-29 14:08:01.132 7fc30fab9700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -55> 2021-04-29 14:08:01.132 7fc30fab9700 10 osd.76 221098 dequeue_op 0x56027842a2c0 prio 196 cost 0 latency 0.000068 osd_repop_reply(client.421492180.0:227642 6.cf e221098/221088) v2 pg pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] -54> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 221098 share_map osd.110 v2:192.168.148.58:6822/1038515 221098 -53> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 221098 should_share_map osd.110 v2:192.168.148.58:6822/1038515 221098 -52> 2021-04-29 14:08:01.132 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _handle_message: 0x56027842a2c0 -51> 2021-04-29 14:08:01.132 7fc30fab9700 7 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_repop_reply: tid 114279197 op ack_type 4 from 110 -50> 2021-04-29 14:08:01.132 7fc30fab9700 10 osd.76 221098 dequeue_op 0x56027842a2c0 finish -49> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process empty q, waiting -48> 2021-04-29 14:08:01.132 7fc32daf5700 15 osd.76 221098 enqueue_op 0x5602129adce0 prio 196 cost 0 latency 0.000004 epoch 221098 osd_repop_reply(client.421492180.0:227672 6.cf e221098/221088) v2 -47> 2021-04-29 14:08:01.132 7fc32daf5700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.cf PGOpItem(op=osd_repop_reply(client.421492180.0:227672 6.cf e221098/221088) v2) prio 196 cost 0 e221098) -46> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process <> waiting <> waiting_peering {} -45> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=osd_repop_reply(client.421492180.0:227672 6.cf e221098/221088) v2) prio 196 cost 0 e221098) queued -44> 2021-04-29 14:08:01.132 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] lock -43> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process waiting <> waiting_peering {} -42> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=osd_repop_reply(client.421492180.0:227672 6.cf e221098/221088) v2) prio 196 cost 0 e221098) pg 0x5601da35b000 -41> 2021-04-29 14:08:01.132 7fc30fab9700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -40> 2021-04-29 14:08:01.132 7fc30fab9700 10 osd.76 221098 dequeue_op 0x5602129adce0 prio 196 cost 0 latency 0.000047 osd_repop_reply(client.421492180.0:227672 6.cf e221098/221088) v2 pg pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] -39> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 221098 share_map osd.110 v2:192.168.148.58:6822/1038515 221098 -38> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 221098 should_share_map osd.110 v2:192.168.148.58:6822/1038515 221098 -37> 2021-04-29 14:08:01.132 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _handle_message: 0x5602129adce0 -36> 2021-04-29 14:08:01.132 7fc30fab9700 7 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] do_repop_reply: tid 114279198 op ack_type 4 from 110 -35> 2021-04-29 14:08:01.132 7fc30fab9700 10 osd.76 221098 dequeue_op 0x5602129adce0 finish -34> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process empty q, waiting -33> 2021-04-29 14:08:01.132 7fc32caf3700 15 osd.76 221098 enqueue_op 0x5602da1d71e0 prio 1 cost 8389608 latency 0.000010 epoch 221098 MOSDPGPushReply(6.cf 221098/221088 [PushReplyOp(6:f382cb1f:::incoming128-85:head)]) v3 -32> 2021-04-29 14:08:01.132 7fc32caf3700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.cf PGOpItem(op=MOSDPGPushReply(6.cf 221098/221088 [PushReplyOp(6:f382cb1f:::incoming128-85:head)]) v3) prio 1 cost 8389608 e221098) -31> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process <> waiting <> waiting_peering {} -30> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=MOSDPGPushReply(6.cf 221098/221088 [PushReplyOp(6:f382cb1f:::incoming128-85:head)]) v3) prio 1 cost 8389608 e221098) queued -29> 2021-04-29 14:08:01.132 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] lock -28> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process 6.cf to_process waiting <> waiting_peering {} -27> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 op_wq(2) _process OpQueueItem(6.cf PGOpItem(op=MOSDPGPushReply(6.cf 221098/221088 [PushReplyOp(6:f382cb1f:::incoming128-85:head)]) v3) prio 1 cost 8389608 e221098) pg 0x5601da35b000 -26> 2021-04-29 14:08:01.132 7fc30fab9700 30 dequeue status: {"OSD:ShardedOpWQ:0":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:1":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:2":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:3":{"high_queues":[],"queues":[]},"OSD:ShardedOpWQ:4":{"high_queues":[],"queues":[]}} -25> 2021-04-29 14:08:01.132 7fc30fab9700 10 osd.76 221098 dequeue_op 0x5602da1d71e0 prio 1 cost 8389608 latency 0.000076 MOSDPGPushReply(6.cf 221098/221088 [PushReplyOp(6:f382cb1f:::incoming128-85:head)]) v3 pg pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] -24> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 221098 share_map osd.58 v2:192.168.148.53:6835/3069374 221098 -23> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 221098 should_share_map osd.58 v2:192.168.148.53:6835/3069374 221098 -22> 2021-04-29 14:08:01.132 7fc30fab9700 10 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _handle_message: 0x5602da1d71e0 -21> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats actingset 76,110 upset 58,76,110 acting_recovery_backfill 58,76,110 -20> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats acting [76,110] up [76,58,110] -19> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats shard 76 primary objects 21718 missing 0 -18> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats shard 58 objects 21718 missing 2 -17> 2021-04-29 14:08:01.132 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats shard 110 objects 21718 missing 0 -16> 2021-04-29 14:08:01.133 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats 6:f33f1367:::incoming128-44:head object_location_counts[76,110]=1 -15> 2021-04-29 14:08:01.133 7fc32caf3700 15 osd.76 221098 enqueue_op 0x5602571a4160 prio 196 cost 0 latency 0.000006 epoch 221098 osd_repop_reply(client.421492180.0:227642 6.cf e221098/221088) v2 -14> 2021-04-29 14:08:01.133 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats 6:f382cb1f:::incoming128-85:head object_location_counts[76,110]=2 -13> 2021-04-29 14:08:01.133 7fc32caf3700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.cf PGOpItem(op=osd_repop_reply(client.421492180.0:227642 6.cf e221098/221088) v2) prio 196 cost 0 e221098) -12> 2021-04-29 14:08:01.133 7fc30fab9700 30 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats object_location_counts[58,76,110]=0 -11> 2021-04-29 14:08:01.133 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats object_location_counts {58,76,110=0,76,110=21716} -10> 2021-04-29 14:08:01.133 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats ml 2 upset size 3 up 2 -9> 2021-04-29 14:08:01.133 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats missing based degraded 2 -8> 2021-04-29 14:08:01.133 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats missing based misplaced 0 -7> 2021-04-29 14:08:01.133 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats degraded 2 -6> 2021-04-29 14:08:01.133 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] _update_calc_stats misplaced 0 -5> 2021-04-29 14:08:01.133 7fc30fab9700 20 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] publish_stats_to_osd reporting purged_snaps [] -4> 2021-04-29 14:08:01.133 7fc30fab9700 15 osd.76 pg_epoch: 221098 pg[6.cf( v 221098'634239126 (221020'634236049,221098'634239126] local-lis/les=221088/221089 n=21718 ec=185965/269 lis/c 221088/214506 les/c/f 221089/214507/0 221087/221088/214152) [76,58,110]/[76,110] async=[58] r=0 lpr=221088 pi=[214506,221088)/1 luod=221098'634239124 rops=1 crt=221098'634239126 lcod 221098'634239123 mlcod 221098'634239119 active+recovering+undersized+degraded+remapped mbc={255={(2+0)=2}}] publish_stats_to_osd 221098:809321299 -3> 2021-04-29 14:08:01.133 7fc32caf3700 15 osd.76 221098 enqueue_op 0x560266cd5600 prio 196 cost 0 latency 0.000004 epoch 221098 osd_repop_reply(client.421492180.0:227672 6.cf e221098/221088) v2 -2> 2021-04-29 14:08:01.133 7fc32caf3700 20 osd.76 op_wq(2) _enqueue OpQueueItem(6.cf PGOpItem(op=osd_repop_reply(client.421492180.0:227672 6.cf e221098/221088) v2) prio 196 cost 0 e221098) -1> 2021-04-29 14:08:01.138 7fc30fab9700 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/gigantic/release/14.2.19/rpm/el7/BUILD/ceph-14.2.19/src/osd/osd_types.h: In function 'void pg_missing_set::got(const hobject_t&, eversion_t) [with bool TrackChanges = false]' thread 7fc30fab9700 time 2021-04-29 14:08:01.134009 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/gigantic/release/14.2.19/rpm/el7/BUILD/ceph-14.2.19/src/osd/osd_types.h: 4418: FAILED ceph_assert(p->second.need <= v || p->second.is_delete()) ceph version 14.2.19 (bb796b9b5bab9463106022eef406373182465d11) nautilus (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x14a) [0x5601cdfcb4a3] 2: (()+0x4d966b) [0x5601cdfcb66b] 3: (()+0x6fce92) [0x5601ce1eee92] 4: (ReplicatedBackend::handle_push_reply(pg_shard_t, PushReplyOp const&, PushOp*)+0x575) [0x5601ce3a6dd5] 5: (ReplicatedBackend::do_push_reply(boost::intrusive_ptr)+0xf2) [0x5601ce3aaea2] 6: (ReplicatedBackend::_handle_message(boost::intrusive_ptr)+0x227) [0x5601ce3ab237] 7: (PGBackend::handle_message(boost::intrusive_ptr)+0x4a) [0x5601ce2bcc8a] 8: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x5b3) [0x5601ce26cdb3] 9: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x362) [0x5601ce0ac462] 10: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x62) [0x5601ce33bc02] 11: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x90f) [0x5601ce0c796f] 12: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b6) [0x5601ce683956] 13: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5601ce686470] 14: (()+0x7ea5) [0x7fc331c35ea5] 15: (clone()+0x6d) [0x7fc330af98dd] 0> 2021-04-29 14:08:01.144 7fc30fab9700 -1 *** Caught signal (Aborted) ** in thread 7fc30fab9700 thread_name:tp_osd_tp ceph version 14.2.19 (bb796b9b5bab9463106022eef406373182465d11) nautilus (stable) 1: (()+0xf630) [0x7fc331c3d630] 2: (gsignal()+0x37) [0x7fc330a31387] 3: (abort()+0x148) [0x7fc330a32a78] 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x199) [0x5601cdfcb4f2] 5: (()+0x4d966b) [0x5601cdfcb66b] 6: (()+0x6fce92) [0x5601ce1eee92] 7: (ReplicatedBackend::handle_push_reply(pg_shard_t, PushReplyOp const&, PushOp*)+0x575) [0x5601ce3a6dd5] 8: (ReplicatedBackend::do_push_reply(boost::intrusive_ptr)+0xf2) [0x5601ce3aaea2] 9: (ReplicatedBackend::_handle_message(boost::intrusive_ptr)+0x227) [0x5601ce3ab237] 10: (PGBackend::handle_message(boost::intrusive_ptr)+0x4a) [0x5601ce2bcc8a] 11: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x5b3) [0x5601ce26cdb3] 12: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x362) [0x5601ce0ac462] 13: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x62) [0x5601ce33bc02] 14: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x90f) [0x5601ce0c796f] 15: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b6) [0x5601ce683956] 16: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5601ce686470] 17: (()+0x7ea5) [0x7fc331c35ea5] 18: (clone()+0x6d) [0x7fc330af98dd] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 0 none 0/ 0 lockdep 0/ 0 context 0/ 0 crush 0/ 0 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 0 buffer 0/ 0 timer 0/ 0 filer 0/ 0 striper 0/ 0 objecter 0/ 0 rados 0/ 0 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 0 journaler 0/ 0 objectcacher 0/ 0 client 30/30 osd 0/ 0 optracker 0/ 0 objclass 0/ 0 filestore 0/ 0 journal 0/ 0 ms 0/ 0 mon 0/ 0 monc 0/ 0 paxos 0/ 0 tp 0/ 0 auth 0/ 0 crypto 0/ 0 finisher 0/ 0 reserver 0/ 0 heartbeatmap 0/ 0 perfcounter 0/ 0 rgw 1/ 5 rgw_sync 0/ 0 civetweb 0/ 0 javaclient 0/ 0 asok 0/ 0 throttle 0/ 0 refs 0/ 0 xio 0/ 0 compressor 0/ 0 bluestore 0/ 0 bluefs 0/ 0 bdev 0/ 0 kstore 0/ 0 rocksdb 0/ 0 leveldb 0/ 0 memdb 0/ 0 kinetic 0/ 0 fuse 0/ 0 mgr 0/ 0 mgrc 0/ 0 dpdk 0/ 0 eventtrace 1/ 5 prioritycache -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-osd.76.log --- end dump of recent events ---