2020-05-06T20:06:37.723+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.38 up_from 5076 peer_clock_delta [1167815.474797874s,1167815.475438770s]) 2020-05-06T20:06:37.723+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.38 up_from 5076 peer_clock_delta [1167815.474772622s,1167815.475438770s]) 2020-05-06T20:06:37.731+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.28 up_from 5167 peer_clock_delta [1167441.804939312s,1167441.805586171s]) 2020-05-06T20:06:37.731+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.28 up_from 5167 peer_clock_delta [1167441.804821405s,1167441.805586171s]) 2020-05-06T20:06:37.775+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.1 up_from 5084 peer_clock_delta [1167782.009984516s,1167782.010235376s]) 2020-05-06T20:06:37.775+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.1 up_from 5084 peer_clock_delta [1167782.009863256s,1167782.010235376s]) 2020-05-06T20:06:37.819+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071769315s,1167579.072397128s]) 2020-05-06T20:06:37.819+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071756150s,1167579.072397128s]) 2020-05-06T20:06:38.047+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.4 up_from 5161 peer_clock_delta [1167474.555849089s,1167474.556214194s]) 2020-05-06T20:06:38.047+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.4 up_from 5161 peer_clock_delta [1167474.555832470s,1167474.556214194s]) 2020-05-06T20:06:38.075+0200 7f79139e0700 10 osd.15 5421 tick_without_osd_lock 2020-05-06T20:06:38.075+0200 7f79139e0700 20 osd.15 5421 can_inc_scrubs == true 0 local + 0 remote < max 1 2020-05-06T20:06:38.075+0200 7f79139e0700 20 osd.15 5421 scrub_time_permit should run between 0 - 24 now 20 = yes 2020-05-06T20:06:38.075+0200 7f79139e0700 20 osd.15 5421 scrub_load_below_threshold loadavg per cpu 0 < max 0.5 = yes 2020-05-06T20:06:38.075+0200 7f79139e0700 20 osd.15 5421 sched_scrub load_is_low=1 2020-05-06T20:06:38.075+0200 7f79139e0700 10 osd.15 5421 sched_scrub 2.1s0 scheduled at 2020-05-06T20:58:35.908491+0200 > 2020-05-06T20:06:38.081925+0200 2020-05-06T20:06:38.075+0200 7f79139e0700 20 osd.15 5421 sched_scrub done 2020-05-06T20:06:38.075+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2020-05-06T20:06:38.075+0200 7f79139e0700 20 osd.15 5421 promote_throttle_recalibrate new_prob 1000 2020-05-06T20:06:38.075+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2020-05-06T20:06:38.131+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.20 up_from 5111 peer_clock_delta [1167678.414983375s,1167678.415610265s]) 2020-05-06T20:06:38.131+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.20 up_from 5111 peer_clock_delta [1167678.414942126s,1167678.415610265s]) 2020-05-06T20:06:38.151+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.13 up_from 5086 peer_clock_delta [1167779.698640937s,1167779.699087443s]) 2020-05-06T20:06:38.151+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.13 up_from 5086 peer_clock_delta [1167779.698527366s,1167779.699087443s]) 2020-05-06T20:06:38.155+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x564478ff4420 prio 127 cost 1021 latency 0.000006 epoch 5421 osd_repop(client.398016.0:41636614 4.f e5421/5414) v3 2020-05-06T20:06:38.155+0200 7f79189ea700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.398016.0:41636614 4.f e5421/5414) v3) prio 127 cost 1021 e5421) 2020-05-06T20:06:38.155+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:38.155+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.398016.0:41636614 4.f e5421/5414) v3) prio 127 cost 1021 e5421) queued 2020-05-06T20:06:38.155+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process waiting <> waiting_peering {} 2020-05-06T20:06:38.155+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.398016.0:41636614 4.f e5421/5414) v3) prio 127 cost 1021 e5421) pg 0x56445e51e000 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x564478ff4420 prio 127 cost 1021 latency 0.000050 osd_repop(client.398016.0:41636614 4.f e5421/5414) v3 pg pg[4.f( v 5421'31367 (5421'28900,5421'31367] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31367 lcod 5421'31366 mlcod 5421'31365 active mbc={}] 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31367 (5421'28900,5421'31367] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31367 lcod 5421'31366 mlcod 5421'31365 active mbc={}] _handle_message: 0x564478ff4420 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31367 (5421'28900,5421'31367] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31367 lcod 5421'31366 mlcod 5421'31365 active mbc={}] do_repop 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31368 (transaction) 228 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31367 (5421'28900,5421'31367] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31367 lcod 5421'31366 mlcod 5421'31365 active mbc={}] append_log log((5421'28900,5421'31367], crt=5421'31367) [5421'31368 (5421'31367) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.398016.0:41636614 2020-05-06T20:06:38.160263+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0] 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31367 lcod 5421'31366 mlcod 5421'31365 active mbc={}] add_log_entry 5421'31368 (5421'31367) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.398016.0:41636614 2020-05-06T20:06:38.160263+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:38.155+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31366 mlcod 5421'31365 active mbc={}] rollforward: entry=5421'31368 (5421'31367) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.398016.0:41636614 2020-05-06T20:06:38.160263+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31366 mlcod 5421'31365 active mbc={}] append_log approx pg log length = 2468 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31366 mlcod 5421'31365 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 trim proposed trim_to = 5421'28900 2020-05-06T20:06:38.155+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'31368, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x564478ff4420 finish 2020-05-06T20:06:38.155+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:38.155+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31366 mlcod 5421'31366 active mbc={}] repop_commit on op osd_repop(client.398016.0:41636614 4.f e5421/5414 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31368, mlcod=5421'31366) v3, sending commit to osd.12 2020-05-06T20:06:38.155+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:38.159+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214055532s,1167820.214591990s]) 2020-05-06T20:06:38.159+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.213999050s,1167820.214591990s]) 2020-05-06T20:06:38.171+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x56447d82e520 prio 127 cost 1253 latency 0.000003 epoch 5421 osd_repop(client.398016.0:41636616 4.f e5421/5414) v3 2020-05-06T20:06:38.171+0200 7f79189ea700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.398016.0:41636616 4.f e5421/5414) v3) prio 127 cost 1253 e5421) 2020-05-06T20:06:38.171+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:38.171+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.398016.0:41636616 4.f e5421/5414) v3) prio 127 cost 1253 e5421) queued 2020-05-06T20:06:38.171+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process waiting <> waiting_peering {} 2020-05-06T20:06:38.171+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.398016.0:41636616 4.f e5421/5414) v3) prio 127 cost 1253 e5421) pg 0x56445e51e000 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x56447d82e520 prio 127 cost 1253 latency 0.000037 osd_repop(client.398016.0:41636616 4.f e5421/5414) v3 pg pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31367 mlcod 5421'31366 active mbc={}] 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31367 mlcod 5421'31366 active mbc={}] _handle_message: 0x56447d82e520 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31367 mlcod 5421'31366 active mbc={}] do_repop 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31369 (transaction) 228 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31368 (5421'28900,5421'31368] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31367 mlcod 5421'31366 active mbc={}] append_log log((5421'28900,5421'31368], crt=5421'31368) [5421'31369 (5421'31368) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.398016.0:41636616 2020-05-06T20:06:38.175902+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0] 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31368 lcod 5421'31367 mlcod 5421'31366 active mbc={}] add_log_entry 5421'31369 (5421'31368) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.398016.0:41636616 2020-05-06T20:06:38.175902+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:38.171+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31367 mlcod 5421'31366 active mbc={}] rollforward: entry=5421'31369 (5421'31368) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.398016.0:41636616 2020-05-06T20:06:38.175902+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31367 mlcod 5421'31366 active mbc={}] append_log approx pg log length = 2469 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31367 mlcod 5421'31366 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 trim proposed trim_to = 5421'28900 2020-05-06T20:06:38.171+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'31369, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x56447d82e520 finish 2020-05-06T20:06:38.171+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:38.171+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31367 mlcod 5421'31367 active mbc={}] repop_commit on op osd_repop(client.398016.0:41636616 4.f e5421/5414 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31369, mlcod=5421'31367) v3, sending commit to osd.12 2020-05-06T20:06:38.171+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:38.199+0200 7f79149e2700 10 osd.15 5421 tick 2020-05-06T20:06:38.199+0200 7f79149e2700 10 osd.15 5421 do_waiters -- start 2020-05-06T20:06:38.199+0200 7f79149e2700 10 osd.15 5421 do_waiters -- finish 2020-05-06T20:06:38.199+0200 7f79149e2700 20 osd.15 5421 tick last_purged_snaps_scrub 2020-05-06T09:01:52.905203+0200 next 2020-05-07T09:01:52.905203+0200 2020-05-06T20:06:38.491+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287167660s,1167782.287519521s]) 2020-05-06T20:06:38.491+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287152392s,1167782.287519521s]) 2020-05-06T20:06:38.663+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.33 up_from 5092 peer_clock_delta [1167747.547785636s,1167747.548381878s]) 2020-05-06T20:06:38.663+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.33 up_from 5092 peer_clock_delta [1167747.547762107s,1167747.548381878s]) 2020-05-06T20:06:38.675+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.21 up_from 5153 peer_clock_delta [1167508.090479436s,1167508.091046016s]) 2020-05-06T20:06:38.675+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.21 up_from 5153 peer_clock_delta [1167508.090456025s,1167508.091046016s]) 2020-05-06T20:06:38.719+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.22 up_from 5074 peer_clock_delta [1167818.481376210s,1167818.481899935s]) 2020-05-06T20:06:38.719+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.22 up_from 5074 peer_clock_delta [1167818.481203302s,1167818.481899935s]) 2020-05-06T20:06:39.019+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.29 up_from 5099 peer_clock_delta [1167715.168893640s,1167715.169311199s]) 2020-05-06T20:06:39.019+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.29 up_from 5099 peer_clock_delta [1167715.168829180s,1167715.169311199s]) 2020-05-06T20:06:39.019+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779296938s,1167509.779711233s]) 2020-05-06T20:06:39.019+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779158235s,1167509.779711233s]) 2020-05-06T20:06:39.099+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.32 up_from 5074 peer_clock_delta [1167817.955891093s,1167817.956434171s]) 2020-05-06T20:06:39.099+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.32 up_from 5074 peer_clock_delta [1167817.955873896s,1167817.956434171s]) 2020-05-06T20:06:39.107+0200 7f79139e0700 10 osd.15 5421 tick_without_osd_lock 2020-05-06T20:06:39.107+0200 7f79139e0700 20 osd.15 5421 scrub_random_backoff lost coin flip, randomly backing off 2020-05-06T20:06:39.107+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2020-05-06T20:06:39.107+0200 7f79139e0700 20 osd.15 5421 promote_throttle_recalibrate new_prob 1000 2020-05-06T20:06:39.107+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2020-05-06T20:06:39.215+0200 7f79149e2700 10 osd.15 5421 tick 2020-05-06T20:06:39.215+0200 7f79149e2700 10 osd.15 5421 do_waiters -- start 2020-05-06T20:06:39.215+0200 7f79149e2700 10 osd.15 5421 do_waiters -- finish 2020-05-06T20:06:39.215+0200 7f79149e2700 20 osd.15 5421 tick last_purged_snaps_scrub 2020-05-06T09:01:52.905203+0200 next 2020-05-07T09:01:52.905203+0200 2020-05-06T20:06:39.391+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.30 up_from 5118 peer_clock_delta [1167646.570370904s,1167646.570697544s]) 2020-05-06T20:06:39.391+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.30 up_from 5118 peer_clock_delta [1167646.570353078s,1167646.570697544s]) 2020-05-06T20:06:39.787+0200 7f79191eb700 10 osd.15 5421 ms_handle_authentication new session 0x56449845dde0 con 0x56447fff5400 entity client.admin addr 2020-05-06T20:06:39.787+0200 7f79191eb700 10 osd.15 5421 ms_handle_authentication session 0x56449845dde0 client.admin has caps osdcap[grant(*)] 'allow *' 2020-05-06T20:06:39.787+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x5644903ab760 prio 63 cost 0 latency 0.000005 epoch 5421 osd_op(client.1360725.0:5 7.3 7.49ff62f3 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.787+0200 7f79191eb700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:5 7.3 7.49ff62f3 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:5 7.3 7.49ff62f3 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) queued 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:5 7.3 7.49ff62f3 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) pg 0x56445f06b400 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644903ab760 prio 63 cost 0 latency 0.000050 osd_op(client.1360725.0:5 7.3 7.49ff62f3 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 5421 maybe_share_map con 0x56447fff5400 [2a00:X:X:6::4]:0/2582123382 map epoch 0 -> 5421 (as per caller) 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] _handle_message: 0x5644903ab760 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: op osd_op(client.1360725.0:5 7.3 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] op_has_sufficient_caps session=0x56449845dde0 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op osd_op(client.1360725.0:5 7.3 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: found obc in cache: 0x56447a145180 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: 0x56447a145180 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head rwstate(none n=0 w=0) oi: 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head(4159'4 osd.15.0:35413 dirty|data_digest|omap_digest s 882 uv 2 dd aae8828f od ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x5644780cb9c0 snapset: 0=[]:{} 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] find_object_context 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head @head oi=7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head(4159'4 osd.15.0:35413 dirty|data_digest|omap_digest s 882 uv 2 dd aae8828f od ffffffff alloc_hint [0 0 0]) 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op obc obc(7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head rwstate(read n=1 w=0)) 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 0x564477b38900 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head [read 0~0] ov 4159'4 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head [read 0~0] 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op read 0~0 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_read 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] read got 882 / 882 bytes from obj 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx alloc reply 0x5644664c2d00 result 0 2020-05-06T20:06:39.787+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op_effects client.1360725 con 0x56447fff5400 2020-05-06T20:06:39.787+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] log_op_stats osd_op(client.1360725.0:5 7.3 7:cf46ff92:::periods.779701a3-99bd-4b30-9796-3925998fd466.1:head [read 0~882 out=882b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 882 lat 0.000213 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:39.787+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] publish_stats_to_osd 5421:733016 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644903ab760 finish 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.787+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x5644984d86e0 prio 63 cost 0 latency 0.000002 epoch 5421 osd_op(client.1360725.0:6 7.3 7.636fdd3 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.787+0200 7f79191eb700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:6 7.3 7.636fdd3 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:6 7.3 7.636fdd3 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) queued 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:6 7.3 7.636fdd3 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) pg 0x56445f06b400 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644984d86e0 prio 63 cost 0 latency 0.000029 osd_op(client.1360725.0:6 7.3 7.636fdd3 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] _handle_message: 0x5644984d86e0 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: op osd_op(client.1360725.0:6 7.3 7:cbbf6c60:::converted:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] op_has_sufficient_caps session=0x56449845dde0 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op osd_op(client.1360725.0:6 7.3 7:cbbf6c60:::converted:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: obc NOT found in cache: 7:cbbf6c60:::converted:head 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: no obc for soid 7:cbbf6c60:::converted:head and !can_create 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: find_object_context got error -2 2020-05-06T20:06:39.787+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644984d86e0 finish 2020-05-06T20:06:39.787+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.795+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x564492548420 prio 63 cost 39 latency 0.000003 epoch 5421 osd_op(client.1360725.0:22 7.3 7.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f79191eb700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:22 7.3 7.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:22 7.3 7.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) queued 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:22 7.3 7.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) pg 0x56445f06b400 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564492548420 prio 63 cost 39 latency 0.000032 osd_op(client.1360725.0:22 7.3 7.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 pg pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] _handle_message: 0x564492548420 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: op osd_op(client.1360725.0:22 7.3 7:c0000000::::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] op_has_sufficient_caps session=0x56449845dde0 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_pg_op osd_op(client.1360725.0:22 7.3 7:c0000000::::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls pg=7.3 count 1024 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls lower_bound 7:c0000000::::head pg_end 7:d0000000::::head 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls candidate 0xf5dc5c23 vs lower bound 0x3 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls item 0xf5dc5c23, rev 0xc43a3baf realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls candidate 0x49ff62f3 vs lower bound 0x3 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls item 0x49ff62f3, rev 0xcf46ff92 periods.779701a3-99bd-4b30-9796-3925998fd466.1 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls candidate 0x0 vs lower bound 0x3 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls handle=7:d0000000::::head 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] pgnls result=1 outdata.length()=170 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564492548420 finish 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.795+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x56449064fb80 prio 63 cost 39 latency 0.000002 epoch 5421 osd_op(client.1360725.0:23 7.b 7.b (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f79191eb700 20 osd.15 op_wq(1) _enqueue OpSchedulerItem(7.b PGOpItem(op=osd_op(client.1360725.0:23 7.b 7.b (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process 7.b to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process OpSchedulerItem(7.b PGOpItem(op=osd_op(client.1360725.0:23 7.b 7.b (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) queued 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process 7.b to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process OpSchedulerItem(7.b PGOpItem(op=osd_op(client.1360725.0:23 7.b 7.b (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) pg 0x56445f068000 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 5421 dequeue_op 0x56449064fb80 prio 63 cost 39 latency 0.000042 osd_op(client.1360725.0:23 7.b 7.b (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 pg pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] _handle_message: 0x56449064fb80 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.1360725.0:23 7.b 7:d0000000::::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x56449845dde0 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_pg_op osd_op(client.1360725.0:23 7.b 7:d0000000::::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls pg=7.b count 1024 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls lower_bound 7:d0000000::::head pg_end 7:e0000000::::head 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls candidate 0xa1b219eb vs lower bound 0xb 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls item 0xa1b219eb, rev 0xd7984d85 periods.6c13a0af-cb06-4137-92f2-7f3112221b27.1 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls candidate 0x112eebeb vs lower bound 0xb 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls item 0x112eebeb, rev 0xd7d77488 default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls candidate 0x0 vs lower bound 0xb 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls handle=7:e0000000::::head 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] pgnls result=1 outdata.length()=173 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 5421 dequeue_op 0x56449064fb80 finish 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process empty q, waiting 2020-05-06T20:06:39.795+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x564492283080 prio 63 cost 0 latency 0.000002 epoch 5421 osd_op(client.1360725.0:30 7.3 7.bd31b503 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f79191eb700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:30 7.3 7.bd31b503 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:30 7.3 7.bd31b503 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) queued 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.1360725.0:30 7.3 7.bd31b503 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) pg 0x56445f06b400 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564492283080 prio 63 cost 0 latency 0.000030 osd_op(client.1360725.0:30 7.3 7.bd31b503 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] _handle_message: 0x564492283080 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: op osd_op(client.1360725.0:30 7.3 7:c0ad8cbd:::region_map:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] op_has_sufficient_caps session=0x56449845dde0 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op osd_op(client.1360725.0:30 7.3 7:c0ad8cbd:::region_map:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: obc NOT found in cache: 7:c0ad8cbd:::region_map:head 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: no obc for soid 7:c0ad8cbd:::region_map:head and !can_create 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: find_object_context got error -2 2020-05-06T20:06:39.795+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564492283080 finish 2020-05-06T20:06:39.795+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.795+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x56449255cdc0 prio 63 cost 0 latency 0.000002 epoch 5421 osd_op(client.1360725.0:33 7.b 7.112eebeb (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f79191eb700 20 osd.15 op_wq(1) _enqueue OpSchedulerItem(7.b PGOpItem(op=osd_op(client.1360725.0:33 7.b 7.112eebeb (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process 7.b to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process OpSchedulerItem(7.b PGOpItem(op=osd_op(client.1360725.0:33 7.b 7.112eebeb (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) queued 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process 7.b to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process OpSchedulerItem(7.b PGOpItem(op=osd_op(client.1360725.0:33 7.b 7.112eebeb (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 0 e5421) pg 0x56445f068000 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 5421 dequeue_op 0x56449255cdc0 prio 63 cost 0 latency 0.000029 osd_op(client.1360725.0:33 7.b 7.112eebeb (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] _handle_message: 0x56449255cdc0 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_op: op osd_op(client.1360725.0:33 7.b 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps session=0x56449845dde0 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.1360725.0:33 7.b 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x564478b26580 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x564478b26580 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head rwstate(none n=0 w=0) oi: 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head(4555'5 osd.15.0:19322 dirty|data_digest|omap_digest s 46 uv 2 dd da512095 od ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x5644789cd080 snapset: 0=[]:{} 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] find_object_context 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head @head oi=7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head(4555'5 osd.15.0:19322 dirty|data_digest|omap_digest s 46 uv 2 dd da512095 od ffffffff alloc_hint [0 0 0]) 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_op obc obc(7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head rwstate(read n=1 w=0)) 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 0x564477b3ad00 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head [read 0~0] ov 4555'5 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head [read 0~0] 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_osd_op read 0~0 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_read 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] read got 46 / 46 bytes from obj 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] execute_ctx alloc reply 0x5644673cf840 result 0 2020-05-06T20:06:39.795+0200 7f78fa9ae700 15 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] do_osd_op_effects client.1360725 con 0x56447fff5400 2020-05-06T20:06:39.795+0200 7f78fa9ae700 15 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] log_op_stats osd_op(client.1360725.0:33 7.b 7:d7d77488:::default.zonegroup.75ee0409-2355-407a-bc60-9db7dcf24d7d:head [read 0~46 out=46b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 46 lat 0.000153 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:39.795+0200 7f78fa9ae700 15 osd.15 pg_epoch: 5421 pg[7.b( v 4555'5 (0'0,4555'5] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.609647806s) [15,26,1] r=0 lpr=5414 crt=4555'5 lcod 0'0 mlcod 0'0 active+clean] publish_stats_to_osd 5421:5962 2020-05-06T20:06:39.795+0200 7f78fa9ae700 10 osd.15 5421 dequeue_op 0x56449255cdc0 finish 2020-05-06T20:06:39.795+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process empty q, waiting 2020-05-06T20:06:39.799+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x564492671b20 prio 127 cost 855 latency 0.000005 epoch 5421 osd_repop(client.1360725.0:44 6.0 e5421/5414) v3 2020-05-06T20:06:39.799+0200 7f79181e9700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:44 6.0 e5421/5414) v3) prio 127 cost 855 e5421) 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 6.0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:44 6.0 e5421/5414) v3) prio 127 cost 855 e5421) queued 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 6.0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:44 6.0 e5421/5414) v3) prio 127 cost 855 e5421) pg 0x56446147e800 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x564492671b20 prio 127 cost 855 latency 0.000041 osd_repop(client.1360725.0:44 6.0 e5421/5414) v3 pg pg[6.0( v 5421'1069 (0'0,5421'1069] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1069 lcod 5421'1068 mlcod 5421'1067 active mbc={}] 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1069 (0'0,5421'1069] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1069 lcod 5421'1068 mlcod 5421'1067 active mbc={}] _handle_message: 0x564492671b20 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1069 (0'0,5421'1069] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1069 lcod 5421'1068 mlcod 5421'1067 active mbc={}] do_repop 6:05bf5b68:::notify.1:head v 5421'1070 (transaction) 184 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1069 (0'0,5421'1069] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1069 lcod 5421'1068 mlcod 5421'1067 active mbc={}] append_log log((0'0,5421'1069], crt=5421'1069) [5421'1070 (5421'1069) modify 6:5bf5b680:::notify.1:head by client.1360725.0:44 2020-05-06T20:06:39.803720+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1069 lcod 5421'1068 mlcod 5421'1067 active mbc={}] add_log_entry 5421'1070 (5421'1069) modify 6:5bf5b680:::notify.1:head by client.1360725.0:44 2020-05-06T20:06:39.803720+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1068 mlcod 5421'1067 active mbc={}] rollforward: entry=5421'1070 (5421'1069) modify 6:5bf5b680:::notify.1:head by client.1360725.0:44 2020-05-06T20:06:39.803720+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1068 mlcod 5421'1067 active mbc={}] append_log approx pg log length = 1070 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1068 mlcod 5421'1067 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 trim proposed trim_to = 0'0 2020-05-06T20:06:39.799+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'1070, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x564492671b20 finish 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1068 mlcod 5421'1068 active mbc={}] repop_commit on op osd_repop(client.1360725.0:44 6.0 e5421/5414 6:05bf5b68:::notify.1:head v 5421'1070, mlcod=5421'1068) v3, sending commit to osd.37 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:39.799+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x564492297340 prio 127 cost 1011 latency 0.000003 epoch 5421 osd_repop(client.1360725.0:45 6.0 e5421/5414) v3 2020-05-06T20:06:39.799+0200 7f79181e9700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:45 6.0 e5421/5414) v3) prio 127 cost 1011 e5421) 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 6.0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:45 6.0 e5421/5414) v3) prio 127 cost 1011 e5421) queued 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 6.0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:45 6.0 e5421/5414) v3) prio 127 cost 1011 e5421) pg 0x56446147e800 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x564492297340 prio 127 cost 1011 latency 0.000027 osd_repop(client.1360725.0:45 6.0 e5421/5414) v3 pg pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1069 mlcod 5421'1068 active mbc={}] 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1069 mlcod 5421'1068 active mbc={}] _handle_message: 0x564492297340 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1069 mlcod 5421'1068 active mbc={}] do_repop 6:05bf5b68:::notify.1:head v 5421'1071 (transaction) 184 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1070 (0'0,5421'1070] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1069 mlcod 5421'1068 active mbc={}] append_log log((0'0,5421'1070], crt=5421'1070) [5421'1071 (5421'1070) modify 6:5bf5b680:::notify.1:head by client.1360725.0:45 2020-05-06T20:06:39.805136+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1070 lcod 5421'1069 mlcod 5421'1068 active mbc={}] add_log_entry 5421'1071 (5421'1070) modify 6:5bf5b680:::notify.1:head by client.1360725.0:45 2020-05-06T20:06:39.805136+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1069 mlcod 5421'1068 active mbc={}] rollforward: entry=5421'1071 (5421'1070) modify 6:5bf5b680:::notify.1:head by client.1360725.0:45 2020-05-06T20:06:39.805136+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1069 mlcod 5421'1068 active mbc={}] append_log approx pg log length = 1071 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1069 mlcod 5421'1068 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 trim proposed trim_to = 0'0 2020-05-06T20:06:39.799+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'1071, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x564492297340 finish 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:39.799+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1069 mlcod 5421'1069 active mbc={}] repop_commit on op osd_repop(client.1360725.0:45 6.0 e5421/5414 6:05bf5b68:::notify.1:head v 5421'1071, mlcod=5421'1069) v3, sending commit to osd.37 2020-05-06T20:06:39.799+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:39.803+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x56447b3eec00 prio 127 cost 855 latency 0.000005 epoch 5421 osd_repop(client.1360725.0:50 6.d e5421/5414) v3 2020-05-06T20:06:39.803+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:50 6.d e5421/5414) v3) prio 127 cost 855 e5421) 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 6.d to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:50 6.d e5421/5414) v3) prio 127 cost 855 e5421) queued 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 6.d to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:50 6.d e5421/5414) v3) prio 127 cost 855 e5421) pg 0x56445f06e800 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x56447b3eec00 prio 127 cost 855 latency 0.000037 osd_repop(client.1360725.0:50 6.d e5421/5414) v3 pg pg[6.d( v 5421'1050 (0'0,5421'1050] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1050 lcod 5421'1049 mlcod 5421'1048 active mbc={}] 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1050 (0'0,5421'1050] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1050 lcod 5421'1049 mlcod 5421'1048 active mbc={}] _handle_message: 0x56447b3eec00 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1050 (0'0,5421'1050] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1050 lcod 5421'1049 mlcod 5421'1048 active mbc={}] do_repop 6:b4812045:::notify.4:head v 5421'1051 (transaction) 184 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1050 (0'0,5421'1050] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1050 lcod 5421'1049 mlcod 5421'1048 active mbc={}] append_log log((0'0,5421'1050], crt=5421'1050) [5421'1051 (5421'1050) modify 6:b4812045:::notify.4:head by client.1360725.0:50 2020-05-06T20:06:39.808364+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1050 lcod 5421'1049 mlcod 5421'1048 active mbc={}] add_log_entry 5421'1051 (5421'1050) modify 6:b4812045:::notify.4:head by client.1360725.0:50 2020-05-06T20:06:39.808364+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1049 mlcod 5421'1048 active mbc={}] rollforward: entry=5421'1051 (5421'1050) modify 6:b4812045:::notify.4:head by client.1360725.0:50 2020-05-06T20:06:39.808364+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1049 mlcod 5421'1048 active mbc={}] append_log approx pg log length = 1051 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1049 mlcod 5421'1048 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 trim proposed trim_to = 0'0 2020-05-06T20:06:39.803+0200 7f78f99ac700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'1051, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x56447b3eec00 finish 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1049 mlcod 5421'1049 active mbc={}] repop_commit on op osd_repop(client.1360725.0:50 6.d e5421/5414 6:b4812045:::notify.4:head v 5421'1051, mlcod=5421'1049) v3, sending commit to osd.38 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.803+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x5644929bf600 prio 127 cost 1011 latency 0.000003 epoch 5421 osd_repop(client.1360725.0:51 6.d e5421/5414) v3 2020-05-06T20:06:39.803+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:51 6.d e5421/5414) v3) prio 127 cost 1011 e5421) 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 6.d to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:51 6.d e5421/5414) v3) prio 127 cost 1011 e5421) queued 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 6.d to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:51 6.d e5421/5414) v3) prio 127 cost 1011 e5421) pg 0x56445f06e800 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644929bf600 prio 127 cost 1011 latency 0.000033 osd_repop(client.1360725.0:51 6.d e5421/5414) v3 pg pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1050 mlcod 5421'1049 active mbc={}] 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1050 mlcod 5421'1049 active mbc={}] _handle_message: 0x5644929bf600 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1050 mlcod 5421'1049 active mbc={}] do_repop 6:b4812045:::notify.4:head v 5421'1052 (transaction) 184 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1051 (0'0,5421'1051] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1050 mlcod 5421'1049 active mbc={}] append_log log((0'0,5421'1051], crt=5421'1051) [5421'1052 (5421'1051) modify 6:b4812045:::notify.4:head by client.1360725.0:51 2020-05-06T20:06:39.809044+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1051 lcod 5421'1050 mlcod 5421'1049 active mbc={}] add_log_entry 5421'1052 (5421'1051) modify 6:b4812045:::notify.4:head by client.1360725.0:51 2020-05-06T20:06:39.809044+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1050 mlcod 5421'1049 active mbc={}] rollforward: entry=5421'1052 (5421'1051) modify 6:b4812045:::notify.4:head by client.1360725.0:51 2020-05-06T20:06:39.809044+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1050 mlcod 5421'1049 active mbc={}] append_log approx pg log length = 1052 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1050 mlcod 5421'1049 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 trim proposed trim_to = 0'0 2020-05-06T20:06:39.803+0200 7f78f99ac700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'1052, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644929bf600 finish 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.803+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1050 mlcod 5421'1050 active mbc={}] repop_commit on op osd_repop(client.1360725.0:51 6.d e5421/5414 6:b4812045:::notify.4:head v 5421'1052, mlcod=5421'1050) v3, sending commit to osd.38 2020-05-06T20:06:39.803+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.811+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.24 up_from 5120 peer_clock_delta [1167645.092379888s,1167645.092967759s]) 2020-05-06T20:06:39.811+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.24 up_from 5120 peer_clock_delta [1167645.092366009s,1167645.092967759s]) 2020-05-06T20:06:39.843+0200 7f79189ea700 10 osd.15 5421 ms_handle_authentication new session 0x56448f479c00 con 0x5644929b6000 entity client.admin addr 2020-05-06T20:06:39.843+0200 7f79189ea700 10 osd.15 5421 ms_handle_authentication session 0x56448f479c00 client.admin has caps osdcap[grant(*)] 'allow *' 2020-05-06T20:06:39.843+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x56445e4ee580 prio 63 cost 153 latency 0.000004 epoch 5421 osd_op(client.1360731.0:1 5.9 5.31bb659 (undecoded) ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:39.843+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:1 5.9 5.31bb659 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:1 5.9 5.31bb659 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) queued 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:1 5.9 5.31bb659 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) pg 0x564467384800 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x56445e4ee580 prio 63 cost 153 latency 0.000043 osd_op(client.1360731.0:1 5.9 5.31bb659 (undecoded) ondisk+write+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 5421 maybe_share_map con 0x5644929b6000 [2a00:X:X:6::4]:0/118656264 map epoch 0 -> 5421 (as per caller) 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] _handle_message: 0x56445e4ee580 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_op: op osd_op(client.1360731.0:1 5.9 5:9a6dd8c0:gc::gc.0:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] op_has_sufficient_caps session=0x56448f479c00 pool=5 (default.rgw.log gc) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=1 classes=[class version method check_conds rd 1 wr 0 wl 1,class rgw_gc method rgw_gc_queue_init rd 1 wr 1 wl 1,class version method set rd 1 wr 1 wl 1] -> yes 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_op osd_op(client.1360731.0:1 5.9 5:9a6dd8c0:gc::gc.0:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] get_object_context: found obc in cache: 0x56447a143600 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] get_object_context: 0x56447a143600 5:9a6dd8c0:gc::gc.0:head rwstate(none n=0 w=0) oi: 5:9a6dd8c0:gc::gc.0:head(5421'489809 client.325552.0:1382590 dirty s 732076 uv 489809 alloc_hint [0 0 0]) exists: 1 ssc: 0x5644780c9600 snapset: 0=[]:{} 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] find_object_context 5:9a6dd8c0:gc::gc.0:head @head oi=5:9a6dd8c0:gc::gc.0:head(5421'489809 client.325552.0:1382590 dirty s 732076 uv 489809 alloc_hint [0 0 0]) 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_op obc obc(5:9a6dd8c0:gc::gc.0:head rwstate(excl n=1 w=0)) 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] execute_ctx 0x5644788cfb00 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] execute_ctx 5:9a6dd8c0:gc::gc.0:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] ov 5421'489809 av 5421'489851 snapc 0=[] snapset 0=[]:{} 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_osd_op create 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_osd_op call version.check_conds in=74b 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] call method version.check_conds 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [getxattr ceph.objclass.version in=21b] 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_osd_op getxattr ceph.objclass.version in=21b 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:73: cls_version: read_version :1 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:105: cls_version: check_version :0 (cond=1) 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:191: cls_version: failed condition check 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] method called response length=0 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] do_osd_ops error: (125) Operation canceled 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] execute_ctx alloc reply 0x564468bf0400 result -125 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] calc_trim_to_aggressive limit = 5421'489848 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] calc_trim_to_aggressive approx pg log length = 3450 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] calc_trim_to_aggressive num_to_trim = 80 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] execute_ctx update_log_only -- result=-125 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] record_write_error r=-125 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] submit_log_entries 5421'489851 (0'0) error 5:9a6dd8c0:gc::gc.0:head by client.1360731.0:1 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] new_repop: repgather(0x564478c89c00 5421'489851 rep_tid=16372 committed?=0 r=-125) 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489848 lcod 5421'489849 mlcod 5421'489849 active+clean] merge_new_log_entries 5421'489851 (0'0) error 5:9a6dd8c0:gc::gc.0:head by client.1360731.0:1 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 update missing, append 5421'489851 (0'0) error 5:9a6dd8c0:gc::gc.0:head by client.1360731.0:1 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489850 (5421'486400,5421'489850] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 (info mismatch, log((5421'486400,5421'489851], crt=5421'489849)) crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] rollforward: entry=5421'489849 (0'0) error 5:954be8d2:gc::gc.5:head by client.1360704.0:6 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 lua=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] append_log_entries_update_missing trim_to bool = 1 trim_to = 5421'486400 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 trim proposed trim_to = 5421'486400 2020-05-06T20:06:39.843+0200 7f78f91ab700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'489851, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 lua=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] merge_new_log_entries peer_missing for 21 = missing(0 may_include_deletes = 1) 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 lua=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] merge_new_log_entries peer_missing for 26 = missing(0 may_include_deletes = 1) 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 lua=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] op_applied version 5421'489851 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] calc_trim_to_aggressive limit = 5421'489849 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] calc_trim_to_aggressive approx pg log length = 3451 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] calc_trim_to_aggressive num_to_trim = 81 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x56445e4ee580 finish 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.843+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x5644874a9020 prio 196 cost 0 latency 0.000004 epoch 5421 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3 2020-05-06T20:06:39.843+0200 7f79191eb700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3) prio 196 cost 0 e5421) 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3) prio 196 cost 0 e5421) queued 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3) prio 196 cost 0 e5421) pg 0x564467384800 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644874a9020 prio 196 cost 0 latency 0.000031 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3 pg pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] _handle_message: 0x5644874a9020 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] do_update_log_missing_reply got reply from 21 2020-05-06T20:06:39.843+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644874a9020 finish 2020-05-06T20:06:39.843+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.859+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x5644905c94a0 prio 196 cost 0 latency 0.000004 epoch 5421 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3 2020-05-06T20:06:39.859+0200 7f79191eb700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3) prio 196 cost 0 e5421) 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3) prio 196 cost 0 e5421) queued 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3) prio 196 cost 0 e5421) pg 0x564467384800 2020-05-06T20:06:39.859+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644905c94a0 prio 196 cost 0 latency 0.000039 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16372 lcod 5421'489851) v3 pg pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] 2020-05-06T20:06:39.859+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] _handle_message: 0x5644905c94a0 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] do_update_log_missing_reply got reply from 26 2020-05-06T20:06:39.859+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489850 crt=5421'489849 lcod 5421'489849 mlcod 5421'489849 active+clean] repop_all_committed: repop tid 16372 all committed 2020-05-06T20:06:39.859+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] eval_repop repgather(0x564478c89c00 5421'489851 rep_tid=16372 committed?=1 r=-125) 2020-05-06T20:06:39.859+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] commit: repgather(0x564478c89c00 5421'489851 rep_tid=16372 committed?=1 r=-125) 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:39.859+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] publish_stats_to_osd 5421:576197 2020-05-06T20:06:39.859+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] removing repgather(0x564478c89c00 5421'489851 rep_tid=16372 committed?=1 r=-125) 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] q front is repgather(0x564478c89c00 5421'489851 rep_tid=16372 committed?=1 r=-125) 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] finished operator() r=-125 2020-05-06T20:06:39.859+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] sending commit on osd_op(client.1360731.0:1 5.9 5:9a6dd8c0:gc::gc.0:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 0x564468bf0400 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] remove_repop repgather(0x564478c89c00 5421'489851 rep_tid=16372 committed?=1 r=-125) 2020-05-06T20:06:39.859+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644905c94a0 finish 2020-05-06T20:06:39.859+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.875+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x564490350580 prio 63 cost 153 latency 0.000003 epoch 5421 osd_op(client.1360731.0:6 5.9 5.4b17d2a9 (undecoded) ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:39.875+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:6 5.9 5.4b17d2a9 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:6 5.9 5.4b17d2a9 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) queued 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:6 5.9 5.4b17d2a9 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) pg 0x564467384800 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564490350580 prio 63 cost 153 latency 0.000034 osd_op(client.1360731.0:6 5.9 5.4b17d2a9 (undecoded) ondisk+write+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] _handle_message: 0x564490350580 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_op: op osd_op(client.1360731.0:6 5.9 5:954be8d2:gc::gc.5:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] op_has_sufficient_caps session=0x56448f479c00 pool=5 (default.rgw.log gc) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=1 classes=[class version method check_conds rd 1 wr 0 wl 1,class rgw_gc method rgw_gc_queue_init rd 1 wr 1 wl 1,class version method set rd 1 wr 1 wl 1] -> yes 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_op osd_op(client.1360731.0:6 5.9 5:954be8d2:gc::gc.5:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] get_object_context: found obc in cache: 0x56447a1438c0 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] get_object_context: 0x56447a1438c0 5:954be8d2:gc::gc.5:head rwstate(none n=0 w=0) oi: 5:954be8d2:gc::gc.5:head(5421'489811 client.325552.0:1382605 dirty s 574258 uv 489811 alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e8db20 snapset: 0=[]:{} 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] find_object_context 5:954be8d2:gc::gc.5:head @head oi=5:954be8d2:gc::gc.5:head(5421'489811 client.325552.0:1382605 dirty s 574258 uv 489811 alloc_hint [0 0 0]) 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_op obc obc(5:954be8d2:gc::gc.5:head rwstate(excl n=1 w=0)) 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] execute_ctx 0x5644788cfb00 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] execute_ctx 5:954be8d2:gc::gc.5:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] ov 5421'489811 av 5421'489852 snapc 0=[] snapset 0=[]:{} 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_osd_op 5:954be8d2:gc::gc.5:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_osd_op create 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_osd_op call version.check_conds in=74b 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] call method version.check_conds 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_osd_op 5:954be8d2:gc::gc.5:head [getxattr ceph.objclass.version in=21b] 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_osd_op getxattr ceph.objclass.version in=21b 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:73: cls_version: read_version :1 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:105: cls_version: check_version :0 (cond=1) 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:191: cls_version: failed condition check 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] method called response length=0 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] do_osd_ops error: (125) Operation canceled 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] execute_ctx alloc reply 0x56446c598b40 result -125 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] calc_trim_to_aggressive limit = 5421'489849 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] calc_trim_to_aggressive approx pg log length = 3451 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] calc_trim_to_aggressive num_to_trim = 81 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] execute_ctx update_log_only -- result=-125 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] record_write_error r=-125 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] submit_log_entries 5421'489852 (0'0) error 5:954be8d2:gc::gc.5:head by client.1360731.0:6 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] new_repop: repgather(0x564478c89c00 5421'489852 rep_tid=16373 committed?=0 r=-125) 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489849 lcod 5421'489850 mlcod 5421'489850 active+clean] merge_new_log_entries 5421'489852 (0'0) error 5:954be8d2:gc::gc.5:head by client.1360731.0:6 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 update missing, append 5421'489852 (0'0) error 5:954be8d2:gc::gc.5:head by client.1360731.0:6 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489851 (5421'486400,5421'489851] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 (info mismatch, log((5421'486400,5421'489852], crt=5421'489850)) crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] rollforward: entry=5421'489850 (0'0) error 5:921ecdb1:gc::gc.25:head by client.1360704.0:26 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 lua=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] append_log_entries_update_missing trim_to bool = 1 trim_to = 5421'486400 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 trim proposed trim_to = 5421'486400 2020-05-06T20:06:39.875+0200 7f78f91ab700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'489852, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 lua=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] merge_new_log_entries peer_missing for 21 = missing(0 may_include_deletes = 1) 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 lua=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] merge_new_log_entries peer_missing for 26 = missing(0 may_include_deletes = 1) 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 lua=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] op_applied version 5421'489852 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] calc_trim_to_aggressive limit = 5421'489850 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] calc_trim_to_aggressive approx pg log length = 3452 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] calc_trim_to_aggressive num_to_trim = 82 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564490350580 finish 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.875+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x564498531760 prio 196 cost 0 latency 0.000002 epoch 5421 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3 2020-05-06T20:06:39.875+0200 7f79191eb700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3) prio 196 cost 0 e5421) 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3) prio 196 cost 0 e5421) queued 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3) prio 196 cost 0 e5421) pg 0x564467384800 2020-05-06T20:06:39.875+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x5644905bb180 prio 196 cost 0 latency 0.000001 epoch 5421 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3 2020-05-06T20:06:39.875+0200 7f79191eb700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3) prio 196 cost 0 e5421) 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564498531760 prio 196 cost 0 latency 0.000026 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3 pg pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] _handle_message: 0x564498531760 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] do_update_log_missing_reply got reply from 21 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564498531760 finish 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3) prio 196 cost 0 e5421) queued 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3) prio 196 cost 0 e5421) pg 0x564467384800 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644905bb180 prio 196 cost 0 latency 0.000027 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16373 lcod 5421'489852) v3 pg pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] _handle_message: 0x5644905bb180 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] do_update_log_missing_reply got reply from 26 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489851 crt=5421'489850 lcod 5421'489850 mlcod 5421'489850 active+clean] repop_all_committed: repop tid 16373 all committed 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] eval_repop repgather(0x564478c89c00 5421'489852 rep_tid=16373 committed?=1 r=-125) 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] commit: repgather(0x564478c89c00 5421'489852 rep_tid=16373 committed?=1 r=-125) 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:39.875+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] publish_stats_to_osd 5421:576198 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] removing repgather(0x564478c89c00 5421'489852 rep_tid=16373 committed?=1 r=-125) 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] q front is repgather(0x564478c89c00 5421'489852 rep_tid=16373 committed?=1 r=-125) 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] finished operator() r=-125 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] sending commit on osd_op(client.1360731.0:6 5.9 5:954be8d2:gc::gc.5:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 0x56446c598b40 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] remove_repop repgather(0x564478c89c00 5421'489852 rep_tid=16373 committed?=1 r=-125) 2020-05-06T20:06:39.875+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644905bb180 finish 2020-05-06T20:06:39.875+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.887+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382454086s,1167816.382898583s]) 2020-05-06T20:06:39.887+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382434123s,1167816.382898583s]) 2020-05-06T20:06:39.911+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x564491b37180 prio 63 cost 153 latency 0.000004 epoch 5421 osd_op(client.1360731.0:26 5.9 5.8db37849 (undecoded) ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:39.911+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:26 5.9 5.8db37849 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:26 5.9 5.8db37849 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) queued 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:26 5.9 5.8db37849 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 153 e5421) pg 0x564467384800 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564491b37180 prio 63 cost 153 latency 0.000045 osd_op(client.1360731.0:26 5.9 5.8db37849 (undecoded) ondisk+write+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] _handle_message: 0x564491b37180 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_op: op osd_op(client.1360731.0:26 5.9 5:921ecdb1:gc::gc.25:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] op_has_sufficient_caps session=0x56448f479c00 pool=5 (default.rgw.log gc) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=1 classes=[class version method check_conds rd 1 wr 0 wl 1,class rgw_gc method rgw_gc_queue_init rd 1 wr 1 wl 1,class version method set rd 1 wr 1 wl 1] -> yes 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_op osd_op(client.1360731.0:26 5.9 5:921ecdb1:gc::gc.25:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 may_write may_read -> write-ordered flags ondisk+write+known_if_redirected 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] get_object_context: found obc in cache: 0x564478cda940 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] get_object_context: 0x564478cda940 5:921ecdb1:gc::gc.25:head rwstate(none n=0 w=0) oi: 5:921ecdb1:gc::gc.25:head(5421'489807 client.325552.0:1382569 dirty s 591061 uv 489807 alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e8b760 snapset: 0=[]:{} 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] find_object_context 5:921ecdb1:gc::gc.25:head @head oi=5:921ecdb1:gc::gc.25:head(5421'489807 client.325552.0:1382569 dirty s 591061 uv 489807 alloc_hint [0 0 0]) 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_op obc obc(5:921ecdb1:gc::gc.25:head rwstate(excl n=1 w=0)) 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] execute_ctx 0x5644788cfb00 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] execute_ctx 5:921ecdb1:gc::gc.25:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] ov 5421'489807 av 5421'489853 snapc 0=[] snapset 0=[]:{} 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_osd_op 5:921ecdb1:gc::gc.25:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_osd_op create 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_osd_op call version.check_conds in=74b 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] call method version.check_conds 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_osd_op 5:921ecdb1:gc::gc.25:head [getxattr ceph.objclass.version in=21b] 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_osd_op getxattr ceph.objclass.version in=21b 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:73: cls_version: read_version :1 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:105: cls_version: check_version :0 (cond=1) 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:191: cls_version: failed condition check 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] method called response length=0 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] do_osd_ops error: (125) Operation canceled 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] execute_ctx alloc reply 0x56446d8d1840 result -125 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] calc_trim_to_aggressive limit = 5421'489850 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] calc_trim_to_aggressive approx pg log length = 3452 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] calc_trim_to_aggressive num_to_trim = 82 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] execute_ctx update_log_only -- result=-125 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] record_write_error r=-125 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] submit_log_entries 5421'489853 (0'0) error 5:921ecdb1:gc::gc.25:head by client.1360731.0:26 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] new_repop: repgather(0x564478c89c00 5421'489853 rep_tid=16374 committed?=0 r=-125) 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489850 lcod 5421'489851 mlcod 5421'489851 active+clean] merge_new_log_entries 5421'489853 (0'0) error 5:921ecdb1:gc::gc.25:head by client.1360731.0:26 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 update missing, append 5421'489853 (0'0) error 5:921ecdb1:gc::gc.25:head by client.1360731.0:26 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489852 (5421'486400,5421'489852] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 (info mismatch, log((5421'486400,5421'489853], crt=5421'489851)) crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] rollforward: entry=5421'489851 (0'0) error 5:9a6dd8c0:gc::gc.0:head by client.1360731.0:1 0.000000 -125 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 lua=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] append_log_entries_update_missing trim_to bool = 1 trim_to = 5421'486400 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 trim proposed trim_to = 5421'486400 2020-05-06T20:06:39.911+0200 7f78f91ab700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'489853, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 lua=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] merge_new_log_entries peer_missing for 21 = missing(0 may_include_deletes = 1) 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 lua=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] merge_new_log_entries peer_missing for 26 = missing(0 may_include_deletes = 1) 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 lua=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] op_applied version 5421'489853 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] calc_trim_to_aggressive limit = 5421'489851 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] calc_trim_to_aggressive approx pg log length = 3453 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] calc_trim_to_aggressive num_to_trim = 83 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564491b37180 finish 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.911+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x5644905bad60 prio 196 cost 0 latency 0.000003 epoch 5421 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3 2020-05-06T20:06:39.911+0200 7f79191eb700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3) prio 196 cost 0 e5421) 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3) prio 196 cost 0 e5421) queued 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3) prio 196 cost 0 e5421) pg 0x564467384800 2020-05-06T20:06:39.911+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x56447a4f19c0 prio 196 cost 0 latency 0.000001 epoch 5421 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3 2020-05-06T20:06:39.911+0200 7f79191eb700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3) prio 196 cost 0 e5421) 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644905bad60 prio 196 cost 0 latency 0.000029 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3 pg pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] _handle_message: 0x5644905bad60 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] do_update_log_missing_reply got reply from 21 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644905bad60 finish 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3) prio 196 cost 0 e5421) queued 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3) prio 196 cost 0 e5421) pg 0x564467384800 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x56447a4f19c0 prio 196 cost 0 latency 0.000024 pg_update_log_missing_reply(5.9 epoch 5421/5414 rep_tid 16374 lcod 5421'489853) v3 pg pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] _handle_message: 0x56447a4f19c0 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] do_update_log_missing_reply got reply from 26 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 luod=5421'489852 crt=5421'489851 lcod 5421'489851 mlcod 5421'489851 active+clean] repop_all_committed: repop tid 16374 all committed 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] eval_repop repgather(0x564478c89c00 5421'489853 rep_tid=16374 committed?=1 r=-125) 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] commit: repgather(0x564478c89c00 5421'489853 rep_tid=16374 committed?=1 r=-125) 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:39.911+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] publish_stats_to_osd 5421:576199 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] removing repgather(0x564478c89c00 5421'489853 rep_tid=16374 committed?=1 r=-125) 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] q front is repgather(0x564478c89c00 5421'489853 rep_tid=16374 committed?=1 r=-125) 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] finished operator() r=-125 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] sending commit on osd_op(client.1360731.0:26 5.9 5:921ecdb1:gc::gc.25:head [create,call version.check_conds in=74b,call rgw_gc.rgw_gc_queue_init in=45b,call version.set in=34b] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 0x56446d8d1840 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] remove_repop repgather(0x564478c89c00 5421'489853 rep_tid=16374 committed?=1 r=-125) 2020-05-06T20:06:39.911+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x56447a4f19c0 finish 2020-05-06T20:06:39.911+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.923+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x564490046940 prio 63 cost 25 latency 0.000003 epoch 5421 osd_op(client.1360731.0:33 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.923+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:33 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 25 e5421) 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:33 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 25 e5421) queued 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:33 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 25 e5421) pg 0x564467384800 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564490046940 prio 63 cost 25 latency 0.000037 osd_op(client.1360731.0:33 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] _handle_message: 0x564490046940 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: op osd_op(client.1360731.0:33 5.9 5:9a6dd8c0:gc::gc.0:head [call rgw.gc_list in=25b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] op_has_sufficient_caps session=0x56448f479c00 pool=5 (default.rgw.log gc) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method gc_list rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op osd_op(client.1360731.0:33 5.9 5:9a6dd8c0:gc::gc.0:head [call rgw.gc_list in=25b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: found obc in cache: 0x56447a143600 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: 0x56447a143600 5:9a6dd8c0:gc::gc.0:head rwstate(none n=0 w=0) oi: 5:9a6dd8c0:gc::gc.0:head(5421'489809 client.325552.0:1382590 dirty s 732076 uv 489809 alloc_hint [0 0 0]) exists: 1 ssc: 0x5644780c9600 snapset: 0=[]:{} 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] find_object_context 5:9a6dd8c0:gc::gc.0:head @head oi=5:9a6dd8c0:gc::gc.0:head(5421'489809 client.325552.0:1382590 dirty s 732076 uv 489809 alloc_hint [0 0 0]) 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op obc obc(5:9a6dd8c0:gc::gc.0:head rwstate(read n=1 w=0)) 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx 0x5644788cfb00 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx 5:9a6dd8c0:gc::gc.0:head [call rgw.gc_list in=25b] ov 5421'489809 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [call rgw.gc_list in=25b] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op call rgw.gc_list in=25b 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] call method rgw.gc_list 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 /build/ceph-15.2.1/src/cls/rgw/cls_rgw.cc:3486: gc_iterate_entries 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 /build/ceph-15.2.1/src/cls/rgw/cls_rgw.cc:3508: gc_iterate_entries end_key=1_01588788399.928270062 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [omap-get-vals in=18b] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op omap-get-vals in=18b 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] method called response length=15 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx alloc reply 0x564468c00400 result 0 2020-05-06T20:06:39.923+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op_effects client.1360731 con 0x5644929b6000 2020-05-06T20:06:39.923+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] log_op_stats osd_op(client.1360731.0:33 5.9 5:9a6dd8c0:gc::gc.0:head [call rgw.gc_list in=25b out=15b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 15 lat 0.000172 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:39.923+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] publish_stats_to_osd 5421:576200 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564490046940 finish 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.923+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x56447d836260 prio 63 cost 11 latency 0.000002 epoch 5421 osd_op(client.1360731.0:34 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.923+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:34 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 11 e5421) 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:34 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 11 e5421) queued 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:34 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 11 e5421) pg 0x564467384800 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x56447d836260 prio 63 cost 11 latency 0.000027 osd_op(client.1360731.0:34 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] _handle_message: 0x56447d836260 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: op osd_op(client.1360731.0:34 5.9 5:9a6dd8c0:gc::gc.0:head [call version.read in=11b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] op_has_sufficient_caps session=0x56448f479c00 pool=5 (default.rgw.log gc) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class version method read rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op osd_op(client.1360731.0:34 5.9 5:9a6dd8c0:gc::gc.0:head [call version.read in=11b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: found obc in cache: 0x56447a143600 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: 0x56447a143600 5:9a6dd8c0:gc::gc.0:head rwstate(none n=0 w=0) oi: 5:9a6dd8c0:gc::gc.0:head(5421'489809 client.325552.0:1382590 dirty s 732076 uv 489809 alloc_hint [0 0 0]) exists: 1 ssc: 0x5644780c9600 snapset: 0=[]:{} 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] find_object_context 5:9a6dd8c0:gc::gc.0:head @head oi=5:9a6dd8c0:gc::gc.0:head(5421'489809 client.325552.0:1382590 dirty s 732076 uv 489809 alloc_hint [0 0 0]) 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op obc obc(5:9a6dd8c0:gc::gc.0:head rwstate(read n=1 w=0)) 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx 0x5644788cfb00 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx 5:9a6dd8c0:gc::gc.0:head [call version.read in=11b] ov 5421'489809 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [call version.read in=11b] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op call version.read in=11b 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] call method version.read 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [getxattr ceph.objclass.version in=21b] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op getxattr ceph.objclass.version in=21b 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/version/cls_version.cc:73: cls_version: read_version :1 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] method called response length=24 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx alloc reply 0x56446f932d00 result 0 2020-05-06T20:06:39.923+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op_effects client.1360731 con 0x5644929b6000 2020-05-06T20:06:39.923+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] log_op_stats osd_op(client.1360731.0:34 5.9 5:9a6dd8c0:gc::gc.0:head [call version.read in=11b out=24b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 24 lat 0.000142 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:39.923+0200 7f78f91ab700 15 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] publish_stats_to_osd 5421:576201 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x56447d836260 finish 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.923+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x5644874a0ec0 prio 63 cost 46 latency 0.000002 epoch 5421 osd_op(client.1360731.0:35 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.923+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:35 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 46 e5421) 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:35 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 46 e5421) queued 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.1360731.0:35 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 46 e5421) pg 0x564467384800 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644874a0ec0 prio 63 cost 46 latency 0.000027 osd_op(client.1360731.0:35 5.9 5.31bb659 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] _handle_message: 0x5644874a0ec0 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: op osd_op(client.1360731.0:35 5.9 5:9a6dd8c0:gc::gc.0:head [call rgw_gc.rgw_gc_queue_list_entries in=46b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] op_has_sufficient_caps session=0x56448f479c00 pool=5 (default.rgw.log gc) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw_gc method rgw_gc_queue_list_entries rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op osd_op(client.1360731.0:35 5.9 5:9a6dd8c0:gc::gc.0:head [call rgw_gc.rgw_gc_queue_list_entries in=46b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: found obc in cache: 0x56447a143600 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: 0x56447a143600 5:9a6dd8c0:gc::gc.0:head rwstate(none n=0 w=0) oi: 5:9a6dd8c0:gc::gc.0:head(5421'489809 client.325552.0:1382590 dirty s 732076 uv 489809 alloc_hint [0 0 0]) exists: 1 ssc: 0x5644780c9600 snapset: 0=[]:{} 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] find_object_context 5:9a6dd8c0:gc::gc.0:head @head oi=5:9a6dd8c0:gc::gc.0:head(5421'489809 client.325552.0:1382590 dirty s 732076 uv 489809 alloc_hint [0 0 0]) 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op obc obc(5:9a6dd8c0:gc::gc.0:head rwstate(read n=1 w=0)) 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx 0x5644788cfb00 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx 5:9a6dd8c0:gc::gc.0:head [call rgw_gc.rgw_gc_queue_list_entries in=46b] ov 5421'489809 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [call rgw_gc.rgw_gc_queue_list_entries in=46b] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op call rgw_gc.rgw_gc_queue_list_entries in=46b 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] call method rgw_gc.rgw_gc_queue_list_entries 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [sync_read 0~1024] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op sync_read 0~1024 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_read 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] read got 1024 / 1024 bytes from obj 5:9a6dd8c0:gc::gc.0:head 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [sync_read 1024~3090 [fadvise_sequential]] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op sync_read 1024~3090 [fadvise_sequential] 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_read 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] read got 3090 / 3090 bytes from obj 5:9a6dd8c0:gc::gc.0:head 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 /build/ceph-15.2.1/src/cls/queue/cls_queue_src.cc:263: INFO: queue_list_entries(): front is: 0/4096, tail is 0/732076 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 /build/ceph-15.2.1/src/cls/queue/cls_queue_src.cc:271: INFO: queue_list_entries(): start_offset is 4096 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 /build/ceph-15.2.1/src/cls/queue/cls_queue_src.cc:280: INFO: queue_list_entries(): size_to_read is 1024 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op 5:9a6dd8c0:gc::gc.0:head [sync_read 4096~1024] 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_op sync_read 4096~1024 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_read 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] read got 1024 / 1024 bytes from obj 5:9a6dd8c0:gc::gc.0:head 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/queue/cls_queue_src.cc:295: INFO: queue_list_entries(): Entry start offset accounting for leftover data is 4096 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 /build/ceph-15.2.1/src/cls/queue/cls_queue_src.cc:299: INFO: queue_list_entries(): size of chunk 1024 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 /build/ceph-15.2.1/src/cls/queue/cls_queue_src.cc:306: INFO: queue_list_entries(): index: 0, size_to_process: 1024 2020-05-06T20:06:39.923+0200 7f78f91ab700 5 /build/ceph-15.2.1/src/cls/queue/cls_queue_src.cc:312: INFO: queue_list_entries(): offset: 0/4096 2020-05-06T20:06:39.923+0200 7f78f91ab700 5 /build/ceph-15.2.1/src/cls/queue/cls_queue_src.cc:326: ERROR: queue_list_entries: invalid entry start 0 2020-05-06T20:06:39.923+0200 7f78f91ab700 5 /build/ceph-15.2.1/src/cls/rgw_gc/cls_rgw_gc.cc:135: ERROR: queue_list_entries(): returned error -22 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] method called response length=0 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_osd_ops error: (22) Invalid argument 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] execute_ctx alloc reply 0x564475613680 result -22 2020-05-06T20:06:39.923+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644874a0ec0 finish 2020-05-06T20:06:39.923+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:39.927+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x56448e0249a0 prio 127 cost 855 latency 0.000004 epoch 5421 osd_repop(client.1360725.0:59 6.0 e5421/5414) v3 2020-05-06T20:06:39.927+0200 7f79181e9700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:59 6.0 e5421/5414) v3) prio 127 cost 855 e5421) 2020-05-06T20:06:39.927+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 6.0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.927+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:59 6.0 e5421/5414) v3) prio 127 cost 855 e5421) queued 2020-05-06T20:06:39.927+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 6.0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.927+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(6.0 PGOpItem(op=osd_repop(client.1360725.0:59 6.0 e5421/5414) v3) prio 127 cost 855 e5421) pg 0x56446147e800 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x56448e0249a0 prio 127 cost 855 latency 0.000043 osd_repop(client.1360725.0:59 6.0 e5421/5414) v3 pg pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1070 mlcod 5421'1069 active mbc={}] 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1070 mlcod 5421'1069 active mbc={}] _handle_message: 0x56448e0249a0 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1070 mlcod 5421'1069 active mbc={}] do_repop 6:05bf5b68:::notify.1:head v 5421'1072 (transaction) 184 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1071 (0'0,5421'1071] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1070 mlcod 5421'1069 active mbc={}] append_log log((0'0,5421'1071], crt=5421'1071) [5421'1072 (5421'1071) modify 6:5bf5b680:::notify.1:head by client.1360725.0:59 2020-05-06T20:06:39.931527+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1071 lcod 5421'1070 mlcod 5421'1069 active mbc={}] add_log_entry 5421'1072 (5421'1071) modify 6:5bf5b680:::notify.1:head by client.1360725.0:59 2020-05-06T20:06:39.931527+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.927+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1070 mlcod 5421'1069 active mbc={}] rollforward: entry=5421'1072 (5421'1071) modify 6:5bf5b680:::notify.1:head by client.1360725.0:59 2020-05-06T20:06:39.931527+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1070 mlcod 5421'1069 active mbc={}] append_log approx pg log length = 1072 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1070 mlcod 5421'1069 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 trim proposed trim_to = 0'0 2020-05-06T20:06:39.927+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'1072, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x56448e0249a0 finish 2020-05-06T20:06:39.927+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:39.927+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1070 mlcod 5421'1070 active mbc={}] repop_commit on op osd_repop(client.1360725.0:59 6.0 e5421/5414 6:05bf5b68:::notify.1:head v 5421'1072, mlcod=5421'1070) v3, sending commit to osd.37 2020-05-06T20:06:39.927+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:39.931+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x564478700000 prio 127 cost 855 latency 0.000004 epoch 5421 osd_repop(client.1360725.0:62 6.d e5421/5414) v3 2020-05-06T20:06:39.931+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:62 6.d e5421/5414) v3) prio 127 cost 855 e5421) 2020-05-06T20:06:39.931+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 6.d to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:39.931+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:62 6.d e5421/5414) v3) prio 127 cost 855 e5421) queued 2020-05-06T20:06:39.931+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 6.d to_process waiting <> waiting_peering {} 2020-05-06T20:06:39.931+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(6.d PGOpItem(op=osd_repop(client.1360725.0:62 6.d e5421/5414) v3) prio 127 cost 855 e5421) pg 0x56445f06e800 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564478700000 prio 127 cost 855 latency 0.000039 osd_repop(client.1360725.0:62 6.d e5421/5414) v3 pg pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1051 mlcod 5421'1050 active mbc={}] 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1051 mlcod 5421'1050 active mbc={}] _handle_message: 0x564478700000 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1051 mlcod 5421'1050 active mbc={}] do_repop 6:b4812045:::notify.4:head v 5421'1053 (transaction) 184 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1052 (0'0,5421'1052] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1051 mlcod 5421'1050 active mbc={}] append_log log((0'0,5421'1052], crt=5421'1052) [5421'1053 (5421'1052) modify 6:b4812045:::notify.4:head by client.1360725.0:62 2020-05-06T20:06:39.933903+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0] 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1052 lcod 5421'1051 mlcod 5421'1050 active mbc={}] add_log_entry 5421'1053 (5421'1052) modify 6:b4812045:::notify.4:head by client.1360725.0:62 2020-05-06T20:06:39.933903+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.931+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1051 mlcod 5421'1050 active mbc={}] rollforward: entry=5421'1053 (5421'1052) modify 6:b4812045:::notify.4:head by client.1360725.0:62 2020-05-06T20:06:39.933903+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1051 mlcod 5421'1050 active mbc={}] append_log approx pg log length = 1053 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1051 mlcod 5421'1050 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 trim proposed trim_to = 0'0 2020-05-06T20:06:39.931+0200 7f78f99ac700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'1053, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564478700000 finish 2020-05-06T20:06:39.931+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.931+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1051 mlcod 5421'1051 active mbc={}] repop_commit on op osd_repop(client.1360725.0:62 6.d e5421/5414 6:b4812045:::notify.4:head v 5421'1053, mlcod=5421'1051) v3, sending commit to osd.38 2020-05-06T20:06:39.931+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:39.931+0200 7f79089e5700 2 osd.15 5421 ms_handle_reset con 0x56447fff5400 session 0x56449845dde0 2020-05-06T20:06:39.935+0200 7f79089e5700 2 osd.15 5421 ms_handle_reset con 0x5644929b6000 session 0x56448f479c00 2020-05-06T20:06:40.003+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597543204s,1167680.597986950s]) 2020-05-06T20:06:40.003+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597530746s,1167680.597986950s]) 2020-05-06T20:06:40.071+0200 7f79139e0700 10 osd.15 5421 tick_without_osd_lock 2020-05-06T20:06:40.071+0200 7f79139e0700 20 osd.15 5421 can_inc_scrubs == true 0 local + 0 remote < max 1 2020-05-06T20:06:40.071+0200 7f79139e0700 20 osd.15 5421 scrub_time_permit should run between 0 - 24 now 20 = yes 2020-05-06T20:06:40.071+0200 7f79139e0700 20 osd.15 5421 scrub_load_below_threshold loadavg per cpu 0 < max 0.5 = yes 2020-05-06T20:06:40.071+0200 7f79139e0700 20 osd.15 5421 sched_scrub load_is_low=1 2020-05-06T20:06:40.071+0200 7f79139e0700 10 osd.15 5421 sched_scrub 2.1s0 scheduled at 2020-05-06T20:58:35.908491+0200 > 2020-05-06T20:06:40.075345+0200 2020-05-06T20:06:40.071+0200 7f79139e0700 20 osd.15 5421 sched_scrub done 2020-05-06T20:06:40.071+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2020-05-06T20:06:40.071+0200 7f79139e0700 20 osd.15 5421 promote_throttle_recalibrate new_prob 1000 2020-05-06T20:06:40.071+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2020-05-06T20:06:40.119+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779231166s,1167509.779688105s]) 2020-05-06T20:06:40.119+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779217172s,1167509.779688105s]) 2020-05-06T20:06:40.191+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287236476s,1167782.287586623s]) 2020-05-06T20:06:40.191+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287219775s,1167782.287586623s]) 2020-05-06T20:06:40.203+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885252791s,1167644.885686954s]) 2020-05-06T20:06:40.203+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885129249s,1167644.885686954s]) 2020-05-06T20:06:40.211+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644600710s,1167441.645318802s]) 2020-05-06T20:06:40.211+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644582899s,1167441.645318802s]) 2020-05-06T20:06:40.215+0200 7f79149e2700 10 osd.15 5421 tick 2020-05-06T20:06:40.215+0200 7f79149e2700 10 osd.15 5421 do_waiters -- start 2020-05-06T20:06:40.215+0200 7f79149e2700 10 osd.15 5421 do_waiters -- finish 2020-05-06T20:06:40.215+0200 7f79149e2700 20 osd.15 5421 tick last_purged_snaps_scrub 2020-05-06T09:01:52.905203+0200 next 2020-05-07T09:01:52.905203+0200 2020-05-06T20:06:40.219+0200 7f78f89aa700 5 osd.15 5421 heartbeat osd_stat(store_statfs(0x56d343b0000/0x40000000/0x581c17fe000, data 0x7cd9f3eeb/0x7cd64c000, compress 0x16a45768/0x1807c000/0x1d698000, omap 0x2a6425, meta 0x3fd59bdb), peers [0,1,2,3,4,5,6,7,9,10,11,12,13,14,16,17,18,19,21,22,23,24,26,27,28,29,30,31,32,33,35,36,37,38,39] op hist []) 2020-05-06T20:06:40.219+0200 7f78f89aa700 20 osd.15 5421 check_full_status cur ratio 0.014401, physical ratio 0.014401, new state none 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.0 up_from 5071 peer_clock_delta [1167821.367721318s,1167821.367884569s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.0 up_from 5071 peer_clock_delta [1167821.367711853s,1167821.367889900s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.1 up_from 5084 peer_clock_delta [1167782.010031239s,1167782.010235338s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.2 up_from 5111 peer_clock_delta [1167678.885537694s,1167678.885838323s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.5 up_from 5136 peer_clock_delta [1167576.395849222s,1167576.396184456s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.4 up_from 5161 peer_clock_delta [1167474.555838639s,1167474.556209808s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.1 up_from 5084 peer_clock_delta [1167782.009834163s,1167782.010240834s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.3 up_from 5126 peer_clock_delta [1167611.710349466s,1167611.710773238s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798063514s,1167440.798498111s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.4 up_from 5161 peer_clock_delta [1167474.555774793s,1167474.556217147s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597449718s,1167680.597903559s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.2 up_from 5111 peer_clock_delta [1167678.885376063s,1167678.885849858s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287033836s,1167782.287519486s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.7 up_from 5103 peer_clock_delta [1167710.312115444s,1167710.312620675s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214099658s,1167820.214612754s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798077882s,1167440.798598748s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.3 up_from 5126 peer_clock_delta [1167611.710253464s,1167611.710784618s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214153819s,1167820.214690978s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.5 up_from 5136 peer_clock_delta [1167576.395649399s,1167576.396196047s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597287445s,1167680.597846985s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132434765s,1167609.133004700s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885029221s,1167644.885609130s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.18 up_from 5161 peer_clock_delta [1167473.938907064s,1167473.939493451s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.286920198s,1167782.287510488s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779166514s,1167509.779764412s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.13 up_from 5086 peer_clock_delta [1167779.698399769s,1167779.699007181s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.21 up_from 5153 peer_clock_delta [1167508.090428808s,1167508.091043342s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.13 up_from 5086 peer_clock_delta [1167779.698412065s,1167779.699030643s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132307160s,1167609.132931562s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.19 up_from 5092 peer_clock_delta [1167747.405816681s,1167747.406452510s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382191009s,1167816.382833960s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779046122s,1167509.779693174s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071729107s,1167579.072382174s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.21 up_from 5153 peer_clock_delta [1167508.090329543s,1167508.090988754s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.24 up_from 5120 peer_clock_delta [1167645.092365525s,1167645.093033198s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.18 up_from 5161 peer_clock_delta [1167473.938753117s,1167473.939430395s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.22 up_from 5074 peer_clock_delta [1167818.481117284s,1167818.481806443s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.19 up_from 5092 peer_clock_delta [1167747.405701975s,1167747.406396319s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385276603s,1167815.385975554s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.22 up_from 5074 peer_clock_delta [1167818.481074012s,1167818.481781675s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.32 up_from 5074 peer_clock_delta [1167817.955842555s,1167817.956561701s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.23 up_from 5093 peer_clock_delta [1167747.079630436s,1167747.080360266s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.23 up_from 5093 peer_clock_delta [1167747.079658428s,1167747.080394906s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.884878026s,1167644.885627439s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.33 up_from 5092 peer_clock_delta [1167747.547611500s,1167747.548367319s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385156000s,1167815.385924189s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.35 up_from 5126 peer_clock_delta [1167611.833140570s,1167611.833919084s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.38 up_from 5076 peer_clock_delta [1167815.474638533s,1167815.475435405s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.28 up_from 5167 peer_clock_delta [1167441.804622446s,1167441.805430438s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656456767s,1167712.657271240s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.24 up_from 5120 peer_clock_delta [1167645.092134044s,1167645.092961188s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382017294s,1167816.382857379s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071470085s,1167579.072315770s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.31 up_from 5159 peer_clock_delta [1167476.584969154s,1167476.585819943s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.7 up_from 5103 peer_clock_delta [1167710.311774110s,1167710.312633293s]) 2020-05-06T20:06:40.219+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.28 up_from 5167 peer_clock_delta [1167441.804725768s,1167441.805593046s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.29 up_from 5099 peer_clock_delta [1167715.168507647s,1167715.169384062s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.29 up_from 5099 peer_clock_delta [1167715.168544829s,1167715.169429663s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.30 up_from 5118 peer_clock_delta [1167646.569962615s,1167646.570857795s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.30 up_from 5118 peer_clock_delta [1167646.570017383s,1167646.570919780s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.32 up_from 5074 peer_clock_delta [1167817.955575571s,1167817.956483423s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.31 up_from 5159 peer_clock_delta [1167476.584896433s,1167476.585813787s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.35 up_from 5126 peer_clock_delta [1167611.833344440s,1167611.834271376s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.33 up_from 5092 peer_clock_delta [1167747.547412957s,1167747.548348877s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032931364s,1167577.033877062s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032539248s,1167577.033489048s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644684407s,1167441.645640031s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.38 up_from 5076 peer_clock_delta [1167815.474466991s,1167815.475428288s]) 2020-05-06T20:06:40.219+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656683154s,1167712.657649535s]) 2020-05-06T20:06:40.219+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644354604s,1167441.645327818s]) 2020-05-06T20:06:40.235+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385323090s,1167815.385975554s]) 2020-05-06T20:06:40.235+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385304557s,1167815.385975554s]) 2020-05-06T20:06:40.279+0200 7f79191eb700 10 osd.15 5421 ping from client.335576 2020-05-06T20:06:40.279+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x5644903be160 prio 63 cost 0 latency 0.000003 epoch 5421 osd_op(client.335576.0:240453 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:40.279+0200 7f79191eb700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.335576.0:240453 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.335576.0:240453 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) queued 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.335576.0:240453 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) pg 0x56445f06b400 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644903be160 prio 63 cost 0 latency 0.000044 osd_op(client.335576.0:240453 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8 pg pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] _handle_message: 0x5644903be160 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: op osd_op(client.335576.0:240453 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] op_has_sufficient_caps session=0x5644777b12c0 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op osd_op(client.335576.0:240453 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 may_read -> write-ordered flags ondisk+write+known_if_redirected 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: found obc in cache: 0x564478128dc0 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: 0x564478128dc0 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head rwstate(none n=0 w=0) oi: 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head(5252'194 client.398034.0:1 dirty|data_digest|omap_digest s 0 uv 1 dd ffffffff od ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e64000 snapset: 0=[]:{} 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] find_object_context 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head @head oi=7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head(5252'194 client.398034.0:1 dirty|data_digest|omap_digest s 0 uv 1 dd ffffffff od ffffffff alloc_hint [0 0 0]) 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op obc obc(7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head rwstate(excl n=1 w=0)) 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 0x564477b38900 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] ov 5252'194 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op watch ping cookie 94107840416624 gen 12 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch ping: ctx->obc=0x564478128dc0 cookie=94107840416624 oi.version=194 ctx->at_version=0'0 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch: oi.user_version=1 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch: peer_addr=[2a00:X:X:6::7]:0/1293198946 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] found existing watch watch(cookie 94107840416624 30s [2a00:X:X:6::7]:0/1293198946) by client.335576 2020-05-06T20:06:40.279+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] -- Watch(94107840416624,client.335576) re-registering callback, timeout: 30 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx alloc reply 0x564467d578c0 result 0 2020-05-06T20:06:40.279+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op_effects client.335576 con 0x5644776c3000 2020-05-06T20:06:40.279+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] log_op_stats osd_op(client.335576.0:240453 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 inb 0 outb 0 lat 0.000198 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:40.279+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] publish_stats_to_osd 5421:733017 2020-05-06T20:06:40.279+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644903be160 finish 2020-05-06T20:06:40.279+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:40.311+0200 7f79189ea700 10 osd.15 5421 ping from client.398034 2020-05-06T20:06:40.311+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x56448fa312e0 prio 63 cost 0 latency 0.000003 epoch 5421 osd_op(client.398034.0:161456 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:40.311+0200 7f79189ea700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.398034.0:161456 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.398034.0:161456 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) queued 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.398034.0:161456 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) pg 0x56445f06b400 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x56448fa312e0 prio 63 cost 0 latency 0.000033 osd_op(client.398034.0:161456 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8 pg pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] _handle_message: 0x56448fa312e0 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: op osd_op(client.398034.0:161456 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] op_has_sufficient_caps session=0x5644777b0d20 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op osd_op(client.398034.0:161456 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 may_read -> write-ordered flags ondisk+write+known_if_redirected 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: found obc in cache: 0x564478128dc0 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: 0x564478128dc0 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head rwstate(none n=0 w=0) oi: 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head(5252'194 client.398034.0:1 dirty|data_digest|omap_digest s 0 uv 1 dd ffffffff od ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e64000 snapset: 0=[]:{} 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] find_object_context 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head @head oi=7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head(5252'194 client.398034.0:1 dirty|data_digest|omap_digest s 0 uv 1 dd ffffffff od ffffffff alloc_hint [0 0 0]) 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op obc obc(7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head rwstate(excl n=1 w=0)) 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 0x564477b38900 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] ov 5252'194 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op watch ping cookie 93832430088736 gen 12 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch ping: ctx->obc=0x564478128dc0 cookie=93832430088736 oi.version=194 ctx->at_version=0'0 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch: oi.user_version=1 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch: peer_addr=[2a00:X:X:6::8]:0/569487430 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] found existing watch watch(cookie 93832430088736 30s [2a00:X:X:6::8]:0/569487430) by client.398034 2020-05-06T20:06:40.311+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] -- Watch(93832430088736,client.398034) re-registering callback, timeout: 30 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx alloc reply 0x56447560db00 result 0 2020-05-06T20:06:40.311+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op_effects client.398034 con 0x5644776c3400 2020-05-06T20:06:40.311+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] log_op_stats osd_op(client.398034.0:161456 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 inb 0 outb 0 lat 0.000153 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:40.311+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] publish_stats_to_osd 5421:733018 2020-05-06T20:06:40.311+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x56448fa312e0 finish 2020-05-06T20:06:40.311+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:40.319+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x564498428ec0 prio 63 cost 60 latency 0.000004 epoch 5421 osd_op(client.398016.0:41636649 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:40.319+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636649 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) 2020-05-06T20:06:40.319+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:40.319+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636649 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) queued 2020-05-06T20:06:40.319+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:40.319+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636649 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) pg 0x564467384800 2020-05-06T20:06:40.319+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564498428ec0 prio 63 cost 60 latency 0.000043 osd_op(client.398016.0:41636649 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] 2020-05-06T20:06:40.319+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] _handle_message: 0x564498428ec0 2020-05-06T20:06:40.319+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: op osd_op(client.398016.0:41636649 5.9 5:993b03e9:usage::usage.18:head [call rgw.user_usage_log_read in=60b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:40.319+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] op_has_sufficient_caps session=0x564479fa2000 pool=5 (default.rgw.log usage) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method user_usage_log_read rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:40.319+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op osd_op(client.398016.0:41636649 5.9 5:993b03e9:usage::usage.18:head [call rgw.user_usage_log_read in=60b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:40.319+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: obc NOT found in cache: 5:993b03e9:usage::usage.18:head 2020-05-06T20:06:40.319+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: no obc for soid 5:993b03e9:usage::usage.18:head and !can_create 2020-05-06T20:06:40.319+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: find_object_context got error -2 2020-05-06T20:06:40.319+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564498428ec0 finish 2020-05-06T20:06:40.319+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:40.335+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656607119s,1167712.657649535s]) 2020-05-06T20:06:40.335+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656540289s,1167712.657649535s]) 2020-05-06T20:06:40.351+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.2 up_from 5111 peer_clock_delta [1167678.885634311s,1167678.885849858s]) 2020-05-06T20:06:40.351+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.2 up_from 5111 peer_clock_delta [1167678.885606803s,1167678.885849858s]) 2020-05-06T20:06:40.367+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x564492671020 prio 63 cost 39 latency 0.000004 epoch 5421 osd_op(client.398016.0:41636675 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:40.367+0200 7f79189ea700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.398016.0:41636675 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 3.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.398016.0:41636675 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) queued 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 3.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.398016.0:41636675 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) pg 0x564475c48000 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564492671020 prio 63 cost 39 latency 0.000047 osd_op(client.398016.0:41636675 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 pg pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] _handle_message: 0x564492671020 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] do_op: op osd_op(client.398016.0:41636675 3.3 3:c0000000:root:::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] op_has_sufficient_caps session=0x564479fa2000 pool=3 (default.rgw.meta root) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] do_pg_op osd_op(client.398016.0:41636675 3.3 3:c0000000:root:::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls pg=3.3 count 1024 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls lower_bound 3:c0000000::::head pg_end 3:d0000000::::head 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xae4b6783 vs lower bound 0x3 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x51d05d43 vs lower bound 0x3 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x1f2e8723 vs lower bound 0x3 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xeb6e0593 vs lower bound 0x3 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls item 0xeb6e0593, rev 0xc9a076d7 ekkono-offsite-backups 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xce7f4ed3 vs lower bound 0x3 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x5166d8f3 vs lower bound 0x3 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls item 0x5166d8f3, rev 0xcf1b668a viskadalskyrkan-owncloud-2 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x0 vs lower bound 0x3 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls handle=3:d0000000::::head 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls result=1 outdata.length()=129 2020-05-06T20:06:40.367+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564492671020 finish 2020-05-06T20:06:40.367+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:40.371+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x564498531600 prio 63 cost 47 latency 0.000002 epoch 5421 osd_op(client.398016.0:41636683 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:40.371+0200 7f79189ea700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.398016.0:41636683 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.398016.0:41636683 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) queued 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.398016.0:41636683 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) pg 0x56445e521400 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x564498531600 prio 63 cost 47 latency 0.000042 osd_op(client.398016.0:41636683 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] _handle_message: 0x564498531600 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op: op osd_op(client.398016.0:41636683 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] op_has_sufficient_caps session=0x564479fa2000 pool=4 (default.rgw.buckets.index ) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method bucket_list rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op osd_op(client.398016.0:41636683 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: found obc in cache: 0x564479b41440 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: 0x564479b41440 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head rwstate(none n=0 w=0) oi: 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head(5421'12660 osd.15.0:15907 dirty|omap|data_digest|omap_digest s 0 uv 12334 dd ffffffff od 423b25ef alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e65080 snapset: 0=[]:{} 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] find_object_context 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head @head oi=4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head(5421'12660 osd.15.0:15907 dirty|omap|data_digest|omap_digest s 0 uv 12334 dd ffffffff od 423b25ef alloc_hint [0 0 0]) 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op obc obc(4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head rwstate(read n=1 w=0)) 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 0x56447a964000 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] ov 5421'12660 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op call rgw.bucket_list in=47b 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] call method rgw.bucket_list 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [omap-get-header] 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op omap-get-header 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] method called response length=110 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx alloc reply 0x56445f725b00 result 0 2020-05-06T20:06:40.371+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op_effects client.398016 con 0x564479675000 2020-05-06T20:06:40.371+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] log_op_stats osd_op(client.398016.0:41636683 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b out=110b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 110 lat 0.000244 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:40.371+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] publish_stats_to_osd 5421:1159095 2020-05-06T20:06:40.371+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x564498531600 finish 2020-05-06T20:06:40.371+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:40.375+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x5644984d6680 prio 63 cost 47 latency 0.000002 epoch 5421 osd_op(client.398016.0:41636745 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:40.375+0200 7f79189ea700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.398016.0:41636745 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.398016.0:41636745 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) queued 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.398016.0:41636745 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) pg 0x56445e521400 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x5644984d6680 prio 63 cost 47 latency 0.000032 osd_op(client.398016.0:41636745 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] _handle_message: 0x5644984d6680 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op: op osd_op(client.398016.0:41636745 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] op_has_sufficient_caps session=0x564479fa2000 pool=4 (default.rgw.buckets.index ) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method bucket_list rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op osd_op(client.398016.0:41636745 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: found obc in cache: 0x564479b41700 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: 0x564479b41700 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head rwstate(none n=0 w=0) oi: 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head(5421'12661 osd.15.0:15908 dirty|omap|data_digest|omap_digest s 0 uv 12261 dd ffffffff od 5ac232fe alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e66520 snapset: 0=[]:{} 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] find_object_context 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head @head oi=4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head(5421'12661 osd.15.0:15908 dirty|omap|data_digest|omap_digest s 0 uv 12261 dd ffffffff od 5ac232fe alloc_hint [0 0 0]) 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op obc obc(4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head rwstate(read n=1 w=0)) 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 0x56447a964000 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] ov 5421'12661 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op call rgw.bucket_list in=47b 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] call method rgw.bucket_list 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [omap-get-header] 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op omap-get-header 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] method called response length=110 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx alloc reply 0x5644707a8480 result 0 2020-05-06T20:06:40.375+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op_effects client.398016 con 0x564479675000 2020-05-06T20:06:40.375+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] log_op_stats osd_op(client.398016.0:41636745 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b out=110b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 110 lat 0.000185 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:40.375+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] publish_stats_to_osd 5421:1159096 2020-05-06T20:06:40.375+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x5644984d6680 finish 2020-05-06T20:06:40.375+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:40.395+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.5 up_from 5136 peer_clock_delta [1167576.395846586s,1167576.396196047s]) 2020-05-06T20:06:40.395+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.5 up_from 5136 peer_clock_delta [1167576.395836467s,1167576.396196047s]) 2020-05-06T20:06:40.395+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032941106s,1167577.033877062s]) 2020-05-06T20:06:40.395+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032791772s,1167577.033877062s]) 2020-05-06T20:06:40.431+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.20 up_from 5111 peer_clock_delta [1167678.414985996s,1167678.415519260s]) 2020-05-06T20:06:40.431+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.20 up_from 5111 peer_clock_delta [1167678.414952304s,1167678.415519260s]) 2020-05-06T20:06:40.459+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214147984s,1167820.214690978s]) 2020-05-06T20:06:40.459+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214130996s,1167820.214690978s]) 2020-05-06T20:06:40.559+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.19 up_from 5092 peer_clock_delta [1167747.405988777s,1167747.406452510s]) 2020-05-06T20:06:40.559+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.19 up_from 5092 peer_clock_delta [1167747.405971845s,1167747.406452510s]) 2020-05-06T20:06:40.655+0200 7f79191eb700 15 osd.15 5421 enqueue_peering_evt 2.1f1s3 epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.24 pg_lease(ru 1203404.515202010s ub 1203412.515330303s int 16s) 2020-05-06T20:06:40.655+0200 7f79191eb700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(2.1f1s3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.24 pg_lease(ru 1203404.515202010s ub 1203412.515330303s int 16s)) prio 255 cost 10 e5421) 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 2.1f1s3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(2.1f1s3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.24 pg_lease(ru 1203404.515202010s ub 1203412.515330303s int 16s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 2.1f1s3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(2.1f1s3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.24 pg_lease(ru 1203404.515202010s ub 1203412.515330303s int 16s)) prio 255 cost 10 e5421) pg 0x564476d96000 2020-05-06T20:06:40.655+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[2.1f1s3( v 5421'6160 (4570'3600,5421'6160] local-lis/les=5414/5415 n=645 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [24,32,1,15,30,39,4,17]p24(0) r=3 lpr=5414 luod=0'0 crt=5421'6159 mlcod 5421'6159 active mbc={}] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.24 pg_lease(ru 1203404.515202010s ub 1203412.515330303s int 16s) 2020-05-06T20:06:40.655+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[2.1f1s3( v 5421'6160 (4570'3600,5421'6160] local-lis/les=5414/5415 n=645 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [24,32,1,15,30,39,4,17]p24(0) r=3 lpr=5414 luod=0'0 crt=5421'6159 mlcod 5421'6159 active mbc={}] proc_lease pg_lease(ru 1203404.515202010s ub 1203412.515330303s int 16s) 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[2.1f1s3( v 5421'6160 (4570'3600,5421'6160] local-lis/les=5414/5415 n=645 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [24,32,1,15,30,39,4,17]p24(0) r=3 lpr=5414 luod=0'0 crt=5421'6159 mlcod 5421'6159 active mbc={}] peer_clock_delta_ub 1167645.092961188s -> ru 35759.422240822s 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[2.1f1s3( v 5421'6160 (4570'3600,5421'6160] local-lis/les=5414/5415 n=645 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [24,32,1,15,30,39,4,17]p24(0) r=3 lpr=5414 luod=0'0 crt=5421'6159 mlcod 5421'6159 active mbc={}] proc_lease readable_until now 35759.422240822s 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[2.1f1s3( v 5421'6160 (4570'3600,5421'6160] local-lis/les=5414/5415 n=645 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [24,32,1,15,30,39,4,17]p24(0) r=3 lpr=5414 luod=0'0 crt=5421'6159 mlcod 5421'6159 active mbc={}] peer_clock_delta_lb 1167645.092134044s -> ruub 35767.423196259s 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[2.1f1s3( v 5421'6160 (4570'3600,5421'6160] local-lis/les=5414/5415 n=645 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [24,32,1,15,30,39,4,17]p24(0) r=3 lpr=5414 luod=0'0 crt=5421'6159 mlcod 5421'6159 active mbc={}] proc_lease readable_until_ub now 35767.423196259s 2020-05-06T20:06:40.655+0200 7f78fa1ad700 10 log is not dirty 2020-05-06T20:06:40.655+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:40.655+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.3 up_from 5126 peer_clock_delta [1167611.710362051s,1167611.710784618s]) 2020-05-06T20:06:40.655+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.3 up_from 5126 peer_clock_delta [1167611.710348402s,1167611.710784618s]) 2020-05-06T20:06:40.735+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385470224s,1167815.385824314s]) 2020-05-06T20:06:40.735+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385408028s,1167815.385824314s]) 2020-05-06T20:06:40.735+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.18 up_from 5161 peer_clock_delta [1167473.938927136s,1167473.939493451s]) 2020-05-06T20:06:40.735+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.18 up_from 5161 peer_clock_delta [1167473.938909941s,1167473.939493451s]) 2020-05-06T20:06:40.863+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.23 up_from 5093 peer_clock_delta [1167747.079801817s,1167747.080394906s]) 2020-05-06T20:06:40.863+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.23 up_from 5093 peer_clock_delta [1167747.079769184s,1167747.080394906s]) 2020-05-06T20:06:41.103+0200 7f79139e0700 10 osd.15 5421 tick_without_osd_lock 2020-05-06T20:06:41.103+0200 7f79139e0700 20 osd.15 5421 scrub_random_backoff lost coin flip, randomly backing off 2020-05-06T20:06:41.103+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2020-05-06T20:06:41.103+0200 7f79139e0700 20 osd.15 5421 promote_throttle_recalibrate new_prob 1000 2020-05-06T20:06:41.103+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2020-05-06T20:06:41.255+0200 7f79149e2700 10 osd.15 5421 tick 2020-05-06T20:06:41.255+0200 7f79149e2700 10 osd.15 5421 do_waiters -- start 2020-05-06T20:06:41.255+0200 7f79149e2700 10 osd.15 5421 do_waiters -- finish 2020-05-06T20:06:41.255+0200 7f79149e2700 20 osd.15 5421 tick last_purged_snaps_scrub 2020-05-06T09:01:52.905203+0200 next 2020-05-07T09:01:52.905203+0200 2020-05-06T20:06:41.287+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798148214s,1167440.798598748s]) 2020-05-06T20:06:41.287+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798041370s,1167440.798598748s]) 2020-05-06T20:06:41.287+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287127330s,1167782.287519486s]) 2020-05-06T20:06:41.287+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287110853s,1167782.287519486s]) 2020-05-06T20:06:41.311+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644558145s,1167441.645640031s]) 2020-05-06T20:06:41.311+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644545868s,1167441.645640031s]) 2020-05-06T20:06:41.331+0200 7f79069e1700 20 osd.15 5421 reports for 0 queries 2020-05-06T20:06:41.531+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.20 up_from 5111 peer_clock_delta [1167678.415039587s,1167678.415519290s]) 2020-05-06T20:06:41.531+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.20 up_from 5111 peer_clock_delta [1167678.414981233s,1167678.415519290s]) 2020-05-06T20:06:41.547+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.4 up_from 5161 peer_clock_delta [1167474.555768026s,1167474.556217147s]) 2020-05-06T20:06:41.547+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.4 up_from 5161 peer_clock_delta [1167474.555756173s,1167474.556217147s]) 2020-05-06T20:06:41.615+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132554821s,1167609.133004700s]) 2020-05-06T20:06:41.615+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132540635s,1167609.133004700s]) 2020-05-06T20:06:41.695+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.30 up_from 5118 peer_clock_delta [1167646.570401034s,1167646.570919780s]) 2020-05-06T20:06:41.695+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.30 up_from 5118 peer_clock_delta [1167646.570382957s,1167646.570919780s]) 2020-05-06T20:06:41.743+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x564490350840 prio 127 cost 0 latency 0.000017 epoch 5421 MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98560, reqid=client.325546.0:51685709, at_version=5421'8604, trim_to=5286'5700, roll_forward_to=5421'8602)) v2 2020-05-06T20:06:41.743+0200 7f79181e9700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(2.d6s7 PGOpItem(op=MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98560, reqid=client.325546.0:51685709, at_version=5421'8604, trim_to=5286'5700, roll_forward_to=5421'8602)) v2) prio 127 cost 0 e5421) 2020-05-06T20:06:41.743+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 2.d6s7 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:41.743+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(2.d6s7 PGOpItem(op=MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98560, reqid=client.325546.0:51685709, at_version=5421'8604, trim_to=5286'5700, roll_forward_to=5421'8602)) v2) prio 127 cost 0 e5421) queued 2020-05-06T20:06:41.743+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 2.d6s7 to_process waiting <> waiting_peering {} 2020-05-06T20:06:41.743+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(2.d6s7 PGOpItem(op=MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98560, reqid=client.325546.0:51685709, at_version=5421'8604, trim_to=5286'5700, roll_forward_to=5421'8602)) v2) prio 127 cost 0 e5421) pg 0x56447626a800 2020-05-06T20:06:41.743+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564490350840 prio 127 cost 0 latency 0.000072 MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98560, reqid=client.325546.0:51685709, at_version=5421'8604, trim_to=5286'5700, roll_forward_to=5421'8602)) v2 pg pg[2.d6s7( v 5421'8603 (5286'5700,5421'8603] local-lis/les=5414/5415 n=627 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8601 mlcod 5421'8596 active mbc={}] 2020-05-06T20:06:41.743+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8603 (5286'5700,5421'8603] local-lis/les=5414/5415 n=627 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8601 mlcod 5421'8596 active mbc={}] _handle_message: MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98560, reqid=client.325546.0:51685709, at_version=5421'8604, trim_to=5286'5700, roll_forward_to=5421'8602)) v2 2020-05-06T20:06:41.743+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8603 (5286'5700,5421'8603] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8601 mlcod 5421'8596 active mbc={}] append_log log((5286'5700,5421'8603], crt=5421'8601) [5421'8604 (0'0) modify 2:6b305394:::5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2__shadow_delivery%2fPE603841.png.2~uGQROkZFqjoAZNqX1xf3_dPLYzZkZts.2_3:head by client.325546.0:51685709 2020-05-06T20:06:41.738993+0200 0 ObjectCleanRegions clean_offsets: [3256616~18446744073706294999], clean_omap: 1, new_object: 0] 2020-05-06T20:06:41.743+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 lua=5421'8603 crt=5421'8601 lcod 5421'8603 mlcod 5421'8596 active mbc={}] add_log_entry 5421'8604 (0'0) modify 2:6b305394:::5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2__shadow_delivery%2fPE603841.png.2~uGQROkZFqjoAZNqX1xf3_dPLYzZkZts.2_3:head by client.325546.0:51685709 2020-05-06T20:06:41.738993+0200 0 ObjectCleanRegions clean_offsets: [3256616~18446744073706294999], clean_omap: 1, new_object: 0 2020-05-06T20:06:41.743+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 lua=5421'8603 crt=5421'8602 lcod 5421'8603 mlcod 5421'8596 active mbc={}] rollforward: entry=5421'8602 (0'0) error 2:6b66f52b:::5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.3__multipart_urn%3aoid%3a1833.2~bWi1-yh2qsD2ZHoOU2mY4YRSD9GsA1F.133:head by client.325552.0:1384928 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:41.743+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 lua=5421'8603 crt=5421'8602 lcod 5421'8603 mlcod 5421'8596 active mbc={}] append_log approx pg log length = 2904 2020-05-06T20:06:41.743+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 lua=5421'8603 crt=5421'8602 lcod 5421'8603 mlcod 5421'8596 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:41.743+0200 7f78f91ab700 10 trim proposed trim_to = 5286'5700 2020-05-06T20:06:41.743+0200 7f78f91ab700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'8604, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:41.747+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 lua=5421'8603 crt=5421'8602 lcod 5421'8603 mlcod 5421'8602 active mbc={}] op_applied version 5421'8604 2020-05-06T20:06:41.747+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564490350840 finish 2020-05-06T20:06:41.747+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:41.763+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x564492673760 prio 127 cost 0 latency 0.000007 epoch 5421 MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98561, reqid=client.325546.0:51685709, at_version=0'0, trim_to=5286'5700, roll_forward_to=5421'8604)) v2 2020-05-06T20:06:41.763+0200 7f79181e9700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(2.d6s7 PGOpItem(op=MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98561, reqid=client.325546.0:51685709, at_version=0'0, trim_to=5286'5700, roll_forward_to=5421'8604)) v2) prio 127 cost 0 e5421) 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 2.d6s7 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(2.d6s7 PGOpItem(op=MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98561, reqid=client.325546.0:51685709, at_version=0'0, trim_to=5286'5700, roll_forward_to=5421'8604)) v2) prio 127 cost 0 e5421) queued 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 2.d6s7 to_process waiting <> waiting_peering {} 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(2.d6s7 PGOpItem(op=MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98561, reqid=client.325546.0:51685709, at_version=0'0, trim_to=5286'5700, roll_forward_to=5421'8604)) v2) prio 127 cost 0 e5421) pg 0x56447626a800 2020-05-06T20:06:41.763+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564492673760 prio 127 cost 0 latency 0.000040 MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98561, reqid=client.325546.0:51685709, at_version=0'0, trim_to=5286'5700, roll_forward_to=5421'8604)) v2 pg pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8602 mlcod 5421'8602 active mbc={}] 2020-05-06T20:06:41.763+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8602 mlcod 5421'8602 active mbc={}] _handle_message: MOSDECSubOpWrite(2.d6s7 5421/5414 ECSubWrite(tid=98561, reqid=client.325546.0:51685709, at_version=0'0, trim_to=5286'5700, roll_forward_to=5421'8604)) v2 2020-05-06T20:06:41.763+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8602 mlcod 5421'8602 active mbc={}] append_log log((5286'5700,5421'8604], crt=5421'8602) [] 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8604 mlcod 5421'8602 active mbc={}] rollforward: entry=5421'8603 (0'0) error 2:6b1ac79d:::5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.3__multipart_urn%3aoid%3a1833.2~bWi1-yh2qsD2ZHoOU2mY4YRSD9GsA1F.243:head by client.325552.0:1385148 0.000000 -2 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 1, new_object: 0 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8604 mlcod 5421'8602 active mbc={}] rollforward: entry=5421'8604 (0'0) modify 2:6b305394:::5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2__shadow_delivery%2fPE603841.png.2~uGQROkZFqjoAZNqX1xf3_dPLYzZkZts.2_3:head by client.325546.0:51685709 2020-05-06T20:06:41.738993+0200 0 ObjectCleanRegions clean_offsets: [3256616~18446744073706294999], clean_omap: 1, new_object: 0 2020-05-06T20:06:41.763+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8604 mlcod 5421'8602 active mbc={}] append_log approx pg log length = 2904 2020-05-06T20:06:41.763+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[2.d6s7( v 5421'8604 (5286'5700,5421'8604] local-lis/les=5414/5415 n=628 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,5,7,32,33,14,9,15]p38(0) r=7 lpr=5414 luod=0'0 crt=5421'8604 mlcod 5421'8602 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:41.763+0200 7f78f91ab700 10 trim proposed trim_to = 5286'5700 2020-05-06T20:06:41.763+0200 7f78f91ab700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 4294967295'18446744073709551615, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:41.763+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x564492673760 finish 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:41.763+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:41.763+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x564491faa9a0 prio 127 cost 1021 latency 0.000006 epoch 5421 osd_repop(client.325546.0:51685710 4.f e5421/5414) v3 2020-05-06T20:06:41.763+0200 7f79189ea700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685710 4.f e5421/5414) v3) prio 127 cost 1021 e5421) 2020-05-06T20:06:41.763+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:41.763+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685710 4.f e5421/5414) v3) prio 127 cost 1021 e5421) queued 2020-05-06T20:06:41.763+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process waiting <> waiting_peering {} 2020-05-06T20:06:41.763+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685710 4.f e5421/5414) v3) prio 127 cost 1021 e5421) pg 0x56445e51e000 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x564491faa9a0 prio 127 cost 1021 latency 0.000047 osd_repop(client.325546.0:51685710 4.f e5421/5414) v3 pg pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31368 mlcod 5421'31367 active mbc={}] 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31368 mlcod 5421'31367 active mbc={}] _handle_message: 0x564491faa9a0 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31368 mlcod 5421'31367 active mbc={}] do_repop 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31370 (transaction) 228 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31369 (5421'28900,5421'31369] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31368 mlcod 5421'31367 active mbc={}] append_log log((5421'28900,5421'31369], crt=5421'31369) [5421'31370 (5421'31369) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685710 2020-05-06T20:06:41.769490+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0] 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31369 lcod 5421'31368 mlcod 5421'31367 active mbc={}] add_log_entry 5421'31370 (5421'31369) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685710 2020-05-06T20:06:41.769490+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:41.763+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31368 mlcod 5421'31367 active mbc={}] rollforward: entry=5421'31370 (5421'31369) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685710 2020-05-06T20:06:41.769490+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31368 mlcod 5421'31367 active mbc={}] append_log approx pg log length = 2470 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31368 mlcod 5421'31367 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 trim proposed trim_to = 5421'28900 2020-05-06T20:06:41.763+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'31370, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x564491faa9a0 finish 2020-05-06T20:06:41.763+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:41.763+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31368 mlcod 5421'31368 active mbc={}] repop_commit on op osd_repop(client.325546.0:51685710 4.f e5421/5414 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31370, mlcod=5421'31368) v3, sending commit to osd.12 2020-05-06T20:06:41.763+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:41.775+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x56447b13d020 prio 127 cost 1253 latency 0.000003 epoch 5421 osd_repop(client.325546.0:51685712 4.f e5421/5414) v3 2020-05-06T20:06:41.775+0200 7f79189ea700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685712 4.f e5421/5414) v3) prio 127 cost 1253 e5421) 2020-05-06T20:06:41.775+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:41.775+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685712 4.f e5421/5414) v3) prio 127 cost 1253 e5421) queued 2020-05-06T20:06:41.775+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process waiting <> waiting_peering {} 2020-05-06T20:06:41.775+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685712 4.f e5421/5414) v3) prio 127 cost 1253 e5421) pg 0x56445e51e000 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x56447b13d020 prio 127 cost 1253 latency 0.000029 osd_repop(client.325546.0:51685712 4.f e5421/5414) v3 pg pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31369 mlcod 5421'31368 active mbc={}] 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31369 mlcod 5421'31368 active mbc={}] _handle_message: 0x56447b13d020 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31369 mlcod 5421'31368 active mbc={}] do_repop 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31371 (transaction) 228 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31370 (5421'28900,5421'31370] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31369 mlcod 5421'31368 active mbc={}] append_log log((5421'28900,5421'31370], crt=5421'31370) [5421'31371 (5421'31370) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685712 2020-05-06T20:06:41.780788+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0] 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31370 lcod 5421'31369 mlcod 5421'31368 active mbc={}] add_log_entry 5421'31371 (5421'31370) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685712 2020-05-06T20:06:41.780788+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:41.775+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31369 mlcod 5421'31368 active mbc={}] rollforward: entry=5421'31371 (5421'31370) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685712 2020-05-06T20:06:41.780788+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31369 mlcod 5421'31368 active mbc={}] append_log approx pg log length = 2471 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31369 mlcod 5421'31368 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 trim proposed trim_to = 5421'28900 2020-05-06T20:06:41.775+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'31371, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x56447b13d020 finish 2020-05-06T20:06:41.775+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:41.775+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31369 mlcod 5421'31369 active mbc={}] repop_commit on op osd_repop(client.325546.0:51685712 4.f e5421/5414 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31371, mlcod=5421'31369) v3, sending commit to osd.12 2020-05-06T20:06:41.775+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:41.791+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798082821s,1167440.798658083s]) 2020-05-06T20:06:41.791+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798063030s,1167440.798658083s]) 2020-05-06T20:06:41.859+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.7 up_from 5103 peer_clock_delta [1167710.312290936s,1167710.312633293s]) 2020-05-06T20:06:41.859+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.7 up_from 5103 peer_clock_delta [1167710.312260884s,1167710.312633293s]) 2020-05-06T20:06:41.875+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.1 up_from 5084 peer_clock_delta [1167782.010015722s,1167782.010240834s]) 2020-05-06T20:06:41.875+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.1 up_from 5084 peer_clock_delta [1167782.009809068s,1167782.010240834s]) 2020-05-06T20:06:41.903+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885168297s,1167644.885627439s]) 2020-05-06T20:06:41.903+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885133003s,1167644.885627439s]) 2020-05-06T20:06:41.919+0200 7f78f89aa700 5 osd.15 5421 heartbeat osd_stat(store_statfs(0x56d343b0000/0x40000000/0x581c17fe000, data 0x7cd9f3eeb/0x7cd64c000, compress 0x16a45768/0x1807c000/0x1d698000, omap 0x2a6425, meta 0x3fd59bdb), peers [0,1,2,3,4,5,6,7,9,10,11,12,13,14,16,17,18,19,21,22,23,24,26,27,28,29,30,31,32,33,35,36,37,38,39] op hist []) 2020-05-06T20:06:41.919+0200 7f78f89aa700 20 osd.15 5421 check_full_status cur ratio 0.014401, physical ratio 0.014401, new state none 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.0 up_from 5071 peer_clock_delta [1167821.367724633s,1167821.367886860s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.1 up_from 5084 peer_clock_delta [1167782.010055775s,1167782.010231194s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.0 up_from 5071 peer_clock_delta [1167821.367705901s,1167821.367897662s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.4 up_from 5161 peer_clock_delta [1167474.555994223s,1167474.556204105s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287280035s,1167782.287498703s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.2 up_from 5111 peer_clock_delta [1167678.885626347s,1167678.885851946s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.4 up_from 5161 peer_clock_delta [1167474.555964475s,1167474.556211360s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.5 up_from 5136 peer_clock_delta [1167576.395926164s,1167576.396188720s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.5 up_from 5136 peer_clock_delta [1167576.395922382s,1167576.396199056s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597603591s,1167680.597892328s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214290551s,1167820.214599725s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.1 up_from 5084 peer_clock_delta [1167782.009913252s,1167782.010237049s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.3 up_from 5126 peer_clock_delta [1167611.710422744s,1167611.710775407s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.6 up_from 5083 peer_clock_delta [1167782.287139236s,1167782.287504037s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.7 up_from 5103 peer_clock_delta [1167710.312204073s,1167710.312591251s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597430335s,1167680.597842412s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798045341s,1167440.798473710s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.13 up_from 5086 peer_clock_delta [1167779.698482105s,1167779.698936293s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.7 up_from 5103 peer_clock_delta [1167710.312156207s,1167710.312619890s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.2 up_from 5111 peer_clock_delta [1167678.885375134s,1167678.885843878s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214096206s,1167820.214568920s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885132919s,1167644.885613413s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.3 up_from 5126 peer_clock_delta [1167611.710274196s,1167611.710763751s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.797999599s,1167440.798495423s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779107637s,1167509.779610516s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.13 up_from 5086 peer_clock_delta [1167779.698531338s,1167779.699039527s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885016174s,1167644.885528384s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132368800s,1167609.132885193s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779073175s,1167509.779596700s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.18 up_from 5161 peer_clock_delta [1167473.938932643s,1167473.939461693s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382291887s,1167816.382827988s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.21 up_from 5153 peer_clock_delta [1167508.090468571s,1167508.091010445s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.21 up_from 5153 peer_clock_delta [1167508.090412455s,1167508.090959344s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132517016s,1167609.133071361s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.23 up_from 5093 peer_clock_delta [1167747.079804382s,1167747.080368481s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382398297s,1167816.382968369s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.22 up_from 5074 peer_clock_delta [1167818.481202443s,1167818.481776362s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.24 up_from 5120 peer_clock_delta [1167645.092418227s,1167645.092996134s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.23 up_from 5093 peer_clock_delta [1167747.079744576s,1167747.080328611s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071766951s,1167579.072358115s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385291798s,1167815.385887600s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.18 up_from 5161 peer_clock_delta [1167473.938965672s,1167473.939564950s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385351135s,1167815.385954283s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.19 up_from 5092 peer_clock_delta [1167747.405919790s,1167747.406529949s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.28 up_from 5167 peer_clock_delta [1167441.804772525s,1167441.805390940s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.24 up_from 5120 peer_clock_delta [1167645.092467184s,1167645.093090747s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.31 up_from 5159 peer_clock_delta [1167476.585172448s,1167476.585799992s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071807420s,1167579.072440924s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.29 up_from 5099 peer_clock_delta [1167715.168743059s,1167715.169385435s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.22 up_from 5074 peer_clock_delta [1167818.481273026s,1167818.481922347s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.30 up_from 5118 peer_clock_delta [1167646.570207068s,1167646.570863306s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.32 up_from 5074 peer_clock_delta [1167817.955882354s,1167817.956545129s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.32 up_from 5074 peer_clock_delta [1167817.955920415s,1167817.956591752s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032749523s,1167577.033430387s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.29 up_from 5099 peer_clock_delta [1167715.168817923s,1167715.169503971s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.33 up_from 5092 peer_clock_delta [1167747.547658359s,1167747.548347967s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.33 up_from 5092 peer_clock_delta [1167747.547606222s,1167747.548300611s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.30 up_from 5118 peer_clock_delta [1167646.570277407s,1167646.570980915s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.19 up_from 5092 peer_clock_delta [1167747.405708617s,1167747.406421019s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.38 up_from 5076 peer_clock_delta [1167815.474642345s,1167815.475364209s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.35 up_from 5126 peer_clock_delta [1167611.833166255s,1167611.833892779s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.35 up_from 5126 peer_clock_delta [1167611.833208194s,1167611.833938390s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.31 up_from 5159 peer_clock_delta [1167476.585018380s,1167476.585753059s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032800478s,1167577.033541143s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.38 up_from 5076 peer_clock_delta [1167815.474666594s,1167815.475414376s]) 2020-05-06T20:06:41.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644516868s,1167441.645274313s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656487072s,1167712.657248290s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644532420s,1167441.645300757s]) 2020-05-06T20:06:41.919+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.28 up_from 5167 peer_clock_delta [1167441.804747746s,1167441.805525190s]) 2020-05-06T20:06:41.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656516792s,1167712.657303457s]) 2020-05-06T20:06:42.035+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656692146s,1167712.657303457s]) 2020-05-06T20:06:42.035+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656675568s,1167712.657303457s]) 2020-05-06T20:06:42.131+0200 7f79139e0700 10 osd.15 5421 tick_without_osd_lock 2020-05-06T20:06:42.131+0200 7f79139e0700 20 osd.15 5421 scrub_random_backoff lost coin flip, randomly backing off 2020-05-06T20:06:42.131+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2020-05-06T20:06:42.131+0200 7f79139e0700 20 osd.15 5421 promote_throttle_recalibrate new_prob 1000 2020-05-06T20:06:42.131+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2020-05-06T20:06:42.163+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.33 up_from 5092 peer_clock_delta [1167747.547817559s,1167747.548347967s]) 2020-05-06T20:06:42.163+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.33 up_from 5092 peer_clock_delta [1167747.547802636s,1167747.548347967s]) 2020-05-06T20:06:42.207+0200 7f79149e2700 10 osd.15 5421 tick 2020-05-06T20:06:42.207+0200 7f79149e2700 10 osd.15 5421 do_waiters -- start 2020-05-06T20:06:42.207+0200 7f79149e2700 10 osd.15 5421 do_waiters -- finish 2020-05-06T20:06:42.207+0200 7f79149e2700 20 osd.15 5421 tick last_purged_snaps_scrub 2020-05-06T09:01:52.905203+0200 next 2020-05-07T09:01:52.905203+0200 2020-05-06T20:06:42.303+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597373721s,1167680.597892328s]) 2020-05-06T20:06:42.303+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597355208s,1167680.597892328s]) 2020-05-06T20:06:42.431+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.28 up_from 5167 peer_clock_delta [1167441.804866092s,1167441.805525190s]) 2020-05-06T20:06:42.431+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.28 up_from 5167 peer_clock_delta [1167441.804832757s,1167441.805525190s]) 2020-05-06T20:06:42.563+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.23 up_from 5093 peer_clock_delta [1167747.079894280s,1167747.080368481s]) 2020-05-06T20:06:42.563+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.23 up_from 5093 peer_clock_delta [1167747.079858660s,1167747.080368481s]) 2020-05-06T20:06:42.699+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032778099s,1167577.033541143s]) 2020-05-06T20:06:42.699+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032721984s,1167577.033541143s]) 2020-05-06T20:06:42.711+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.24 up_from 5120 peer_clock_delta [1167645.092507827s,1167645.093090747s]) 2020-05-06T20:06:42.711+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.24 up_from 5120 peer_clock_delta [1167645.092489843s,1167645.093090747s]) 2020-05-06T20:06:42.791+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382448643s,1167816.382968369s]) 2020-05-06T20:06:42.791+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382437130s,1167816.382968369s]) 2020-05-06T20:06:42.887+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798089120s,1167440.798495423s]) 2020-05-06T20:06:42.887+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798072503s,1167440.798495423s]) 2020-05-06T20:06:42.955+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.3 up_from 5126 peer_clock_delta [1167611.710413111s,1167611.710775407s]) 2020-05-06T20:06:42.955+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.3 up_from 5126 peer_clock_delta [1167611.710399961s,1167611.710775407s]) 2020-05-06T20:06:43.039+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.18 up_from 5161 peer_clock_delta [1167473.938871882s,1167473.939564950s]) 2020-05-06T20:06:43.039+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.18 up_from 5161 peer_clock_delta [1167473.938855206s,1167473.939564950s]) 2020-05-06T20:06:43.135+0200 7f79139e0700 10 osd.15 5421 tick_without_osd_lock 2020-05-06T20:06:43.135+0200 7f79139e0700 20 osd.15 5421 scrub_random_backoff lost coin flip, randomly backing off 2020-05-06T20:06:43.135+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2020-05-06T20:06:43.135+0200 7f79139e0700 20 osd.15 5421 promote_throttle_recalibrate new_prob 1000 2020-05-06T20:06:43.135+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2020-05-06T20:06:43.163+0200 7f79149e2700 10 osd.15 5421 tick 2020-05-06T20:06:43.163+0200 7f79149e2700 10 osd.15 5421 do_waiters -- start 2020-05-06T20:06:43.163+0200 7f79149e2700 10 osd.15 5421 do_waiters -- finish 2020-05-06T20:06:43.163+0200 7f79149e2700 20 osd.15 5421 tick last_purged_snaps_scrub 2020-05-06T09:01:52.905203+0200 next 2020-05-07T09:01:52.905203+0200 2020-05-06T20:06:43.375+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.0 up_from 5071 peer_clock_delta [1167821.367651220s,1167821.367897662s]) 2020-05-06T20:06:43.375+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.0 up_from 5071 peer_clock_delta [1167821.367492754s,1167821.367897662s]) 2020-05-06T20:06:43.387+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798173209s,1167440.798744888s]) 2020-05-06T20:06:43.387+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.9 up_from 5168 peer_clock_delta [1167440.798161577s,1167440.798744888s]) 2020-05-06T20:06:43.623+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.38 up_from 5076 peer_clock_delta [1167815.474839977s,1167815.475414376s]) 2020-05-06T20:06:43.623+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.38 up_from 5076 peer_clock_delta [1167815.474817700s,1167815.475414376s]) 2020-05-06T20:06:43.635+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385387093s,1167815.385954283s]) 2020-05-06T20:06:43.635+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.27 up_from 5077 peer_clock_delta [1167815.385371310s,1167815.385954283s]) 2020-05-06T20:06:43.719+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071894370s,1167579.072440924s]) 2020-05-06T20:06:43.719+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071878120s,1167579.072440924s]) 2020-05-06T20:06:43.735+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656710098s,1167712.657184809s]) 2020-05-06T20:06:43.735+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.39 up_from 5101 peer_clock_delta [1167712.656694499s,1167712.657184809s]) 2020-05-06T20:06:43.799+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.32 up_from 5074 peer_clock_delta [1167817.955952241s,1167817.956591752s]) 2020-05-06T20:06:43.799+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.32 up_from 5074 peer_clock_delta [1167817.955930335s,1167817.956591752s]) 2020-05-06T20:06:43.799+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032872526s,1167577.033334509s]) 2020-05-06T20:06:43.799+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032634853s,1167577.033334509s]) 2020-05-06T20:06:43.911+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132464304s,1167609.133071361s]) 2020-05-06T20:06:43.911+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132449369s,1167609.133071361s]) 2020-05-06T20:06:43.935+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 4.5 epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.28 pg_lease(ru 1203204.509171000s ub 1203212.509271398s int 16s) 2020-05-06T20:06:43.935+0200 7f79181e9700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(4.5 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.28 pg_lease(ru 1203204.509171000s ub 1203212.509271398s int 16s)) prio 255 cost 10 e5421) 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.5 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.5 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.28 pg_lease(ru 1203204.509171000s ub 1203212.509271398s int 16s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.5 to_process waiting <> waiting_peering {} 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.5 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.28 pg_lease(ru 1203204.509171000s ub 1203212.509271398s int 16s)) prio 255 cost 10 e5421) pg 0x56445e524800 2020-05-06T20:06:43.935+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.5( v 5421'12103 (5421'9300,5421'12103] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [28,15,32] r=1 lpr=5414 luod=0'0 crt=5421'12101 mlcod 5421'12094 active mbc={}] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.28 pg_lease(ru 1203204.509171000s ub 1203212.509271398s int 16s) 2020-05-06T20:06:43.935+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.5( v 5421'12103 (5421'9300,5421'12103] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [28,15,32] r=1 lpr=5414 luod=0'0 crt=5421'12101 mlcod 5421'12094 active mbc={}] proc_lease pg_lease(ru 1203204.509171000s ub 1203212.509271398s int 16s) 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.5( v 5421'12103 (5421'9300,5421'12103] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [28,15,32] r=1 lpr=5414 luod=0'0 crt=5421'12101 mlcod 5421'12094 active mbc={}] peer_clock_delta_ub 1167441.805525190s -> ru 35762.703645810s 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.5( v 5421'12103 (5421'9300,5421'12103] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [28,15,32] r=1 lpr=5414 luod=0'0 crt=5421'12101 mlcod 5421'12094 active mbc={}] proc_lease readable_until now 35762.703645810s 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.5( v 5421'12103 (5421'9300,5421'12103] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [28,15,32] r=1 lpr=5414 luod=0'0 crt=5421'12101 mlcod 5421'12094 active mbc={}] peer_clock_delta_lb 1167441.804832757s -> ruub 35770.704438641s 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.5( v 5421'12103 (5421'9300,5421'12103] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [28,15,32] r=1 lpr=5414 luod=0'0 crt=5421'12101 mlcod 5421'12094 active mbc={}] proc_lease readable_until_ub now 35770.704438641s 2020-05-06T20:06:43.935+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:43.935+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:43.959+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214163812s,1167820.214599725s]) 2020-05-06T20:06:43.959+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.11 up_from 5072 peer_clock_delta [1167820.214144643s,1167820.214599725s]) 2020-05-06T20:06:43.971+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.21 up_from 5153 peer_clock_delta [1167508.090552783s,1167508.091010445s]) 2020-05-06T20:06:43.971+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.21 up_from 5153 peer_clock_delta [1167508.090478047s,1167508.091010445s]) 2020-05-06T20:06:43.999+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597465228s,1167680.598110207s]) 2020-05-06T20:06:43.999+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597450826s,1167680.598110207s]) 2020-05-06T20:06:44.023+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.22 up_from 5074 peer_clock_delta [1167818.481383503s,1167818.481922347s]) 2020-05-06T20:06:44.023+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.22 up_from 5074 peer_clock_delta [1167818.481210424s,1167818.481922347s]) 2020-05-06T20:06:44.051+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.13 up_from 5086 peer_clock_delta [1167779.698501569s,1167779.699039527s]) 2020-05-06T20:06:44.051+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.13 up_from 5086 peer_clock_delta [1167779.698483215s,1167779.699039527s]) 2020-05-06T20:06:44.107+0200 7f79139e0700 10 osd.15 5421 tick_without_osd_lock 2020-05-06T20:06:44.107+0200 7f79139e0700 20 osd.15 5421 can_inc_scrubs == true 0 local + 0 remote < max 1 2020-05-06T20:06:44.107+0200 7f79139e0700 20 osd.15 5421 scrub_time_permit should run between 0 - 24 now 20 = yes 2020-05-06T20:06:44.107+0200 7f79139e0700 20 osd.15 5421 scrub_load_below_threshold loadavg per cpu 0 < max 0.5 = yes 2020-05-06T20:06:44.107+0200 7f79139e0700 20 osd.15 5421 sched_scrub load_is_low=1 2020-05-06T20:06:44.107+0200 7f79139e0700 10 osd.15 5421 sched_scrub 2.1s0 scheduled at 2020-05-06T20:58:35.908491+0200 > 2020-05-06T20:06:44.111034+0200 2020-05-06T20:06:44.107+0200 7f79139e0700 20 osd.15 5421 sched_scrub done 2020-05-06T20:06:44.107+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2020-05-06T20:06:44.107+0200 7f79139e0700 20 osd.15 5421 promote_throttle_recalibrate new_prob 1000 2020-05-06T20:06:44.107+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2020-05-06T20:06:44.139+0200 7f79149e2700 10 osd.15 5421 tick 2020-05-06T20:06:44.139+0200 7f79149e2700 10 osd.15 5421 do_waiters -- start 2020-05-06T20:06:44.139+0200 7f79149e2700 10 osd.15 5421 do_waiters -- finish 2020-05-06T20:06:44.139+0200 7f79149e2700 20 osd.15 5421 tick last_purged_snaps_scrub 2020-05-06T09:01:52.905203+0200 next 2020-05-07T09:01:52.905203+0200 2020-05-06T20:06:44.415+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132463212s,1167609.132991249s]) 2020-05-06T20:06:44.415+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.16 up_from 5128 peer_clock_delta [1167609.132444393s,1167609.132991249s]) 2020-05-06T20:06:44.431+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.20 up_from 5111 peer_clock_delta [1167678.414934205s,1167678.415502948s]) 2020-05-06T20:06:44.431+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.20 up_from 5111 peer_clock_delta [1167678.414901392s,1167678.415502948s]) 2020-05-06T20:06:44.487+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382321414s,1167816.382896846s]) 2020-05-06T20:06:44.487+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.17 up_from 5076 peer_clock_delta [1167816.382308667s,1167816.382896846s]) 2020-05-06T20:06:44.503+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597422703s,1167680.598014873s]) 2020-05-06T20:06:44.503+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.10 up_from 5109 peer_clock_delta [1167680.597403943s,1167680.598014873s]) 2020-05-06T20:06:44.659+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.19 up_from 5092 peer_clock_delta [1167747.405985090s,1167747.406529949s]) 2020-05-06T20:06:44.659+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.19 up_from 5092 peer_clock_delta [1167747.405970206s,1167747.406529949s]) 2020-05-06T20:06:44.819+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779164199s,1167509.779610516s]) 2020-05-06T20:06:44.819+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.14 up_from 5152 peer_clock_delta [1167509.779147386s,1167509.779610516s]) 2020-05-06T20:06:44.899+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032931043s,1167577.033328929s]) 2020-05-06T20:06:44.899+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.36 up_from 5136 peer_clock_delta [1167577.032768845s,1167577.033328929s]) 2020-05-06T20:06:44.919+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.29 up_from 5099 peer_clock_delta [1167715.168943224s,1167715.169503971s]) 2020-05-06T20:06:44.919+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.29 up_from 5099 peer_clock_delta [1167715.168739877s,1167715.169503971s]) 2020-05-06T20:06:45.039+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x5644925632e0 prio 63 cost 60 latency 0.000005 epoch 5421 osd_op(client.398016.0:41636799 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.039+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636799 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) 2020-05-06T20:06:45.039+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.039+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636799 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) queued 2020-05-06T20:06:45.039+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.039+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636799 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) pg 0x564467384800 2020-05-06T20:06:45.039+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644925632e0 prio 63 cost 60 latency 0.000056 osd_op(client.398016.0:41636799 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] 2020-05-06T20:06:45.039+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] _handle_message: 0x5644925632e0 2020-05-06T20:06:45.039+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: op osd_op(client.398016.0:41636799 5.9 5:993b03e9:usage::usage.18:head [call rgw.user_usage_log_read in=60b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.039+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] op_has_sufficient_caps session=0x564479fa2000 pool=5 (default.rgw.log usage) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method user_usage_log_read rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:45.039+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op osd_op(client.398016.0:41636799 5.9 5:993b03e9:usage::usage.18:head [call rgw.user_usage_log_read in=60b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:45.039+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: obc NOT found in cache: 5:993b03e9:usage::usage.18:head 2020-05-06T20:06:45.039+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: no obc for soid 5:993b03e9:usage::usage.18:head and !can_create 2020-05-06T20:06:45.039+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: find_object_context got error -2 2020-05-06T20:06:45.039+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644925632e0 finish 2020-05-06T20:06:45.039+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:45.047+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.2 up_from 5111 peer_clock_delta [1167678.885589928s,1167678.885851946s]) 2020-05-06T20:06:45.047+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.2 up_from 5111 peer_clock_delta [1167678.885565450s,1167678.885851946s]) 2020-05-06T20:06:45.091+0200 7f79139e0700 10 osd.15 5421 tick_without_osd_lock 2020-05-06T20:06:45.091+0200 7f79139e0700 20 osd.15 5421 can_inc_scrubs == true 0 local + 0 remote < max 1 2020-05-06T20:06:45.091+0200 7f79139e0700 20 osd.15 5421 scrub_time_permit should run between 0 - 24 now 20 = yes 2020-05-06T20:06:45.091+0200 7f79139e0700 20 osd.15 5421 scrub_load_below_threshold loadavg per cpu 0 < max 0.5 = yes 2020-05-06T20:06:45.091+0200 7f79139e0700 20 osd.15 5421 sched_scrub load_is_low=1 2020-05-06T20:06:45.091+0200 7f79139e0700 10 osd.15 5421 sched_scrub 2.1s0 scheduled at 2020-05-06T20:58:35.908491+0200 > 2020-05-06T20:06:45.095104+0200 2020-05-06T20:06:45.091+0200 7f79139e0700 20 osd.15 5421 sched_scrub done 2020-05-06T20:06:45.091+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate 0 attempts, promoted 0 objects and 0 B; target 25 obj/sec or 5 MiB/sec 2020-05-06T20:06:45.091+0200 7f79139e0700 20 osd.15 5421 promote_throttle_recalibrate new_prob 1000 2020-05-06T20:06:45.091+0200 7f79139e0700 10 osd.15 5421 promote_throttle_recalibrate actual 0, actual/prob ratio 1, adjusted new_prob 1000, prob 1000 -> 1000 2020-05-06T20:06:45.091+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x564489de66e0 prio 63 cost 39 latency 0.000005 epoch 5421 osd_op(client.325546.0:51685745 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:45.091+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.325546.0:51685745 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 3.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.325546.0:51685745 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) queued 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 3.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.325546.0:51685745 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) pg 0x564475c48000 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564489de66e0 prio 63 cost 39 latency 0.000052 osd_op(client.325546.0:51685745 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 pg pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] _handle_message: 0x564489de66e0 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] do_op: op osd_op(client.325546.0:51685745 3.3 3:c0000000:root:::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] op_has_sufficient_caps session=0x5644779650c0 pool=3 (default.rgw.meta root) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] do_pg_op osd_op(client.325546.0:51685745 3.3 3:c0000000:root:::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls pg=3.3 count 1024 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls lower_bound 3:c0000000::::head pg_end 3:d0000000::::head 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xae4b6783 vs lower bound 0x3 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x51d05d43 vs lower bound 0x3 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x1f2e8723 vs lower bound 0x3 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xeb6e0593 vs lower bound 0x3 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls item 0xeb6e0593, rev 0xc9a076d7 ekkono-offsite-backups 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xce7f4ed3 vs lower bound 0x3 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x5166d8f3 vs lower bound 0x3 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls item 0x5166d8f3, rev 0xcf1b668a viskadalskyrkan-owncloud-2 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x0 vs lower bound 0x3 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls handle=3:d0000000::::head 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls result=1 outdata.length()=129 2020-05-06T20:06:45.091+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x564489de66e0 finish 2020-05-06T20:06:45.091+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:45.095+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x5644941955a0 prio 63 cost 47 latency 0.000004 epoch 5421 osd_op(client.325546.0:51685751 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.095+0200 7f79181e9700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685751 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685751 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) queued 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685751 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) pg 0x56445e521400 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x5644941955a0 prio 63 cost 47 latency 0.000049 osd_op(client.325546.0:51685751 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] _handle_message: 0x5644941955a0 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op: op osd_op(client.325546.0:51685751 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] op_has_sufficient_caps session=0x5644779650c0 pool=4 (default.rgw.buckets.index ) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method bucket_list rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op osd_op(client.325546.0:51685751 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: found obc in cache: 0x564479b41440 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: 0x564479b41440 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head rwstate(none n=0 w=0) oi: 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head(5421'12660 osd.15.0:15907 dirty|omap|data_digest|omap_digest s 0 uv 12334 dd ffffffff od 423b25ef alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e65080 snapset: 0=[]:{} 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] find_object_context 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head @head oi=4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head(5421'12660 osd.15.0:15907 dirty|omap|data_digest|omap_digest s 0 uv 12334 dd ffffffff od 423b25ef alloc_hint [0 0 0]) 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op obc obc(4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head rwstate(read n=1 w=0)) 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 0x56447a964000 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] ov 5421'12660 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op call rgw.bucket_list in=47b 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] call method rgw.bucket_list 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [omap-get-header] 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op omap-get-header 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] method called response length=110 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx alloc reply 0x56446f576900 result 0 2020-05-06T20:06:45.095+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op_effects client.325546 con 0x5644781f7000 2020-05-06T20:06:45.095+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] log_op_stats osd_op(client.325546.0:51685751 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b out=110b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 110 lat 0.000261 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:45.095+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] publish_stats_to_osd 5421:1159097 2020-05-06T20:06:45.095+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x5644941955a0 finish 2020-05-06T20:06:45.095+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:45.095+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.5 up_from 5136 peer_clock_delta [1167576.395765757s,1167576.396199056s]) 2020-05-06T20:06:45.095+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.5 up_from 5136 peer_clock_delta [1167576.395728098s,1167576.396199056s]) 2020-05-06T20:06:45.099+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x56449255e680 prio 63 cost 47 latency 0.000004 epoch 5421 osd_op(client.325546.0:51685815 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.099+0200 7f79181e9700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685815 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685815 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) queued 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685815 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) pg 0x56445e521400 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x56449255e680 prio 63 cost 47 latency 0.000035 osd_op(client.325546.0:51685815 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] _handle_message: 0x56449255e680 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op: op osd_op(client.325546.0:51685815 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] op_has_sufficient_caps session=0x5644779650c0 pool=4 (default.rgw.buckets.index ) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method bucket_list rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op osd_op(client.325546.0:51685815 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: found obc in cache: 0x564479b41700 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: 0x564479b41700 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head rwstate(none n=0 w=0) oi: 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head(5421'12661 osd.15.0:15908 dirty|omap|data_digest|omap_digest s 0 uv 12261 dd ffffffff od 5ac232fe alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e66520 snapset: 0=[]:{} 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] find_object_context 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head @head oi=4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head(5421'12661 osd.15.0:15908 dirty|omap|data_digest|omap_digest s 0 uv 12261 dd ffffffff od 5ac232fe alloc_hint [0 0 0]) 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op obc obc(4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head rwstate(read n=1 w=0)) 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 0x56447a964000 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] ov 5421'12661 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op call rgw.bucket_list in=47b 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] call method rgw.bucket_list 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [omap-get-header] 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op omap-get-header 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] method called response length=110 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx alloc reply 0x5644707a8000 result 0 2020-05-06T20:06:45.099+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op_effects client.325546 con 0x5644781f7000 2020-05-06T20:06:45.099+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] log_op_stats osd_op(client.325546.0:51685815 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b out=110b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 110 lat 0.000192 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:45.099+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] publish_stats_to_osd 5421:1159098 2020-05-06T20:06:45.099+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x56449255e680 finish 2020-05-06T20:06:45.099+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:45.107+0200 7f79149e2700 10 osd.15 5421 tick 2020-05-06T20:06:45.107+0200 7f79149e2700 10 osd.15 5421 do_waiters -- start 2020-05-06T20:06:45.107+0200 7f79149e2700 10 osd.15 5421 do_waiters -- finish 2020-05-06T20:06:45.107+0200 7f79149e2700 20 osd.15 5421 tick last_purged_snaps_scrub 2020-05-06T09:01:52.905203+0200 next 2020-05-07T09:01:52.905203+0200 2020-05-06T20:06:45.283+0200 7f79191eb700 10 osd.15 5421 ping from client.335576 2020-05-06T20:06:45.283+0200 7f79191eb700 15 osd.15 5421 enqueue_op 0x5644913ff340 prio 63 cost 0 latency 0.000003 epoch 5421 osd_op(client.335576.0:240454 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:45.283+0200 7f79191eb700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.335576.0:240454 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.335576.0:240454 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) queued 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.335576.0:240454 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) pg 0x56445f06b400 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644913ff340 prio 63 cost 0 latency 0.000036 osd_op(client.335576.0:240454 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8 pg pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] _handle_message: 0x5644913ff340 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: op osd_op(client.335576.0:240454 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] op_has_sufficient_caps session=0x5644777b12c0 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op osd_op(client.335576.0:240454 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 may_read -> write-ordered flags ondisk+write+known_if_redirected 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: found obc in cache: 0x564478128dc0 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: 0x564478128dc0 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head rwstate(none n=0 w=0) oi: 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head(5252'194 client.398034.0:1 dirty|data_digest|omap_digest s 0 uv 1 dd ffffffff od ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e64000 snapset: 0=[]:{} 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] find_object_context 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head @head oi=7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head(5252'194 client.398034.0:1 dirty|data_digest|omap_digest s 0 uv 1 dd ffffffff od ffffffff alloc_hint [0 0 0]) 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op obc obc(7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head rwstate(excl n=1 w=0)) 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 0x564477b38900 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] ov 5252'194 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op watch ping cookie 94107840416624 gen 12 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch ping: ctx->obc=0x564478128dc0 cookie=94107840416624 oi.version=194 ctx->at_version=0'0 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch: oi.user_version=1 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch: peer_addr=[2a00:X:X:6::7]:0/1293198946 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] found existing watch watch(cookie 94107840416624 30s [2a00:X:X:6::7]:0/1293198946) by client.335576 2020-05-06T20:06:45.283+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] -- Watch(94107840416624,client.335576) re-registering callback, timeout: 30 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx alloc reply 0x56447ca03f80 result 0 2020-05-06T20:06:45.283+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op_effects client.335576 con 0x5644776c3000 2020-05-06T20:06:45.283+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] log_op_stats osd_op(client.335576.0:240454 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 94107840416624 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 inb 0 outb 0 lat 0.000187 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:45.283+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] publish_stats_to_osd 5421:733019 2020-05-06T20:06:45.283+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x5644913ff340 finish 2020-05-06T20:06:45.283+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:45.315+0200 7f79189ea700 10 osd.15 5421 ping from client.398034 2020-05-06T20:06:45.315+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x56447dabffa0 prio 63 cost 0 latency 0.000003 epoch 5421 osd_op(client.398034.0:161457 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:45.315+0200 7f79189ea700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.398034.0:161457 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.398034.0:161457 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) queued 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGOpItem(op=osd_op(client.398034.0:161457 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8) prio 63 cost 0 e5421) pg 0x56445f06b400 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x56447dabffa0 prio 63 cost 0 latency 0.000035 osd_op(client.398034.0:161457 7.3 7.f5dc5c23 (undecoded) ondisk+write+known_if_redirected e5421) v8 pg pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] _handle_message: 0x56447dabffa0 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op: op osd_op(client.398034.0:161457 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] op_has_sufficient_caps session=0x5644777b0d20 pool=7 (.rgw.root ) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op osd_op(client.398034.0:161457 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 may_read -> write-ordered flags ondisk+write+known_if_redirected 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: found obc in cache: 0x564478128dc0 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] get_object_context: 0x564478128dc0 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head rwstate(none n=0 w=0) oi: 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head(5252'194 client.398034.0:1 dirty|data_digest|omap_digest s 0 uv 1 dd ffffffff od ffffffff alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e64000 snapset: 0=[]:{} 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] find_object_context 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head @head oi=7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head(5252'194 client.398034.0:1 dirty|data_digest|omap_digest s 0 uv 1 dd ffffffff od ffffffff alloc_hint [0 0 0]) 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_op obc obc(7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head rwstate(excl n=1 w=0)) 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 0x564477b38900 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] ov 5252'194 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op watch ping cookie 93832430088736 gen 12 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch ping: ctx->obc=0x564478128dc0 cookie=93832430088736 oi.version=194 ctx->at_version=0'0 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch: oi.user_version=1 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] watch: peer_addr=[2a00:X:X:6::8]:0/569487430 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] found existing watch watch(cookie 93832430088736 30s [2a00:X:X:6::8]:0/569487430) by client.398034 2020-05-06T20:06:45.315+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] -- Watch(93832430088736,client.398034) re-registering callback, timeout: 30 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] execute_ctx alloc reply 0x564471313b00 result 0 2020-05-06T20:06:45.315+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_osd_op_effects client.398034 con 0x5644776c3400 2020-05-06T20:06:45.315+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] log_op_stats osd_op(client.398034.0:161457 7.3 7:c43a3baf:::realms.75ee0409-2355-407a-bc60-9db7dcf24d7d.control:head [watch ping cookie 93832430088736 gen 12] snapc 0=[] ondisk+write+known_if_redirected e5421) v8 inb 0 outb 0 lat 0.000179 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:45.315+0200 7f78f99ac700 15 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] publish_stats_to_osd 5421:733020 2020-05-06T20:06:45.315+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x56447dabffa0 finish 2020-05-06T20:06:45.315+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:45.331+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x5644914c4520 prio 63 cost 60 latency 0.000005 epoch 5421 osd_op(client.398016.0:41636831 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.331+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636831 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) 2020-05-06T20:06:45.331+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.331+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636831 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) queued 2020-05-06T20:06:45.331+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 5.9 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.331+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(5.9 PGOpItem(op=osd_op(client.398016.0:41636831 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 60 e5421) pg 0x564467384800 2020-05-06T20:06:45.331+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644914c4520 prio 63 cost 60 latency 0.000051 osd_op(client.398016.0:41636831 5.9 5.97c0dc99 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] 2020-05-06T20:06:45.331+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] _handle_message: 0x5644914c4520 2020-05-06T20:06:45.331+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: op osd_op(client.398016.0:41636831 5.9 5:993b03e9:usage::usage.18:head [call rgw.user_usage_log_read in=60b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.331+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] op_has_sufficient_caps session=0x564479fa2000 pool=5 (default.rgw.log usage) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method user_usage_log_read rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:45.331+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op osd_op(client.398016.0:41636831 5.9 5:993b03e9:usage::usage.18:head [call rgw.user_usage_log_read in=60b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:45.331+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: obc NOT found in cache: 5:993b03e9:usage::usage.18:head 2020-05-06T20:06:45.331+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] get_object_context: no obc for soid 5:993b03e9:usage::usage.18:head and !can_create 2020-05-06T20:06:45.331+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[5.9( v 5421'489853 (5421'486400,5421'489853] local-lis/les=5414/5415 n=59 ec=67/67 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,21,26] r=0 lpr=5414 crt=5421'489851 lcod 5421'489852 mlcod 5421'489852 active+clean] do_op: find_object_context got error -2 2020-05-06T20:06:45.331+0200 7f78f91ab700 10 osd.15 5421 dequeue_op 0x5644914c4520 finish 2020-05-06T20:06:45.331+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:45.359+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.7 up_from 5103 peer_clock_delta [1167710.312205878s,1167710.312619890s]) 2020-05-06T20:06:45.359+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.7 up_from 5103 peer_clock_delta [1167710.312146900s,1167710.312619890s]) 2020-05-06T20:06:45.371+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x56446c144000 prio 127 cost 1021 latency 0.000006 epoch 5421 osd_repop(client.325546.0:51685841 4.f e5421/5414) v3 2020-05-06T20:06:45.371+0200 7f79189ea700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685841 4.f e5421/5414) v3) prio 127 cost 1021 e5421) 2020-05-06T20:06:45.371+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.371+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685841 4.f e5421/5414) v3) prio 127 cost 1021 e5421) queued 2020-05-06T20:06:45.371+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.371+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685841 4.f e5421/5414) v3) prio 127 cost 1021 e5421) pg 0x56445e51e000 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x56446c144000 prio 127 cost 1021 latency 0.000059 osd_repop(client.325546.0:51685841 4.f e5421/5414) v3 pg pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31370 mlcod 5421'31369 active mbc={}] 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31370 mlcod 5421'31369 active mbc={}] _handle_message: 0x56446c144000 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31370 mlcod 5421'31369 active mbc={}] do_repop 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31372 (transaction) 228 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31371 (5421'28900,5421'31371] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31370 mlcod 5421'31369 active mbc={}] append_log log((5421'28900,5421'31371], crt=5421'31371) [5421'31372 (5421'31371) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685841 2020-05-06T20:06:45.378033+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0] 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31371 lcod 5421'31370 mlcod 5421'31369 active mbc={}] add_log_entry 5421'31372 (5421'31371) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685841 2020-05-06T20:06:45.378033+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:45.375+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31370 mlcod 5421'31369 active mbc={}] rollforward: entry=5421'31372 (5421'31371) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685841 2020-05-06T20:06:45.378033+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31370 mlcod 5421'31369 active mbc={}] append_log approx pg log length = 2472 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31370 mlcod 5421'31369 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 trim proposed trim_to = 5421'28900 2020-05-06T20:06:45.375+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'31372, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x56446c144000 finish 2020-05-06T20:06:45.375+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:45.375+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31370 mlcod 5421'31370 active mbc={}] repop_commit on op osd_repop(client.325546.0:51685841 4.f e5421/5414 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31372, mlcod=5421'31370) v3, sending commit to osd.12 2020-05-06T20:06:45.375+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:45.379+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x56447a4efb80 prio 63 cost 39 latency 0.000004 epoch 5421 osd_op(client.325546.0:51685853 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:45.379+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.325546.0:51685853 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 3.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.325546.0:51685853 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) queued 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 3.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(3.3 PGOpItem(op=osd_op(client.325546.0:51685853 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8) prio 63 cost 39 e5421) pg 0x564475c48000 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x56447a4efb80 prio 63 cost 39 latency 0.000051 osd_op(client.325546.0:51685853 3.3 3.3 (undecoded) ondisk+read+ignore_overlay+known_if_redirected e5421) v8 pg pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] _handle_message: 0x56447a4efb80 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] do_op: op osd_op(client.325546.0:51685853 3.3 3:c0000000:root:::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] op_has_sufficient_caps session=0x5644779650c0 pool=3 (default.rgw.meta root) pool_app_metadata={rgw={}} need_read_cap=1 need_write_cap=0 classes=[] -> yes 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] do_pg_op osd_op(client.325546.0:51685853 3.3 3:c0000000:root:::head [pgnls start_epoch 5421 in=39b] snapc 0=[] ondisk+read+ignore_overlay+known_if_redirected e5421) v8 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls pg=3.3 count 1024 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls lower_bound 3:c0000000::::head pg_end 3:d0000000::::head 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xae4b6783 vs lower bound 0x3 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x51d05d43 vs lower bound 0x3 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x1f2e8723 vs lower bound 0x3 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xeb6e0593 vs lower bound 0x3 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls item 0xeb6e0593, rev 0xc9a076d7 ekkono-offsite-backups 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0xce7f4ed3 vs lower bound 0x3 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x5166d8f3 vs lower bound 0x3 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls item 0x5166d8f3, rev 0xcf1b668a viskadalskyrkan-owncloud-2 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls candidate 0x0 vs lower bound 0x3 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls handle=3:d0000000::::head 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] pgnls result=1 outdata.length()=129 2020-05-06T20:06:45.379+0200 7f78f99ac700 10 osd.15 5421 dequeue_op 0x56447a4efb80 finish 2020-05-06T20:06:45.379+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:45.379+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x564498520dc0 prio 63 cost 47 latency 0.000003 epoch 5421 osd_op(client.325546.0:51685859 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.379+0200 7f79181e9700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685859 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685859 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) queued 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685859 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) pg 0x56445e521400 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x564498520dc0 prio 63 cost 47 latency 0.000046 osd_op(client.325546.0:51685859 4.2 4.bdd264a2 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] _handle_message: 0x564498520dc0 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op: op osd_op(client.325546.0:51685859 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] op_has_sufficient_caps session=0x5644779650c0 pool=4 (default.rgw.buckets.index ) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method bucket_list rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op osd_op(client.325546.0:51685859 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: found obc in cache: 0x564479b41440 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: 0x564479b41440 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head rwstate(none n=0 w=0) oi: 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head(5421'12660 osd.15.0:15907 dirty|omap|data_digest|omap_digest s 0 uv 12334 dd ffffffff od 423b25ef alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e65080 snapset: 0=[]:{} 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] find_object_context 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head @head oi=4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head(5421'12660 osd.15.0:15907 dirty|omap|data_digest|omap_digest s 0 uv 12334 dd ffffffff od 423b25ef alloc_hint [0 0 0]) 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op obc obc(4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head rwstate(read n=1 w=0)) 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 0x56447a964000 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] ov 5421'12660 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b] 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op call rgw.bucket_list in=47b 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] call method rgw.bucket_list 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [omap-get-header] 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op omap-get-header 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] method called response length=110 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx alloc reply 0x5644698d4480 result 0 2020-05-06T20:06:45.379+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op_effects client.325546 con 0x5644781f7000 2020-05-06T20:06:45.379+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] log_op_stats osd_op(client.325546.0:51685859 4.2 4:45264bbd:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.1.2:head [call rgw.bucket_list in=47b out=110b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 110 lat 0.000242 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:45.379+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] publish_stats_to_osd 5421:1159099 2020-05-06T20:06:45.379+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x564498520dc0 finish 2020-05-06T20:06:45.379+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:45.399+0200 7f79181e9700 15 osd.15 5421 enqueue_op 0x56449095a9a0 prio 63 cost 47 latency 0.000004 epoch 5421 osd_op(client.325546.0:51685922 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.399+0200 7f79181e9700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685922 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685922 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) queued 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGOpItem(op=osd_op(client.325546.0:51685922 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8) prio 63 cost 47 e5421) pg 0x56445e521400 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x56449095a9a0 prio 63 cost 47 latency 0.000043 osd_op(client.325546.0:51685922 4.2 4.c7d07e2 (undecoded) ondisk+read+known_if_redirected e5421) v8 pg pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] _handle_message: 0x56449095a9a0 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op: op osd_op(client.325546.0:51685922 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] op_has_sufficient_caps session=0x5644779650c0 pool=4 (default.rgw.buckets.index ) pool_app_metadata={rgw={}} need_read_cap=0 need_write_cap=0 classes=[class rgw method bucket_list rd 1 wr 0 wl 1] -> yes 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op osd_op(client.325546.0:51685922 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 may_read -> read-ordered flags ondisk+read+known_if_redirected 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: found obc in cache: 0x564479b41700 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] get_object_context: 0x564479b41700 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head rwstate(none n=0 w=0) oi: 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head(5421'12661 osd.15.0:15908 dirty|omap|data_digest|omap_digest s 0 uv 12261 dd ffffffff od 5ac232fe alloc_hint [0 0 0]) exists: 1 ssc: 0x564479e66520 snapset: 0=[]:{} 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] find_object_context 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head @head oi=4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head(5421'12661 osd.15.0:15908 dirty|omap|data_digest|omap_digest s 0 uv 12261 dd ffffffff od 5ac232fe alloc_hint [0 0 0]) 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_op obc obc(4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head rwstate(read n=1 w=0)) 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 0x56447a964000 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] ov 5421'12661 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b] 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op call rgw.bucket_list in=47b 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] call method rgw.bucket_list 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [omap-get-header] 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op omap-get-header 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] method called response length=110 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] execute_ctx alloc reply 0x56447560ad00 result 0 2020-05-06T20:06:45.399+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_osd_op_effects client.325546 con 0x5644781f7000 2020-05-06T20:06:45.399+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] log_op_stats osd_op(client.325546.0:51685922 4.2 4:47e0be30:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.398022.3.9:head [call rgw.bucket_list in=47b out=110b] snapc 0=[] ondisk+read+known_if_redirected e5421) v8 inb 0 outb 110 lat 0.000228 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] prepare_stats_for_publish reporting purged_snaps [] 2020-05-06T20:06:45.399+0200 7f78fa1ad700 15 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] publish_stats_to_osd 5421:1159100 2020-05-06T20:06:45.399+0200 7f78fa1ad700 10 osd.15 5421 dequeue_op 0x56449095a9a0 finish 2020-05-06T20:06:45.399+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:45.403+0200 7f79191eb700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885206185s,1167644.885613413s]) 2020-05-06T20:06:45.403+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.12 up_from 5120 peer_clock_delta [1167644.885189936s,1167644.885613413s]) 2020-05-06T20:06:45.407+0200 7f79189ea700 15 osd.15 5421 enqueue_op 0x5644922a6580 prio 127 cost 1253 latency 0.000005 epoch 5421 osd_repop(client.325546.0:51685946 4.f e5421/5414) v3 2020-05-06T20:06:45.407+0200 7f79189ea700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685946 4.f e5421/5414) v3) prio 127 cost 1253 e5421) 2020-05-06T20:06:45.407+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.407+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685946 4.f e5421/5414) v3) prio 127 cost 1253 e5421) queued 2020-05-06T20:06:45.407+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.407+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGOpItem(op=osd_repop(client.325546.0:51685946 4.f e5421/5414) v3) prio 127 cost 1253 e5421) pg 0x56445e51e000 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x5644922a6580 prio 127 cost 1253 latency 0.000043 osd_repop(client.325546.0:51685946 4.f e5421/5414) v3 pg pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31371 mlcod 5421'31370 active mbc={}] 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31371 mlcod 5421'31370 active mbc={}] _handle_message: 0x5644922a6580 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31371 mlcod 5421'31370 active mbc={}] do_repop 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31373 (transaction) 228 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31372 (5421'28900,5421'31372] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31371 mlcod 5421'31370 active mbc={}] append_log log((5421'28900,5421'31372], crt=5421'31372) [5421'31373 (5421'31372) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685946 2020-05-06T20:06:45.411454+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0] 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31372 lcod 5421'31371 mlcod 5421'31370 active mbc={}] add_log_entry 5421'31373 (5421'31372) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685946 2020-05-06T20:06:45.411454+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:45.407+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31371 mlcod 5421'31370 active mbc={}] rollforward: entry=5421'31373 (5421'31372) modify 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head by client.325546.0:51685946 2020-05-06T20:06:45.411454+0200 0 ObjectCleanRegions clean_offsets: [0~18446744073709551615], clean_omap: 0, new_object: 0 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31371 mlcod 5421'31370 active mbc={}] append_log approx pg log length = 2473 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31371 mlcod 5421'31370 active mbc={}] append_log transaction_applied = 1 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 trim proposed trim_to = 5421'28900 2020-05-06T20:06:45.407+0200 7f78fb1af700 6 write_log_and_missing with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, writeout_from: 5421'31373, trimmed: , trimmed_dups: , clear_divergent_priors: 0 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 5421 dequeue_op 0x5644922a6580 finish 2020-05-06T20:06:45.407+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:45.407+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31371 mlcod 5421'31371 active mbc={}] repop_commit on op osd_repop(client.325546.0:51685946 4.f e5421/5414 4:f6e9cd68:::.dir.5a5c812a-3d31-4d79-87e6-1a17206228ac.325552.2.0:head v 5421'31373, mlcod=5421'31371) v3, sending commit to osd.12 2020-05-06T20:06:45.407+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:45.411+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644610003s,1167441.645300757s]) 2020-05-06T20:06:45.411+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.37 up_from 5167 peer_clock_delta [1167441.644586800s,1167441.645300757s]) 2020-05-06T20:06:45.419+0200 7f79189ea700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071901301s,1167579.072351949s]) 2020-05-06T20:06:45.419+0200 7f79181e9700 20 osd.15 5421 handle_osd_ping new stamps hbstamp(osd.26 up_from 5134 peer_clock_delta [1167579.071889603s,1167579.072351949s]) 2020-05-06T20:06:45.431+0200 7f79189ea700 15 osd.15 5421 enqueue_peering_evt 4.f epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.12 pg_lease(ru 1203409.083829117s ub 1203417.083910708s int 16s) 2020-05-06T20:06:45.431+0200 7f79189ea700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(4.f PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.12 pg_lease(ru 1203409.083829117s ub 1203417.083910708s int 16s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.12 pg_lease(ru 1203409.083829117s ub 1203417.083910708s int 16s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 4.f to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(4.f PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.12 pg_lease(ru 1203409.083829117s ub 1203417.083910708s int 16s)) prio 255 cost 10 e5421) pg 0x56445e51e000 2020-05-06T20:06:45.431+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31372 mlcod 5421'31371 active mbc={}] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.12 pg_lease(ru 1203409.083829117s ub 1203417.083910708s int 16s) 2020-05-06T20:06:45.431+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31372 mlcod 5421'31371 active mbc={}] proc_lease pg_lease(ru 1203409.083829117s ub 1203417.083910708s int 16s) 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31372 mlcod 5421'31371 active mbc={}] peer_clock_delta_ub 1167644.885613413s -> ru 35764.198215704s 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31372 mlcod 5421'31371 active mbc={}] proc_lease readable_until now 35764.198215704s 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31372 mlcod 5421'31371 active mbc={}] peer_clock_delta_lb 1167644.885189936s -> ruub 35772.198720772s 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[4.f( v 5421'31373 (5421'28900,5421'31373] local-lis/les=5414/5415 n=4 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [12,15,32] r=1 lpr=5414 luod=0'0 lua=5421'31309 crt=5421'31373 lcod 5421'31372 mlcod 5421'31371 active mbc={}] proc_lease readable_until_ub now 35772.198720772s 2020-05-06T20:06:45.431+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.431+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:45.487+0200 7f79189ea700 15 osd.15 5421 enqueue_peering_evt 4.13 epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.30 pg_lease(ru 1203410.824689739s ub 1203418.824777992s int 16s) 2020-05-06T20:06:45.487+0200 7f79189ea700 20 osd.15 op_wq(4) _enqueue OpSchedulerItem(4.13 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.30 pg_lease(ru 1203410.824689739s ub 1203418.824777992s int 16s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.487+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 6.d epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.38 pg_lease(ru 1203579.729155667s ub 1203587.729236415s int 16s) 2020-05-06T20:06:45.487+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(6.d PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.38 pg_lease(ru 1203579.729155667s ub 1203587.729236415s int 16s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 4.13 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 6.d to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(4.13 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.30 pg_lease(ru 1203410.824689739s ub 1203418.824777992s int 16s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(6.d PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.38 pg_lease(ru 1203579.729155667s ub 1203587.729236415s int 16s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 op_wq(4) _process 4.13 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 op_wq(4) _process OpSchedulerItem(4.13 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.30 pg_lease(ru 1203410.824689739s ub 1203418.824777992s int 16s)) prio 255 cost 10 e5421) pg 0x56445f4b2800 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 6.d to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(6.d PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.38 pg_lease(ru 1203579.729155667s ub 1203587.729236415s int 16s)) prio 255 cost 10 e5421) pg 0x56445f06e800 2020-05-06T20:06:45.487+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[4.13( v 5421'5899 (5189'3400,5421'5899] local-lis/les=5414/5415 n=5 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [30,4,15] r=2 lpr=5414 luod=0'0 crt=5421'5895 mlcod 5421'5877 active mbc={}] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.30 pg_lease(ru 1203410.824689739s ub 1203418.824777992s int 16s) 2020-05-06T20:06:45.487+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1052 mlcod 5421'1051 active mbc={}] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.38 pg_lease(ru 1203579.729155667s ub 1203587.729236415s int 16s) 2020-05-06T20:06:45.487+0200 7f78f91ab700 10 osd.15 pg_epoch: 5421 pg[4.13( v 5421'5899 (5189'3400,5421'5899] local-lis/les=5414/5415 n=5 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [30,4,15] r=2 lpr=5414 luod=0'0 crt=5421'5895 mlcod 5421'5877 active mbc={}] proc_lease pg_lease(ru 1203410.824689739s ub 1203418.824777992s int 16s) 2020-05-06T20:06:45.487+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1052 mlcod 5421'1051 active mbc={}] proc_lease pg_lease(ru 1203579.729155667s ub 1203587.729236415s int 16s) 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[4.13( v 5421'5899 (5189'3400,5421'5899] local-lis/les=5414/5415 n=5 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [30,4,15] r=2 lpr=5414 luod=0'0 crt=5421'5895 mlcod 5421'5877 active mbc={}] peer_clock_delta_ub 1167646.570980915s -> ru 35764.253708824s 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1052 mlcod 5421'1051 active mbc={}] peer_clock_delta_ub 1167815.475414376s -> ru 35764.253741291s 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[4.13( v 5421'5899 (5189'3400,5421'5899] local-lis/les=5414/5415 n=5 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [30,4,15] r=2 lpr=5414 luod=0'0 crt=5421'5895 mlcod 5421'5877 active mbc={}] proc_lease readable_until now 35764.253708824s 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1052 mlcod 5421'1051 active mbc={}] proc_lease readable_until now 35764.253741291s 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[4.13( v 5421'5899 (5189'3400,5421'5899] local-lis/les=5414/5415 n=5 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [30,4,15] r=2 lpr=5414 luod=0'0 crt=5421'5895 mlcod 5421'5877 active mbc={}] peer_clock_delta_lb 1167646.570277407s -> ruub 35772.254500585s 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1052 mlcod 5421'1051 active mbc={}] peer_clock_delta_lb 1167815.474817700s -> ruub 35772.254418715s 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 pg_epoch: 5421 pg[4.13( v 5421'5899 (5189'3400,5421'5899] local-lis/les=5414/5415 n=5 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [30,4,15] r=2 lpr=5414 luod=0'0 crt=5421'5895 mlcod 5421'5877 active mbc={}] proc_lease readable_until_ub now 35772.254500585s 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[6.d( v 5421'1053 (0'0,5421'1053] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [38,1,15] r=2 lpr=5414 luod=0'0 lua=5415'912 crt=5421'1053 lcod 5421'1052 mlcod 5421'1051 active mbc={}] proc_lease readable_until_ub now 35772.254418715s 2020-05-06T20:06:45.487+0200 7f78f91ab700 10 log is not dirty 2020-05-06T20:06:45.487+0200 7f78f99ac700 10 log is not dirty 2020-05-06T20:06:45.487+0200 7f78f91ab700 20 osd.15 op_wq(4) _process empty q, waiting 2020-05-06T20:06:45.487+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:45.491+0200 7f78fc1b1700 15 osd.15 5421 enqueue_peering_evt 7.3 epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.491+0200 7f78fc1b1700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) queued 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) pg 0x56445f06b400 2020-05-06T20:06:45.491+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_peering_event: epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until peer osd.29 ruub 35764.260494946s 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until peer osd.31 ruub 35764.260494946s 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until readable_until[_ub] 35764.260494946s (sent 35772.260608261s) 2020-05-06T20:06:45.491+0200 7f78f99ac700 10 log is not dirty 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:45.491+0200 7f79189ea700 15 osd.15 5421 enqueue_peering_evt 7.3 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.31 pg_lease_ack(ruub 35772.260608261s) 2020-05-06T20:06:45.491+0200 7f79189ea700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.31 pg_lease_ack(ruub 35772.260608261s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.491+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 7.3 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.29 pg_lease_ack(ruub 35772.260608261s) 2020-05-06T20:06:45.491+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.29 pg_lease_ack(ruub 35772.260608261s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.31 pg_lease_ack(ruub 35772.260608261s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.31 pg_lease_ack(ruub 35772.260608261s)) prio 255 cost 10 e5421) pg 0x56445f06b400 2020-05-06T20:06:45.491+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.31 pg_lease_ack(ruub 35772.260608261s) 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until peer osd.29 ruub 35764.260494946s 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until peer osd.31 ruub 35772.260608261s 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until readable_until[_ub] 35764.260494946s (sent 35772.260608261s) 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.491+0200 7f78f99ac700 10 log is not dirty 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.29 pg_lease_ack(ruub 35772.260608261s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 7.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(7.3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.29 pg_lease_ack(ruub 35772.260608261s)) prio 255 cost 10 e5421) pg 0x56445f06b400 2020-05-06T20:06:45.491+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.29 pg_lease_ack(ruub 35772.260608261s) 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until peer osd.29 ruub 35772.260608261s 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until peer osd.31 ruub 35772.260608261s 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recalc_readable_until readable_until[_ub] 35772.260608261s (sent 35772.260608261s) 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[7.3( v 5416'210 (0'0,5416'210] local-lis/les=5414/5415 n=2 ec=74/74 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,29,31] r=0 lpr=5414 crt=5413'208 lcod 5416'209 mlcod 5416'209 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.491+0200 7f78f99ac700 10 log is not dirty 2020-05-06T20:06:45.491+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:45.491+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 6.0 epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.37 pg_lease(ru 1203205.906342790s ub 1203213.906423685s int 16s) 2020-05-06T20:06:45.491+0200 7f79181e9700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(6.0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.37 pg_lease(ru 1203205.906342790s ub 1203213.906423685s int 16s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 6.0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(6.0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.37 pg_lease(ru 1203205.906342790s ub 1203213.906423685s int 16s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 6.0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(6.0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.37 pg_lease(ru 1203205.906342790s ub 1203213.906423685s int 16s)) prio 255 cost 10 e5421) pg 0x56446147e800 2020-05-06T20:06:45.491+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1071 mlcod 5421'1070 active mbc={}] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.37 pg_lease(ru 1203205.906342790s ub 1203213.906423685s int 16s) 2020-05-06T20:06:45.491+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1071 mlcod 5421'1070 active mbc={}] proc_lease pg_lease(ru 1203205.906342790s ub 1203213.906423685s int 16s) 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1071 mlcod 5421'1070 active mbc={}] peer_clock_delta_ub 1167441.645300757s -> ru 35764.261042033s 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1071 mlcod 5421'1070 active mbc={}] proc_lease readable_until now 35764.261042033s 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1071 mlcod 5421'1070 active mbc={}] peer_clock_delta_lb 1167441.644586800s -> ruub 35772.261836885s 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[6.0( v 5421'1072 (0'0,5421'1072] local-lis/les=5414/5415 n=1 ec=68/68 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [37,38,15] r=2 lpr=5414 luod=0'0 lua=5415'931 crt=5421'1072 lcod 5421'1071 mlcod 5421'1070 active mbc={}] proc_lease readable_until_ub now 35772.261836885s 2020-05-06T20:06:45.491+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.491+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:45.495+0200 7f78fc1b1700 15 osd.15 5421 enqueue_peering_evt 4.2 epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.495+0200 7f78fc1b1700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) queued 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) pg 0x56445e521400 2020-05-06T20:06:45.495+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_peering_event: epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until peer osd.10 ruub 35764.263205668s 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until peer osd.11 ruub 35764.263205668s 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until readable_until[_ub] 35764.263205668s (sent 35772.263320556s) 2020-05-06T20:06:45.495+0200 7f78fa1ad700 10 log is not dirty 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:45.495+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 4.2 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.263320556s) 2020-05-06T20:06:45.495+0200 7f79181e9700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.263320556s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.495+0200 7f79189ea700 15 osd.15 5421 enqueue_peering_evt 4.2 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.11 pg_lease_ack(ruub 35772.263320556s) 2020-05-06T20:06:45.495+0200 7f79189ea700 20 osd.15 op_wq(2) _enqueue OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.11 pg_lease_ack(ruub 35772.263320556s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.263320556s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.263320556s)) prio 255 cost 10 e5421) pg 0x56445e521400 2020-05-06T20:06:45.495+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.263320556s) 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until peer osd.10 ruub 35772.263320556s 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until peer osd.11 ruub 35764.263205668s 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until readable_until[_ub] 35764.263205668s (sent 35772.263320556s) 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.495+0200 7f78fa1ad700 10 log is not dirty 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.11 pg_lease_ack(ruub 35772.263320556s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process 4.2 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process OpSchedulerItem(4.2 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.11 pg_lease_ack(ruub 35772.263320556s)) prio 255 cost 10 e5421) pg 0x56445e521400 2020-05-06T20:06:45.495+0200 7f78fa1ad700 10 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.11 pg_lease_ack(ruub 35772.263320556s) 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until peer osd.10 ruub 35772.263320556s 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until peer osd.11 ruub 35772.263320556s 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recalc_readable_until readable_until[_ub] 35772.263320556s (sent 35772.263320556s) 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 pg_epoch: 5421 pg[4.2( v 5421'12661 (5252'9200,5421'12661] local-lis/les=5414/5415 n=2 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,10,11] r=0 lpr=5414 crt=5421'12661 lcod 5421'12660 mlcod 5421'12660 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.495+0200 7f78fa1ad700 10 log is not dirty 2020-05-06T20:06:45.495+0200 7f78fa1ad700 20 osd.15 op_wq(2) _process empty q, waiting 2020-05-06T20:06:45.495+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 4.1 epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.4 pg_lease(ru 1203238.821673326s ub 1203246.821766786s int 16s) 2020-05-06T20:06:45.495+0200 7f79181e9700 20 osd.15 op_wq(1) _enqueue OpSchedulerItem(4.1 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.4 pg_lease(ru 1203238.821673326s ub 1203246.821766786s int 16s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process 4.1 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process OpSchedulerItem(4.1 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.4 pg_lease(ru 1203238.821673326s ub 1203246.821766786s int 16s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process 4.1 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process OpSchedulerItem(4.1 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.4 pg_lease(ru 1203238.821673326s ub 1203246.821766786s int 16s)) prio 255 cost 10 e5421) pg 0x56445f4ac000 2020-05-06T20:06:45.495+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[4.1( v 5383'13647 (5252'11100,5383'13647] local-lis/les=5414/5415 n=0 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [4,2,15] r=2 lpr=5414 luod=0'0 crt=5383'13647 lcod 0'0 mlcod 0'0 active mbc={}] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLease epoch 5421 from osd.4 pg_lease(ru 1203238.821673326s ub 1203246.821766786s int 16s) 2020-05-06T20:06:45.495+0200 7f78fa9ae700 10 osd.15 pg_epoch: 5421 pg[4.1( v 5383'13647 (5252'11100,5383'13647] local-lis/les=5414/5415 n=0 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [4,2,15] r=2 lpr=5414 luod=0'0 crt=5383'13647 lcod 0'0 mlcod 0'0 active mbc={}] proc_lease pg_lease(ru 1203238.821673326s ub 1203246.821766786s int 16s) 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[4.1( v 5383'13647 (5252'11100,5383'13647] local-lis/les=5414/5415 n=0 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [4,2,15] r=2 lpr=5414 luod=0'0 crt=5383'13647 lcod 0'0 mlcod 0'0 active mbc={}] peer_clock_delta_ub 1167474.556211360s -> ru 35764.265461966s 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[4.1( v 5383'13647 (5252'11100,5383'13647] local-lis/les=5414/5415 n=0 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [4,2,15] r=2 lpr=5414 luod=0'0 crt=5383'13647 lcod 0'0 mlcod 0'0 active mbc={}] proc_lease readable_until now 35764.265461966s 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[4.1( v 5383'13647 (5252'11100,5383'13647] local-lis/les=5414/5415 n=0 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [4,2,15] r=2 lpr=5414 luod=0'0 crt=5383'13647 lcod 0'0 mlcod 0'0 active mbc={}] peer_clock_delta_lb 1167474.555964475s -> ruub 35772.265802311s 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 pg_epoch: 5421 pg[4.1( v 5383'13647 (5252'11100,5383'13647] local-lis/les=5414/5415 n=0 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [4,2,15] r=2 lpr=5414 luod=0'0 crt=5383'13647 lcod 0'0 mlcod 0'0 active mbc={}] proc_lease readable_until_ub now 35772.265802311s 2020-05-06T20:06:45.495+0200 7f78fa9ae700 10 log is not dirty 2020-05-06T20:06:45.495+0200 7f78fa9ae700 20 osd.15 op_wq(1) _process empty q, waiting 2020-05-06T20:06:45.503+0200 7f78fc1b1700 15 osd.15 5421 enqueue_peering_evt 2.1f9s0 epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.503+0200 7f78fc1b1700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) queued 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) pg 0x56446d702800 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] do_peering_event: epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.24 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.1 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.36 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.38 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.21 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.10 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.28 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until readable_until[_ub] 35764.270379653s (sent 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:45.503+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 2.1f9s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.28 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f79191eb700 15 osd.15 5421 enqueue_peering_evt 2.1f9s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.21 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f79181e9700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.28 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f79191eb700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.21 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.28 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.28 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) pg 0x56446d702800 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.28 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.24 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.1 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f79191eb700 15 osd.15 5421 enqueue_peering_evt 2.1f9s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.36 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f79191eb700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.36 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.36 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.38 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f79191eb700 15 osd.15 5421 enqueue_peering_evt 2.1f9s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.1 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f79191eb700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.1 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.21 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 2.1f9s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f79181e9700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.10 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.28 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until readable_until[_ub] 35764.270379653s (sent 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 2.1f9s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.38 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f79181e9700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.38 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f79191eb700 15 osd.15 5421 enqueue_peering_evt 2.1f9s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.24 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.21 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f79191eb700 20 osd.15 op_wq(0) _enqueue OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.24 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.21 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) pg 0x56446d702800 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.21 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.24 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.1 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.36 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.38 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fc1b1700 15 osd.15 5421 enqueue_peering_evt 2.58s0 epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.503+0200 7f78fc1b1700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.21 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fc1b1700 15 osd.15 5421 enqueue_peering_evt 3.3 epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.10 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fc1b1700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(3.3 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.28 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 2.58s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) queued 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until readable_until[_ub] 35764.270379653s (sent 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 2.58s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) pg 0x564475034000 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.36 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.36 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) pg 0x56446d702800 2020-05-06T20:06:45.503+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] do_peering_event: epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.36 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.24 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.7 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.1 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.30 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.36 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.31 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.38 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.27 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.21 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.9 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.10 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.1 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.28 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.3 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until readable_until[_ub] 35764.270379653s (sent 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until readable_until[_ub] 35764.270704836s (sent 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.1 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.1 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) pg 0x56446d702800 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.1 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.24 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.1 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.36 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.38 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 3.3 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.21 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(3.3 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) queued 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.10 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 3.3 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(3.3 PGPeeringEvent(epoch_sent: 5414 epoch_requested: 5414 RenewLease) prio 255 cost 10 e5414) pg 0x564475c48000 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.28 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until readable_until[_ub] 35764.270379653s (sent 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] do_peering_event: epoch_sent: 5414 epoch_requested: 5414 RenewLease 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] recalc_readable_until peer osd.2 ruub 35764.270739306s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) pg 0x56446d702800 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] recalc_readable_until peer osd.29 ruub 35764.270739306s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[3.3( v 5421'151 (0'0,5421'151] local-lis/les=5414/5415 n=6 ec=66/66 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.597593370s) [15,2,29] r=0 lpr=5414 crt=5421'151 lcod 5387'150 mlcod 5387'150 active+clean] recalc_readable_until readable_until[_ub] 35764.270739306s (sent 35772.270938856s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.10 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.24 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78f99ac700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process empty q, waiting 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.1 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.36 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.38 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.21 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.10 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.28 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 2.58s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.7 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.7 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until readable_until[_ub] 35764.270379653s (sent 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f79189ea700 15 osd.15 5421 enqueue_peering_evt 2.58s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.9 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f79189ea700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.9 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 2.58s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.7 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.24 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 2.58s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.24 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) pg 0x56446d702800 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.7 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) pg 0x564475034000 2020-05-06T20:06:45.503+0200 7f79189ea700 15 osd.15 5421 enqueue_peering_evt 2.58s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.31 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f79189ea700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.31 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.24 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f79191eb700 15 osd.15 5421 enqueue_peering_evt 2.58s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.1 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.7 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.24 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f79191eb700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.1 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.1 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.7 ruub 35772.270826429s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.36 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 2.58s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.27 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.27 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.38 ruub 35764.270379653s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.30 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.21 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f79189ea700 15 osd.15 5421 enqueue_peering_evt 2.58s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.30 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f79189ea700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.30 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.10 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.31 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.28 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until readable_until[_ub] 35764.270379653s (sent 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.27 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 3.3 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.29 pg_lease_ack(ruub 35772.270938856s) 2020-05-06T20:06:45.503+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(3.3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.29 pg_lease_ack(ruub 35772.270938856s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.38 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.9 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process 2.1f9s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process OpSchedulerItem(2.1f9s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.38 pg_lease_ack(ruub 35772.270495202s)) prio 255 cost 10 e5421) pg 0x56446d702800 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.38 pg_lease_ack(ruub 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.1 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f79191eb700 15 osd.15 5421 enqueue_peering_evt 3.3 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.2 pg_lease_ack(ruub 35772.270938856s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.24 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f79191eb700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(3.3 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.2 pg_lease_ack(ruub 35772.270938856s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.3 ruub 35764.270704836s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.1 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.36 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until readable_until[_ub] 35764.270704836s (sent 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f79181e9700 15 osd.15 5421 enqueue_peering_evt 2.58s0 epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.3 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.38 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f79181e9700 20 osd.15 op_wq(3) _enqueue OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.3 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.21 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.503+0200 7f78f99ac700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.10 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 2.58s0 to_process <> waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until peer osd.28 ruub 35772.270495202s 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.9 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) queued 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process 2.58s0 to_process waiting <> waiting_peering {} 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recalc_readable_until readable_until[_ub] 35772.270495202s (sent 35772.270495202s) 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 op_wq(3) _process OpSchedulerItem(2.58s0 PGPeeringEvent(epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.9 pg_lease_ack(ruub 35772.270826429s)) prio 255 cost 10 e5421) pg 0x564475034000 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 pg_epoch: 5421 pg[2.1f9s0( v 5421'9694 (5236'6300,5421'9694] local-lis/les=5414/5415 n=660 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414) [15,24,1,36,38,21,10,28]p15(0) r=0 lpr=5414 crt=5421'9694 lcod 5421'9693 mlcod 5421'9693 active+clean] recheck_readable wasn't wait or laggy 2020-05-06T20:06:45.503+0200 7f78fb1af700 10 log is not dirty 2020-05-06T20:06:45.503+0200 7f78fb1af700 20 osd.15 op_wq(0) _process empty q, waiting 2020-05-06T20:06:45.503+0200 7f78f99ac700 10 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] do_peering_event: epoch_sent: 5421 epoch_requested: 5421 MLeaseAck epoch 5421 from osd.9 pg_lease_ack(ruub 35772.270826429s) 2020-05-06T20:06:45.503+0200 7f78f99ac700 20 osd.15 pg_epoch: 5421 pg[2.58s0( v 5421'9146 (5236'5700,5421'9146] local-lis/les=5414/5415 n=651 ec=3990/64 lis/c=5414/5414 les/c/f=5415/5415/0 sis=5414 pruub=8.617241760s) [15,7,30,31,27,9,1,3]p15(0) r=0 lpr=5414 crt=5421'9146 lcod 5421'9145 mlcod 5421'9145 active+clean] recalc_readable_until peer osd.7 ruub 35772.270826429s