Project

General

Profile

Bug #38745 » osd5_perf_log.log

Chris Callegari, 03/25/2019 12:15 AM

 
# /usr/bin/ceph daemon osd.${OSD_ID} perf dump | /usr/bin/jq .
{
"AsyncMessenger::Worker-0": {
"msgr_recv_messages": 530,
"msgr_send_messages": 482,
"msgr_recv_bytes": 617463,
"msgr_send_bytes": 864448,
"msgr_created_connections": 8850,
"msgr_active_connections": 6,
"msgr_running_total_time": 0.631865665,
"msgr_running_send_time": 0.007946493,
"msgr_running_recv_time": 0.196621975,
"msgr_running_fast_dispatch_time": 0.011263747
},
"AsyncMessenger::Worker-1": {
"msgr_recv_messages": 113,
"msgr_send_messages": 108,
"msgr_recv_bytes": 162602,
"msgr_send_bytes": 158624,
"msgr_created_connections": 9886,
"msgr_active_connections": 7,
"msgr_running_total_time": 0.448096262,
"msgr_running_send_time": 0.003640945,
"msgr_running_recv_time": 0.220514381,
"msgr_running_fast_dispatch_time": 0.00376902
},
"AsyncMessenger::Worker-2": {
"msgr_recv_messages": 697,
"msgr_send_messages": 641,
"msgr_recv_bytes": 781623,
"msgr_send_bytes": 761168,
"msgr_created_connections": 9950,
"msgr_active_connections": 15,
"msgr_running_total_time": 0.472736893,
"msgr_running_send_time": 0.013016242,
"msgr_running_recv_time": 0.223604199,
"msgr_running_fast_dispatch_time": 0.010814783
},
"bluefs": {
"gift_bytes": 0,
"reclaim_bytes": 0,
"db_total_bytes": 49998192640,
"db_used_bytes": 8279547904,
"wal_total_bytes": 997191680,
"wal_used_bytes": 11530240,
"slow_total_bytes": 74758225920,
"slow_used_bytes": 69206016,
"num_files": 145,
"log_bytes": 8065024,
"log_compactions": 0,
"logged_bytes": 942080,
"files_written_wal": 1,
"files_written_sst": 0,
"bytes_written_wal": 1909116,
"bytes_written_sst": 0,
"bytes_written_slow": 0,
"max_bytes_wal": 11530240,
"max_bytes_db": 8281645056,
"max_bytes_slow": 0
},
"bluestore": {
"kv_flush_lat": {
"avgcount": 226,
"sum": 5.1764e-05,
"avgtime": 2.29e-07
},
"kv_commit_lat": {
"avgcount": 226,
"sum": 12.712259745,
"avgtime": 0.056248936
},
"kv_sync_lat": {
"avgcount": 226,
"sum": 12.712311509,
"avgtime": 0.056249165
},
"kv_final_lat": {
"avgcount": 226,
"sum": 0.004775756,
"avgtime": 2.1131e-05
},
"state_prepare_lat": {
"avgcount": 1445,
"sum": 2.087611502,
"avgtime": 0.001444713
},
"state_aio_wait_lat": {
"avgcount": 1445,
"sum": 0.000158765,
"avgtime": 1.09e-07
},
"state_io_done_lat": {
"avgcount": 1445,
"sum": 0.000825901,
"avgtime": 5.71e-07
},
"state_kv_queued_lat": {
"avgcount": 1445,
"sum": 72.228754255,
"avgtime": 0.049985297
},
"state_kv_commiting_lat": {
"avgcount": 1445,
"sum": 90.730693218,
"avgtime": 0.062789407
},
"state_kv_done_lat": {
"avgcount": 1445,
"sum": 0.00056139,
"avgtime": 3.88e-07
},
"state_deferred_queued_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"state_deferred_aio_wait_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"state_deferred_cleanup_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"state_finishing_lat": {
"avgcount": 1442,
"sum": 0.000158383,
"avgtime": 1.09e-07
},
"state_done_lat": {
"avgcount": 1442,
"sum": 0.000700141,
"avgtime": 4.85e-07
},
"throttle_lat": {
"avgcount": 1445,
"sum": 2.074492362,
"avgtime": 0.001435634
},
"submit_lat": {
"avgcount": 1445,
"sum": 2.091780569,
"avgtime": 0.001447599
},
"commit_lat": {
"avgcount": 1445,
"sum": 165.048480539,
"avgtime": 0.114220401
},
"read_lat": {
"avgcount": 1976,
"sum": 0.35395076,
"avgtime": 0.000179124
},
"read_onode_meta_lat": {
"avgcount": 3931,
"sum": 0.088629562,
"avgtime": 2.2546e-05
},
"read_wait_aio_lat": {
"avgcount": 1955,
"sum": 0.25733333,
"avgtime": 0.000131628
},
"compress_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"decompress_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"csum_lat": {
"avgcount": 1888,
"sum": 0.001122053,
"avgtime": 5.94e-07
},
"compress_success_count": 0,
"compress_rejected_count": 0,
"write_pad_bytes": 169410,
"deferred_write_ops": 0,
"deferred_write_bytes": 0,
"write_penalty_read_ops": 0,
"bluestore_allocated": 4961992482816,
"bluestore_stored": 4960992061745,
"bluestore_compressed": 0,
"bluestore_compressed_allocated": 0,
"bluestore_compressed_original": 0,
"bluestore_onodes": 1200,
"bluestore_onode_hits": 3076,
"bluestore_onode_misses": 1221,
"bluestore_onode_shard_hits": 0,
"bluestore_onode_shard_misses": 0,
"bluestore_extents": 1962,
"bluestore_blobs": 1962,
"bluestore_buffers": 1886,
"bluestore_buffer_bytes": 7725056,
"bluestore_buffer_hit_bytes": 5679876,
"bluestore_buffer_miss_bytes": 5321466,
"bluestore_write_big": 26,
"bluestore_write_big_bytes": 106496,
"bluestore_write_big_blobs": 26,
"bluestore_write_small": 54,
"bluestore_write_small_bytes": 51774,
"bluestore_write_small_unused": 0,
"bluestore_write_small_deferred": 6,
"bluestore_write_small_pre_read": 6,
"bluestore_write_small_new": 48,
"bluestore_txc": 1445,
"bluestore_onode_reshard": 0,
"bluestore_blob_split": 0,
"bluestore_extent_compress": 0,
"bluestore_gc_merged": 0,
"bluestore_read_eio": 0,
"bluestore_reads_with_retries": 0,
"bluestore_fragmentation_micros": 858,
"omap_seek_to_first_lat": {
"avgcount": 206,
"sum": 0.003339307,
"avgtime": 1.621e-05
},
"omap_upper_bound_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"omap_lower_bound_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"omap_next_lat": {
"avgcount": 627128,
"sum": 10.558333834,
"avgtime": 1.6836e-05
}
},
"cct": {
"total_workers": 6,
"unhealthy_workers": 0
},
"finisher-commit_finisher": {
"queue_len": 0,
"complete_latency": {
"avgcount": 3,
"sum": 0.001894844,
"avgtime": 0.000631614
}
},
"finisher-defered_finisher": {
"queue_len": 0,
"complete_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"finisher-objecter-finisher-0": {
"queue_len": 0,
"complete_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"mempool": {
"bloom_filter_bytes": 0,
"bloom_filter_items": 0,
"bluestore_alloc_bytes": 183034760,
"bluestore_alloc_items": 22879345,
"bluestore_cache_data_bytes": 7725056,
"bluestore_cache_data_items": 1886,
"bluestore_cache_onode_bytes": 672000,
"bluestore_cache_onode_items": 1200,
"bluestore_cache_other_bytes": 1262183,
"bluestore_cache_other_items": 126830,
"bluestore_fsck_bytes": 0,
"bluestore_fsck_items": 0,
"bluestore_txc_bytes": 2160,
"bluestore_txc_items": 3,
"bluestore_writing_deferred_bytes": 458208,
"bluestore_writing_deferred_items": 109,
"bluestore_writing_bytes": 0,
"bluestore_writing_items": 0,
"bluefs_bytes": 54048,
"bluefs_items": 929,
"buffer_anon_bytes": 2173364,
"buffer_anon_items": 32,
"buffer_meta_bytes": 166232,
"buffer_meta_items": 1889,
"osd_bytes": 2391248,
"osd_items": 206,
"osd_mapbl_bytes": 267900,
"osd_mapbl_items": 50,
"osd_pglog_bytes": 275392480,
"osd_pglog_items": 625892,
"osdmap_bytes": 290264,
"osdmap_items": 5543,
"osdmap_mapping_bytes": 0,
"osdmap_mapping_items": 0,
"pgmap_bytes": 0,
"pgmap_items": 0,
"mds_co_bytes": 0,
"mds_co_items": 0,
"unittest_1_bytes": 0,
"unittest_1_items": 0,
"unittest_2_bytes": 0,
"unittest_2_items": 0
},
"objecter": {
"op_active": 0,
"op_laggy": 0,
"op_send": 0,
"op_send_bytes": 0,
"op_resend": 0,
"op_reply": 0,
"op": 0,
"op_r": 0,
"op_w": 0,
"op_rmw": 0,
"op_pg": 0,
"osdop_stat": 0,
"osdop_create": 0,
"osdop_read": 0,
"osdop_write": 0,
"osdop_writefull": 0,
"osdop_writesame": 0,
"osdop_append": 0,
"osdop_zero": 0,
"osdop_truncate": 0,
"osdop_delete": 0,
"osdop_mapext": 0,
"osdop_sparse_read": 0,
"osdop_clonerange": 0,
"osdop_getxattr": 0,
"osdop_setxattr": 0,
"osdop_cmpxattr": 0,
"osdop_rmxattr": 0,
"osdop_resetxattrs": 0,
"osdop_call": 0,
"osdop_watch": 0,
"osdop_notify": 0,
"osdop_src_cmpxattr": 0,
"osdop_pgls": 0,
"osdop_pgls_filter": 0,
"osdop_other": 0,
"linger_active": 0,
"linger_send": 0,
"linger_resend": 0,
"linger_ping": 0,
"poolop_active": 0,
"poolop_send": 0,
"poolop_resend": 0,
"poolstat_active": 0,
"poolstat_send": 0,
"poolstat_resend": 0,
"statfs_active": 0,
"statfs_send": 0,
"statfs_resend": 0,
"command_active": 0,
"command_send": 0,
"command_resend": 0,
"map_epoch": 45656,
"map_full": 0,
"map_inc": 24,
"osd_sessions": 0,
"osd_session_open": 0,
"osd_session_close": 0,
"osd_laggy": 0,
"omap_wr": 0,
"omap_rd": 0,
"omap_del": 0
},
"osd": {
"op_wip": 0,
"op": 0,
"op_in_bytes": 0,
"op_out_bytes": 0,
"op_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_process_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_prepare_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_r": 0,
"op_r_out_bytes": 0,
"op_r_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_r_process_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_r_prepare_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_w": 0,
"op_w_in_bytes": 0,
"op_w_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_w_process_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_w_prepare_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_rw": 0,
"op_rw_in_bytes": 0,
"op_rw_out_bytes": 0,
"op_rw_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_rw_process_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_rw_prepare_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"op_before_queue_op_lat": {
"avgcount": 4,
"sum": 2.7781e-05,
"avgtime": 6.945e-06
},
"op_before_dequeue_op_lat": {
"avgcount": 4,
"sum": 6.3745e-05,
"avgtime": 1.5936e-05
},
"subop": 0,
"subop_in_bytes": 0,
"subop_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"subop_w": 0,
"subop_w_in_bytes": 0,
"subop_w_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"subop_pull": 0,
"subop_pull_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"subop_push": 0,
"subop_push_in_bytes": 0,
"subop_push_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"pull": 0,
"push": 0,
"push_out_bytes": 0,
"recovery_ops": 0,
"recovery_bytes": 0,
"loadavg": 404,
"cached_crc": 0,
"cached_crc_adjusted": 0,
"missed_crc": 0,
"numpg": 206,
"numpg_primary": 83,
"numpg_replica": 123,
"numpg_stray": 0,
"numpg_removing": 0,
"heartbeat_to_peers": 5,
"map_messages": 5,
"map_message_epochs": 48,
"map_message_epoch_dups": 24,
"messages_delayed_for_map": 2,
"osd_map_cache_hit": 9096,
"osd_map_cache_miss": 1948,
"osd_map_cache_miss_low": 2,
"osd_map_cache_miss_low_avg": {
"avgcount": 2,
"sum": 1836
},
"osd_map_bl_cache_hit": 1,
"osd_map_bl_cache_miss": 1971,
"stat_bytes": 5999170674688,
"stat_bytes_used": 5013064425472,
"stat_bytes_avail": 986106249216,
"copyfrom": 0,
"tier_promote": 0,
"tier_flush": 0,
"tier_flush_fail": 0,
"tier_try_flush": 0,
"tier_try_flush_fail": 0,
"tier_evict": 0,
"tier_whiteout": 0,
"tier_dirty": 0,
"tier_clean": 0,
"tier_delay": 0,
"tier_proxy_read": 0,
"tier_proxy_write": 0,
"agent_wake": 0,
"agent_skip": 0,
"agent_flush": 0,
"agent_evict": 0,
"object_ctx_cache_hit": 0,
"object_ctx_cache_total": 0,
"op_cache_hit": 0,
"osd_tier_flush_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"osd_tier_promote_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"osd_tier_r_lat": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"osd_pg_info": 1030,
"osd_pg_fastinfo": 0,
"osd_pg_biginfo": 641
},
"recoverystate_perf": {
"initial_latency": {
"avgcount": 206,
"sum": 16.847598694,
"avgtime": 0.081784459
},
"started_latency": {
"avgcount": 229,
"sum": 351.921917879,
"avgtime": 1.536776933
},
"reset_latency": {
"avgcount": 435,
"sum": 1858.250395024,
"avgtime": 4.271839988
},
"start_latency": {
"avgcount": 435,
"sum": 0.000387076,
"avgtime": 8.89e-07
},
"primary_latency": {
"avgcount": 91,
"sum": 141.759351572,
"avgtime": 1.557795072
},
"peering_latency": {
"avgcount": 174,
"sum": 224.908361015,
"avgtime": 1.292576787
},
"backfilling_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"waitremotebackfillreserved_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"waitlocalbackfillreserved_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"notbackfilling_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"repnotrecovering_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"repwaitrecoveryreserved_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"repwaitbackfillreserved_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"reprecovering_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"activating_latency": {
"avgcount": 83,
"sum": 15.768899389,
"avgtime": 0.189986739
},
"waitlocalrecoveryreserved_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"waitremoterecoveryreserved_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"recovering_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"recovered_latency": {
"avgcount": 83,
"sum": 0.000266823,
"avgtime": 3.214e-06
},
"clean_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"active_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"replicaactive_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"stray_latency": {
"avgcount": 261,
"sum": 333.641836096,
"avgtime": 1.278321211
},
"getinfo_latency": {
"avgcount": 174,
"sum": 142.003068274,
"avgtime": 0.816109587
},
"getlog_latency": {
"avgcount": 83,
"sum": 0.162026635,
"avgtime": 0.001952128
},
"waitactingchange_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"incomplete_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"down_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"getmissing_latency": {
"avgcount": 83,
"sum": 0.000414861,
"avgtime": 4.998e-06
},
"waitupthru_latency": {
"avgcount": 83,
"sum": 82.742230385,
"avgtime": 0.996894341
},
"notrecovering_latency": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"rocksdb": {
"get": 2663,
"submit_transaction": 1445,
"submit_transaction_sync": 226,
"get_latency": {
"avgcount": 2663,
"sum": 2.667627958,
"avgtime": 0.001001737
},
"submit_latency": {
"avgcount": 1445,
"sum": 0.008577888,
"avgtime": 5.936e-06
},
"submit_sync_latency": {
"avgcount": 226,
"sum": 12.700672239,
"avgtime": 0.056197664
},
"compact": 0,
"compact_range": 0,
"compact_queue_merge": 0,
"compact_queue_len": 0,
"rocksdb_write_wal_time": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"rocksdb_write_memtable_time": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"rocksdb_write_delay_time": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
},
"rocksdb_write_pre_and_post_time": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-bluestore_throttle_bytes": {
"val": 0,
"max": 67108864,
"get_started": 1445,
"get": 1445,
"get_sum": 969621805,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 226,
"put_sum": 969621805,
"wait": {
"avgcount": 57,
"sum": 2.07373288,
"avgtime": 0.036381278
}
},
"throttle-bluestore_throttle_deferred_bytes": {
"val": 2176202,
"max": 201326592,
"get_started": 0,
"get": 3,
"get_sum": 2176202,
"get_or_fail_fail": 0,
"get_or_fail_success": 3,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-msgr_dispatch_throttler-client": {
"val": 0,
"max": 1048576000,
"get_started": 0,
"get": 14,
"get_sum": 69485,
"get_or_fail_fail": 0,
"get_or_fail_success": 14,
"take": 0,
"take_sum": 0,
"put": 14,
"put_sum": 69485,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-msgr_dispatch_throttler-cluster": {
"val": 0,
"max": 1048576000,
"get_started": 0,
"get": 1181,
"get_sum": 1108079,
"get_or_fail_fail": 0,
"get_or_fail_success": 1181,
"take": 0,
"take_sum": 0,
"put": 1181,
"put_sum": 1108079,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-msgr_dispatch_throttler-hb_back_client": {
"val": 0,
"max": 1048576000,
"get_started": 0,
"get": 35,
"get_sum": 70140,
"get_or_fail_fail": 0,
"get_or_fail_success": 35,
"take": 0,
"take_sum": 0,
"put": 35,
"put_sum": 70140,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-msgr_dispatch_throttler-hb_back_server": {
"val": 0,
"max": 1048576000,
"get_started": 0,
"get": 36,
"get_sum": 72144,
"get_or_fail_fail": 0,
"get_or_fail_success": 36,
"take": 0,
"take_sum": 0,
"put": 36,
"put_sum": 72144,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-msgr_dispatch_throttler-hb_front_client": {
"val": 0,
"max": 1048576000,
"get_started": 0,
"get": 35,
"get_sum": 70140,
"get_or_fail_fail": 0,
"get_or_fail_success": 35,
"take": 0,
"take_sum": 0,
"put": 35,
"put_sum": 70140,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-msgr_dispatch_throttler-hb_front_server": {
"val": 0,
"max": 1048576000,
"get_started": 0,
"get": 36,
"get_sum": 72144,
"get_or_fail_fail": 0,
"get_or_fail_success": 36,
"take": 0,
"take_sum": 0,
"put": 36,
"put_sum": 72144,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-msgr_dispatch_throttler-ms_objecter": {
"val": 0,
"max": 1048576000,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-objecter_bytes": {
"val": 0,
"max": 1048576000,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-objecter_ops": {
"val": 0,
"max": 10240,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
},
"throttle-osd_client_bytes": {
"val": 0,
"max": 0,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0,
"avgtime": 0
}
}
}
(1-1/5)