{ "AsyncMessenger::Worker-0": { "msgr_recv_messages": 343797, "msgr_send_messages": 343796, "msgr_recv_bytes": 714418560, "msgr_send_bytes": 714751884, "msgr_created_connections": 10, "msgr_active_connections": 5, "msgr_running_total_time": 42.170223715, "msgr_running_send_time": 11.568272698, "msgr_running_recv_time": 18.333187572, "msgr_running_fast_dispatch_time": 10.863557324 }, "AsyncMessenger::Worker-1": { "msgr_recv_messages": 344328, "msgr_send_messages": 454358, "msgr_recv_bytes": 715507953, "msgr_send_bytes": 775327437, "msgr_created_connections": 8, "msgr_active_connections": 6, "msgr_running_total_time": 51.113111923, "msgr_running_send_time": 15.879598463, "msgr_running_recv_time": 21.984463604, "msgr_running_fast_dispatch_time": 11.201722681 }, "AsyncMessenger::Worker-2": { "msgr_recv_messages": 344447, "msgr_send_messages": 345360, "msgr_recv_bytes": 715456214, "msgr_send_bytes": 715831075, "msgr_created_connections": 9, "msgr_active_connections": 7, "msgr_running_total_time": 47.763720232, "msgr_running_send_time": 14.176455071, "msgr_running_recv_time": 19.992492119, "msgr_running_fast_dispatch_time": 11.346713302 }, "bluefs": { "gift_bytes": 0, "reclaim_bytes": 0, "db_total_bytes": 78938374144, "db_used_bytes": 47811002368, "wal_total_bytes": 0, "wal_used_bytes": 0, "slow_total_bytes": 0, "slow_used_bytes": 0, "num_files": 649, "log_bytes": 1134911488, "log_compactions": 0, "logged_bytes": 1133858816, "files_written_wal": 1, "files_written_sst": 0, "bytes_written_wal": 573212245, "bytes_written_sst": 0 }, "bluestore": { "kv_flush_lat": { "avgcount": 276817, "sum": 0.669491621, "avgtime": 0.000002418 }, "kv_commit_lat": { "avgcount": 276817, "sum": 630.135281538, "avgtime": 0.002276360 }, "kv_lat": { "avgcount": 276817, "sum": 630.804773159, "avgtime": 0.002278779 }, "state_prepare_lat": { "avgcount": 1, "sum": 1.021651006, "avgtime": 1.021651006 }, "state_aio_wait_lat": { "avgcount": 1, "sum": 0.000613626, "avgtime": 0.000613626 }, "state_io_done_lat": { "avgcount": 1, "sum": 0.000023781, "avgtime": 0.000023781 }, "state_kv_queued_lat": { "avgcount": 1, "sum": 0.000219339, "avgtime": 0.000219339 }, "state_kv_commiting_lat": { "avgcount": 1, "sum": 0.000826672, "avgtime": 0.000826672 }, "state_kv_done_lat": { "avgcount": 1, "sum": 0.000000141, "avgtime": 0.000000141 }, "state_deferred_queued_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "state_deferred_aio_wait_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "state_deferred_cleanup_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "state_finishing_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "state_done_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "throttle_lat": { "avgcount": 1, "sum": 0.000002089, "avgtime": 0.000002089 }, "submit_lat": { "avgcount": 1, "sum": 1.021842606, "avgtime": 1.021842606 }, "commit_lat": { "avgcount": 1, "sum": 1.023326102, "avgtime": 1.023326102 }, "read_lat": { "avgcount": 2, "sum": 0.000487266, "avgtime": 0.000243633 }, "read_onode_meta_lat": { "avgcount": 4, "sum": 0.000100650, "avgtime": 0.000025162 }, "read_wait_aio_lat": { "avgcount": 2, "sum": 0.000336390, "avgtime": 0.000168195 }, "compress_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "decompress_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "csum_lat": { "avgcount": 2, "sum": 0.000008043, "avgtime": 0.000004021 }, "compress_success_count": 0, "compress_rejected_count": 0, "write_pad_bytes": 78275, "deferred_write_ops": 0, "deferred_write_bytes": 0, "write_penalty_read_ops": 0, "bluestore_allocated": 23478272, "bluestore_stored": 5025350, "bluestore_compressed": 0, "bluestore_compressed_allocated": 0, "bluestore_compressed_original": 0, "bluestore_onodes": 25, "bluestore_onode_hits": 0, "bluestore_onode_misses": 28, "bluestore_onode_shard_hits": 0, "bluestore_onode_shard_misses": 0, "bluestore_extents": 25, "bluestore_blobs": 25, "bluestore_buffers": 0, "bluestore_buffer_bytes": 0, "bluestore_buffer_hit_bytes": 0, "bluestore_buffer_miss_bytes": 5719, "bluestore_write_big": 0, "bluestore_write_big_bytes": 0, "bluestore_write_big_blobs": 0, "bluestore_write_small": 25, "bluestore_write_small_bytes": 73277, "bluestore_write_small_unused": 0, "bluestore_write_small_deferred": 1, "bluestore_write_small_pre_read": 1, "bluestore_write_small_new": 24, "bluestore_txc": 1, "bluestore_onode_reshard": 0, "bluestore_blob_split": 0, "bluestore_extent_compress": 0, "bluestore_gc_merged": 0, "bluestore_read_eio": 0 }, "finisher-defered_finisher": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "finisher-finisher-0": { "queue_len": 0, "complete_latency": { "avgcount": 1, "sum": 0.000209084, "avgtime": 0.000209084 } }, "mutex-OSD:ShardedOpWQ:.0": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.1": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.2": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.3": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.4": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.5": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.6": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.7": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.0": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.1": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.2": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.3": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.4": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.5": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.6": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.7": { "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 0, "op_send_bytes": 0, "op_resend": 0, "op_reply": 0, "op": 0, "op_r": 0, "op_w": 0, "op_rmw": 0, "op_pg": 0, "osdop_stat": 0, "osdop_create": 0, "osdop_read": 0, "osdop_write": 0, "osdop_writefull": 0, "osdop_writesame": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_tmap_up": 0, "osdop_tmap_put": 0, "osdop_tmap_get": 0, "osdop_call": 0, "osdop_watch": 0, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 0, "linger_active": 0, "linger_send": 0, "linger_resend": 0, "linger_ping": 0, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 1218, "map_full": 0, "map_inc": 12, "osd_sessions": 0, "osd_session_open": 0, "osd_session_close": 0, "osd_laggy": 0, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "osd": { "op_wip": 0, "op": 0, "op_in_bytes": 0, "op_out_bytes": 0, "op_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_process_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_prepare_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_r": 0, "op_r_out_bytes": 0, "op_r_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_r_process_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_r_prepare_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_w": 0, "op_w_in_bytes": 0, "op_w_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_w_process_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_w_prepare_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_rw": 0, "op_rw_in_bytes": 0, "op_rw_out_bytes": 0, "op_rw_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_rw_process_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_rw_prepare_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_before_queue_op_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "op_before_dequeue_op_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "subop": 0, "subop_in_bytes": 0, "subop_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "subop_w": 0, "subop_w_in_bytes": 0, "subop_w_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "subop_pull": 0, "subop_pull_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "subop_push": 0, "subop_push_in_bytes": 0, "subop_push_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "pull": 0, "push": 0, "push_out_bytes": 0, "recovery_ops": 0, "loadavg": 28, "buffer_bytes": 0, "history_alloc_Mbytes": 0, "history_alloc_num": 0, "cached_crc": 0, "cached_crc_adjusted": 0, "missed_crc": 0, "numpg": 0, "numpg_primary": 0, "numpg_replica": 0, "numpg_stray": 0, "numpg_removing": 0, "heartbeat_to_peers": 3, "map_messages": 2, "map_message_epochs": 24, "map_message_epoch_dups": 12, "messages_delayed_for_map": 0, "osd_map_cache_hit": 12, "osd_map_cache_miss": 0, "osd_map_cache_miss_low": 0, "osd_map_cache_miss_low_avg": { "avgcount": 0, "sum": 0 }, "osd_map_bl_cache_hit": 12, "osd_map_bl_cache_miss": 0, "stat_bytes": 120032591872, "stat_bytes_used": 47834497024, "stat_bytes_avail": 72198094848, "copyfrom": 0, "tier_promote": 0, "tier_flush": 0, "tier_flush_fail": 0, "tier_try_flush": 0, "tier_try_flush_fail": 0, "tier_evict": 0, "tier_whiteout": 0, "tier_dirty": 0, "tier_clean": 0, "tier_delay": 0, "tier_proxy_read": 0, "tier_proxy_write": 0, "agent_wake": 0, "agent_skip": 0, "agent_flush": 0, "agent_evict": 0, "object_ctx_cache_hit": 0, "object_ctx_cache_total": 0, "op_cache_hit": 0, "osd_tier_flush_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_tier_promote_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_tier_r_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_pg_info": 0, "osd_pg_fastinfo": 0, "osd_pg_biginfo": 0 }, "recoverystate_perf": { "initial_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "started_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "reset_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "start_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "primary_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "peering_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "backfilling_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "waitremotebackfillreserved_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "waitlocalbackfillreserved_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "notbackfilling_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "repnotrecovering_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "repwaitrecoveryreserved_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "repwaitbackfillreserved_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "reprecovering_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "activating_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "waitlocalrecoveryreserved_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "waitremoterecoveryreserved_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "recovering_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "recovered_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "clean_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "active_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "replicaactive_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "stray_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "getinfo_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "getlog_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "waitactingchange_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "incomplete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "down_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "getmissing_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "waitupthru_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "notrecovering_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "rocksdb": { "get": 36, "submit_transaction": 1, "submit_transaction_sync": 276817, "get_latency": { "avgcount": 36, "sum": 1.070204707, "avgtime": 0.029727908 }, "submit_latency": { "avgcount": 1, "sum": 0.000108310, "avgtime": 0.000108310 }, "submit_sync_latency": { "avgcount": 276817, "sum": 626.597788752, "avgtime": 0.002263581 }, "compact": 0, "compact_range": 0, "compact_queue_merge": 0, "compact_queue_len": 0, "rocksdb_write_wal_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_memtable_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_delay_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_pre_and_post_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-bluestore_throttle_bytes": { "val": 0, "max": 67108864, "get_started": 1, "get": 1, "get_sum": 184898, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 1, "put_sum": 184898, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-bluestore_throttle_deferred_bytes": { "val": 184898, "max": 201326592, "get_started": 0, "get": 1, "get_sum": 184898, "get_or_fail_fail": 0, "get_or_fail_success": 1, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-client": { "val": 0, "max": 104857600, "get_started": 0, "get": 199, "get_sum": 88513, "get_or_fail_fail": 0, "get_or_fail_success": 199, "take": 0, "take_sum": 0, "put": 199, "put_sum": 88513, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-cluster": { "val": 0, "max": 104857600, "get_started": 0, "get": 1, "get_sum": 10398, "get_or_fail_fail": 0, "get_or_fail_success": 1, "take": 0, "take_sum": 0, "put": 1, "put_sum": 10398, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_back_client": { "val": 0, "max": 104857600, "get_started": 0, "get": 258249, "get_sum": 517530996, "get_or_fail_fail": 0, "get_or_fail_success": 258249, "take": 0, "take_sum": 0, "put": 258249, "put_sum": 517530996, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0, "max": 104857600, "get_started": 0, "get": 257937, "get_sum": 516905748, "get_or_fail_fail": 0, "get_or_fail_success": 257937, "take": 0, "take_sum": 0, "put": 257937, "put_sum": 516905748, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_front_client": { "val": 0, "max": 104857600, "get_started": 0, "get": 258249, "get_sum": 517530996, "get_or_fail_fail": 0, "get_or_fail_success": 258249, "take": 0, "take_sum": 0, "put": 258249, "put_sum": 517530996, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0, "max": 104857600, "get_started": 0, "get": 257937, "get_sum": 516905748, "get_or_fail_fail": 0, "get_or_fail_success": 257937, "take": 0, "take_sum": 0, "put": 257937, "put_sum": 516905748, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-ms_objecter": { "val": 0, "max": 104857600, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_bytes": { "val": 0, "max": 104857600, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_ops": { "val": 0, "max": 1024, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-osd_client_bytes": { "val": 0, "max": 524288000, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } } }