# /usr/bin/ceph daemon osd.${OSD_ID} perf dump | /usr/bin/jq . { "AsyncMessenger::Worker-0": { "msgr_recv_messages": 530, "msgr_send_messages": 482, "msgr_recv_bytes": 617463, "msgr_send_bytes": 864448, "msgr_created_connections": 8850, "msgr_active_connections": 6, "msgr_running_total_time": 0.631865665, "msgr_running_send_time": 0.007946493, "msgr_running_recv_time": 0.196621975, "msgr_running_fast_dispatch_time": 0.011263747 }, "AsyncMessenger::Worker-1": { "msgr_recv_messages": 113, "msgr_send_messages": 108, "msgr_recv_bytes": 162602, "msgr_send_bytes": 158624, "msgr_created_connections": 9886, "msgr_active_connections": 7, "msgr_running_total_time": 0.448096262, "msgr_running_send_time": 0.003640945, "msgr_running_recv_time": 0.220514381, "msgr_running_fast_dispatch_time": 0.00376902 }, "AsyncMessenger::Worker-2": { "msgr_recv_messages": 697, "msgr_send_messages": 641, "msgr_recv_bytes": 781623, "msgr_send_bytes": 761168, "msgr_created_connections": 9950, "msgr_active_connections": 15, "msgr_running_total_time": 0.472736893, "msgr_running_send_time": 0.013016242, "msgr_running_recv_time": 0.223604199, "msgr_running_fast_dispatch_time": 0.010814783 }, "bluefs": { "gift_bytes": 0, "reclaim_bytes": 0, "db_total_bytes": 49998192640, "db_used_bytes": 8279547904, "wal_total_bytes": 997191680, "wal_used_bytes": 11530240, "slow_total_bytes": 74758225920, "slow_used_bytes": 69206016, "num_files": 145, "log_bytes": 8065024, "log_compactions": 0, "logged_bytes": 942080, "files_written_wal": 1, "files_written_sst": 0, "bytes_written_wal": 1909116, "bytes_written_sst": 0, "bytes_written_slow": 0, "max_bytes_wal": 11530240, "max_bytes_db": 8281645056, "max_bytes_slow": 0 }, "bluestore": { "kv_flush_lat": { "avgcount": 226, "sum": 5.1764e-05, "avgtime": 2.29e-07 }, "kv_commit_lat": { "avgcount": 226, "sum": 12.712259745, "avgtime": 0.056248936 }, "kv_sync_lat": { "avgcount": 226, "sum": 12.712311509, "avgtime": 0.056249165 }, "kv_final_lat": { "avgcount": 226, "sum": 0.004775756, "avgtime": 2.1131e-05 }, "state_prepare_lat": { "avgcount": 1445, "sum": 2.087611502, "avgtime": 0.001444713 }, "state_aio_wait_lat": { "avgcount": 1445, "sum": 0.000158765, "avgtime": 1.09e-07 }, "state_io_done_lat": { "avgcount": 1445, "sum": 0.000825901, "avgtime": 5.71e-07 }, "state_kv_queued_lat": { "avgcount": 1445, "sum": 72.228754255, "avgtime": 0.049985297 }, "state_kv_commiting_lat": { "avgcount": 1445, "sum": 90.730693218, "avgtime": 0.062789407 }, "state_kv_done_lat": { "avgcount": 1445, "sum": 0.00056139, "avgtime": 3.88e-07 }, "state_deferred_queued_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "state_deferred_aio_wait_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "state_deferred_cleanup_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "state_finishing_lat": { "avgcount": 1442, "sum": 0.000158383, "avgtime": 1.09e-07 }, "state_done_lat": { "avgcount": 1442, "sum": 0.000700141, "avgtime": 4.85e-07 }, "throttle_lat": { "avgcount": 1445, "sum": 2.074492362, "avgtime": 0.001435634 }, "submit_lat": { "avgcount": 1445, "sum": 2.091780569, "avgtime": 0.001447599 }, "commit_lat": { "avgcount": 1445, "sum": 165.048480539, "avgtime": 0.114220401 }, "read_lat": { "avgcount": 1976, "sum": 0.35395076, "avgtime": 0.000179124 }, "read_onode_meta_lat": { "avgcount": 3931, "sum": 0.088629562, "avgtime": 2.2546e-05 }, "read_wait_aio_lat": { "avgcount": 1955, "sum": 0.25733333, "avgtime": 0.000131628 }, "compress_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "decompress_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "csum_lat": { "avgcount": 1888, "sum": 0.001122053, "avgtime": 5.94e-07 }, "compress_success_count": 0, "compress_rejected_count": 0, "write_pad_bytes": 169410, "deferred_write_ops": 0, "deferred_write_bytes": 0, "write_penalty_read_ops": 0, "bluestore_allocated": 4961992482816, "bluestore_stored": 4960992061745, "bluestore_compressed": 0, "bluestore_compressed_allocated": 0, "bluestore_compressed_original": 0, "bluestore_onodes": 1200, "bluestore_onode_hits": 3076, "bluestore_onode_misses": 1221, "bluestore_onode_shard_hits": 0, "bluestore_onode_shard_misses": 0, "bluestore_extents": 1962, "bluestore_blobs": 1962, "bluestore_buffers": 1886, "bluestore_buffer_bytes": 7725056, "bluestore_buffer_hit_bytes": 5679876, "bluestore_buffer_miss_bytes": 5321466, "bluestore_write_big": 26, "bluestore_write_big_bytes": 106496, "bluestore_write_big_blobs": 26, "bluestore_write_small": 54, "bluestore_write_small_bytes": 51774, "bluestore_write_small_unused": 0, "bluestore_write_small_deferred": 6, "bluestore_write_small_pre_read": 6, "bluestore_write_small_new": 48, "bluestore_txc": 1445, "bluestore_onode_reshard": 0, "bluestore_blob_split": 0, "bluestore_extent_compress": 0, "bluestore_gc_merged": 0, "bluestore_read_eio": 0, "bluestore_reads_with_retries": 0, "bluestore_fragmentation_micros": 858, "omap_seek_to_first_lat": { "avgcount": 206, "sum": 0.003339307, "avgtime": 1.621e-05 }, "omap_upper_bound_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "omap_lower_bound_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "omap_next_lat": { "avgcount": 627128, "sum": 10.558333834, "avgtime": 1.6836e-05 } }, "cct": { "total_workers": 6, "unhealthy_workers": 0 }, "finisher-commit_finisher": { "queue_len": 0, "complete_latency": { "avgcount": 3, "sum": 0.001894844, "avgtime": 0.000631614 } }, "finisher-defered_finisher": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "finisher-objecter-finisher-0": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluestore_alloc_bytes": 183034760, "bluestore_alloc_items": 22879345, "bluestore_cache_data_bytes": 7725056, "bluestore_cache_data_items": 1886, "bluestore_cache_onode_bytes": 672000, "bluestore_cache_onode_items": 1200, "bluestore_cache_other_bytes": 1262183, "bluestore_cache_other_items": 126830, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_txc_bytes": 2160, "bluestore_txc_items": 3, "bluestore_writing_deferred_bytes": 458208, "bluestore_writing_deferred_items": 109, "bluestore_writing_bytes": 0, "bluestore_writing_items": 0, "bluefs_bytes": 54048, "bluefs_items": 929, "buffer_anon_bytes": 2173364, "buffer_anon_items": 32, "buffer_meta_bytes": 166232, "buffer_meta_items": 1889, "osd_bytes": 2391248, "osd_items": 206, "osd_mapbl_bytes": 267900, "osd_mapbl_items": 50, "osd_pglog_bytes": 275392480, "osd_pglog_items": 625892, "osdmap_bytes": 290264, "osdmap_items": 5543, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "mds_co_bytes": 0, "mds_co_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 0, "op_send_bytes": 0, "op_resend": 0, "op_reply": 0, "op": 0, "op_r": 0, "op_w": 0, "op_rmw": 0, "op_pg": 0, "osdop_stat": 0, "osdop_create": 0, "osdop_read": 0, "osdop_write": 0, "osdop_writefull": 0, "osdop_writesame": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_call": 0, "osdop_watch": 0, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 0, "linger_active": 0, "linger_send": 0, "linger_resend": 0, "linger_ping": 0, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 45656, "map_full": 0, "map_inc": 24, "osd_sessions": 0, "osd_session_open": 0, "osd_session_close": 0, "osd_laggy": 0, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "osd": { "op_wip": 0, "op": 0, "op_in_bytes": 0, "op_out_bytes": 0, "op_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_process_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_prepare_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_r": 0, "op_r_out_bytes": 0, "op_r_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_r_process_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_r_prepare_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_w": 0, "op_w_in_bytes": 0, "op_w_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_w_process_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_w_prepare_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_rw": 0, "op_rw_in_bytes": 0, "op_rw_out_bytes": 0, "op_rw_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_rw_process_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_rw_prepare_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "op_before_queue_op_lat": { "avgcount": 4, "sum": 2.7781e-05, "avgtime": 6.945e-06 }, "op_before_dequeue_op_lat": { "avgcount": 4, "sum": 6.3745e-05, "avgtime": 1.5936e-05 }, "subop": 0, "subop_in_bytes": 0, "subop_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "subop_w": 0, "subop_w_in_bytes": 0, "subop_w_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "subop_pull": 0, "subop_pull_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "subop_push": 0, "subop_push_in_bytes": 0, "subop_push_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "pull": 0, "push": 0, "push_out_bytes": 0, "recovery_ops": 0, "recovery_bytes": 0, "loadavg": 404, "cached_crc": 0, "cached_crc_adjusted": 0, "missed_crc": 0, "numpg": 206, "numpg_primary": 83, "numpg_replica": 123, "numpg_stray": 0, "numpg_removing": 0, "heartbeat_to_peers": 5, "map_messages": 5, "map_message_epochs": 48, "map_message_epoch_dups": 24, "messages_delayed_for_map": 2, "osd_map_cache_hit": 9096, "osd_map_cache_miss": 1948, "osd_map_cache_miss_low": 2, "osd_map_cache_miss_low_avg": { "avgcount": 2, "sum": 1836 }, "osd_map_bl_cache_hit": 1, "osd_map_bl_cache_miss": 1971, "stat_bytes": 5999170674688, "stat_bytes_used": 5013064425472, "stat_bytes_avail": 986106249216, "copyfrom": 0, "tier_promote": 0, "tier_flush": 0, "tier_flush_fail": 0, "tier_try_flush": 0, "tier_try_flush_fail": 0, "tier_evict": 0, "tier_whiteout": 0, "tier_dirty": 0, "tier_clean": 0, "tier_delay": 0, "tier_proxy_read": 0, "tier_proxy_write": 0, "agent_wake": 0, "agent_skip": 0, "agent_flush": 0, "agent_evict": 0, "object_ctx_cache_hit": 0, "object_ctx_cache_total": 0, "op_cache_hit": 0, "osd_tier_flush_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "osd_tier_promote_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "osd_tier_r_lat": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "osd_pg_info": 1030, "osd_pg_fastinfo": 0, "osd_pg_biginfo": 641 }, "recoverystate_perf": { "initial_latency": { "avgcount": 206, "sum": 16.847598694, "avgtime": 0.081784459 }, "started_latency": { "avgcount": 229, "sum": 351.921917879, "avgtime": 1.536776933 }, "reset_latency": { "avgcount": 435, "sum": 1858.250395024, "avgtime": 4.271839988 }, "start_latency": { "avgcount": 435, "sum": 0.000387076, "avgtime": 8.89e-07 }, "primary_latency": { "avgcount": 91, "sum": 141.759351572, "avgtime": 1.557795072 }, "peering_latency": { "avgcount": 174, "sum": 224.908361015, "avgtime": 1.292576787 }, "backfilling_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "waitremotebackfillreserved_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "waitlocalbackfillreserved_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "notbackfilling_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "repnotrecovering_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "repwaitrecoveryreserved_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "repwaitbackfillreserved_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "reprecovering_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "activating_latency": { "avgcount": 83, "sum": 15.768899389, "avgtime": 0.189986739 }, "waitlocalrecoveryreserved_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "waitremoterecoveryreserved_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "recovering_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "recovered_latency": { "avgcount": 83, "sum": 0.000266823, "avgtime": 3.214e-06 }, "clean_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "active_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "replicaactive_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "stray_latency": { "avgcount": 261, "sum": 333.641836096, "avgtime": 1.278321211 }, "getinfo_latency": { "avgcount": 174, "sum": 142.003068274, "avgtime": 0.816109587 }, "getlog_latency": { "avgcount": 83, "sum": 0.162026635, "avgtime": 0.001952128 }, "waitactingchange_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "incomplete_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "down_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "getmissing_latency": { "avgcount": 83, "sum": 0.000414861, "avgtime": 4.998e-06 }, "waitupthru_latency": { "avgcount": 83, "sum": 82.742230385, "avgtime": 0.996894341 }, "notrecovering_latency": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "rocksdb": { "get": 2663, "submit_transaction": 1445, "submit_transaction_sync": 226, "get_latency": { "avgcount": 2663, "sum": 2.667627958, "avgtime": 0.001001737 }, "submit_latency": { "avgcount": 1445, "sum": 0.008577888, "avgtime": 5.936e-06 }, "submit_sync_latency": { "avgcount": 226, "sum": 12.700672239, "avgtime": 0.056197664 }, "compact": 0, "compact_range": 0, "compact_queue_merge": 0, "compact_queue_len": 0, "rocksdb_write_wal_time": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "rocksdb_write_memtable_time": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "rocksdb_write_delay_time": { "avgcount": 0, "sum": 0, "avgtime": 0 }, "rocksdb_write_pre_and_post_time": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-bluestore_throttle_bytes": { "val": 0, "max": 67108864, "get_started": 1445, "get": 1445, "get_sum": 969621805, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 226, "put_sum": 969621805, "wait": { "avgcount": 57, "sum": 2.07373288, "avgtime": 0.036381278 } }, "throttle-bluestore_throttle_deferred_bytes": { "val": 2176202, "max": 201326592, "get_started": 0, "get": 3, "get_sum": 2176202, "get_or_fail_fail": 0, "get_or_fail_success": 3, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-msgr_dispatch_throttler-client": { "val": 0, "max": 1048576000, "get_started": 0, "get": 14, "get_sum": 69485, "get_or_fail_fail": 0, "get_or_fail_success": 14, "take": 0, "take_sum": 0, "put": 14, "put_sum": 69485, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-msgr_dispatch_throttler-cluster": { "val": 0, "max": 1048576000, "get_started": 0, "get": 1181, "get_sum": 1108079, "get_or_fail_fail": 0, "get_or_fail_success": 1181, "take": 0, "take_sum": 0, "put": 1181, "put_sum": 1108079, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-msgr_dispatch_throttler-hb_back_client": { "val": 0, "max": 1048576000, "get_started": 0, "get": 35, "get_sum": 70140, "get_or_fail_fail": 0, "get_or_fail_success": 35, "take": 0, "take_sum": 0, "put": 35, "put_sum": 70140, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0, "max": 1048576000, "get_started": 0, "get": 36, "get_sum": 72144, "get_or_fail_fail": 0, "get_or_fail_success": 36, "take": 0, "take_sum": 0, "put": 36, "put_sum": 72144, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-msgr_dispatch_throttler-hb_front_client": { "val": 0, "max": 1048576000, "get_started": 0, "get": 35, "get_sum": 70140, "get_or_fail_fail": 0, "get_or_fail_success": 35, "take": 0, "take_sum": 0, "put": 35, "put_sum": 70140, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0, "max": 1048576000, "get_started": 0, "get": 36, "get_sum": 72144, "get_or_fail_fail": 0, "get_or_fail_success": 36, "take": 0, "take_sum": 0, "put": 36, "put_sum": 72144, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-msgr_dispatch_throttler-ms_objecter": { "val": 0, "max": 1048576000, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-objecter_bytes": { "val": 0, "max": 1048576000, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-objecter_ops": { "val": 0, "max": 10240, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } }, "throttle-osd_client_bytes": { "val": 0, "max": 0, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0, "avgtime": 0 } } }