# HELP ceph_bluestore_state_aio_wait_lat_sum Average aio_wait state latency Total # TYPE ceph_bluestore_state_aio_wait_lat_sum counter ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.6"} 851.414265088 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.9"} 8607.0029356 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.8"} 8882.422716122 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.0"} 10165.099574285 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.3"} 5184.906618761 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.11"} 8056.066786292 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.2"} 8084.530353929 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.10"} 6461.998379443 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.5"} 2552.032893843 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.4"} 11100.913903896 ceph_bluestore_state_aio_wait_lat_sum{ceph_daemon="osd.7"} 1832.787438386 # HELP ceph_paxos_store_state_latency_sum Storing state latency Total # TYPE ceph_paxos_store_state_latency_sum counter ceph_paxos_store_state_latency_sum{ceph_daemon="mon.weil04"} 1337.743401206 ceph_paxos_store_state_latency_sum{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_store_state_latency_sum{ceph_daemon="mon.weil02"} 30222.377641656 # HELP ceph_rocksdb_compact_queue_merge Mergings of ranges in compaction queue # TYPE ceph_rocksdb_compact_queue_merge counter ceph_rocksdb_compact_queue_merge{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_compact_queue_merge{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_compact_queue_merge{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_osd_op_out_bytes Client operations total read size # TYPE ceph_osd_op_out_bytes counter ceph_osd_op_out_bytes{ceph_daemon="osd.6"} 59261244.0 ceph_osd_op_out_bytes{ceph_daemon="osd.9"} 17335625.0 ceph_osd_op_out_bytes{ceph_daemon="osd.8"} 21777277.0 ceph_osd_op_out_bytes{ceph_daemon="osd.0"} 11492639.0 ceph_osd_op_out_bytes{ceph_daemon="osd.3"} 12900321.0 ceph_osd_op_out_bytes{ceph_daemon="osd.11"} 22828904.0 ceph_osd_op_out_bytes{ceph_daemon="osd.2"} 7761479.0 ceph_osd_op_out_bytes{ceph_daemon="osd.10"} 10152150.0 ceph_osd_op_out_bytes{ceph_daemon="osd.5"} 4482896.0 ceph_osd_op_out_bytes{ceph_daemon="osd.4"} 7679481.0 ceph_osd_op_out_bytes{ceph_daemon="osd.7"} 19280212.0 # HELP ceph_pg_incomplete PG incomplete # TYPE ceph_pg_incomplete gauge ceph_pg_incomplete 0.0 # HELP ceph_bluestore_submit_lat_sum Average submit latency Total # TYPE ceph_bluestore_submit_lat_sum counter ceph_bluestore_submit_lat_sum{ceph_daemon="osd.6"} 49327.452132768 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.9"} 91366.978936145 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.8"} 52316.491460701 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.0"} 185973.437470404 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.3"} 45619.940753143 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.11"} 67779.800647928 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.2"} 129036.484671279 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.10"} 91567.722574798 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.5"} 34700.998987923 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.4"} 40082.307993278 ceph_bluestore_submit_lat_sum{ceph_daemon="osd.7"} 93089.893185091 # HELP ceph_rbd_mirror_metadata RBD Mirror Metadata # TYPE ceph_rbd_mirror_metadata untyped ceph_rbd_mirror_metadata{ceph_daemon="rbd-mirror.222864335",id="mirrorweil",instance_id="222864335",hostname="weil01",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 # HELP ceph_paxos_commit Commits # TYPE ceph_paxos_commit counter ceph_paxos_commit{ceph_daemon="mon.weil04"} 1431825.0 ceph_paxos_commit{ceph_daemon="mon.weil01"} 1574765.0 ceph_paxos_commit{ceph_daemon="mon.weil02"} 1571390.0 # HELP ceph_paxos_new_pn_latency_count New proposal number getting latency Count # TYPE ceph_paxos_new_pn_latency_count counter ceph_paxos_new_pn_latency_count{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_new_pn_latency_count{ceph_daemon="mon.weil01"} 42.0 ceph_paxos_new_pn_latency_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_osd_op_r_process_latency_count Latency of read operation (excluding queue time) Count # TYPE ceph_osd_op_r_process_latency_count counter ceph_osd_op_r_process_latency_count{ceph_daemon="osd.6"} 2988325.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.9"} 2312093.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.8"} 4301253.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.0"} 1598947.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.3"} 1204124.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.11"} 2042112.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.2"} 971926.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.10"} 1026017.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.5"} 940628.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.4"} 725056.0 ceph_osd_op_r_process_latency_count{ceph_daemon="osd.7"} 1852834.0 # HELP ceph_osd_flag_norebalance OSD Flag norebalance # TYPE ceph_osd_flag_norebalance untyped ceph_osd_flag_norebalance 0.0 # HELP ceph_pool_num_bytes_recovered OSD POOL STATS: num_bytes_recovered # TYPE ceph_pool_num_bytes_recovered gauge ceph_pool_num_bytes_recovered{pool_id="5"} 0.0 ceph_pool_num_bytes_recovered{pool_id="3"} 0.0 ceph_pool_num_bytes_recovered{pool_id="4"} 0.0 # HELP ceph_bluestore_submit_lat_count Average submit latency Count # TYPE ceph_bluestore_submit_lat_count counter ceph_bluestore_submit_lat_count{ceph_daemon="osd.6"} 16633450.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.9"} 22463308.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.8"} 21391351.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.0"} 38579398.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.3"} 15597014.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.11"} 15937565.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.2"} 28701056.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.10"} 22985083.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.5"} 13706683.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.4"} 14910397.0 ceph_bluestore_submit_lat_count{ceph_daemon="osd.7"} 16292552.0 # HELP ceph_rbd_mirror_replay_latency_count Replay latency Count # TYPE ceph_rbd_mirror_replay_latency_count counter ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335"} 175509171.0 # HELP ceph_osd_in OSD status in # TYPE ceph_osd_in untyped ceph_osd_in{ceph_daemon="osd.6"} 1.0 ceph_osd_in{ceph_daemon="osd.9"} 1.0 ceph_osd_in{ceph_daemon="osd.8"} 1.0 ceph_osd_in{ceph_daemon="osd.0"} 1.0 ceph_osd_in{ceph_daemon="osd.3"} 1.0 ceph_osd_in{ceph_daemon="osd.11"} 1.0 ceph_osd_in{ceph_daemon="osd.2"} 1.0 ceph_osd_in{ceph_daemon="osd.10"} 1.0 ceph_osd_in{ceph_daemon="osd.5"} 1.0 ceph_osd_in{ceph_daemon="osd.4"} 1.0 ceph_osd_in{ceph_daemon="osd.7"} 1.0 # HELP ceph_bluestore_kv_final_lat_sum Average kv_finalize thread latency Total # TYPE ceph_bluestore_kv_final_lat_sum counter ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.6"} 878.929935729 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.9"} 935.842373408 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.8"} 917.572768503 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.0"} 1481.025816969 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.3"} 578.617004097 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.11"} 732.932282843 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.2"} 981.507044376 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.10"} 961.277873939 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.5"} 745.319665926 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.4"} 575.504659623 ceph_bluestore_kv_final_lat_sum{ceph_daemon="osd.7"} 817.382979196 # HELP ceph_paxos_collect_keys_sum Keys in transaction on peon collect Total # TYPE ceph_paxos_collect_keys_sum counter ceph_paxos_collect_keys_sum{ceph_daemon="mon.weil04"} 38.0 ceph_paxos_collect_keys_sum{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_collect_keys_sum{ceph_daemon="mon.weil02"} 27.0 # HELP ceph_paxos_accept_timeout Accept timeouts # TYPE ceph_paxos_accept_timeout counter ceph_paxos_accept_timeout{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_accept_timeout{ceph_daemon="mon.weil01"} 7.0 ceph_paxos_accept_timeout{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_paxos_begin_latency_count Latency of begin operation Count # TYPE ceph_paxos_begin_latency_count counter ceph_paxos_begin_latency_count{ceph_daemon="mon.weil04"} 1431846.0 ceph_paxos_begin_latency_count{ceph_daemon="mon.weil01"} 1574788.0 ceph_paxos_begin_latency_count{ceph_daemon="mon.weil02"} 1571384.0 # HELP ceph_bluefs_wal_total_bytes Total bytes (wal device) # TYPE ceph_bluefs_wal_total_bytes gauge ceph_bluefs_wal_total_bytes{ceph_daemon="osd.6"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.9"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.8"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.0"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.3"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.11"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.2"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.10"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.5"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.4"} 0.0 ceph_bluefs_wal_total_bytes{ceph_daemon="osd.7"} 0.0 # HELP ceph_osd_flag_nobackfill OSD Flag nobackfill # TYPE ceph_osd_flag_nobackfill untyped ceph_osd_flag_nobackfill 0.0 # HELP ceph_paxos_refresh Refreshes # TYPE ceph_paxos_refresh counter ceph_paxos_refresh{ceph_daemon="mon.weil04"} 1431825.0 ceph_paxos_refresh{ceph_daemon="mon.weil01"} 1574784.0 ceph_paxos_refresh{ceph_daemon="mon.weil02"} 1571390.0 # HELP ceph_bluestore_read_lat_count Average read latency Count # TYPE ceph_bluestore_read_lat_count counter ceph_bluestore_read_lat_count{ceph_daemon="osd.6"} 1858967.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.9"} 3834489.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.8"} 3429348.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.0"} 4510960.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.3"} 1831359.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.11"} 2477772.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.2"} 3171526.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.10"} 3142094.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.5"} 1971452.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.4"} 1915142.0 ceph_bluestore_read_lat_count{ceph_daemon="osd.7"} 2058325.0 # HELP ceph_mgr_metadata MGR metadata # TYPE ceph_mgr_metadata gauge ceph_mgr_metadata{ceph_daemon="mgr.weil04",hostname="weil04",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_mgr_metadata{ceph_daemon="mgr.weil02",hostname="weil02",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_mgr_metadata{ceph_daemon="mgr.weil01",hostname="weil01",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 # HELP ceph_pg_degraded PG degraded # TYPE ceph_pg_degraded gauge ceph_pg_degraded 0.0 # HELP ceph_mon_num_sessions Open sessions # TYPE ceph_mon_num_sessions gauge ceph_mon_num_sessions{ceph_daemon="mon.weil04"} 12.0 ceph_mon_num_sessions{ceph_daemon="mon.weil01"} 16.0 ceph_mon_num_sessions{ceph_daemon="mon.weil02"} 3.0 # HELP ceph_rbd_mirror_replay_bytes Replayed data # TYPE ceph_rbd_mirror_replay_bytes counter ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-138-disk-1"} 131229958952.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-101-disk-1"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-171-disk-1"} 159065005348.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-170-disk-1"} 220801827424.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-163-disk-1"} 2301801820.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-143-disk-1"} 24770299772.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-109-disk-1"} 11362459344.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-160-disk-1"} 30635585520.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-1"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-108-disk-1"} 180210691713.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-158-disk-2"} 36498139356.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-114-disk-1"} 2782285736.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-153-disk-1"} 54851097920.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-108-disk-2"} 135461258274.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-106-disk-1"} 5723163733.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-163-disk-2"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-105-disk-1"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-151-disk-1"} 14278221444.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-149-disk-3"} 21343244956.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-131-disk-1"} 114743201052.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-3"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-107-disk-1"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-148-disk-1"} 3638733276.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-171-disk-2"} 1048025538.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-102-disk-2"} 1155124.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-137-disk-1"} 152168779749.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-167-disk-2"} 67437167292.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-120-disk-1"} 193725816.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-128-disk-1"} 59360473464.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-167-disk-1"} 26717465844.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-116-disk-2"} 4919368.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-158-disk-1"} 18882873020.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-116-disk-1"} 22827112860.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-133-state-inobioProd"} 0.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-151-disk-2"} 5065898680.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-179-disk-1"} 22800073920.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-136-disk-1"} 33079284308.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-157-disk-1"} 42104455748.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-149-disk-1"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-157-disk-2"} 865261608.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-111-disk-2"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-169-disk-2"} 9207945216.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-169-disk-1"} 20138073708.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-113-disk-1"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-113-disk-2"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-2"} 400.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-178-disk-1"} 14834539375.0 ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-102-disk-1"} 3473217688.0 # HELP ceph_bluefs_bytes_written_wal Bytes written to WAL # TYPE ceph_bluefs_bytes_written_wal counter ceph_bluefs_bytes_written_wal{ceph_daemon="osd.6"} 337974151758.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.9"} 485695734328.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.8"} 445707915913.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.0"} 812871442868.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.3"} 315315266493.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.11"} 305763469581.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.2"} 544901160091.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.10"} 494132824932.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.5"} 301808169456.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.4"} 318658568213.0 ceph_bluefs_bytes_written_wal{ceph_daemon="osd.7"} 354206541423.0 # HELP ceph_mon_num_elections Elections participated in # TYPE ceph_mon_num_elections counter ceph_mon_num_elections{ceph_daemon="mon.weil04"} 144.0 ceph_mon_num_elections{ceph_daemon="mon.weil01"} 171.0 ceph_mon_num_elections{ceph_daemon="mon.weil02"} 130.0 # HELP ceph_rocksdb_compact Compactions # TYPE ceph_rocksdb_compact counter ceph_rocksdb_compact{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_compact{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_compact{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_bluestore_kv_sync_lat_sum Average kv_sync thread latency Total # TYPE ceph_bluestore_kv_sync_lat_sum counter ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.6"} 196154.467067781 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.9"} 269931.634110177 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.8"} 234359.012635236 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.0"} 464781.576569019 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.3"} 181252.939464544 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.11"} 255804.630407505 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.2"} 331345.082479098 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.10"} 262753.528054723 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.5"} 169832.340898253 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.4"} 200614.883536986 ceph_bluestore_kv_sync_lat_sum{ceph_daemon="osd.7"} 251813.070534443 # HELP ceph_osd_op_process_latency_count Latency of client operations (excluding queue time) Count # TYPE ceph_osd_op_process_latency_count counter ceph_osd_op_process_latency_count{ceph_daemon="osd.6"} 11206764.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.9"} 12910399.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.8"} 14385689.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.0"} 20034061.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.3"} 8112878.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.11"} 9119822.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.2"} 15940265.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.10"} 9592656.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.5"} 6684605.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.4"} 8815787.0 ceph_osd_op_process_latency_count{ceph_daemon="osd.7"} 10038988.0 # HELP ceph_fs_metadata FS Metadata # TYPE ceph_fs_metadata untyped # HELP ceph_osd_op_w_prepare_latency_count Latency of write operations (excluding queue time and wait for finished) Count # TYPE ceph_osd_op_w_prepare_latency_count counter ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.6"} 7389117.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.9"} 10567371.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.8"} 9979557.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.0"} 18356269.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.3"} 6881790.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.11"} 6946025.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.2"} 14910800.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.10"} 8550842.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.5"} 5713290.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.4"} 8078669.0 ceph_osd_op_w_prepare_latency_count{ceph_daemon="osd.7"} 8098829.0 # HELP ceph_pool_stored DF pool stored # TYPE ceph_pool_stored gauge ceph_pool_stored{pool_id="5"} 96906.0 ceph_pool_stored{pool_id="3"} 3076487765933.0 ceph_pool_stored{pool_id="4"} 1649821898145.0 # HELP ceph_rbd_mirror_replay_latency_count Replay latency Count # TYPE ceph_rbd_mirror_replay_latency_count counter ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-138-disk-1"} 15473541.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-167-disk-1"} 2524722.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-171-disk-1"} 19901911.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-109-disk-1"} 1065208.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-163-disk-1"} 272384.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-143-disk-1"} 4918521.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-160-disk-1"} 3036822.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-1"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-108-disk-1"} 16537981.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-169-disk-1"} 1882556.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-102-disk-2"} 199.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-153-disk-1"} 5799236.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-108-disk-2"} 19751202.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-171-disk-2"} 91206.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-163-disk-2"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-105-disk-1"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-151-disk-1"} 2503404.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-149-disk-3"} 2053622.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-131-disk-1"} 10419436.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-3"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-107-disk-1"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-148-disk-1"} 628972.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-113-disk-1"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-137-disk-1"} 13264209.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-167-disk-2"} 5664503.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-120-disk-1"} 13665.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-128-disk-1"} 5502818.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-101-disk-1"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-116-disk-2"} 466.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-158-disk-2"} 5950691.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-158-disk-1"} 3603101.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-116-disk-1"} 2093321.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-133-state-inobioProd"} 0.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-2"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-151-disk-2"} 1298803.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-136-disk-1"} 4168553.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-157-disk-1"} 4465245.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-149-disk-1"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-157-disk-2"} 69306.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-111-disk-2"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-114-disk-1"} 594541.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-106-disk-1"} 754835.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-179-disk-1"} 2458948.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-113-disk-2"} 8.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-169-disk-2"} 739467.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-170-disk-1"} 16377912.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-178-disk-1"} 1408997.0 ceph_rbd_mirror_replay_latency_count{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-102-disk-1"} 218779.0 # HELP ceph_mon_election_win Elections won # TYPE ceph_mon_election_win counter ceph_mon_election_win{ceph_daemon="mon.weil04"} 0.0 ceph_mon_election_win{ceph_daemon="mon.weil01"} 42.0 ceph_mon_election_win{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pg_backfill_unfound PG backfill_unfound # TYPE ceph_pg_backfill_unfound gauge ceph_pg_backfill_unfound 0.0 # HELP ceph_num_objects_degraded Number of degraded objects # TYPE ceph_num_objects_degraded gauge ceph_num_objects_degraded 0.0 # HELP ceph_osd_flag_nodeep_scrub OSD Flag nodeep-scrub # TYPE ceph_osd_flag_nodeep_scrub untyped ceph_osd_flag_nodeep_scrub 0.0 # HELP ceph_osd_apply_latency_ms OSD stat apply_latency_ms # TYPE ceph_osd_apply_latency_ms gauge ceph_osd_apply_latency_ms{ceph_daemon="osd.6"} 19.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.9"} 10.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.8"} 9.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.0"} 18.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.3"} 10.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.11"} 21.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.2"} 34.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.10"} 16.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.5"} 13.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.4"} 19.0 ceph_osd_apply_latency_ms{ceph_daemon="osd.7"} 19.0 # HELP ceph_paxos_begin_latency_sum Latency of begin operation Total # TYPE ceph_paxos_begin_latency_sum counter ceph_paxos_begin_latency_sum{ceph_daemon="mon.weil04"} 1994.287175276 ceph_paxos_begin_latency_sum{ceph_daemon="mon.weil01"} 47044.627733924 ceph_paxos_begin_latency_sum{ceph_daemon="mon.weil02"} 32350.574532529 # HELP ceph_osd_flag_noin OSD Flag noin # TYPE ceph_osd_flag_noin untyped ceph_osd_flag_noin 0.0 # HELP ceph_osd_op_r Client read operations # TYPE ceph_osd_op_r counter ceph_osd_op_r{ceph_daemon="osd.6"} 2988325.0 ceph_osd_op_r{ceph_daemon="osd.9"} 2312093.0 ceph_osd_op_r{ceph_daemon="osd.8"} 4301253.0 ceph_osd_op_r{ceph_daemon="osd.0"} 1598947.0 ceph_osd_op_r{ceph_daemon="osd.3"} 1204124.0 ceph_osd_op_r{ceph_daemon="osd.11"} 2042112.0 ceph_osd_op_r{ceph_daemon="osd.2"} 971926.0 ceph_osd_op_r{ceph_daemon="osd.10"} 1026017.0 ceph_osd_op_r{ceph_daemon="osd.5"} 940628.0 ceph_osd_op_r{ceph_daemon="osd.4"} 725056.0 ceph_osd_op_r{ceph_daemon="osd.7"} 1852834.0 # HELP ceph_bluefs_read_random_buffer_bytes Bytes read from prefetch buffer in random read mode # TYPE ceph_bluefs_read_random_buffer_bytes counter ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.6"} 3985938391.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.9"} 7038730322.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.8"} 6550513150.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.0"} 8946112055.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.3"} 4177833416.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.11"} 4561348621.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.2"} 5997670262.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.10"} 5430660337.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.5"} 4144286369.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.4"} 4381088496.0 ceph_bluefs_read_random_buffer_bytes{ceph_daemon="osd.7"} 4368925931.0 # HELP ceph_paxos_new_pn New proposal number queries # TYPE ceph_paxos_new_pn counter ceph_paxos_new_pn{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_new_pn{ceph_daemon="mon.weil01"} 42.0 ceph_paxos_new_pn{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_rocksdb_get_latency_count Get latency Count # TYPE ceph_rocksdb_get_latency_count counter ceph_rocksdb_get_latency_count{ceph_daemon="mon.weil04"} 55603073.0 ceph_rocksdb_get_latency_count{ceph_daemon="mon.weil01"} 61993436.0 ceph_rocksdb_get_latency_count{ceph_daemon="mon.weil02"} 61430343.0 # HELP ceph_pool_max_avail DF pool max_avail # TYPE ceph_pool_max_avail gauge ceph_pool_max_avail{pool_id="5"} 6825071083520.0 ceph_pool_max_avail{pool_id="3"} 6825071083520.0 ceph_pool_max_avail{pool_id="4"} 6825071083520.0 # HELP ceph_pool_rd DF pool rd # TYPE ceph_pool_rd gauge ceph_pool_rd{pool_id="5"} 85.0 ceph_pool_rd{pool_id="3"} 1494932295.0 ceph_pool_rd{pool_id="4"} 2046817.0 # HELP ceph_paxos_refresh_latency_sum Refresh latency Total # TYPE ceph_paxos_refresh_latency_sum counter ceph_paxos_refresh_latency_sum{ceph_daemon="mon.weil04"} 4086.421205842 ceph_paxos_refresh_latency_sum{ceph_daemon="mon.weil01"} 4133.502008712 ceph_paxos_refresh_latency_sum{ceph_daemon="mon.weil02"} 12472.872514726 # HELP ceph_paxos_commit_latency_count Commit latency Count # TYPE ceph_paxos_commit_latency_count counter ceph_paxos_commit_latency_count{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_commit_latency_count{ceph_daemon="mon.weil01"} 1574765.0 ceph_paxos_commit_latency_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_bluestore_throttle_lat_count Average submit throttle latency Count # TYPE ceph_bluestore_throttle_lat_count counter ceph_bluestore_throttle_lat_count{ceph_daemon="osd.6"} 16633450.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.9"} 22463308.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.8"} 21391351.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.0"} 38579398.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.3"} 15597014.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.11"} 15937565.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.2"} 28701056.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.10"} 22985083.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.5"} 13706683.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.4"} 14910397.0 ceph_bluestore_throttle_lat_count{ceph_daemon="osd.7"} 16292552.0 # HELP ceph_paxos_lease_ack_timeout Lease acknowledgement timeouts # TYPE ceph_paxos_lease_ack_timeout counter ceph_paxos_lease_ack_timeout{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_lease_ack_timeout{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_lease_ack_timeout{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_bluestore_commit_lat_sum Average commit latency Total # TYPE ceph_bluestore_commit_lat_sum counter ceph_bluestore_commit_lat_sum{ceph_daemon="osd.6"} 442434.83696263 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.9"} 537573.050470032 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.8"} 384911.932846041 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.0"} 1193865.75698551 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.3"} 363114.922451094 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.11"} 571842.213503093 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.2"} 1177825.220400532 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.10"} 531416.406329495 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.5"} 295643.884879297 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.4"} 394285.674198471 ceph_bluestore_commit_lat_sum{ceph_daemon="osd.7"} 778896.012611544 # HELP ceph_paxos_collect_bytes_sum Data in transaction on peon collect Total # TYPE ceph_paxos_collect_bytes_sum counter ceph_paxos_collect_bytes_sum{ceph_daemon="mon.weil04"} 912.0 ceph_paxos_collect_bytes_sum{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_collect_bytes_sum{ceph_daemon="mon.weil02"} 648.0 # HELP ceph_cluster_total_used_raw_bytes DF total_used_raw_bytes # TYPE ceph_cluster_total_used_raw_bytes gauge ceph_cluster_total_used_raw_bytes 7729696145408.0 # HELP ceph_pg_stale PG stale # TYPE ceph_pg_stale gauge ceph_pg_stale 0.0 # HELP ceph_health_status Cluster health status # TYPE ceph_health_status untyped ceph_health_status 0.0 # HELP ceph_pool_wr_bytes DF pool wr_bytes # TYPE ceph_pool_wr_bytes gauge ceph_pool_wr_bytes{pool_id="5"} 167936.0 ceph_pool_wr_bytes{pool_id="3"} 65796709497856.0 ceph_pool_wr_bytes{pool_id="4"} 1520149735424.0 # HELP ceph_bluestore_kv_flush_lat_count Average kv_thread flush latency Count # TYPE ceph_bluestore_kv_flush_lat_count counter ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.6"} 15344063.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.9"} 21065806.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.8"} 20518535.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.0"} 33446557.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.3"} 14177316.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.11"} 14729922.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.2"} 19270180.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.10"} 21361503.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.5"} 12885773.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.4"} 13779209.0 ceph_bluestore_kv_flush_lat_count{ceph_daemon="osd.7"} 13941359.0 # HELP ceph_paxos_collect_uncommitted Uncommitted values in started and handled collects # TYPE ceph_paxos_collect_uncommitted counter ceph_paxos_collect_uncommitted{ceph_daemon="mon.weil04"} 21.0 ceph_paxos_collect_uncommitted{ceph_daemon="mon.weil01"} 23.0 ceph_paxos_collect_uncommitted{ceph_daemon="mon.weil02"} 5.0 # HELP ceph_rbd_mirror_replay_latency_sum Replay latency Total # TYPE ceph_rbd_mirror_replay_latency_sum counter ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-138-disk-1"} 1080663.642760252 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-167-disk-1"} 267296.580384702 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-171-disk-1"} 1194270.425267457 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-109-disk-1"} 132454.149395761 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-163-disk-1"} 23113.706539605 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-143-disk-1"} 501049.026075561 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-160-disk-1"} 342861.154914396 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-1"} 20.147813603 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-108-disk-1"} 1044127.987116461 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-169-disk-1"} 182406.840204755 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-102-disk-2"} 144.240956332 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-153-disk-1"} 509267.014979715 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-108-disk-2"} 1203213.011221244 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-171-disk-2"} 13758.827989541 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-163-disk-2"} 16.903080081 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-105-disk-1"} 10.553142715 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-151-disk-1"} 237980.680275699 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-149-disk-3"} 224086.298834621 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-131-disk-1"} 795380.997804681 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-3"} 16.863394719 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-107-disk-1"} 10.934132387 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-148-disk-1"} 77529.542082759 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-113-disk-1"} 16.459783018 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-137-disk-1"} 781629.164606375 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-167-disk-2"} 471286.756234466 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-120-disk-1"} 319900.739474779 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-128-disk-1"} 467977.843759837 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-101-disk-1"} 6.974942773 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-116-disk-2"} 141.973856424 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-158-disk-2"} 464312.489459313 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-158-disk-1"} 218269.986495747 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-116-disk-1"} 263053.394837122 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-133-state-inobioProd"} 0.0 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-2"} 15.349184286 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-151-disk-2"} 78061.59934554 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-136-disk-1"} 415981.935035221 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-157-disk-1"} 389919.969026685 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-149-disk-1"} 10.610951887 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-157-disk-2"} 8984.887977843 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-111-disk-2"} 13.095691366 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-114-disk-1"} 59599.699638123 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-106-disk-1"} 61522.535166546 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-179-disk-1"} 196744.796253064 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-113-disk-2"} 16.117969879 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-169-disk-2"} 63888.786987343 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-170-disk-1"} 1066993.561156268 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-178-disk-1"} 145386.172141069 ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-102-disk-1"} 14634.855046048 # HELP ceph_osd_op_rw_latency_sum Latency of read-modify-write operation (including queue time) Total # TYPE ceph_osd_op_rw_latency_sum counter ceph_osd_op_rw_latency_sum{ceph_daemon="osd.6"} 142195.596532687 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.9"} 3983.140557987 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.8"} 9119.005629579 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.0"} 15987.906863202 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.3"} 1549.867668718 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.11"} 52151.98572109 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.2"} 3341.486785764 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.10"} 2071.655631174 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.5"} 1693.709100474 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.4"} 718.698340681 ceph_osd_op_rw_latency_sum{ceph_daemon="osd.7"} 6489.169432905 # HELP ceph_paxos_share_state Sharings of state # TYPE ceph_paxos_share_state counter ceph_paxos_share_state{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_share_state{ceph_daemon="mon.weil01"} 18.0 ceph_paxos_share_state{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pool_stored_raw DF pool stored_raw # TYPE ceph_pool_stored_raw gauge ceph_pool_stored_raw{pool_id="5"} 96906.0 ceph_pool_stored_raw{pool_id="3"} 6152975146925.0 ceph_pool_stored_raw{pool_id="4"} 3299643774369.0 # HELP ceph_osd_op_r_prepare_latency_sum Latency of read operations (excluding queue time and wait for finished) Total # TYPE ceph_osd_op_r_prepare_latency_sum counter ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.6"} 260.128107199 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.9"} 148.901322625 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.8"} 249.237912326 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.0"} 108.157828965 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.3"} 76.248849461 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.11"} 158.514600379 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.2"} 68.08647613 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.10"} 67.473259824 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.5"} 73.445340273 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.4"} 48.761507672 ceph_osd_op_r_prepare_latency_sum{ceph_daemon="osd.7"} 188.507156697 # HELP ceph_bluestore_kv_flush_lat_sum Average kv_thread flush latency Total # TYPE ceph_bluestore_kv_flush_lat_sum counter ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.6"} 1191.554454932 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.9"} 6436.387751891 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.8"} 7359.370438949 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.0"} 6131.020340543 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.3"} 5412.511175608 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.11"} 11862.392771696 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.2"} 4929.62516145 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.10"} 4310.581780351 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.5"} 2.33160408 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.4"} 3635.670413584 ceph_bluestore_kv_flush_lat_sum{ceph_daemon="osd.7"} 2045.457957417 # HELP ceph_osd_op_rw_process_latency_sum Latency of read-modify-write operation (excluding queue time) Total # TYPE ceph_osd_op_rw_process_latency_sum counter ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.6"} 73542.994773595 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.9"} 3944.41805338 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.8"} 9079.959240914 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.0"} 10528.623686328 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.3"} 1539.119630094 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.11"} 21499.352460115 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.2"} 3288.194997773 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.10"} 2003.190670729 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.5"} 1619.464703048 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.4"} 712.506705068 ceph_osd_op_rw_process_latency_sum{ceph_daemon="osd.7"} 6075.63939817 # HELP ceph_osd_metadata OSD Metadata # TYPE ceph_osd_metadata untyped ceph_osd_metadata{back_iface="enp13s0",ceph_daemon="osd.3",cluster_addr="192.168.0.151",device_class="hdd",front_iface="eno1",hostname="weil01",objectstore="bluestore",public_addr="10.10.50.151",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="ens5",ceph_daemon="osd.6",cluster_addr="192.168.0.152",device_class="hdd",front_iface="eno1",hostname="weil02",objectstore="bluestore",public_addr="10.10.50.152",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="ens5",ceph_daemon="osd.11",cluster_addr="192.168.0.152",device_class="hdd",front_iface="eno1",hostname="weil02",objectstore="bluestore",public_addr="10.10.50.152",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="enp13s0",ceph_daemon="osd.4",cluster_addr="192.168.0.151",device_class="hdd",front_iface="eno1",hostname="weil01",objectstore="bluestore",public_addr="10.10.50.151",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="enp6s0",ceph_daemon="osd.8",cluster_addr="192.168.0.154",device_class="hdd",front_iface="eno1",hostname="weil04",objectstore="bluestore",public_addr="10.10.50.154",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="enp6s0",ceph_daemon="osd.10",cluster_addr="192.168.0.154",device_class="hdd",front_iface="eno1",hostname="weil04",objectstore="bluestore",public_addr="10.10.50.154",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="enp6s0",ceph_daemon="osd.0",cluster_addr="192.168.0.154",device_class="hdd",front_iface="eno1",hostname="weil04",objectstore="bluestore",public_addr="10.10.50.154",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="ens5",ceph_daemon="osd.7",cluster_addr="192.168.0.152",device_class="hdd",front_iface="eno1",hostname="weil02",objectstore="bluestore",public_addr="10.10.50.152",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="ens5",ceph_daemon="osd.5",cluster_addr="192.168.0.152",device_class="hdd",front_iface="eno1",hostname="weil02",objectstore="bluestore",public_addr="10.10.50.152",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="enp6s0",ceph_daemon="osd.9",cluster_addr="192.168.0.154",device_class="hdd",front_iface="eno1",hostname="weil04",objectstore="bluestore",public_addr="10.10.50.154",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_osd_metadata{back_iface="enp13s0",ceph_daemon="osd.2",cluster_addr="192.168.0.151",device_class="hdd",front_iface="eno1",hostname="weil01",objectstore="bluestore",public_addr="10.10.50.151",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 # HELP ceph_rocksdb_rocksdb_write_memtable_time_count Rocksdb write memtable time Count # TYPE ceph_rocksdb_rocksdb_write_memtable_time_count counter ceph_rocksdb_rocksdb_write_memtable_time_count{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_rocksdb_write_memtable_time_count{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_rocksdb_write_memtable_time_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_paxos_collect_latency_count Peon collect latency Count # TYPE ceph_paxos_collect_latency_count counter ceph_paxos_collect_latency_count{ceph_daemon="mon.weil04"} 38.0 ceph_paxos_collect_latency_count{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_collect_latency_count{ceph_daemon="mon.weil02"} 27.0 # HELP ceph_pg_undersized PG undersized # TYPE ceph_pg_undersized gauge ceph_pg_undersized 0.0 # HELP ceph_osd_op_rw_prepare_latency_count Latency of read-modify-write operations (excluding queue time and wait for finished) Count # TYPE ceph_osd_op_rw_prepare_latency_count counter ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.6"} 829433.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.9"} 31132.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.8"} 105169.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.0"} 79038.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.3"} 27002.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.11"} 131847.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.2"} 57556.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.10"} 15883.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.5"} 30753.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.4"} 12119.0 ceph_osd_op_rw_prepare_latency_count{ceph_daemon="osd.7"} 87412.0 # HELP ceph_paxos_collect_latency_sum Peon collect latency Total # TYPE ceph_paxos_collect_latency_sum counter ceph_paxos_collect_latency_sum{ceph_daemon="mon.weil04"} 0.091999527 ceph_paxos_collect_latency_sum{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_collect_latency_sum{ceph_daemon="mon.weil02"} 5.471969341 # HELP ceph_rocksdb_rocksdb_write_delay_time_count Rocksdb write delay time Count # TYPE ceph_rocksdb_rocksdb_write_delay_time_count counter ceph_rocksdb_rocksdb_write_delay_time_count{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_rocksdb_write_delay_time_count{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_rocksdb_write_delay_time_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_objecter_op_rmw Read-modify-write operations # TYPE ceph_objecter_op_rmw counter ceph_objecter_op_rmw{ceph_daemon="osd.6"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.9"} 0.0 ceph_objecter_op_rmw{ceph_daemon="rbd-mirror.222864335"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.8"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.0"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.3"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.11"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.2"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.10"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.5"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.4"} 0.0 ceph_objecter_op_rmw{ceph_daemon="osd.7"} 0.0 # HELP ceph_paxos_begin_bytes_sum Data in transaction on begin Total # TYPE ceph_paxos_begin_bytes_sum counter ceph_paxos_begin_bytes_sum{ceph_daemon="mon.weil04"} 32948966856.0 ceph_paxos_begin_bytes_sum{ceph_daemon="mon.weil01"} 36734845189.0 ceph_paxos_begin_bytes_sum{ceph_daemon="mon.weil02"} 36651701682.0 # HELP ceph_bluefs_bytes_written_slow Bytes written to WAL/SSTs at slow device # TYPE ceph_bluefs_bytes_written_slow counter ceph_bluefs_bytes_written_slow{ceph_daemon="osd.6"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.9"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.8"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.0"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.3"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.11"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.2"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.10"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.5"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.4"} 0.0 ceph_bluefs_bytes_written_slow{ceph_daemon="osd.7"} 0.0 # HELP ceph_pg_peering PG peering # TYPE ceph_pg_peering gauge ceph_pg_peering 0.0 # HELP ceph_osd_numpg Placement groups # TYPE ceph_osd_numpg gauge ceph_osd_numpg{ceph_daemon="osd.6"} 128.0 ceph_osd_numpg{ceph_daemon="osd.9"} 134.0 ceph_osd_numpg{ceph_daemon="osd.8"} 119.0 ceph_osd_numpg{ceph_daemon="osd.0"} 142.0 ceph_osd_numpg{ceph_daemon="osd.3"} 131.0 ceph_osd_numpg{ceph_daemon="osd.11"} 98.0 ceph_osd_numpg{ceph_daemon="osd.2"} 280.0 ceph_osd_numpg{ceph_daemon="osd.10"} 122.0 ceph_osd_numpg{ceph_daemon="osd.5"} 131.0 ceph_osd_numpg{ceph_daemon="osd.4"} 118.0 ceph_osd_numpg{ceph_daemon="osd.7"} 135.0 # HELP ceph_osd_flag_noout OSD Flag noout # TYPE ceph_osd_flag_noout untyped ceph_osd_flag_noout 0.0 # HELP ceph_pg_inconsistent PG inconsistent # TYPE ceph_pg_inconsistent gauge ceph_pg_inconsistent 0.0 # HELP ceph_rbd_mirror_replay Replays # TYPE ceph_rbd_mirror_replay counter ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-138-disk-1"} 15473541.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-167-disk-1"} 2524722.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-171-disk-1"} 19901911.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-109-disk-1"} 1065208.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-105-disk-1"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-143-disk-1"} 4918521.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-151-disk-1"} 2503404.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-160-disk-1"} 3036822.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-169-disk-1"} 1882556.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-108-disk-1"} 16537981.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-1"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-102-disk-2"} 199.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-153-disk-1"} 5799236.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-108-disk-2"} 19751202.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-171-disk-2"} 91206.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-2"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-163-disk-1"} 272384.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="ssd",namespace="",image="vm-170-disk-1"} 16377912.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-149-disk-3"} 2053622.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-131-disk-1"} 10419436.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-163-disk-2"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-116-disk-1"} 2093321.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-107-disk-1"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-148-disk-1"} 628972.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-114-disk-1"} 594541.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-137-disk-1"} 13264209.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-167-disk-2"} 5664503.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-120-disk-1"} 13665.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-102-disk-1"} 218779.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-101-disk-1"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-116-disk-2"} 466.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-158-disk-2"} 5950691.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-158-disk-1"} 3603101.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-125-disk-3"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-133-state-inobioProd"} 0.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-151-disk-2"} 1298803.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-179-disk-1"} 2458948.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-136-disk-1"} 4168553.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-157-disk-1"} 4465245.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-149-disk-1"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-157-disk-2"} 69306.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-111-disk-2"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-106-disk-1"} 754835.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-113-disk-1"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-113-disk-2"} 8.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-169-disk-2"} 739467.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-178-disk-1"} 1408997.0 ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335",pool="cephrescue",namespace="",image="vm-128-disk-1"} 5502818.0 # HELP ceph_osd_op_rw_prepare_latency_sum Latency of read-modify-write operations (excluding queue time and wait for finished) Total # TYPE ceph_osd_op_rw_prepare_latency_sum counter ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.6"} 289.099047349 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.9"} 9.800328769 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.8"} 159.264419864 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.0"} 234.438770871 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.3"} 40.848765435 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.11"} 459.78729391 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.2"} 347.94818276 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.10"} 9.988959169 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.5"} 45.733275209 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.4"} 29.893689413 ceph_osd_op_rw_prepare_latency_sum{ceph_daemon="osd.7"} 32.307734658 # HELP ceph_rocksdb_rocksdb_write_pre_and_post_time_count total time spent on writing a record, excluding write process Count # TYPE ceph_rocksdb_rocksdb_write_pre_and_post_time_count counter ceph_rocksdb_rocksdb_write_pre_and_post_time_count{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_rocksdb_write_pre_and_post_time_count{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_rocksdb_write_pre_and_post_time_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_bluefs_read_prefetch_bytes Bytes requested in prefetch read mode # TYPE ceph_bluefs_read_prefetch_bytes counter ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.6"} 18562596851.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.9"} 45122093867.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.8"} 50743581910.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.0"} 65539045197.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.3"} 28824638274.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.11"} 45199198051.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.2"} 21446512581.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.10"} 31783845888.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.5"} 18070210651.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.4"} 36909428282.0 ceph_bluefs_read_prefetch_bytes{ceph_daemon="osd.7"} 21051653845.0 # HELP ceph_paxos_collect_bytes_count Data in transaction on peon collect Count # TYPE ceph_paxos_collect_bytes_count counter ceph_paxos_collect_bytes_count{ceph_daemon="mon.weil04"} 38.0 ceph_paxos_collect_bytes_count{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_collect_bytes_count{ceph_daemon="mon.weil02"} 27.0 # HELP ceph_osd_op Client operations # TYPE ceph_osd_op counter ceph_osd_op{ceph_daemon="osd.6"} 11206764.0 ceph_osd_op{ceph_daemon="osd.9"} 12910399.0 ceph_osd_op{ceph_daemon="osd.8"} 14385689.0 ceph_osd_op{ceph_daemon="osd.0"} 20034061.0 ceph_osd_op{ceph_daemon="osd.3"} 8112878.0 ceph_osd_op{ceph_daemon="osd.11"} 9119822.0 ceph_osd_op{ceph_daemon="osd.2"} 15940265.0 ceph_osd_op{ceph_daemon="osd.10"} 9592656.0 ceph_osd_op{ceph_daemon="osd.5"} 6684605.0 ceph_osd_op{ceph_daemon="osd.4"} 8815787.0 ceph_osd_op{ceph_daemon="osd.7"} 10038988.0 # HELP ceph_paxos_commit_keys_sum Keys in transaction on commit Total # TYPE ceph_paxos_commit_keys_sum counter ceph_paxos_commit_keys_sum{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_commit_keys_sum{ceph_daemon="mon.weil01"} 12444619.0 ceph_paxos_commit_keys_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_osd_op_rw_in_bytes Client read-modify-write operations write in # TYPE ceph_osd_op_rw_in_bytes counter ceph_osd_op_rw_in_bytes{ceph_daemon="osd.6"} 16080.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.9"} 130476.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.8"} 96651476.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.0"} 41475086.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.3"} 15811838.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.11"} 82474182.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.2"} 114743214.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.10"} 534028.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.5"} 14757842.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.4"} 8466274.0 ceph_osd_op_rw_in_bytes{ceph_daemon="osd.7"} 201020.0 # HELP ceph_osd_op_rw_out_bytes Client read-modify-write operations read out # TYPE ceph_osd_op_rw_out_bytes counter ceph_osd_op_rw_out_bytes{ceph_daemon="osd.6"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.9"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.8"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.0"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.3"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.11"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.2"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.10"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.5"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.4"} 0.0 ceph_osd_op_rw_out_bytes{ceph_daemon="osd.7"} 0.0 # HELP ceph_bluefs_bytes_written_sst Bytes written to SSTs # TYPE ceph_bluefs_bytes_written_sst counter ceph_bluefs_bytes_written_sst{ceph_daemon="osd.6"} 68943545074.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.9"} 103178423445.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.8"} 97779033365.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.0"} 168754707527.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.3"} 78050727868.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.11"} 69233351601.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.2"} 179211031250.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.10"} 97458404292.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.5"} 61697854978.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.4"} 76014278983.0 ceph_bluefs_bytes_written_sst{ceph_daemon="osd.7"} 67832048721.0 # HELP ceph_osd_op_rw_process_latency_count Latency of read-modify-write operation (excluding queue time) Count # TYPE ceph_osd_op_rw_process_latency_count counter ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.6"} 829329.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.9"} 30943.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.8"} 104892.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.0"} 78860.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.3"} 26965.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.11"} 131689.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.2"} 57540.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.10"} 15809.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.5"} 30688.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.4"} 12067.0 ceph_osd_op_rw_process_latency_count{ceph_daemon="osd.7"} 87335.0 # HELP ceph_rocksdb_compact_queue_len Length of compaction queue # TYPE ceph_rocksdb_compact_queue_len gauge ceph_rocksdb_compact_queue_len{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_compact_queue_len{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_compact_queue_len{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pool_wr DF pool wr # TYPE ceph_pool_wr gauge ceph_pool_wr{pool_id="5"} 68.0 ceph_pool_wr{pool_id="3"} 7600400348.0 ceph_pool_wr{pool_id="4"} 142717475.0 # HELP ceph_bluestore_throttle_lat_sum Average submit throttle latency Total # TYPE ceph_bluestore_throttle_lat_sum counter ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.6"} 23.702100794 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.9"} 32.669413334 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.8"} 28.495068461 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.0"} 55.869131134 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.3"} 20.181567375 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.11"} 28.798872971 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.2"} 37.55058513 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.10"} 30.040020184 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.5"} 19.339967145 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.4"} 20.882094874 ceph_bluestore_throttle_lat_sum{ceph_daemon="osd.7"} 24.411061798 # HELP ceph_bluefs_slow_used_bytes Used bytes (slow device) # TYPE ceph_bluefs_slow_used_bytes gauge ceph_bluefs_slow_used_bytes{ceph_daemon="osd.6"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.9"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.8"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.0"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.3"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.11"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.2"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.10"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.5"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.4"} 0.0 ceph_bluefs_slow_used_bytes{ceph_daemon="osd.7"} 0.0 # HELP ceph_osd_op_r_latency_sum Latency of read operation (including queue time) Total # TYPE ceph_osd_op_r_latency_sum counter ceph_osd_op_r_latency_sum{ceph_daemon="osd.6"} 176445.417329897 ceph_osd_op_r_latency_sum{ceph_daemon="osd.9"} 4753.777656237 ceph_osd_op_r_latency_sum{ceph_daemon="osd.8"} 5004.7199743 ceph_osd_op_r_latency_sum{ceph_daemon="osd.0"} 11700.425492062 ceph_osd_op_r_latency_sum{ceph_daemon="osd.3"} 901.700138333 ceph_osd_op_r_latency_sum{ceph_daemon="osd.11"} 63433.159154959 ceph_osd_op_r_latency_sum{ceph_daemon="osd.2"} 1434.853562904 ceph_osd_op_r_latency_sum{ceph_daemon="osd.10"} 2478.06699214 ceph_osd_op_r_latency_sum{ceph_daemon="osd.5"} 1183.417528764 ceph_osd_op_r_latency_sum{ceph_daemon="osd.4"} 766.752380189 ceph_osd_op_r_latency_sum{ceph_daemon="osd.7"} 10077.838831263 # HELP ceph_osd_op_rw_latency_count Latency of read-modify-write operation (including queue time) Count # TYPE ceph_osd_op_rw_latency_count counter ceph_osd_op_rw_latency_count{ceph_daemon="osd.6"} 829329.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.9"} 30943.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.8"} 104892.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.0"} 78860.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.3"} 26965.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.11"} 131689.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.2"} 57540.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.10"} 15809.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.5"} 30688.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.4"} 12067.0 ceph_osd_op_rw_latency_count{ceph_daemon="osd.7"} 87335.0 # HELP ceph_rocksdb_compact_range Compactions by range # TYPE ceph_rocksdb_compact_range counter ceph_rocksdb_compact_range{ceph_daemon="mon.weil04"} 18285.0 ceph_rocksdb_compact_range{ceph_daemon="mon.weil01"} 20161.0 ceph_rocksdb_compact_range{ceph_daemon="mon.weil02"} 20130.0 # HELP ceph_osd_op_latency_sum Latency of client operations (including queue time) Total # TYPE ceph_osd_op_latency_sum counter ceph_osd_op_latency_sum{ceph_daemon="osd.6"} 697102.83003455 ceph_osd_op_latency_sum{ceph_daemon="osd.9"} 657365.169175785 ceph_osd_op_latency_sum{ceph_daemon="osd.8"} 582442.504024808 ceph_osd_op_latency_sum{ceph_daemon="osd.0"} 1357243.179684299 ceph_osd_op_latency_sum{ceph_daemon="osd.3"} 268400.923673934 ceph_osd_op_latency_sum{ceph_daemon="osd.11"} 578772.536291184 ceph_osd_op_latency_sum{ceph_daemon="osd.2"} 797491.092282921 ceph_osd_op_latency_sum{ceph_daemon="osd.10"} 566312.300650041 ceph_osd_op_latency_sum{ceph_daemon="osd.5"} 266128.776717641 ceph_osd_op_latency_sum{ceph_daemon="osd.4"} 328967.91006423 ceph_osd_op_latency_sum{ceph_daemon="osd.7"} 602903.598256378 # HELP ceph_osd_op_w_process_latency_count Latency of write operation (excluding queue time) Count # TYPE ceph_osd_op_w_process_latency_count counter ceph_osd_op_w_process_latency_count{ceph_daemon="osd.6"} 7389110.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.9"} 10567363.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.8"} 9979544.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.0"} 18356254.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.3"} 6881789.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.11"} 6946021.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.2"} 14910799.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.10"} 8550830.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.5"} 5713289.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.4"} 8078664.0 ceph_osd_op_w_process_latency_count{ceph_daemon="osd.7"} 8098819.0 # HELP ceph_mon_session_add Created sessions # TYPE ceph_mon_session_add counter ceph_mon_session_add{ceph_daemon="mon.weil04"} 335348.0 ceph_mon_session_add{ceph_daemon="mon.weil01"} 1173582.0 ceph_mon_session_add{ceph_daemon="mon.weil02"} 1082671.0 # HELP ceph_paxos_share_state_keys_count Keys in shared state Count # TYPE ceph_paxos_share_state_keys_count counter ceph_paxos_share_state_keys_count{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_share_state_keys_count{ceph_daemon="mon.weil01"} 18.0 ceph_paxos_share_state_keys_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pool_recovering_bytes_per_sec OSD POOL STATS: recovering_bytes_per_sec # TYPE ceph_pool_recovering_bytes_per_sec gauge ceph_pool_recovering_bytes_per_sec{pool_id="5"} 0.0 ceph_pool_recovering_bytes_per_sec{pool_id="3"} 0.0 ceph_pool_recovering_bytes_per_sec{pool_id="4"} 0.0 # HELP ceph_num_objects_misplaced Number of misplaced objects # TYPE ceph_num_objects_misplaced gauge ceph_num_objects_misplaced 0.0 # HELP ceph_paxos_collect Peon collects # TYPE ceph_paxos_collect counter ceph_paxos_collect{ceph_daemon="mon.weil04"} 38.0 ceph_paxos_collect{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_collect{ceph_daemon="mon.weil02"} 27.0 # HELP ceph_mgr_status MGR status (0=standby, 1=active) # TYPE ceph_mgr_status gauge ceph_mgr_status{ceph_daemon="mgr.weil02"} 0.0 ceph_mgr_status{ceph_daemon="mgr.weil04"} 0.0 ceph_mgr_status{ceph_daemon="mgr.weil01"} 1.0 # HELP ceph_objecter_op_active Operations active # TYPE ceph_objecter_op_active gauge ceph_objecter_op_active{ceph_daemon="osd.6"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.9"} 0.0 ceph_objecter_op_active{ceph_daemon="rbd-mirror.222864335"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.8"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.0"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.3"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.11"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.2"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.10"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.5"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.4"} 0.0 ceph_objecter_op_active{ceph_daemon="osd.7"} 0.0 # HELP ceph_osd_op_r_process_latency_sum Latency of read operation (excluding queue time) Total # TYPE ceph_osd_op_r_process_latency_sum counter ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.6"} 211.011045214 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.9"} 114.75157775 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.8"} 191.916706319 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.0"} 80.330719399 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.3"} 57.674320937 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.11"} 122.818054665 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.2"} 51.485651416 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.10"} 48.965041078 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.5"} 53.614379378 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.4"} 36.200302274 ceph_osd_op_r_process_latency_sum{ceph_daemon="osd.7"} 156.236745071 # HELP ceph_paxos_start_peon Starts in peon role # TYPE ceph_paxos_start_peon counter ceph_paxos_start_peon{ceph_daemon="mon.weil04"} 38.0 ceph_paxos_start_peon{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_start_peon{ceph_daemon="mon.weil02"} 27.0 # HELP ceph_cluster_total_bytes DF total_bytes # TYPE ceph_cluster_total_bytes gauge ceph_cluster_total_bytes 28005334253568.0 # HELP ceph_mon_session_trim Trimmed sessions # TYPE ceph_mon_session_trim counter ceph_mon_session_trim{ceph_daemon="mon.weil04"} 31.0 ceph_mon_session_trim{ceph_daemon="mon.weil01"} 93.0 ceph_mon_session_trim{ceph_daemon="mon.weil02"} 202.0 # HELP ceph_pg_recovery_wait PG recovery_wait # TYPE ceph_pg_recovery_wait gauge ceph_pg_recovery_wait 0.0 # HELP ceph_rocksdb_get_latency_sum Get latency Total # TYPE ceph_rocksdb_get_latency_sum counter ceph_rocksdb_get_latency_sum{ceph_daemon="mon.weil04"} 484.418679917 ceph_rocksdb_get_latency_sum{ceph_daemon="mon.weil01"} 445.701841976 ceph_rocksdb_get_latency_sum{ceph_daemon="mon.weil02"} 1187.797364473 # HELP ceph_rocksdb_submit_transaction_sync Submit transactions sync # TYPE ceph_rocksdb_submit_transaction_sync counter ceph_rocksdb_submit_transaction_sync{ceph_daemon="mon.weil04"} 2872225.0 ceph_rocksdb_submit_transaction_sync{ceph_daemon="mon.weil01"} 3158524.0 ceph_rocksdb_submit_transaction_sync{ceph_daemon="mon.weil02"} 3151751.0 # HELP ceph_osd_op_rw Client read-modify-write operations # TYPE ceph_osd_op_rw counter ceph_osd_op_rw{ceph_daemon="osd.6"} 829329.0 ceph_osd_op_rw{ceph_daemon="osd.9"} 30943.0 ceph_osd_op_rw{ceph_daemon="osd.8"} 104892.0 ceph_osd_op_rw{ceph_daemon="osd.0"} 78860.0 ceph_osd_op_rw{ceph_daemon="osd.3"} 26965.0 ceph_osd_op_rw{ceph_daemon="osd.11"} 131689.0 ceph_osd_op_rw{ceph_daemon="osd.2"} 57540.0 ceph_osd_op_rw{ceph_daemon="osd.10"} 15809.0 ceph_osd_op_rw{ceph_daemon="osd.5"} 30688.0 ceph_osd_op_rw{ceph_daemon="osd.4"} 12067.0 ceph_osd_op_rw{ceph_daemon="osd.7"} 87335.0 # HELP ceph_paxos_store_state_keys_count Keys in transaction in stored state Count # TYPE ceph_paxos_store_state_keys_count counter ceph_paxos_store_state_keys_count{ceph_daemon="mon.weil04"} 1431825.0 ceph_paxos_store_state_keys_count{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_store_state_keys_count{ceph_daemon="mon.weil02"} 1571390.0 # HELP ceph_rocksdb_rocksdb_write_delay_time_sum Rocksdb write delay time Total # TYPE ceph_rocksdb_rocksdb_write_delay_time_sum counter ceph_rocksdb_rocksdb_write_delay_time_sum{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_rocksdb_write_delay_time_sum{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_rocksdb_write_delay_time_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pool_objects DF pool objects # TYPE ceph_pool_objects gauge ceph_pool_objects{pool_id="5"} 11.0 ceph_pool_objects{pool_id="3"} 740665.0 ceph_pool_objects{pool_id="4"} 397253.0 # HELP ceph_pg_backfill_wait PG backfill_wait # TYPE ceph_pg_backfill_wait gauge ceph_pg_backfill_wait 0.0 # HELP ceph_bluefs_read_random_disk_bytes Bytes read from disk in random read mode # TYPE ceph_bluefs_read_random_disk_bytes counter ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.6"} 70933023525.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.9"} 104594665310.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.8"} 98906124992.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.0"} 172687912320.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.3"} 79186589464.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.11"} 68958681545.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.2"} 184816225031.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.10"} 98171492273.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.5"} 62892573274.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.4"} 76047995334.0 ceph_bluefs_read_random_disk_bytes{ceph_daemon="osd.7"} 69950628257.0 # HELP ceph_mgr_module_status MGR module status (0=disabled, 1=enabled, 2=auto-enabled) # TYPE ceph_mgr_module_status gauge ceph_mgr_module_status{name="pg_autoscaler"} 0.0 ceph_mgr_module_status{name="restful"} 1.0 ceph_mgr_module_status{name="rook"} 0.0 ceph_mgr_module_status{name="influx"} 0.0 ceph_mgr_module_status{name="balancer"} 2.0 ceph_mgr_module_status{name="devicehealth"} 2.0 ceph_mgr_module_status{name="zabbix"} 0.0 ceph_mgr_module_status{name="localpool"} 0.0 ceph_mgr_module_status{name="status"} 2.0 ceph_mgr_module_status{name="volumes"} 2.0 ceph_mgr_module_status{name="crash"} 2.0 ceph_mgr_module_status{name="insights"} 0.0 ceph_mgr_module_status{name="ssh"} 0.0 ceph_mgr_module_status{name="rbd_support"} 2.0 ceph_mgr_module_status{name="dashboard"} 1.0 ceph_mgr_module_status{name="deepsea"} 0.0 ceph_mgr_module_status{name="iostat"} 0.0 ceph_mgr_module_status{name="test_orchestrator"} 0.0 ceph_mgr_module_status{name="ansible"} 0.0 ceph_mgr_module_status{name="diskprediction_local"} 0.0 ceph_mgr_module_status{name="selftest"} 0.0 ceph_mgr_module_status{name="telegraf"} 0.0 ceph_mgr_module_status{name="diskprediction_cloud"} 0.0 ceph_mgr_module_status{name="telemetry"} 0.0 ceph_mgr_module_status{name="prometheus"} 1.0 ceph_mgr_module_status{name="orchestrator_cli"} 2.0 ceph_mgr_module_status{name="progress"} 2.0 # HELP ceph_osd_recovery_ops Started recovery operations # TYPE ceph_osd_recovery_ops counter ceph_osd_recovery_ops{ceph_daemon="osd.6"} 120958.0 ceph_osd_recovery_ops{ceph_daemon="osd.9"} 337179.0 ceph_osd_recovery_ops{ceph_daemon="osd.8"} 285638.0 ceph_osd_recovery_ops{ceph_daemon="osd.0"} 580302.0 ceph_osd_recovery_ops{ceph_daemon="osd.3"} 47791.0 ceph_osd_recovery_ops{ceph_daemon="osd.11"} 118928.0 ceph_osd_recovery_ops{ceph_daemon="osd.2"} 0.0 ceph_osd_recovery_ops{ceph_daemon="osd.10"} 266652.0 ceph_osd_recovery_ops{ceph_daemon="osd.5"} 107029.0 ceph_osd_recovery_ops{ceph_daemon="osd.4"} 81489.0 ceph_osd_recovery_ops{ceph_daemon="osd.7"} 124979.0 # HELP ceph_bluefs_logged_bytes Bytes written to the metadata log # TYPE ceph_bluefs_logged_bytes counter ceph_bluefs_logged_bytes{ceph_daemon="osd.6"} 197832704.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.9"} 587821056.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.8"} 573698048.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.0"} 286044160.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.3"} 377204736.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.11"} 586776576.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.2"} 217387008.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.10"} 664473600.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.5"} 180449280.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.4"} 283525120.0 ceph_bluefs_logged_bytes{ceph_daemon="osd.7"} 172953600.0 # HELP ceph_rocksdb_get Gets # TYPE ceph_rocksdb_get counter ceph_rocksdb_get{ceph_daemon="mon.weil04"} 55603073.0 ceph_rocksdb_get{ceph_daemon="mon.weil01"} 61993436.0 ceph_rocksdb_get{ceph_daemon="mon.weil02"} 61430343.0 # HELP ceph_pool_metadata POOL Metadata # TYPE ceph_pool_metadata untyped ceph_pool_metadata{pool_id="4",name="ssd"} 1.0 ceph_pool_metadata{pool_id="5",name="device_health_metrics"} 1.0 ceph_pool_metadata{pool_id="3",name="cephrescue"} 1.0 # HELP ceph_bluefs_db_total_bytes Total bytes (main db device) # TYPE ceph_bluefs_db_total_bytes gauge ceph_bluefs_db_total_bytes{ceph_daemon="osd.6"} 160030523392.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.9"} 80015785984.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.8"} 80015785984.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.0"} 80015785984.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.3"} 80015785984.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.11"} 80015785984.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.2"} 160030523392.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.10"} 80015785984.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.5"} 80015785984.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.4"} 80015785984.0 ceph_bluefs_db_total_bytes{ceph_daemon="osd.7"} 160030523392.0 # HELP ceph_pg_recovery_toofull PG recovery_toofull # TYPE ceph_pg_recovery_toofull gauge ceph_pg_recovery_toofull 0.0 # HELP ceph_osd_op_w_latency_count Latency of write operation (including queue time) Count # TYPE ceph_osd_op_w_latency_count counter ceph_osd_op_w_latency_count{ceph_daemon="osd.6"} 7389110.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.9"} 10567363.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.8"} 9979544.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.0"} 18356254.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.3"} 6881789.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.11"} 6946021.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.2"} 14910799.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.10"} 8550830.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.5"} 5713289.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.4"} 8078664.0 ceph_osd_op_w_latency_count{ceph_daemon="osd.7"} 8098819.0 # HELP ceph_mon_metadata MON Metadata # TYPE ceph_mon_metadata untyped ceph_mon_metadata{ceph_daemon="mon.weil02",hostname="weil02",public_addr="10.10.50.152",rank="1",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_mon_metadata{ceph_daemon="mon.weil04",hostname="weil04",public_addr="10.10.50.154",rank="2",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 ceph_mon_metadata{ceph_daemon="mon.weil01",hostname="weil01",public_addr="10.10.50.151",rank="0",ceph_version="ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)"} 1.0 # HELP ceph_bluestore_commit_lat_count Average commit latency Count # TYPE ceph_bluestore_commit_lat_count counter ceph_bluestore_commit_lat_count{ceph_daemon="osd.6"} 16633447.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.9"} 22463305.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.8"} 21391350.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.0"} 38579397.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.3"} 15597014.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.11"} 15937565.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.2"} 28701054.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.10"} 22985082.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.5"} 13706683.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.4"} 14910397.0 ceph_bluestore_commit_lat_count{ceph_daemon="osd.7"} 16292552.0 # HELP ceph_bluestore_state_aio_wait_lat_count Average aio_wait state latency Count # TYPE ceph_bluestore_state_aio_wait_lat_count counter ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.6"} 16633450.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.9"} 22463308.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.8"} 21391351.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.0"} 38579398.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.3"} 15597014.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.11"} 15937565.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.2"} 28701056.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.10"} 22985083.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.5"} 13706683.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.4"} 14910397.0 ceph_bluestore_state_aio_wait_lat_count{ceph_daemon="osd.7"} 16292552.0 # HELP ceph_pg_unknown PG unknown # TYPE ceph_pg_unknown gauge ceph_pg_unknown 0.0 # HELP ceph_paxos_begin_bytes_count Data in transaction on begin Count # TYPE ceph_paxos_begin_bytes_count counter ceph_paxos_begin_bytes_count{ceph_daemon="mon.weil04"} 1431846.0 ceph_paxos_begin_bytes_count{ceph_daemon="mon.weil01"} 1574788.0 ceph_paxos_begin_bytes_count{ceph_daemon="mon.weil02"} 1571384.0 # HELP ceph_pg_recovery_unfound PG recovery_unfound # TYPE ceph_pg_recovery_unfound gauge ceph_pg_recovery_unfound 0.0 # HELP ceph_pool_quota_bytes DF pool quota_bytes # TYPE ceph_pool_quota_bytes gauge ceph_pool_quota_bytes{pool_id="5"} 0.0 ceph_pool_quota_bytes{pool_id="3"} 0.0 ceph_pool_quota_bytes{pool_id="4"} 0.0 # HELP ceph_pg_snaptrim_wait PG snaptrim_wait # TYPE ceph_pg_snaptrim_wait gauge ceph_pg_snaptrim_wait 4.0 # HELP ceph_paxos_start_leader Starts in leader role # TYPE ceph_paxos_start_leader counter ceph_paxos_start_leader{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_start_leader{ceph_daemon="mon.weil01"} 42.0 ceph_paxos_start_leader{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pg_creating PG creating # TYPE ceph_pg_creating gauge ceph_pg_creating 0.0 # HELP ceph_mon_election_call Elections started # TYPE ceph_mon_election_call counter ceph_mon_election_call{ceph_daemon="mon.weil04"} 32.0 ceph_mon_election_call{ceph_daemon="mon.weil01"} 50.0 ceph_mon_election_call{ceph_daemon="mon.weil02"} 32.0 # HELP ceph_osd_op_w_prepare_latency_sum Latency of write operations (excluding queue time and wait for finished) Total # TYPE ceph_osd_op_w_prepare_latency_sum counter ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.6"} 28416.056252753 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.9"} 53137.297378721 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.8"} 30973.737755312 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.0"} 102049.346012002 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.3"} 22485.955344432 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.11"} 33900.257963705 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.2"} 77141.418608831 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.10"} 41624.120037441 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.5"} 15400.392074979 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.4"} 23655.682297837 ceph_osd_op_w_prepare_latency_sum{ceph_daemon="osd.7"} 42783.184145378 # HELP ceph_rbd_mirror_replay_latency_sum Replay latency Total # TYPE ceph_rbd_mirror_replay_latency_sum counter ceph_rbd_mirror_replay_latency_sum{ceph_daemon="rbd-mirror.222864335"} 13318049.283388069 # HELP ceph_mon_session_rm Removed sessions # TYPE ceph_mon_session_rm counter ceph_mon_session_rm{ceph_daemon="mon.weil04"} 520577.0 ceph_mon_session_rm{ceph_daemon="mon.weil01"} 1369926.0 ceph_mon_session_rm{ceph_daemon="mon.weil02"} 1186636.0 # HELP ceph_cluster_total_used_bytes DF total_used_bytes # TYPE ceph_cluster_total_used_bytes gauge ceph_cluster_total_used_bytes 7707400273920.0 # HELP ceph_pg_active PG active # TYPE ceph_pg_active gauge ceph_pg_active 769.0 # HELP ceph_paxos_store_state Store a shared state on disk # TYPE ceph_paxos_store_state counter ceph_paxos_store_state{ceph_daemon="mon.weil04"} 1431825.0 ceph_paxos_store_state{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_store_state{ceph_daemon="mon.weil02"} 1571390.0 # HELP ceph_pg_activating PG activating # TYPE ceph_pg_activating gauge ceph_pg_activating 0.0 # HELP ceph_scrape_duration_secs Time taken to gather metrics from Ceph (secs) # TYPE ceph_scrape_duration_secs gauge ceph_scrape_duration_secs 1.913534164428711 # HELP ceph_paxos_store_state_bytes_count Data in transaction in stored state Count # TYPE ceph_paxos_store_state_bytes_count counter ceph_paxos_store_state_bytes_count{ceph_daemon="mon.weil04"} 1431825.0 ceph_paxos_store_state_bytes_count{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_store_state_bytes_count{ceph_daemon="mon.weil02"} 1571390.0 # HELP ceph_osd_op_w_latency_sum Latency of write operation (including queue time) Total # TYPE ceph_osd_op_w_latency_sum counter ceph_osd_op_w_latency_sum{ceph_daemon="osd.6"} 378461.816171966 ceph_osd_op_w_latency_sum{ceph_daemon="osd.9"} 648628.250961561 ceph_osd_op_w_latency_sum{ceph_daemon="osd.8"} 568318.778420929 ceph_osd_op_w_latency_sum{ceph_daemon="osd.0"} 1329554.847329035 ceph_osd_op_w_latency_sum{ceph_daemon="osd.3"} 265949.355866883 ceph_osd_op_w_latency_sum{ceph_daemon="osd.11"} 463187.391415135 ceph_osd_op_w_latency_sum{ceph_daemon="osd.2"} 792714.751934253 ceph_osd_op_w_latency_sum{ceph_daemon="osd.10"} 561762.578026727 ceph_osd_op_w_latency_sum{ceph_daemon="osd.5"} 263251.650088403 ceph_osd_op_w_latency_sum{ceph_daemon="osd.4"} 327482.45934336 ceph_osd_op_w_latency_sum{ceph_daemon="osd.7"} 586336.58999221 # HELP ceph_rocksdb_submit_latency_count Submit Latency Count # TYPE ceph_rocksdb_submit_latency_count counter ceph_rocksdb_submit_latency_count{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_submit_latency_count{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_submit_latency_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pool_dirty DF pool dirty # TYPE ceph_pool_dirty gauge ceph_pool_dirty{pool_id="5"} 11.0 ceph_pool_dirty{pool_id="3"} 740665.0 ceph_pool_dirty{pool_id="4"} 397253.0 # HELP ceph_paxos_commit_latency_sum Commit latency Total # TYPE ceph_paxos_commit_latency_sum counter ceph_paxos_commit_latency_sum{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_commit_latency_sum{ceph_daemon="mon.weil01"} 32793.23950733 ceph_paxos_commit_latency_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_rocksdb_rocksdb_write_memtable_time_sum Rocksdb write memtable time Total # TYPE ceph_rocksdb_rocksdb_write_memtable_time_sum counter ceph_rocksdb_rocksdb_write_memtable_time_sum{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_rocksdb_write_memtable_time_sum{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_rocksdb_write_memtable_time_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_rgw_metadata RGW Metadata # TYPE ceph_rgw_metadata untyped # HELP ceph_paxos_share_state_bytes_sum Data in shared state Total # TYPE ceph_paxos_share_state_bytes_sum counter ceph_paxos_share_state_bytes_sum{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_share_state_bytes_sum{ceph_daemon="mon.weil01"} 2863519.0 ceph_paxos_share_state_bytes_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pool_num_objects_recovered OSD POOL STATS: num_objects_recovered # TYPE ceph_pool_num_objects_recovered gauge ceph_pool_num_objects_recovered{pool_id="5"} 0.0 ceph_pool_num_objects_recovered{pool_id="3"} 0.0 ceph_pool_num_objects_recovered{pool_id="4"} 0.0 # HELP ceph_osd_op_process_latency_sum Latency of client operations (excluding queue time) Total # TYPE ceph_osd_op_process_latency_sum counter ceph_osd_op_process_latency_sum{ceph_daemon="osd.6"} 447533.249269723 ceph_osd_op_process_latency_sum{ceph_daemon="osd.9"} 648993.497101591 ceph_osd_op_process_latency_sum{ceph_daemon="osd.8"} 575852.971042465 ceph_osd_op_process_latency_sum{ceph_daemon="osd.0"} 1327416.488064675 ceph_osd_op_process_latency_sum{ceph_daemon="osd.3"} 266171.695896197 ceph_osd_op_process_latency_sum{ceph_daemon="osd.11"} 479564.576105981 ceph_osd_op_process_latency_sum{ceph_daemon="osd.2"} 788978.275206645 ceph_osd_op_process_latency_sum{ceph_daemon="osd.10"} 560358.426374838 ceph_osd_op_process_latency_sum{ceph_daemon="osd.5"} 263446.173270321 ceph_osd_op_process_latency_sum{ceph_daemon="osd.4"} 326614.517557759 ceph_osd_op_process_latency_sum{ceph_daemon="osd.7"} 582264.024466003 # HELP ceph_paxos_begin_keys_sum Keys in transaction on begin Total # TYPE ceph_paxos_begin_keys_sum counter ceph_paxos_begin_keys_sum{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_begin_keys_sum{ceph_daemon="mon.weil01"} 4724364.0 ceph_paxos_begin_keys_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pg_snaptrim_error PG snaptrim_error # TYPE ceph_pg_snaptrim_error gauge ceph_pg_snaptrim_error 0.0 # HELP ceph_pg_backfilling PG backfilling # TYPE ceph_pg_backfilling gauge ceph_pg_backfilling 0.0 # HELP ceph_rocksdb_rocksdb_write_pre_and_post_time_sum total time spent on writing a record, excluding write process Total # TYPE ceph_rocksdb_rocksdb_write_pre_and_post_time_sum counter ceph_rocksdb_rocksdb_write_pre_and_post_time_sum{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_rocksdb_write_pre_and_post_time_sum{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_rocksdb_write_pre_and_post_time_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_bluefs_wal_used_bytes Used bytes (wal device) # TYPE ceph_bluefs_wal_used_bytes gauge ceph_bluefs_wal_used_bytes{ceph_daemon="osd.6"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.9"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.8"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.0"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.3"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.11"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.2"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.10"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.5"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.4"} 0.0 ceph_bluefs_wal_used_bytes{ceph_daemon="osd.7"} 0.0 # HELP ceph_pool_rd_bytes DF pool rd_bytes # TYPE ceph_pool_rd_bytes gauge ceph_pool_rd_bytes{pool_id="5"} 214016.0 ceph_pool_rd_bytes{pool_id="3"} 1387120968704.0 ceph_pool_rd_bytes{pool_id="4"} 2207959040.0 # HELP ceph_pg_deep PG deep # TYPE ceph_pg_deep gauge ceph_pg_deep 0.0 # HELP ceph_rocksdb_rocksdb_write_wal_time_sum Rocksdb write wal time Total # TYPE ceph_rocksdb_rocksdb_write_wal_time_sum counter ceph_rocksdb_rocksdb_write_wal_time_sum{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_rocksdb_write_wal_time_sum{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_rocksdb_write_wal_time_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pool_recovering_keys_per_sec OSD POOL STATS: recovering_keys_per_sec # TYPE ceph_pool_recovering_keys_per_sec gauge ceph_pool_recovering_keys_per_sec{pool_id="5"} 0.0 ceph_pool_recovering_keys_per_sec{pool_id="3"} 0.0 ceph_pool_recovering_keys_per_sec{pool_id="4"} 0.0 # HELP ceph_pg_backfill_toofull PG backfill_toofull # TYPE ceph_pg_backfill_toofull gauge ceph_pg_backfill_toofull 0.0 # HELP ceph_osd_flag_noup OSD Flag noup # TYPE ceph_osd_flag_noup untyped ceph_osd_flag_noup 0.0 # HELP ceph_pg_scrubbing PG scrubbing # TYPE ceph_pg_scrubbing gauge ceph_pg_scrubbing 0.0 # HELP ceph_num_objects_unfound Number of unfound objects # TYPE ceph_num_objects_unfound gauge ceph_num_objects_unfound 0.0 # HELP ceph_mon_quorum_status Monitors in quorum # TYPE ceph_mon_quorum_status gauge ceph_mon_quorum_status{ceph_daemon="mon.weil04"} 1.0 ceph_mon_quorum_status{ceph_daemon="mon.weil01"} 1.0 ceph_mon_quorum_status{ceph_daemon="mon.weil02"} 1.0 # HELP ceph_paxos_lease_timeout Lease timeouts # TYPE ceph_paxos_lease_timeout counter ceph_paxos_lease_timeout{ceph_daemon="mon.weil04"} 17.0 ceph_paxos_lease_timeout{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_lease_timeout{ceph_daemon="mon.weil02"} 19.0 # HELP ceph_osd_op_r_out_bytes Client data read # TYPE ceph_osd_op_r_out_bytes counter ceph_osd_op_r_out_bytes{ceph_daemon="osd.6"} 59261244.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.9"} 17335625.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.8"} 21777277.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.0"} 11492639.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.3"} 12900321.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.11"} 22828904.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.2"} 7761479.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.10"} 10152150.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.5"} 4482896.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.4"} 7679481.0 ceph_osd_op_r_out_bytes{ceph_daemon="osd.7"} 19280212.0 # HELP ceph_paxos_begin_keys_count Keys in transaction on begin Count # TYPE ceph_paxos_begin_keys_count counter ceph_paxos_begin_keys_count{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_begin_keys_count{ceph_daemon="mon.weil01"} 1574788.0 ceph_paxos_begin_keys_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_bluestore_kv_sync_lat_count Average kv_sync thread latency Count # TYPE ceph_bluestore_kv_sync_lat_count counter ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.6"} 15344063.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.9"} 21065806.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.8"} 20518535.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.0"} 33446557.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.3"} 14177316.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.11"} 14729922.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.2"} 19270180.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.10"} 21361503.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.5"} 12885773.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.4"} 13779209.0 ceph_bluestore_kv_sync_lat_count{ceph_daemon="osd.7"} 13941359.0 # HELP ceph_osd_op_prepare_latency_count Latency of client operations (excluding queue time and wait for finished) Count # TYPE ceph_osd_op_prepare_latency_count counter ceph_osd_op_prepare_latency_count{ceph_daemon="osd.6"} 11207139.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.9"} 12910634.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.8"} 14386069.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.0"} 20034313.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.3"} 8112925.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.11"} 9120316.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.2"} 15940282.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.10"} 9592758.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.5"} 6685032.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.4"} 8815846.0 ceph_osd_op_prepare_latency_count{ceph_daemon="osd.7"} 10039208.0 # HELP ceph_pg_recovering PG recovering # TYPE ceph_pg_recovering gauge ceph_pg_recovering 0.0 # HELP ceph_rocksdb_submit_latency_sum Submit Latency Total # TYPE ceph_rocksdb_submit_latency_sum counter ceph_rocksdb_submit_latency_sum{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_submit_latency_sum{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_submit_latency_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pg_repair PG repair # TYPE ceph_pg_repair gauge ceph_pg_repair 0.0 # HELP ceph_osd_op_r_latency_count Latency of read operation (including queue time) Count # TYPE ceph_osd_op_r_latency_count counter ceph_osd_op_r_latency_count{ceph_daemon="osd.6"} 2988325.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.9"} 2312093.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.8"} 4301253.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.0"} 1598947.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.3"} 1204124.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.11"} 2042112.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.2"} 971926.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.10"} 1026017.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.5"} 940628.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.4"} 725056.0 ceph_osd_op_r_latency_count{ceph_daemon="osd.7"} 1852834.0 # HELP ceph_paxos_share_state_keys_sum Keys in shared state Total # TYPE ceph_paxos_share_state_keys_sum counter ceph_paxos_share_state_keys_sum{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_share_state_keys_sum{ceph_daemon="mon.weil01"} 167.0 ceph_paxos_share_state_keys_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_paxos_store_state_bytes_sum Data in transaction in stored state Total # TYPE ceph_paxos_store_state_bytes_sum counter ceph_paxos_store_state_bytes_sum{ceph_daemon="mon.weil04"} 65522691700.0 ceph_paxos_store_state_bytes_sum{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_store_state_bytes_sum{ceph_daemon="mon.weil02"} 72896113524.0 # HELP ceph_osd_op_latency_count Latency of client operations (including queue time) Count # TYPE ceph_osd_op_latency_count counter ceph_osd_op_latency_count{ceph_daemon="osd.6"} 11206764.0 ceph_osd_op_latency_count{ceph_daemon="osd.9"} 12910399.0 ceph_osd_op_latency_count{ceph_daemon="osd.8"} 14385689.0 ceph_osd_op_latency_count{ceph_daemon="osd.0"} 20034061.0 ceph_osd_op_latency_count{ceph_daemon="osd.3"} 8112878.0 ceph_osd_op_latency_count{ceph_daemon="osd.11"} 9119822.0 ceph_osd_op_latency_count{ceph_daemon="osd.2"} 15940265.0 ceph_osd_op_latency_count{ceph_daemon="osd.10"} 9592656.0 ceph_osd_op_latency_count{ceph_daemon="osd.5"} 6684605.0 ceph_osd_op_latency_count{ceph_daemon="osd.4"} 8815787.0 ceph_osd_op_latency_count{ceph_daemon="osd.7"} 10038988.0 # HELP ceph_paxos_commit_bytes_count Data in transaction on commit Count # TYPE ceph_paxos_commit_bytes_count counter ceph_paxos_commit_bytes_count{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_commit_bytes_count{ceph_daemon="mon.weil01"} 1574765.0 ceph_paxos_commit_bytes_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_paxos_restart Restarts # TYPE ceph_paxos_restart counter ceph_paxos_restart{ceph_daemon="mon.weil04"} 167.0 ceph_paxos_restart{ceph_daemon="mon.weil01"} 179.0 ceph_paxos_restart{ceph_daemon="mon.weil02"} 212.0 # HELP ceph_rbd_mirror_replay Replays # TYPE ceph_rbd_mirror_replay counter ceph_rbd_mirror_replay{ceph_daemon="rbd-mirror.222864335"} 175509171.0 # HELP ceph_pg_down PG down # TYPE ceph_pg_down gauge ceph_pg_down 0.0 # HELP ceph_bluefs_slow_total_bytes Total bytes (slow device) # TYPE ceph_bluefs_slow_total_bytes gauge ceph_bluefs_slow_total_bytes{ceph_daemon="osd.6"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.9"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.8"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.0"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.3"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.11"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.2"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.10"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.5"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.4"} 0.0 ceph_bluefs_slow_total_bytes{ceph_daemon="osd.7"} 0.0 # HELP ceph_paxos_collect_timeout Collect timeouts # TYPE ceph_paxos_collect_timeout counter ceph_paxos_collect_timeout{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_collect_timeout{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_collect_timeout{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pg_peered PG peered # TYPE ceph_pg_peered gauge ceph_pg_peered 0.0 # HELP ceph_osd_commit_latency_ms OSD stat commit_latency_ms # TYPE ceph_osd_commit_latency_ms gauge ceph_osd_commit_latency_ms{ceph_daemon="osd.6"} 19.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.9"} 10.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.8"} 9.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.0"} 18.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.3"} 10.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.11"} 21.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.2"} 34.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.10"} 16.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.5"} 13.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.4"} 19.0 ceph_osd_commit_latency_ms{ceph_daemon="osd.7"} 19.0 # HELP ceph_osd_op_w_process_latency_sum Latency of write operation (excluding queue time) Total # TYPE ceph_osd_op_w_process_latency_sum counter ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.6"} 373779.243450914 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.9"} 644934.327470461 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.8"} 566581.095095232 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.0"} 1316807.533658948 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.3"} 264574.901945166 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.11"} 457942.405591201 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.2"} 785638.594557456 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.10"} 558306.270663031 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.5"} 261773.094187895 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.4"} 325865.810550417 ceph_osd_op_w_process_latency_sum{ceph_daemon="osd.7"} 576032.148322762 # HELP ceph_osd_weight OSD status weight # TYPE ceph_osd_weight untyped ceph_osd_weight{ceph_daemon="osd.6"} 1.0 ceph_osd_weight{ceph_daemon="osd.9"} 1.0 ceph_osd_weight{ceph_daemon="osd.8"} 1.0 ceph_osd_weight{ceph_daemon="osd.0"} 1.0 ceph_osd_weight{ceph_daemon="osd.3"} 1.0 ceph_osd_weight{ceph_daemon="osd.11"} 1.0 ceph_osd_weight{ceph_daemon="osd.2"} 1.0 ceph_osd_weight{ceph_daemon="osd.10"} 1.0 ceph_osd_weight{ceph_daemon="osd.5"} 1.0 ceph_osd_weight{ceph_daemon="osd.4"} 1.0 ceph_osd_weight{ceph_daemon="osd.7"} 1.0 # HELP ceph_paxos_collect_keys_count Keys in transaction on peon collect Count # TYPE ceph_paxos_collect_keys_count counter ceph_paxos_collect_keys_count{ceph_daemon="mon.weil04"} 38.0 ceph_paxos_collect_keys_count{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_collect_keys_count{ceph_daemon="mon.weil02"} 27.0 # HELP ceph_paxos_share_state_bytes_count Data in shared state Count # TYPE ceph_paxos_share_state_bytes_count counter ceph_paxos_share_state_bytes_count{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_share_state_bytes_count{ceph_daemon="mon.weil01"} 18.0 ceph_paxos_share_state_bytes_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_bluefs_read_random_bytes Bytes requested in random read mode # TYPE ceph_bluefs_read_random_bytes counter ceph_bluefs_read_random_bytes{ceph_daemon="osd.6"} 74918961916.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.9"} 111633395632.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.8"} 105456638142.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.0"} 181634024375.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.3"} 83364422880.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.11"} 73520030166.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.2"} 190813895293.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.10"} 103602152610.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.5"} 67036859643.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.4"} 80429083830.0 ceph_bluefs_read_random_bytes{ceph_daemon="osd.7"} 74319554188.0 # HELP ceph_bluestore_read_lat_sum Average read latency Total # TYPE ceph_bluestore_read_lat_sum counter ceph_bluestore_read_lat_sum{ceph_daemon="osd.6"} 21354.164026554 ceph_bluestore_read_lat_sum{ceph_daemon="osd.9"} 37967.211000321 ceph_bluestore_read_lat_sum{ceph_daemon="osd.8"} 24340.67618387 ceph_bluestore_read_lat_sum{ceph_daemon="osd.0"} 68585.225772869 ceph_bluestore_read_lat_sum{ceph_daemon="osd.3"} 9811.025031418 ceph_bluestore_read_lat_sum{ceph_daemon="osd.11"} 31826.258086929 ceph_bluestore_read_lat_sum{ceph_daemon="osd.2"} 16591.822887491 ceph_bluestore_read_lat_sum{ceph_daemon="osd.10"} 29396.600614709 ceph_bluestore_read_lat_sum{ceph_daemon="osd.5"} 13444.984068359 ceph_bluestore_read_lat_sum{ceph_daemon="osd.4"} 13411.233161952 ceph_bluestore_read_lat_sum{ceph_daemon="osd.7"} 28999.806245867 # HELP ceph_osd_flag_noscrub OSD Flag noscrub # TYPE ceph_osd_flag_noscrub untyped ceph_osd_flag_noscrub 0.0 # HELP ceph_osd_stat_bytes_used Used space # TYPE ceph_osd_stat_bytes_used gauge ceph_osd_stat_bytes_used{ceph_daemon="osd.6"} 646485180416.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.9"} 660592852992.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.8"} 594984042496.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.0"} 714320576512.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.3"} 676032479232.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.11"} 485244796928.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.2"} 1437834543104.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.10"} 594482757632.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.5"} 658621399040.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.4"} 585999777792.0 ceph_osd_stat_bytes_used{ceph_daemon="osd.7"} 675103244288.0 # HELP ceph_osd_flag_norecover OSD Flag norecover # TYPE ceph_osd_flag_norecover untyped ceph_osd_flag_norecover 0.0 # HELP ceph_pg_clean PG clean # TYPE ceph_pg_clean gauge ceph_pg_clean 769.0 # HELP ceph_paxos_begin Started and handled begins # TYPE ceph_paxos_begin counter ceph_paxos_begin{ceph_daemon="mon.weil04"} 1431846.0 ceph_paxos_begin{ceph_daemon="mon.weil01"} 1574788.0 ceph_paxos_begin{ceph_daemon="mon.weil02"} 1571384.0 # HELP ceph_pool_recovering_objects_per_sec OSD POOL STATS: recovering_objects_per_sec # TYPE ceph_pool_recovering_objects_per_sec gauge ceph_pool_recovering_objects_per_sec{pool_id="5"} 0.0 ceph_pool_recovering_objects_per_sec{pool_id="3"} 0.0 ceph_pool_recovering_objects_per_sec{pool_id="4"} 0.0 # HELP ceph_rbd_mirror_replay_bytes Replayed data # TYPE ceph_rbd_mirror_replay_bytes counter ceph_rbd_mirror_replay_bytes{ceph_daemon="rbd-mirror.222864335"} 1649907428366.0 # HELP ceph_rocksdb_rocksdb_write_wal_time_count Rocksdb write wal time Count # TYPE ceph_rocksdb_rocksdb_write_wal_time_count counter ceph_rocksdb_rocksdb_write_wal_time_count{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_rocksdb_write_wal_time_count{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_rocksdb_write_wal_time_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_osd_stat_bytes OSD size # TYPE ceph_osd_stat_bytes gauge ceph_osd_stat_bytes{ceph_daemon="osd.6"} 4000762036224.0 ceph_osd_stat_bytes{ceph_daemon="osd.9"} 2000381018112.0 ceph_osd_stat_bytes{ceph_daemon="osd.8"} 2000381018112.0 ceph_osd_stat_bytes{ceph_daemon="osd.0"} 2000381018112.0 ceph_osd_stat_bytes{ceph_daemon="osd.3"} 2000381018112.0 ceph_osd_stat_bytes{ceph_daemon="osd.11"} 2000381018112.0 ceph_osd_stat_bytes{ceph_daemon="osd.2"} 4000762036224.0 ceph_osd_stat_bytes{ceph_daemon="osd.10"} 2000381018112.0 ceph_osd_stat_bytes{ceph_daemon="osd.5"} 2000381018112.0 ceph_osd_stat_bytes{ceph_daemon="osd.4"} 2000381018112.0 ceph_osd_stat_bytes{ceph_daemon="osd.7"} 4000762036224.0 # HELP ceph_disk_occupation Associate Ceph daemon with disk used # TYPE ceph_disk_occupation untyped ceph_disk_occupation{ceph_daemon="osd.5",device="/dev/dm-3",db_device="",wal_device="",instance="weil02"} 1.0 ceph_disk_occupation{ceph_daemon="osd.3",device="/dev/dm-1",db_device="",wal_device="",instance="weil01"} 1.0 ceph_disk_occupation{ceph_daemon="osd.10",device="/dev/dm-3",db_device="",wal_device="",instance="weil04"} 1.0 ceph_disk_occupation{ceph_daemon="osd.7",device="/dev/dm-1",db_device="",wal_device="",instance="weil02"} 1.0 ceph_disk_occupation{ceph_daemon="osd.2",device="/dev/dm-2",db_device="",wal_device="",instance="weil01"} 1.0 ceph_disk_occupation{ceph_daemon="osd.6",device="/dev/dm-2",db_device="",wal_device="",instance="weil02"} 1.0 ceph_disk_occupation{ceph_daemon="osd.4",device="/dev/dm-0",db_device="",wal_device="",instance="weil01"} 1.0 ceph_disk_occupation{ceph_daemon="osd.0",device="/dev/dm-0",db_device="",wal_device="",instance="weil04"} 1.0 ceph_disk_occupation{ceph_daemon="osd.9",device="/dev/dm-2",db_device="",wal_device="",instance="weil04"} 1.0 ceph_disk_occupation{ceph_daemon="osd.8",device="/dev/dm-1",db_device="",wal_device="",instance="weil04"} 1.0 ceph_disk_occupation{ceph_daemon="osd.11",device="/dev/dm-0",db_device="",wal_device="",instance="weil02"} 1.0 # HELP ceph_pg_snaptrim PG snaptrim # TYPE ceph_pg_snaptrim gauge ceph_pg_snaptrim 2.0 # HELP ceph_mds_metadata MDS Metadata # TYPE ceph_mds_metadata untyped # HELP ceph_paxos_store_state_keys_sum Keys in transaction in stored state Total # TYPE ceph_paxos_store_state_keys_sum counter ceph_paxos_store_state_keys_sum{ceph_daemon="mon.weil04"} 12686183.0 ceph_paxos_store_state_keys_sum{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_store_state_keys_sum{ceph_daemon="mon.weil02"} 13990409.0 # HELP ceph_bluefs_read_bytes Bytes requested in buffered read mode # TYPE ceph_bluefs_read_bytes counter ceph_bluefs_read_bytes{ceph_daemon="osd.6"} 18911349067.0 ceph_bluefs_read_bytes{ceph_daemon="osd.9"} 45122173218.0 ceph_bluefs_read_bytes{ceph_daemon="osd.8"} 50743661261.0 ceph_bluefs_read_bytes{ceph_daemon="osd.0"} 66040906765.0 ceph_bluefs_read_bytes{ceph_daemon="osd.3"} 28824717625.0 ceph_bluefs_read_bytes{ceph_daemon="osd.11"} 45199277402.0 ceph_bluefs_read_bytes{ceph_daemon="osd.2"} 21446591932.0 ceph_bluefs_read_bytes{ceph_daemon="osd.10"} 31783925239.0 ceph_bluefs_read_bytes{ceph_daemon="osd.5"} 18353613579.0 ceph_bluefs_read_bytes{ceph_daemon="osd.4"} 36909507633.0 ceph_bluefs_read_bytes{ceph_daemon="osd.7"} 21396828313.0 # HELP ceph_osd_numpg_removing Placement groups queued for local deletion # TYPE ceph_osd_numpg_removing gauge ceph_osd_numpg_removing{ceph_daemon="osd.6"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.9"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.8"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.0"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.3"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.11"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.2"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.10"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.5"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.4"} 0.0 ceph_osd_numpg_removing{ceph_daemon="osd.7"} 0.0 # HELP ceph_pg_remapped PG remapped # TYPE ceph_pg_remapped gauge ceph_pg_remapped 0.0 # HELP ceph_mgr_module_can_run MGR module runnable state i.e. can it run (0=no, 1=yes) # TYPE ceph_mgr_module_can_run gauge ceph_mgr_module_can_run{name="pg_autoscaler"} 1.0 ceph_mgr_module_can_run{name="restful"} 1.0 ceph_mgr_module_can_run{name="rook"} 0.0 ceph_mgr_module_can_run{name="influx"} 0.0 ceph_mgr_module_can_run{name="balancer"} 1.0 ceph_mgr_module_can_run{name="devicehealth"} 1.0 ceph_mgr_module_can_run{name="zabbix"} 1.0 ceph_mgr_module_can_run{name="localpool"} 1.0 ceph_mgr_module_can_run{name="status"} 1.0 ceph_mgr_module_can_run{name="volumes"} 1.0 ceph_mgr_module_can_run{name="crash"} 1.0 ceph_mgr_module_can_run{name="insights"} 1.0 ceph_mgr_module_can_run{name="ssh"} 0.0 ceph_mgr_module_can_run{name="rbd_support"} 1.0 ceph_mgr_module_can_run{name="dashboard"} 1.0 ceph_mgr_module_can_run{name="deepsea"} 1.0 ceph_mgr_module_can_run{name="iostat"} 1.0 ceph_mgr_module_can_run{name="test_orchestrator"} 1.0 ceph_mgr_module_can_run{name="ansible"} 1.0 ceph_mgr_module_can_run{name="diskprediction_local"} 1.0 ceph_mgr_module_can_run{name="selftest"} 1.0 ceph_mgr_module_can_run{name="telegraf"} 1.0 ceph_mgr_module_can_run{name="diskprediction_cloud"} 1.0 ceph_mgr_module_can_run{name="telemetry"} 1.0 ceph_mgr_module_can_run{name="prometheus"} 1.0 ceph_mgr_module_can_run{name="orchestrator_cli"} 1.0 ceph_mgr_module_can_run{name="progress"} 1.0 # HELP ceph_paxos_commit_keys_count Keys in transaction on commit Count # TYPE ceph_paxos_commit_keys_count counter ceph_paxos_commit_keys_count{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_commit_keys_count{ceph_daemon="mon.weil01"} 1574765.0 ceph_paxos_commit_keys_count{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_pg_forced_backfill PG forced_backfill # TYPE ceph_pg_forced_backfill gauge ceph_pg_forced_backfill 0.0 # HELP ceph_paxos_new_pn_latency_sum New proposal number getting latency Total # TYPE ceph_paxos_new_pn_latency_sum counter ceph_paxos_new_pn_latency_sum{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_new_pn_latency_sum{ceph_daemon="mon.weil01"} 1.091998271 ceph_paxos_new_pn_latency_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_osd_op_in_bytes Client operations total write size # TYPE ceph_osd_op_in_bytes counter ceph_osd_op_in_bytes{ceph_daemon="osd.6"} 77538105994.0 ceph_osd_op_in_bytes{ceph_daemon="osd.9"} 110757094750.0 ceph_osd_op_in_bytes{ceph_daemon="osd.8"} 103904395058.0 ceph_osd_op_in_bytes{ceph_daemon="osd.0"} 168915230740.0 ceph_osd_op_in_bytes{ceph_daemon="osd.3"} 68078564738.0 ceph_osd_op_in_bytes{ceph_daemon="osd.11"} 71295569544.0 ceph_osd_op_in_bytes{ceph_daemon="osd.2"} 159131214682.0 ceph_osd_op_in_bytes{ceph_daemon="osd.10"} 89435836484.0 ceph_osd_op_in_bytes{ceph_daemon="osd.5"} 55331744056.0 ceph_osd_op_in_bytes{ceph_daemon="osd.4"} 74972082018.0 ceph_osd_op_in_bytes{ceph_daemon="osd.7"} 75639961332.0 # HELP ceph_paxos_store_state_latency_count Storing state latency Count # TYPE ceph_paxos_store_state_latency_count counter ceph_paxos_store_state_latency_count{ceph_daemon="mon.weil04"} 1431825.0 ceph_paxos_store_state_latency_count{ceph_daemon="mon.weil01"} 0.0 ceph_paxos_store_state_latency_count{ceph_daemon="mon.weil02"} 1571390.0 # HELP ceph_paxos_refresh_latency_count Refresh latency Count # TYPE ceph_paxos_refresh_latency_count counter ceph_paxos_refresh_latency_count{ceph_daemon="mon.weil04"} 1431825.0 ceph_paxos_refresh_latency_count{ceph_daemon="mon.weil01"} 1574784.0 ceph_paxos_refresh_latency_count{ceph_daemon="mon.weil02"} 1571390.0 # HELP ceph_pg_total PG Total Count # TYPE ceph_pg_total gauge ceph_pg_total 769.0 # HELP ceph_osd_op_r_prepare_latency_count Latency of read operations (excluding queue time and wait for finished) Count # TYPE ceph_osd_op_r_prepare_latency_count counter ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.6"} 2988589.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.9"} 2312131.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.8"} 4301343.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.0"} 1599006.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.3"} 1204133.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.11"} 2042444.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.2"} 971926.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.10"} 1026033.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.5"} 940989.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.4"} 725058.0 ceph_osd_op_r_prepare_latency_count{ceph_daemon="osd.7"} 1852967.0 # HELP ceph_objecter_op_w Write operations # TYPE ceph_objecter_op_w counter ceph_objecter_op_w{ceph_daemon="osd.6"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.9"} 0.0 ceph_objecter_op_w{ceph_daemon="rbd-mirror.222864335"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.8"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.0"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.3"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.11"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.2"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.10"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.5"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.4"} 0.0 ceph_objecter_op_w{ceph_daemon="osd.7"} 0.0 # HELP ceph_rocksdb_submit_transaction Submit transactions # TYPE ceph_rocksdb_submit_transaction counter ceph_rocksdb_submit_transaction{ceph_daemon="mon.weil04"} 0.0 ceph_rocksdb_submit_transaction{ceph_daemon="mon.weil01"} 0.0 ceph_rocksdb_submit_transaction{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_objecter_op_r Read operations # TYPE ceph_objecter_op_r counter ceph_objecter_op_r{ceph_daemon="osd.6"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.9"} 0.0 ceph_objecter_op_r{ceph_daemon="rbd-mirror.222864335"} 170981.0 ceph_objecter_op_r{ceph_daemon="osd.8"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.0"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.3"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.11"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.2"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.10"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.5"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.4"} 0.0 ceph_objecter_op_r{ceph_daemon="osd.7"} 0.0 # HELP ceph_rocksdb_submit_sync_latency_sum Submit Sync Latency Total # TYPE ceph_rocksdb_submit_sync_latency_sum counter ceph_rocksdb_submit_sync_latency_sum{ceph_daemon="mon.weil04"} 3482.454621602 ceph_rocksdb_submit_sync_latency_sum{ceph_daemon="mon.weil01"} 79887.233253111 ceph_rocksdb_submit_sync_latency_sum{ceph_daemon="mon.weil02"} 62450.691540933 # HELP ceph_bluefs_num_files File count # TYPE ceph_bluefs_num_files gauge ceph_bluefs_num_files{ceph_daemon="osd.6"} 31.0 ceph_bluefs_num_files{ceph_daemon="osd.9"} 33.0 ceph_bluefs_num_files{ceph_daemon="osd.8"} 31.0 ceph_bluefs_num_files{ceph_daemon="osd.0"} 33.0 ceph_bluefs_num_files{ceph_daemon="osd.3"} 32.0 ceph_bluefs_num_files{ceph_daemon="osd.11"} 27.0 ceph_bluefs_num_files{ceph_daemon="osd.2"} 54.0 ceph_bluefs_num_files{ceph_daemon="osd.10"} 31.0 ceph_bluefs_num_files{ceph_daemon="osd.5"} 34.0 ceph_bluefs_num_files{ceph_daemon="osd.4"} 30.0 ceph_bluefs_num_files{ceph_daemon="osd.7"} 34.0 # HELP ceph_osd_up OSD status up # TYPE ceph_osd_up untyped ceph_osd_up{ceph_daemon="osd.6"} 1.0 ceph_osd_up{ceph_daemon="osd.9"} 1.0 ceph_osd_up{ceph_daemon="osd.8"} 1.0 ceph_osd_up{ceph_daemon="osd.0"} 1.0 ceph_osd_up{ceph_daemon="osd.3"} 1.0 ceph_osd_up{ceph_daemon="osd.11"} 1.0 ceph_osd_up{ceph_daemon="osd.2"} 1.0 ceph_osd_up{ceph_daemon="osd.10"} 1.0 ceph_osd_up{ceph_daemon="osd.5"} 1.0 ceph_osd_up{ceph_daemon="osd.4"} 1.0 ceph_osd_up{ceph_daemon="osd.7"} 1.0 # HELP ceph_mon_election_lose Elections lost # TYPE ceph_mon_election_lose counter ceph_mon_election_lose{ceph_daemon="mon.weil04"} 38.0 ceph_mon_election_lose{ceph_daemon="mon.weil01"} 0.0 ceph_mon_election_lose{ceph_daemon="mon.weil02"} 27.0 # HELP ceph_osd_op_prepare_latency_sum Latency of client operations (excluding queue time and wait for finished) Total # TYPE ceph_osd_op_prepare_latency_sum counter ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.6"} 28965.283407301 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.9"} 53295.999030115 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.8"} 31382.240087502 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.0"} 102391.942611838 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.3"} 22603.052959328 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.11"} 34518.559857994 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.2"} 77557.453267721 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.10"} 41701.582256434 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.5"} 15519.570690461 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.4"} 23734.337494922 ceph_osd_op_prepare_latency_sum{ceph_daemon="osd.7"} 43003.999036733 # HELP ceph_bluefs_db_used_bytes Used bytes (main db device) # TYPE ceph_bluefs_db_used_bytes gauge ceph_bluefs_db_used_bytes{ceph_daemon="osd.6"} 1855979520.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.9"} 1983905792.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.8"} 1875902464.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.0"} 2045771776.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.3"} 1937768448.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.11"} 1652555776.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.2"} 3319791616.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.10"} 1823473664.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.5"} 1982857216.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.4"} 1816133632.0 ceph_bluefs_db_used_bytes{ceph_daemon="osd.7"} 2001731584.0 # HELP ceph_bluestore_kv_final_lat_count Average kv_finalize thread latency Count # TYPE ceph_bluestore_kv_final_lat_count counter ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.6"} 15343639.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.9"} 21065775.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.8"} 20518494.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.0"} 33446491.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.3"} 14177306.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.11"} 14729823.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.2"} 19270168.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.10"} 21361387.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.5"} 12885397.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.4"} 13779176.0 ceph_bluestore_kv_final_lat_count{ceph_daemon="osd.7"} 13940839.0 # HELP ceph_pool_quota_objects DF pool quota_objects # TYPE ceph_pool_quota_objects gauge ceph_pool_quota_objects{pool_id="5"} 0.0 ceph_pool_quota_objects{pool_id="3"} 0.0 ceph_pool_quota_objects{pool_id="4"} 0.0 # HELP ceph_osd_flag_nodown OSD Flag nodown # TYPE ceph_osd_flag_nodown untyped ceph_osd_flag_nodown 0.0 # HELP ceph_pg_forced_recovery PG forced_recovery # TYPE ceph_pg_forced_recovery gauge ceph_pg_forced_recovery 0.0 # HELP ceph_osd_op_wip Replication operations currently being processed (primary) # TYPE ceph_osd_op_wip gauge ceph_osd_op_wip{ceph_daemon="osd.6"} 1.0 ceph_osd_op_wip{ceph_daemon="osd.9"} 2.0 ceph_osd_op_wip{ceph_daemon="osd.8"} 0.0 ceph_osd_op_wip{ceph_daemon="osd.0"} 0.0 ceph_osd_op_wip{ceph_daemon="osd.3"} 0.0 ceph_osd_op_wip{ceph_daemon="osd.11"} 0.0 ceph_osd_op_wip{ceph_daemon="osd.2"} 1.0 ceph_osd_op_wip{ceph_daemon="osd.10"} 1.0 ceph_osd_op_wip{ceph_daemon="osd.5"} 0.0 ceph_osd_op_wip{ceph_daemon="osd.4"} 1.0 ceph_osd_op_wip{ceph_daemon="osd.7"} 0.0 # HELP ceph_osd_recovery_bytes recovery bytes # TYPE ceph_osd_recovery_bytes counter ceph_osd_recovery_bytes{ceph_daemon="osd.6"} 479045137753.0 ceph_osd_recovery_bytes{ceph_daemon="osd.9"} 1330641465250.0 ceph_osd_recovery_bytes{ceph_daemon="osd.8"} 1127721044496.0 ceph_osd_recovery_bytes{ceph_daemon="osd.0"} 2291463896337.0 ceph_osd_recovery_bytes{ceph_daemon="osd.3"} 188070161339.0 ceph_osd_recovery_bytes{ceph_daemon="osd.11"} 470804268763.0 ceph_osd_recovery_bytes{ceph_daemon="osd.2"} 0.0 ceph_osd_recovery_bytes{ceph_daemon="osd.10"} 1056214885498.0 ceph_osd_recovery_bytes{ceph_daemon="osd.5"} 421298833463.0 ceph_osd_recovery_bytes{ceph_daemon="osd.4"} 321454079833.0 ceph_osd_recovery_bytes{ceph_daemon="osd.7"} 494071383477.0 # HELP ceph_osd_op_w_in_bytes Client data written # TYPE ceph_osd_op_w_in_bytes counter ceph_osd_op_w_in_bytes{ceph_daemon="osd.6"} 77538089914.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.9"} 110756964274.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.8"} 103807743582.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.0"} 168873755654.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.3"} 68062752900.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.11"} 71213095362.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.2"} 159016471468.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.10"} 89435302456.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.5"} 55316986214.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.4"} 74963615744.0 ceph_osd_op_w_in_bytes{ceph_daemon="osd.7"} 75639760312.0 # HELP ceph_osd_op_w Client write operations # TYPE ceph_osd_op_w counter ceph_osd_op_w{ceph_daemon="osd.6"} 7389110.0 ceph_osd_op_w{ceph_daemon="osd.9"} 10567363.0 ceph_osd_op_w{ceph_daemon="osd.8"} 9979544.0 ceph_osd_op_w{ceph_daemon="osd.0"} 18356254.0 ceph_osd_op_w{ceph_daemon="osd.3"} 6881789.0 ceph_osd_op_w{ceph_daemon="osd.11"} 6946021.0 ceph_osd_op_w{ceph_daemon="osd.2"} 14910799.0 ceph_osd_op_w{ceph_daemon="osd.10"} 8550830.0 ceph_osd_op_w{ceph_daemon="osd.5"} 5713289.0 ceph_osd_op_w{ceph_daemon="osd.4"} 8078664.0 ceph_osd_op_w{ceph_daemon="osd.7"} 8098819.0 # HELP ceph_paxos_commit_bytes_sum Data in transaction on commit Total # TYPE ceph_paxos_commit_bytes_sum counter ceph_paxos_commit_bytes_sum{ceph_daemon="mon.weil04"} 0.0 ceph_paxos_commit_bytes_sum{ceph_daemon="mon.weil01"} 36392006990.0 ceph_paxos_commit_bytes_sum{ceph_daemon="mon.weil02"} 0.0 # HELP ceph_bluefs_log_bytes Size of the metadata log # TYPE ceph_bluefs_log_bytes gauge ceph_bluefs_log_bytes{ceph_daemon="osd.6"} 16642048.0 ceph_bluefs_log_bytes{ceph_daemon="osd.9"} 1122304.0 ceph_bluefs_log_bytes{ceph_daemon="osd.8"} 8556544.0 ceph_bluefs_log_bytes{ceph_daemon="osd.0"} 11304960.0 ceph_bluefs_log_bytes{ceph_daemon="osd.3"} 4534272.0 ceph_bluefs_log_bytes{ceph_daemon="osd.11"} 12926976.0 ceph_bluefs_log_bytes{ceph_daemon="osd.2"} 8544256.0 ceph_bluefs_log_bytes{ceph_daemon="osd.10"} 8847360.0 ceph_bluefs_log_bytes{ceph_daemon="osd.5"} 13778944.0 ceph_bluefs_log_bytes{ceph_daemon="osd.4"} 6213632.0 ceph_bluefs_log_bytes{ceph_daemon="osd.7"} 9691136.0 # HELP ceph_rocksdb_submit_sync_latency_count Submit Sync Latency Count # TYPE ceph_rocksdb_submit_sync_latency_count counter ceph_rocksdb_submit_sync_latency_count{ceph_daemon="mon.weil04"} 2872225.0 ceph_rocksdb_submit_sync_latency_count{ceph_daemon="mon.weil01"} 3158524.0 ceph_rocksdb_submit_sync_latency_count{ceph_daemon="mon.weil02"} 3151751.0