Feature #64167
openosd: add mclock queue perfcounter
0%
Description
asy to visualize the number of ops in each subqueue of mclock. Add request statistics for each op_scheduler_class ceph daemon osd.0 perf dump osd { "osd": { "mclock_immdiate_op": 640, "mclock_client_op": 1229, "mclock_recovery_op": 887, "mclock_best_effort_op": 925 } }
https://github.com/ceph/ceph/pull/55295
The reasons are as follows:
1. op request type
enum class op_scheduler_class : uint8_t { background_recovery = 0, background_best_effort, immediate, client, };
2. Current tools for viewing mclock queue
- Only the number of clients and requests of shard-queue can be displayed.
- Problem: There is no way to distinguish the number of handles for each request type
# ceph tell osd.10 dump_op_pq_state | jq { "OSD:ShardedOpWQ:0": { "queue_sizes": { "immediate": 0, "scheduler": 0 }, "mClockClients": { "client_count": 2, "clients": "{ PriorityQueue:: { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none } } { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.0831 p:225897.1451 l:149134.0831 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none } } }" }, "mClockQueues": { "queues": "RESER:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.0831 p:225897.1451 l:149134.0831 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.0831 p:225897.1451 l:149134.0831 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.0831 p:225897.1451 l:149134.0831 } req_count:0 top_req:none }" } }, "OSD:ShardedOpWQ:1": { "queue_sizes": { "immediate": 0, "scheduler": 0 }, "mClockClients": { "client_count": 2, "clients": "{ PriorityQueue:: { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.8296 p:248969.0061 l:149134.2010 } req_count:0 top_req:none } } { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none } } }" }, "mClockQueues": { "queues": "RESER:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.8296 p:248969.0061 l:149134.2010 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.8296 p:248969.0061 l:149134.2010 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.8296 p:248969.0061 l:149134.2010 } req_count:0 top_req:none }" } }, "OSD:ShardedOpWQ:2": { "queue_sizes": { "immediate": 0, "scheduler": 0 }, "mClockClients": { "client_count": 2, "clients": "{ PriorityQueue:: { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none } } { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1760 p:225219.9794 l:149134.1760 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none } } }" }, "mClockQueues": { "queues": "RESER:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1760 p:225219.9794 l:149134.1760 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1760 p:225219.9794 l:149134.1760 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1760 p:225219.9794 l:149134.1760 } req_count:0 top_req:none }" } }, "OSD:ShardedOpWQ:3": { "queue_sizes": { "immediate": 0, "scheduler": 0 }, "mClockClients": { "client_count": 2, "clients": "{ PriorityQueue:: { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none } } { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1993 p:213865.4879 l:149134.1993 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none } } }" }, "mClockQueues": { "queues": "RESER:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1993 p:213865.4879 l:149134.1993 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1993 p:213865.4879 l:149134.1993 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1993 p:213865.4879 l:149134.1993 } req_count:0 top_req:none }" } }, "OSD:ShardedOpWQ:4": { "queue_sizes": { "immediate": 0, "scheduler": 0 }, "mClockClients": { "client_count": 2, "clients": "{ PriorityQueue:: { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none } } { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1605 p:211845.4401 l:149134.1695 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none } } }" }, "mClockQueues": { "queues": "RESER:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1605 p:211845.4401 l:149134.1695 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1605 p:211845.4401 l:149134.1695 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1605 p:211845.4401 l:149134.1695 } req_count:0 top_req:none }" } } }
The design of this plan:
1. By adding mclock perfcounter to osd,
2. Before requesting to join the team,
3. Count requests for background_recovery / background_best_effort / immediate / client respectively.
benefit:
1. You can quickly export the processing counts of various requests through ceph tell osd.x perf dump osd.
2. It is convenient for the monitoring component to trace the history of the number of various requests processed by mclock.
3. When analyzing problems, it helps to analyze the source of disk pressure.
No data to display