Ceph

I'm just spitting in the wind here, but maybe we're leaking the xattrs that get shipped over the wire? That occurred to me recently as something that bugs some of our users but that we haven't observed as a problem in the lab.

and possibly also 'log max recent = big number' (100000 by default).

Dropping the default down, and closing this bug. The pg log trimming stuff is already fixed.

as Sam requested, here is the perf dump for each osds on the cluster

on burnupi06:

ubuntu@burnupi06:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.0.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":998367,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":782502690274,"journal_latency":{"avgcount":998367,"sum":28080.685740000},"journal_wr":822064,"journal_wr_bytes":{"avgcount":822064,"sum":785951854592},"op_queue_max_ops":50,"op_queue_ops":0,"ops":998367,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":782493403164,"apply_latency":{"avgcount":998367,"sum":61367.800782000},"committing":0,"commitcycle":29003,"commitcycle_interval":{"avgcount":29003,"sum":148912.317293000},"commitcycle_latency":{"avgcount":29003,"sum":3791.629337000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":570114,"op_in_bytes":396232834645,"op_out_bytes":944659334,"op_latency":{"avgcount":570114,"sum":147677.229038000},"op_r":47544,"op_r_out_bytes":944628728,"op_r_latency":{"avgcount":47544,"sum":1396.782964000},"op_w":521478,"op_w_in_bytes":395453341595,"op_w_rlat":{"avgcount":521478,"sum":8781.429102000},"op_w_latency":{"avgcount":521478,"sum":145879.128186000},"op_rw":1092,"op_rw_in_bytes":779493050,"op_rw_out_bytes":30606,"op_rw_rlat":{"avgcount":1092,"sum":131.950932000},"op_rw_latency":{"avgcount":1092,"sum":401.317888000},"subop":422513,"subop_in_bytes":391063966081,"subop_latency":{"avgcount":422513,"sum":26267.816650000},"subop_w":0,"subop_w_in_bytes":391063966081,"subop_w_latency":{"avgcount":422513,"sum":26267.816650000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":3774,"push_out_bytes":12336812429,"push_in":0,"push_in_bytes":0,"recovery_ops":3776,"loadavg":17,"buffer_bytes":0,"numpg":389,"numpg_primary":204,"numpg_replica":185,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":2941,"map_message_epochs":19071,"map_message_epoch_dups":18768},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":998367,"take_sum":782502690274,"put":822064,"put_sum":782502690274,"wait":{"avgcount":8,"sum":21.122997000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":998367,"take_sum":998367,"put":822064,"put_sum":998367,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2567538,"get_sum":390437860712,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2567538,"put_sum":390437860712,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1178040,"get_sum":437155611483,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1178040,"put_sum":437155611483,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2672166,"get_sum":125591802,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2672166,"put_sum":125591802,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2695424,"get_sum":126684928,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2695424,"put_sum":126684928,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2527169,"get_sum":390421875228,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2964148,"put_sum":390421875228,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi06:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.1.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":960262,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":765872880232,"journal_latency":{"avgcount":960262,"sum":22874.848507000},"journal_wr":788949,"journal_wr_bytes":{"avgcount":788949,"sum":769256820736},"op_queue_max_ops":50,"op_queue_ops":0,"ops":960262,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":765863912777,"apply_latency":{"avgcount":960262,"sum":43505.299078000},"committing":0,"commitcycle":29060,"commitcycle_interval":{"avgcount":29060,"sum":148990.942643000},"commitcycle_latency":{"avgcount":29060,"sum":3608.326023000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":499963,"op_in_bytes":340561151436,"op_out_bytes":806902334,"op_latency":{"avgcount":499963,"sum":140402.492477000},"op_r":53202,"op_r_out_bytes":806858518,"op_r_latency":{"avgcount":53202,"sum":537.139874000},"op_w":445162,"op_w_in_bytes":339928977803,"op_w_rlat":{"avgcount":445162,"sum":7575.826943000},"op_w_latency":{"avgcount":445162,"sum":139282.321266000},"op_rw":1599,"op_rw_in_bytes":632173633,"op_rw_out_bytes":43816,"op_rw_rlat":{"avgcount":1599,"sum":142.612245000},"op_rw_latency":{"avgcount":1599,"sum":583.031337000},"subop":462319,"subop_in_bytes":429027743099,"subop_latency":{"avgcount":462319,"sum":28810.183374000},"subop_w":0,"subop_w_in_bytes":429027743099,"subop_w_latency":{"avgcount":462319,"sum":28810.183374000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":0,"buffer_bytes":0,"numpg":408,"numpg_primary":183,"numpg_replica":225,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3930,"map_message_epochs":20339,"map_message_epoch_dups":20014},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":960262,"take_sum":765872880232,"put":788949,"put_sum":765872880232,"wait":{"avgcount":15,"sum":7.511155000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":960262,"take_sum":960262,"put":788949,"put_sum":960262,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2206466,"get_sum":335946728381,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2206466,"put_sum":335946728381,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1104101,"get_sum":472844784747,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1104101,"put_sum":472844784747,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2704695,"get_sum":127120665,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2704695,"put_sum":127120665,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2700602,"get_sum":126928294,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2700602,"put_sum":126928294,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2157565,"get_sum":335931843530,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2528959,"put_sum":335931843530,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi06:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.2.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1028403,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":822348078490,"journal_latency":{"avgcount":1028403,"sum":23407.661627000},"journal_wr":850366,"journal_wr_bytes":{"avgcount":850366,"sum":825937768448},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1028403,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":822338427700,"apply_latency":{"avgcount":1028403,"sum":45940.947635000},"committing":0,"commitcycle":28942,"commitcycle_interval":{"avgcount":28942,"sum":148515.980642000},"commitcycle_latency":{"avgcount":28942,"sum":3714.417922000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":566759,"op_in_bytes":385354332813,"op_out_bytes":1167903605,"op_latency":{"avgcount":566759,"sum":186032.930317000},"op_r":72714,"op_r_out_bytes":1167879647,"op_r_latency":{"avgcount":72714,"sum":1008.429935000},"op_w":493043,"op_w_in_bytes":384589907024,"op_w_rlat":{"avgcount":493043,"sum":7991.858544000},"op_w_latency":{"avgcount":493043,"sum":184867.757569000},"op_rw":1002,"op_rw_in_bytes":764425789,"op_rw_out_bytes":23958,"op_rw_rlat":{"avgcount":1002,"sum":45.484770000},"op_rw_latency":{"avgcount":1002,"sum":156.742813000},"subop":485313,"subop_in_bytes":440697218095,"subop_latency":{"avgcount":485313,"sum":29397.187169000},"subop_w":0,"subop_w_in_bytes":440697218095,"subop_w_latency":{"avgcount":485313,"sum":29397.187169000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":0,"buffer_bytes":0,"numpg":402,"numpg_primary":191,"numpg_replica":211,"numpg_stray":0,"heartbeat_to_peers":13,"heartbeat_from_peers":0,"map_messages":3993,"map_message_epochs":25269,"map_message_epoch_dups":24955},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1028403,"take_sum":822348078490,"put":850366,"put_sum":822348078490,"wait":{"avgcount":1,"sum":0.564661000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1028403,"take_sum":1028403,"put":850366,"put_sum":1028403,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2483906,"get_sum":380715309309,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2483906,"put_sum":380715309309,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1186005,"get_sum":493606372828,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1186005,"put_sum":493606372828,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2514744,"get_sum":118192968,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2514744,"put_sum":118192968,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2700644,"get_sum":126930268,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2700644,"put_sum":126930268,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2434931,"get_sum":380698701688,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2851586,"put_sum":380698701688,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi06:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.3.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1050519,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":833845024775,"journal_latency":{"avgcount":1050519,"sum":22887.995658000},"journal_wr":868067,"journal_wr_bytes":{"avgcount":868067,"sum":837470330880},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1050519,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":833835174125,"apply_latency":{"avgcount":1050519,"sum":45749.919700000},"committing":0,"commitcycle":29088,"commitcycle_interval":{"avgcount":29088,"sum":149311.920069000},"commitcycle_latency":{"avgcount":29088,"sum":3767.624298000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":571723,"op_in_bytes":397802214418,"op_out_bytes":1283418462,"op_latency":{"avgcount":571723,"sum":181797.146348000},"op_r":56702,"op_r_out_bytes":1283397938,"op_r_latency":{"avgcount":56702,"sum":990.902338000},"op_w":514093,"op_w_in_bytes":396985599653,"op_w_rlat":{"avgcount":514093,"sum":7636.570330000},"op_w_latency":{"avgcount":514093,"sum":180634.005935000},"op_rw":928,"op_rw_in_bytes":816614765,"op_rw_out_bytes":20524,"op_rw_rlat":{"avgcount":928,"sum":51.252302000},"op_rw_latency":{"avgcount":928,"sum":172.238075000},"subop":489339,"subop_in_bytes":440366926405,"subop_latency":{"avgcount":489339,"sum":28917.987346000},"subop_w":0,"subop_w_in_bytes":440366926405,"subop_w_latency":{"avgcount":489339,"sum":28917.987346000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":0,"buffer_bytes":0,"numpg":414,"numpg_primary":192,"numpg_replica":222,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":2830,"map_message_epochs":17190,"map_message_epoch_dups":16860},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1050519,"take_sum":833845024775,"put":868067,"put_sum":833845024775,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1050519,"take_sum":1050519,"put":868067,"put_sum":1050519,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2569206,"get_sum":392483532991,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2569206,"put_sum":392483532991,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1210015,"get_sum":495931444746,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1210015,"put_sum":495931444746,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2705079,"get_sum":127138713,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2705079,"put_sum":127138713,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2700336,"get_sum":126915792,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2700336,"put_sum":126915792,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2528979,"get_sum":392469333020,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2959166,"put_sum":392469333020,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi06:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.4.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":823581,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":662219222471,"journal_latency":{"avgcount":823581,"sum":18612.974454000},"journal_wr":691520,"journal_wr_bytes":{"avgcount":691520,"sum":665108504576},"op_queue_max_ops":50,"op_queue_ops":0,"ops":823581,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":662211494646,"apply_latency":{"avgcount":823581,"sum":33671.790773000},"committing":0,"commitcycle":28746,"commitcycle_interval":{"avgcount":28746,"sum":146906.002984000},"commitcycle_latency":{"avgcount":28746,"sum":3107.340996000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":480486,"op_in_bytes":341565540896,"op_out_bytes":716915774,"op_latency":{"avgcount":480486,"sum":170334.022063000},"op_r":44275,"op_r_out_bytes":716897624,"op_r_latency":{"avgcount":44275,"sum":507.875764000},"op_w":435395,"op_w_in_bytes":340972620581,"op_w_rlat":{"avgcount":435395,"sum":6960.162008000},"op_w_latency":{"avgcount":435395,"sum":169654.789052000},"op_rw":816,"op_rw_in_bytes":592920315,"op_rw_out_bytes":18150,"op_rw_rlat":{"avgcount":816,"sum":50.462317000},"op_rw_latency":{"avgcount":816,"sum":171.357247000},"subop":351706,"subop_in_bytes":322640766245,"subop_latency":{"avgcount":351706,"sum":21501.727409000},"subop_w":0,"subop_w_in_bytes":322640766245,"subop_w_latency":{"avgcount":351706,"sum":21501.727409000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":1206,"push_out_bytes":3838901756,"push_in":0,"push_in_bytes":0,"recovery_ops":1207,"loadavg":0,"buffer_bytes":0,"numpg":330,"numpg_primary":171,"numpg_replica":159,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":2759,"map_message_epochs":17025,"map_message_epoch_dups":16689},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":823581,"take_sum":662219222471,"put":691520,"put_sum":662219222471,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":823581,"take_sum":823581,"put":691520,"put_sum":823581,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2250193,"get_sum":338828024124,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2250193,"put_sum":338828024124,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":955524,"get_sum":369661060550,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":955524,"put_sum":369661060550,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2705069,"get_sum":127138243,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2705069,"put_sum":127138243,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2700667,"get_sum":126931349,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2700667,"put_sum":126931349,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2210379,"get_sum":338815143290,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2580825,"put_sum":338815143290,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi06:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.5.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":922288,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":730324159538,"journal_latency":{"avgcount":922288,"sum":19159.263153000},"journal_wr":773145,"journal_wr_bytes":{"avgcount":773145,"sum":733531480064},"op_queue_max_ops":50,"op_queue_ops":0,"ops":922288,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":730315532543,"apply_latency":{"avgcount":922288,"sum":37011.464065000},"committing":0,"commitcycle":28983,"commitcycle_interval":{"avgcount":28983,"sum":148379.918671000},"commitcycle_latency":{"avgcount":28983,"sum":3383.415827000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":587566,"op_in_bytes":405332614414,"op_out_bytes":1024971345,"op_latency":{"avgcount":587566,"sum":163673.049973000},"op_r":70814,"op_r_out_bytes":1024844776,"op_r_latency":{"avgcount":70814,"sum":1001.099183000},"op_w":515583,"op_w_in_bytes":404574230582,"op_w_rlat":{"avgcount":515583,"sum":7080.888414000},"op_w_latency":{"avgcount":515583,"sum":162334.447527000},"op_rw":1169,"op_rw_in_bytes":758383832,"op_rw_out_bytes":126569,"op_rw_rlat":{"avgcount":1169,"sum":118.761890000},"op_rw_latency":{"avgcount":1169,"sum":337.503263000},"subop":366008,"subop_in_bytes":328520418689,"subop_latency":{"avgcount":366008,"sum":21056.322549000},"subop_w":0,"subop_w_in_bytes":328520418689,"subop_w_latency":{"avgcount":366008,"sum":21056.322549000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":0,"buffer_bytes":0,"numpg":381,"numpg_primary":195,"numpg_replica":186,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3936,"map_message_epochs":16297,"map_message_epoch_dups":15966},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":922288,"take_sum":730324159538,"put":773145,"put_sum":730324159538,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":922288,"take_sum":922288,"put":773145,"put_sum":922288,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2625709,"get_sum":400983495889,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2625709,"put_sum":400983495889,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1058107,"get_sum":385766152051,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1058107,"put_sum":385766152051,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2704137,"get_sum":127094439,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2704137,"put_sum":127094439,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2700686,"get_sum":126932242,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2700686,"put_sum":126932242,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2576552,"get_sum":400968602893,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3014186,"put_sum":400968602893,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi06:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.6.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1124237,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":914838804038,"journal_latency":{"avgcount":1124237,"sum":25805.287594000},"journal_wr":924062,"journal_wr_bytes":{"avgcount":924062,"sum":918786035712},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1124237,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":914828264858,"apply_latency":{"avgcount":1124237,"sum":51725.118482000},"committing":0,"commitcycle":29253,"commitcycle_interval":{"avgcount":29253,"sum":150500.964351000},"commitcycle_latency":{"avgcount":29253,"sum":4138.780995000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":676680,"op_in_bytes":466518475211,"op_out_bytes":1323236081,"op_latency":{"avgcount":676680,"sum":191140.181120000},"op_r":74479,"op_r_out_bytes":1323172990,"op_r_latency":{"avgcount":74479,"sum":1787.979874000},"op_w":599382,"op_w_in_bytes":465605268600,"op_w_rlat":{"avgcount":599382,"sum":9550.309306000},"op_w_latency":{"avgcount":599382,"sum":188522.700157000},"op_rw":2819,"op_rw_in_bytes":913206611,"op_rw_out_bytes":63091,"op_rw_rlat":{"avgcount":2819,"sum":297.787484000},"op_rw_latency":{"avgcount":2819,"sum":829.501089000},"subop":484618,"subop_in_bytes":454430416673,"subop_latency":{"avgcount":484618,"sum":29901.230331000},"subop_w":0,"subop_w_in_bytes":454430416673,"subop_w_latency":{"avgcount":484618,"sum":29901.230331000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":3719,"push_out_bytes":12359347978,"push_in":0,"push_in_bytes":0,"recovery_ops":3721,"loadavg":0,"buffer_bytes":0,"numpg":453,"numpg_primary":222,"numpg_replica":231,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3971,"map_message_epochs":20058,"map_message_epoch_dups":19744},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1124237,"take_sum":914838804038,"put":924062,"put_sum":914838804038,"wait":{"avgcount":2,"sum":0.065886000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1124237,"take_sum":1124237,"put":924062,"put_sum":1124237,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":3031751,"get_sum":459396231913,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3031751,"put_sum":459396231913,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1302931,"get_sum":519278532906,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1302931,"put_sum":519278532906,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2704036,"get_sum":127089692,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2704036,"put_sum":127089692,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2700700,"get_sum":126932900,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2700700,"put_sum":126932900,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2981959,"get_sum":459378829670,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3482130,"put_sum":459378829670,"wait":{"avgcount":0,"sum":0.000000000}}}

burnupi07:
ubuntu@burnupi07:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.7.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":844207,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":675554973492,"journal_latency":{"avgcount":844207,"sum":48487.858864000},"journal_wr":678576,"journal_wr_bytes":{"avgcount":678576,"sum":678506606592},"op_queue_max_ops":50,"op_queue_ops":0,"ops":844207,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":675547052257,"apply_latency":{"avgcount":844207,"sum":101701.215333000},"committing":0,"commitcycle":26735,"commitcycle_interval":{"avgcount":26735,"sum":146994.353742000},"commitcycle_latency":{"avgcount":26735,"sum":12884.550519000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":552231,"op_in_bytes":354647368680,"op_out_bytes":769249754,"op_latency":{"avgcount":552231,"sum":140507.402696000},"op_r":52047,"op_r_out_bytes":769216327,"op_r_latency":{"avgcount":52047,"sum":275.807283000},"op_w":499059,"op_w_in_bytes":354144120694,"op_w_rlat":{"avgcount":499059,"sum":33564.874479000},"op_w_latency":{"avgcount":499059,"sum":140113.495944000},"op_rw":1125,"op_rw_in_bytes":503247986,"op_rw_out_bytes":33427,"op_rw_rlat":{"avgcount":1125,"sum":20.122159000},"op_rw_latency":{"avgcount":1125,"sum":118.099469000},"subop":400948,"subop_in_bytes":371308879462,"subop_latency":{"avgcount":400948,"sum":46580.627508000},"subop_w":0,"subop_w_in_bytes":371308879462,"subop_w_latency":{"avgcount":400948,"sum":46580.627508000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":11,"buffer_bytes":0,"numpg":342,"numpg_primary":150,"numpg_replica":192,"numpg_stray":0,"heartbeat_to_peers":13,"heartbeat_from_peers":0,"map_messages":3890,"map_message_epochs":21408,"map_message_epoch_dups":21057},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":844207,"take_sum":675554973492,"put":678576,"put_sum":675554973492,"wait":{"avgcount":497,"sum":661.515846000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":844207,"take_sum":844207,"put":678576,"put_sum":844207,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":1985254,"get_sum":303456176474,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1985254,"put_sum":303456176474,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":973881,"get_sum":412535678196,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":973881,"put_sum":412535678196,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2509701,"get_sum":117955947,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2509701,"put_sum":117955947,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2701388,"get_sum":126965236,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701388,"put_sum":126965236,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":1937038,"get_sum":303442477896,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2276734,"put_sum":303442477896,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi07:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.8.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":961550,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":768637333083,"journal_latency":{"avgcount":961550,"sum":96670.383311000},"journal_wr":739268,"journal_wr_bytes":{"avgcount":739268,"sum":771997945856},"op_queue_max_ops":50,"op_queue_ops":0,"ops":961550,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":768628322328,"apply_latency":{"avgcount":961550,"sum":231585.047170000},"committing":0,"commitcycle":25467,"commitcycle_interval":{"avgcount":25467,"sum":148197.975636000},"commitcycle_latency":{"avgcount":25467,"sum":19850.462019000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":631917,"op_in_bytes":439060977927,"op_out_bytes":939390995,"op_latency":{"avgcount":631917,"sum":494592.880522000},"op_r":55463,"op_r_out_bytes":939373130,"op_r_latency":{"avgcount":55463,"sum":2043.947316000},"op_w":575681,"op_w_in_bytes":438504367474,"op_w_rlat":{"avgcount":575681,"sum":166480.438858000},"op_w_latency":{"avgcount":575681,"sum":492346.295104000},"op_rw":773,"op_rw_in_bytes":556610453,"op_rw_out_bytes":17865,"op_rw_rlat":{"avgcount":773,"sum":31.024660000},"op_rw_latency":{"avgcount":773,"sum":202.638102000},"subop":458960,"subop_in_bytes":414480828203,"subop_latency":{"avgcount":458960,"sum":101730.146826000},"subop_w":0,"subop_w_in_bytes":414480828203,"subop_w_latency":{"avgcount":457691,"sum":101538.427061000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":1269,"sum":191.719765000},"pull":0,"push":0,"push_out_bytes":0,"push_in":1269,"push_in_bytes":4169321754,"recovery_ops":0,"loadavg":10,"buffer_bytes":0,"numpg":390,"numpg_primary":174,"numpg_replica":216,"numpg_stray":0,"heartbeat_to_peers":15,"heartbeat_from_peers":0,"map_messages":2757,"map_message_epochs":19556,"map_message_epoch_dups":19207},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":961550,"take_sum":768637333083,"put":739268,"put_sum":768637333083,"wait":{"avgcount":2396,"sum":3156.240349000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":961550,"take_sum":961550,"put":739268,"put_sum":961550,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2310794,"get_sum":349095129344,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2310794,"put_sum":349095129344,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1107627,"get_sum":466342350114,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1107627,"put_sum":466342350114,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2734270,"get_sum":128510690,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2734270,"put_sum":128510690,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2701417,"get_sum":126966599,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701417,"put_sum":126966599,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2270992,"get_sum":349081323410,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2654056,"put_sum":349081323410,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi07:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.9.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1017961,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":809726980153,"journal_latency":{"avgcount":1017961,"sum":105515.974270000},"journal_wr":772022,"journal_wr_bytes":{"avgcount":772022,"sum":813260963840},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1017961,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":809717452723,"apply_latency":{"avgcount":1017961,"sum":281669.666295000},"committing":0,"commitcycle":25165,"commitcycle_interval":{"avgcount":25165,"sum":149584.657488000},"commitcycle_latency":{"avgcount":25165,"sum":22686.727007000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":734419,"op_in_bytes":510570297343,"op_out_bytes":1153950721,"op_latency":{"avgcount":734419,"sum":497831.303734000},"op_r":58767,"op_r_out_bytes":1153920593,"op_r_latency":{"avgcount":58767,"sum":497.564103000},"op_w":674549,"op_w_in_bytes":509885441990,"op_w_rlat":{"avgcount":674549,"sum":163196.625809000},"op_w_latency":{"avgcount":674549,"sum":497173.529276000},"op_rw":1103,"op_rw_in_bytes":684855353,"op_rw_out_bytes":30128,"op_rw_rlat":{"avgcount":1103,"sum":29.738439000},"op_rw_latency":{"avgcount":1103,"sum":160.210355000},"subop":448316,"subop_in_bytes":400504995517,"subop_latency":{"avgcount":448316,"sum":95662.422962000},"subop_w":0,"subop_w_in_bytes":400504995517,"subop_w_latency":{"avgcount":444542,"sum":94299.453081000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":3774,"sum":1362.969881000},"pull":0,"push":0,"push_out_bytes":0,"push_in":3774,"push_in_bytes":12336812429,"recovery_ops":0,"loadavg":10,"buffer_bytes":0,"numpg":411,"numpg_primary":201,"numpg_replica":210,"numpg_stray":0,"heartbeat_to_peers":15,"heartbeat_from_peers":0,"map_messages":3816,"map_message_epochs":19945,"map_message_epoch_dups":19566},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1017961,"take_sum":809726980153,"put":772022,"put_sum":809726980153,"wait":{"avgcount":2504,"sum":3081.767744000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1017961,"take_sum":1017961,"put":772022,"put_sum":1017961,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2638459,"get_sum":395964196070,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2638459,"put_sum":395964196070,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1169633,"get_sum":467054469929,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1169633,"put_sum":467054469929,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2737224,"get_sum":128649528,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2737224,"put_sum":128649528,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2701432,"get_sum":126967304,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701432,"put_sum":126967304,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2589295,"get_sum":395948204240,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3032650,"put_sum":395948204240,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi07:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.10.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":994500,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":807893934875,"journal_latency":{"avgcount":994500,"sum":123102.665708000},"journal_wr":752803,"journal_wr_bytes":{"avgcount":752803,"sum":811415359488},"op_queue_max_ops":50,"op_queue_ops":0,"ops":994500,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":807884639435,"apply_latency":{"avgcount":994500,"sum":313555.601525000},"committing":0,"commitcycle":24963,"commitcycle_interval":{"avgcount":24963,"sum":149761.604221000},"commitcycle_latency":{"avgcount":24963,"sum":23856.144568000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":734162,"op_in_bytes":524123028539,"op_out_bytes":1284099316,"op_latency":{"avgcount":734162,"sum":1971077.858424000},"op_r":76786,"op_r_out_bytes":1284063122,"op_r_latency":{"avgcount":76786,"sum":3790.567936000},"op_w":656068,"op_w_in_bytes":523327755969,"op_w_rlat":{"avgcount":656068,"sum":789215.808199000},"op_w_latency":{"avgcount":656068,"sum":1966775.031892000},"op_rw":1308,"op_rw_in_bytes":795272570,"op_rw_out_bytes":36194,"op_rw_rlat":{"avgcount":1308,"sum":170.743443000},"op_rw_latency":{"avgcount":1308,"sum":512.258596000},"subop":433533,"subop_in_bytes":392466775883,"subop_latency":{"avgcount":433533,"sum":122592.571466000},"subop_w":0,"subop_w_in_bytes":392466775883,"subop_w_latency":{"avgcount":430977,"sum":122311.475441000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":2556,"sum":281.096025000},"pull":0,"push":0,"push_out_bytes":0,"push_in":2556,"push_in_bytes":8433463737,"recovery_ops":0,"loadavg":9,"buffer_bytes":0,"numpg":411,"numpg_primary":207,"numpg_replica":204,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":2756,"map_message_epochs":14897,"map_message_epoch_dups":14559},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":994500,"take_sum":807893934875,"put":752803,"put_sum":807893934875,"wait":{"avgcount":3729,"sum":6487.718862000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":994500,"take_sum":994500,"put":752803,"put_sum":994500,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2677864,"get_sum":406121934400,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2677864,"put_sum":406121934400,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1145004,"get_sum":455781987161,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1145004,"put_sum":455781987161,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2704611,"get_sum":127116717,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2704611,"put_sum":127116717,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2512909,"get_sum":118106723,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2512909,"put_sum":118106723,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2637224,"get_sum":406107214279,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3071071,"put_sum":406107214279,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi07:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.11.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1047996,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":834803003008,"journal_latency":{"avgcount":1047996,"sum":173962.203335000},"journal_wr":628989,"journal_wr_bytes":{"avgcount":628989,"sum":838464704512},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1047996,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":834793135313,"apply_latency":{"avgcount":1047996,"sum":563111.767607000},"committing":0,"commitcycle":22938,"commitcycle_interval":{"avgcount":22938,"sum":148497.340188000},"commitcycle_latency":{"avgcount":22938,"sum":32319.365094000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":789343,"op_in_bytes":527693016123,"op_out_bytes":1096545902,"op_latency":{"avgcount":789343,"sum":1310865.645280000},"op_r":62874,"op_r_out_bytes":1096511282,"op_r_latency":{"avgcount":62874,"sum":447.730406000},"op_w":725298,"op_w_in_bytes":527066944119,"op_w_rlat":{"avgcount":725298,"sum":501858.283292000},"op_w_latency":{"avgcount":725298,"sum":1310261.591882000},"op_rw":1171,"op_rw_in_bytes":626072004,"op_rw_out_bytes":34620,"op_rw_rlat":{"avgcount":1171,"sum":23.598872000},"op_rw_latency":{"avgcount":1171,"sum":156.322992000},"subop":488027,"subop_in_bytes":440764019600,"subop_latency":{"avgcount":488027,"sum":177385.170170000},"subop_w":0,"subop_w_in_bytes":440764019600,"subop_w_latency":{"avgcount":488027,"sum":177385.170170000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":8,"buffer_bytes":0,"numpg":399,"numpg_primary":180,"numpg_replica":219,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3917,"map_message_epochs":20565,"map_message_epoch_dups":20230},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1047996,"take_sum":834803003008,"put":628989,"put_sum":834803003008,"wait":{"avgcount":4313,"sum":4936.881017000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1047996,"take_sum":1047996,"put":628989,"put_sum":1047996,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2566472,"get_sum":393065113158,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2566472,"put_sum":393065113158,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1229245,"get_sum":498062677572,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1229245,"put_sum":498062677572,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2701728,"get_sum":126981216,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701728,"put_sum":126981216,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2701470,"get_sum":126969090,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701470,"put_sum":126969090,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2517825,"get_sum":393049285070,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2955605,"put_sum":393049285070,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi07:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.12.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1055073,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":853004048887,"journal_latency":{"avgcount":1055073,"sum":139881.027751000},"journal_wr":791882,"journal_wr_bytes":{"avgcount":791882,"sum":856740306944},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1055073,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":852994205287,"apply_latency":{"avgcount":1055073,"sum":388408.305781000},"committing":0,"commitcycle":24583,"commitcycle_interval":{"avgcount":24583,"sum":151523.637724000},"commitcycle_latency":{"avgcount":24583,"sum":27148.454902000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":895611,"op_in_bytes":662015196869,"op_out_bytes":1289088798,"op_latency":{"avgcount":895611,"sum":3889600.493276000},"op_r":72174,"op_r_out_bytes":1289043753,"op_r_latency":{"avgcount":72174,"sum":1000.760164000},"op_w":821829,"op_w_in_bytes":661153315694,"op_w_rlat":{"avgcount":821829,"sum":1655151.901806000},"op_w_latency":{"avgcount":821829,"sum":3887930.271097000},"op_rw":1608,"op_rw_in_bytes":861881175,"op_rw_out_bytes":45045,"op_rw_rlat":{"avgcount":1608,"sum":231.040710000},"op_rw_latency":{"avgcount":1608,"sum":669.462015000},"subop":381728,"subop_in_bytes":337064987180,"subop_latency":{"avgcount":381728,"sum":123483.099204000},"subop_w":0,"subop_w_in_bytes":337064987180,"subop_w_latency":{"avgcount":375586,"sum":119281.371759000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":6142,"sum":4201.727445000},"pull":0,"push":0,"push_out_bytes":0,"push_in":6142,"push_in_bytes":20313635486,"recovery_ops":0,"loadavg":8,"buffer_bytes":0,"numpg":439,"numpg_primary":264,"numpg_replica":175,"numpg_stray":0,"heartbeat_to_peers":15,"heartbeat_from_peers":0,"map_messages":2822,"map_message_epochs":14992,"map_message_epoch_dups":14631},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1055073,"take_sum":853004048887,"put":791882,"put_sum":853004048887,"wait":{"avgcount":3384,"sum":4299.544839000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1055073,"take_sum":1055073,"put":791882,"put_sum":1055073,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":3257425,"get_sum":494728283764,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3257425,"put_sum":494728283764,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1213835,"get_sum":422590506354,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1213835,"put_sum":422590506354,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2731394,"get_sum":128375518,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2731394,"put_sum":128375518,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2706364,"get_sum":127199108,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2706364,"put_sum":127199108,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":3216219,"get_sum":494710727405,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3754769,"put_sum":494710727405,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi07:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.13.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1120712,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":941442719074,"journal_latency":{"avgcount":1120712,"sum":342681.982353000},"journal_wr":740930,"journal_wr_bytes":{"avgcount":740930,"sum":945402691584},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1120712,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":941432184469,"apply_latency":{"avgcount":1120712,"sum":1653565.941907000},"committing":0,"commitcycle":17013,"commitcycle_interval":{"avgcount":17013,"sum":151085.750806000},"commitcycle_latency":{"avgcount":17013,"sum":61679.687306000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":837340,"op_in_bytes":786678383460,"op_out_bytes":1311317814,"op_latency":{"avgcount":837340,"sum":69428531.506788000},"op_r":72319,"op_r_out_bytes":1311275780,"op_r_latency":{"avgcount":72319,"sum":1075.920746000},"op_w":763456,"op_w_in_bytes":785795780551,"op_w_rlat":{"avgcount":763456,"sum":33515976.929772000},"op_w_latency":{"avgcount":763456,"sum":69426972.581162000},"op_rw":1565,"op_rw_in_bytes":882602909,"op_rw_out_bytes":42034,"op_rw_rlat":{"avgcount":1565,"sum":147.849432000},"op_rw_latency":{"avgcount":1565,"sum":483.004880000},"subop":533050,"subop_in_bytes":487787897594,"subop_latency":{"avgcount":533050,"sum":381168.653471000},"subop_w":0,"subop_w_in_bytes":487787897594,"subop_w_latency":{"avgcount":533050,"sum":381168.653471000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":5042,"push_out_bytes":16718171243,"push_in":0,"push_in_bytes":0,"recovery_ops":5045,"loadavg":8,"buffer_bytes":0,"numpg":427,"numpg_primary":199,"numpg_replica":227,"numpg_stray":1,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3984,"map_message_epochs":17689,"map_message_epoch_dups":17358},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1120712,"take_sum":941442719074,"put":740930,"put_sum":941442719074,"wait":{"avgcount":16891,"sum":17085.411037000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1120712,"take_sum":1120712,"put":740930,"put_sum":1120712,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2813502,"get_sum":453247308314,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2813502,"put_sum":453247308314,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1322598,"get_sum":547267712561,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1322598,"put_sum":547267712561,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2701113,"get_sum":126952311,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701113,"put_sum":126952311,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2793429,"get_sum":131291163,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2793429,"put_sum":131291163,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2762071,"get_sum":453229542910,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3223609,"put_sum":453229542910,"wait":{"avgcount":13517,"sum":21965.928171000}}}

burnupi13:

ubuntu@burnupi13:/var/run/ceph$  sudo ceph --admin-daemon ceph-osd.14.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1029094,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":823777527413,"journal_latency":{"avgcount":1029094,"sum":141622.756467000},"journal_wr":755678,"journal_wr_bytes":{"avgcount":755678,"sum":827410341888},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1029094,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":823767889188,"apply_latency":{"avgcount":1029094,"sum":394172.223134000},"committing":0,"commitcycle":24232,"commitcycle_interval":{"avgcount":24232,"sum":149326.760964000},"commitcycle_latency":{"avgcount":24232,"sum":26720.722204000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":711766,"op_in_bytes":560539725308,"op_out_bytes":1314455817,"op_latency":{"avgcount":711766,"sum":2157673.460708000},"op_r":71973,"op_r_out_bytes":1314432130,"op_r_latency":{"avgcount":71973,"sum":217.391886000},"op_w":638855,"op_w_in_bytes":559730660046,"op_w_rlat":{"avgcount":638855,"sum":870507.060735000},"op_w_latency":{"avgcount":638855,"sum":2157168.356961000},"op_rw":938,"op_rw_in_bytes":809065262,"op_rw_out_bytes":23687,"op_rw_rlat":{"avgcount":938,"sum":72.490867000},"op_rw_latency":{"avgcount":938,"sum":287.711861000},"subop":438880,"subop_in_bytes":397494403398,"subop_latency":{"avgcount":438880,"sum":130296.341257000},"subop_w":0,"subop_w_in_bytes":397494403398,"subop_w_latency":{"avgcount":438880,"sum":130296.341257000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":1,"buffer_bytes":0,"numpg":429,"numpg_primary":225,"numpg_replica":204,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3904,"map_message_epochs":15521,"map_message_epoch_dups":15184},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1029094,"take_sum":823777527413,"put":755678,"put_sum":823777527413,"wait":{"avgcount":4577,"sum":5377.483839000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1029094,"take_sum":1029094,"put":755678,"put_sum":1029094,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2822627,"get_sum":425388068263,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2822627,"put_sum":425388068263,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1195439,"get_sum":451865064280,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1195439,"put_sum":451865064280,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2701057,"get_sum":126949679,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701057,"put_sum":126949679,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2702081,"get_sum":126997807,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2702081,"put_sum":126997807,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2772980,"get_sum":425371781359,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3232837,"put_sum":425371781359,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi13:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.15.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":911740,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":730384494443,"journal_latency":{"avgcount":911740,"sum":65577.104167000},"journal_wr":709180,"journal_wr_bytes":{"avgcount":709180,"sum":733591408640},"op_queue_max_ops":50,"op_queue_ops":0,"ops":911740,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":730375942683,"apply_latency":{"avgcount":911740,"sum":134087.832389000},"committing":0,"commitcycle":26464,"commitcycle_interval":{"avgcount":26464,"sum":148049.395365000},"commitcycle_latency":{"avgcount":26464,"sum":15161.387933000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":638213,"op_in_bytes":462172652380,"op_out_bytes":1052036634,"op_latency":{"avgcount":638213,"sum":305315.775633000},"op_r":52584,"op_r_out_bytes":1052003532,"op_r_latency":{"avgcount":52584,"sum":156.539591000},"op_w":584418,"op_w_in_bytes":461425963154,"op_w_rlat":{"avgcount":584418,"sum":81842.241330000},"op_w_latency":{"avgcount":584418,"sum":304909.313021000},"op_rw":1211,"op_rw_in_bytes":746689226,"op_rw_out_bytes":33102,"op_rw_rlat":{"avgcount":1211,"sum":28.868552000},"op_rw_latency":{"avgcount":1211,"sum":249.923021000},"subop":359304,"subop_in_bytes":335771939271,"subop_latency":{"avgcount":359304,"sum":51026.247939000},"subop_w":0,"subop_w_in_byt

ubuntu@burnupi13:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.16.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":853390,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":681415749143,"journal_latency":{"avgcount":853390,"sum":51504.536600000},"journal_wr":678659,"journal_wr_bytes":{"avgcount":678659,"sum":684411797504},"op_queue_max_ops":50,"op_queue_ops":0,"ops":853390,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":681407742683,"apply_latency":{"avgcount":853390,"sum":101738.947100000},"committing":0,"commitcycle":26846,"commitcycle_interval":{"avgcount":26846,"sum":147093.830852000},"commitcycle_latency":{"avgcount":26846,"sum":12436.155086000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":577183,"op_in_bytes":407880855155,"op_out_bytes":849241073,"op_latency":{"avgcount":577183,"sum":303818.456145000},"op_r":61240,"op_r_out_bytes":849218979,"op_r_latency":{"avgcount":61240,"sum":103.494484000},"op_w":514970,"op_w_in_bytes":407220662611,"op_w_rlat":{"avgcount":514970,"sum":61545.504545000},"op_w_latency":{"avgcount":514970,"sum":303566.093880000},"op_rw":973,"op_rw_in_bytes":660192544,"op_rw_out_bytes":22094,"op_rw_rlat":{"avgcount":973,"sum":29.378941000},"op_rw_latency":{"avgcount":973,"sum":148.867781000},"subop":349606,"subop_in_bytes":323603829669,"subop_latency":{"avgcount":349606,"sum":42670.239578000},"subop_w":0,"subop_w_in_bytes":323603829669,"subop_w_latency":{"avgcount":349606,"sum":42670.239578000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":1,"buffer_bytes":0,"numpg":343,"numpg_primary":190,"numpg_replica":153,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":2711,"map_message_epochs":17196,"map_message_epoch_dups":16926},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":853390,"take_sum":681415749143,"put":678659,"put_sum":681415749143,"wait":{"avgcount":836,"sum":1223.759171000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":853390,"take_sum":853390,"put":678659,"put_sum":853390,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2400720,"get_sum":357032926385,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2400720,"put_sum":357032926385,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":983909,"get_sum":369501031341,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":983909,"put_sum":369501031341,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2705004,"get_sum":127135188,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2705004,"put_sum":127135188,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2702163,"get_sum":127001661,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2702163,"put_sum":127001661,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2360707,"get_sum":357019108181,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2759157,"put_sum":357019108181,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi13:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.17.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":970251,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":786827179629,"journal_latency":{"avgcount":970251,"sum":95980.160607000},"journal_wr":740539,"journal_wr_bytes":{"avgcount":740539,"sum":790232547328},"op_queue_max_ops":50,"op_queue_ops":0,"ops":970251,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":786818063439,"apply_latency":{"avgcount":970251,"sum":210072.051732000},"committing":0,"commitcycle":25520,"commitcycle_interval":{"avgcount":25520,"sum":148205.682495000},"commitcycle_latency":{"avgcount":25520,"sum":19684.003198000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":712046,"op_in_bytes":527251703985,"op_out_bytes":1282965520,"op_latency":{"avgcount":712046,"sum":588254.755165000},"op_r":74003,"op_r_out_bytes":1282934625,"op_r_latency":{"avgcount":74003,"sum":183.442581000},"op_w":636836,"op_w_in_bytes":526378759747,"op_w_rlat":{"avgcount":636836,"sum":204016.997204000},"op_w_latency":{"avgcount":636836,"sum":587916.995838000},"op_rw":1207,"op_rw_in_bytes":872944238,"op_rw_out_bytes":30895,"op_rw_rlat":{"avgcount":1207,"sum":22.162491000},"op_rw_latency":{"avgcount":1207,"sum":154.316746000},"subop":385404,"subop_in_bytes":359932493755,"subop_latency":{"avgcount":385404,"sum":91683.205910000},"subop_w":0,"subop_w_in_bytes":359932493755,"subop_w_latency":{"avgcount":385404,"sum":91683.205910000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":1,"buffer_bytes":0,"numpg":384,"numpg_primary":198,"numpg_replica":186,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3881,"map_message_epochs":29864,"map_message_epoch_dups":29591},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":970251,"take_sum":786827179629,"put":740539,"put_sum":786827179629,"wait":{"avgcount":2586,"sum":3510.651422000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":970251,"take_sum":970251,"put":740539,"put_sum":970251,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2827835,"get_sum":426033911515,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2827835,"put_sum":426033911515,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1128439,"get_sum":420048970122,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1128439,"put_sum":420048970122,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2699145,"get_sum":126859815,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2699145,"put_sum":126859815,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2702177,"get_sum":127002319,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2702177,"put_sum":127002319,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2778600,"get_sum":426015230343,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3246763,"put_sum":426015230343,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi13:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.18.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1060961,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":849064386694,"journal_latency":{"avgcount":1060961,"sum":150769.102059000},"journal_wr":783250,"journal_wr_bytes":{"avgcount":783250,"sum":852765450240},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1060961,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":849054449824,"apply_latency":{"avgcount":1060961,"sum":450698.977919000},"committing":0,"commitcycle":23554,"commitcycle_interval":{"avgcount":23554,"sum":149520.493751000},"commitcycle_latency":{"avgcount":23554,"sum":30108.595879000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":720550,"op_in_bytes":583041236521,"op_out_bytes":978431888,"op_latency":{"avgcount":720550,"sum":1086271.256987000},"op_r":64882,"op_r_out_bytes":978390568,"op_r_latency":{"avgcount":64882,"sum":354.426976000},"op_w":654155,"op_w_in_bytes":582334057461,"op_w_rlat":{"avgcount":654155,"sum":395744.492220000},"op_w_latency":{"avgcount":654155,"sum":1085649.867329000},"op_rw":1513,"op_rw_in_bytes":707179060,"op_rw_out_bytes":41320,"op_rw_rlat":{"avgcount":1513,"sum":67.130053000},"op_rw_latency":{"avgcount":1513,"sum":266.962682000},"subop":456946,"subop_in_bytes":420929290766,"subop_latency":{"avgcount":456946,"sum":131654.072551000},"subop_w":0,"subop_w_in_bytes":420929290766,"subop_w_latency":{"avgcount":456946,"sum":131654.072551000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":1,"buffer_bytes":0,"numpg":438,"numpg_primary":222,"numpg_replica":216,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3865,"map_message_epochs":20930,"map_message_epoch_dups":20617},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1060961,"take_sum":849064386694,"put":783250,"put_sum":849064386694,"wait":{"avgcount":5542,"sum":6162.831376000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1060961,"take_sum":1060961,"put":783250,"put_sum":1060961,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2811262,"get_sum":427166513363,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2811262,"put_sum":427166513363,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1221013,"get_sum":477029131588,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1221013,"put_sum":477029131588,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2701684,"get_sum":126979148,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701684,"put_sum":126979148,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2702196,"get_sum":127003212,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2702196,"put_sum":127003212,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2761609,"get_sum":427149138752,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3231956,"put_sum":427149138752,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi13:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.19.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1164286,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":929949961473,"journal_latency":{"avgcount":1164286,"sum":306582.004810000},"journal_wr":774390,"journal_wr_bytes":{"avgcount":774390,"sum":934007685120},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1164286,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":929939025748,"apply_latency":{"avgcount":1164286,"sum":1147438.311798000},"committing":0,"commitcycle":19256,"commitcycle_interval":{"avgcount":19256,"sum":150538.283864000},"commitcycle_latency":{"avgcount":19256,"sum":51372.443612000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":808016,"op_in_bytes":731654588299,"op_out_bytes":1048521448,"op_latency":{"avgcount":808016,"sum":37008098.934118000},"op_r":58776,"op_r_out_bytes":1048482585,"op_r_latency":{"avgcount":58776,"sum":320.057231000},"op_w":747799,"op_w_in_bytes":730863379574,"op_w_rlat":{"avgcount":747799,"sum":17631529.226122000},"op_w_latency":{"avgcount":747799,"sum":37007500.815919000},"op_rw":1441,"op_rw_in_bytes":791208725,"op_rw_out_bytes":38863,"op_rw_rlat":{"avgcount":1441,"sum":60.134019000},"op_rw_latency":{"avgcount":1441,"sum":278.060968000},"subop":520361,"subop_in_bytes":471691513399,"subop_latency":{"avgcount":520361,"sum":323379.927322000},"subop_w":0,"subop_w_in_bytes":471691513399,"subop_w_latency":{"avgcount":520361,"sum":323379.927322000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":1,"buffer_bytes":0,"numpg":468,"numpg_primary":240,"numpg_replica":228,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":2706,"map_message_epochs":16859,"map_message_epoch_dups":16520},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1164286,"take_sum":929949961473,"put":774390,"put_sum":929949961473,"wait":{"avgcount":15973,"sum":16805.388961000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1164286,"take_sum":1164286,"put":774390,"put_sum":1164286,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2986579,"get_sum":457276510183,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2986579,"put_sum":457276510183,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1356710,"get_sum":530322765121,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1356710,"put_sum":530322765121,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2701553,"get_sum":126972991,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701553,"put_sum":126972991,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2509116,"get_sum":117928452,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2509116,"put_sum":117928452,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2944887,"get_sum":457259196629,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3450174,"put_sum":457259196629,"wait":{"avgcount":0,"sum":0.000000000}}}
ubuntu@burnupi13:/var/run/ceph$ sudo ceph --admin-daemon ceph-osd.20.asok perf dump
{"filestore":{"journal_queue_max_ops":300,"journal_queue_ops":0,"journal_ops":1044024,"journal_queue_max_bytes":33554432,"journal_queue_bytes":0,"journal_bytes":830251514733,"journal_latency":{"avgcount":1044024,"sum":134195.657377000},"journal_wr":782142,"journal_wr_bytes":{"avgcount":782142,"sum":833892941824},"op_queue_max_ops":50,"op_queue_ops":0,"ops":1044024,"op_queue_max_bytes":104857600,"op_queue_bytes":0,"bytes":830241715998,"apply_latency":{"avgcount":1044024,"sum":389403.335172000},"committing":0,"commitcycle":24220,"commitcycle_interval":{"avgcount":24220,"sum":149017.001702000},"commitcycle_latency":{"avgcount":24220,"sum":26610.274773000},"journal_full":0},"mutex-FileJournal::completions_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::finisher_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::write_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-FileJournal::writeq_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::apply_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::ApplyManager::com_lock":{"wait":{"avgcount":0,"sum":0.000000000}},"mutex-JOS::SubmitManager::lock":{"wait":{"avgcount":0,"sum":0.000000000}},"osd":{"opq":0,"op_wip":0,"op":670208,"op_in_bytes":535602739213,"op_out_bytes":1194458446,"op_latency":{"avgcount":670208,"sum":2344290.085185000},"op_r":60325,"op_r_out_bytes":1194437582,"op_r_latency":{"avgcount":60325,"sum":1676.213459000},"op_w":608950,"op_w_in_bytes":534792791297,"op_w_rlat":{"avgcount":608950,"sum":922962.966947000},"op_w_latency":{"avgcount":608950,"sum":2342403.938388000},"op_rw":933,"op_rw_in_bytes":809947916,"op_rw_out_bytes":20864,"op_rw_rlat":{"avgcount":933,"sum":41.897589000},"op_rw_latency":{"avgcount":933,"sum":209.933338000},"subop":468634,"subop_in_bytes":422210743872,"subop_latency":{"avgcount":468634,"sum":133312.195337000},"subop_w":0,"subop_w_in_bytes":422210743872,"subop_w_latency":{"avgcount":468634,"sum":133312.195337000},"subop_pull":0,"subop_pull_latency":{"avgcount":0,"sum":0.000000000},"subop_push":0,"subop_push_in_bytes":0,"subop_push_latency":{"avgcount":0,"sum":0.000000000},"pull":0,"push":0,"push_out_bytes":0,"push_in":0,"push_in_bytes":0,"recovery_ops":0,"loadavg":1,"buffer_bytes":0,"numpg":422,"numpg_primary":215,"numpg_replica":207,"numpg_stray":0,"heartbeat_to_peers":14,"heartbeat_from_peers":0,"map_messages":3924,"map_message_epochs":20589,"map_message_epoch_dups":20281},"throttle-filestore_bytes":{"val":0,"max":33554432,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1044024,"take_sum":830251514733,"put":782142,"put_sum":830251514733,"wait":{"avgcount":4318,"sum":5093.855039000}},"throttle-filestore_ops":{"val":0,"max":300,"get":0,"get_sum":0,"get_or_fail_fail":0,"get_or_fail_success":0,"take":1044024,"take_sum":1044024,"put":782142,"put_sum":1044024,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-client":{"val":0,"max":104857600,"get":2719647,"get_sum":407106283502,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2719647,"put_sum":407106283502,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-cluster":{"val":0,"max":104857600,"get":1202461,"get_sum":475054046078,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":1202461,"put_sum":475054046078,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbclient":{"val":0,"max":104857600,"get":2701444,"get_sum":126967868,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2701444,"put_sum":126967868,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-msgr_dispatch_throttler-hbserver":{"val":0,"max":104857600,"get":2702223,"get_sum":127004481,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":2702223,"put_sum":127004481,"wait":{"avgcount":0,"sum":0.000000000}},"throttle-osd_client_bytes":{"val":0,"max":524288000,"get":2670180,"get_sum":407089730064,"get_or_fail_fail":0,"get_or_fail_success":0,"take":0,"take_sum":0,"put":3117122,"put_sum":407089730064,"wait":{"avgcount":0,"sum":0.000000000}}}

2013-06-05 17:23:08.820204 7f4ca0787700  1 -- 10.10.10.18:6801/3130 <== osd.15 10.10.10.32:6804/1864 9696082 ==== osd_sub_op(unknown.0.0:0 121.19 0//0//-1 [scrub-map] v 0'0 snapset=0=[]:[] snapc=0=[]) v7 ==== 955+0+935002390 (788881636 0 3376632837) 0x268a8400 con 0x18d399a0

found it (probably):

2013-06-05 17:23:20.453250 7f4c93f6e700 10 osd.17 pg_epoch: 55980 pg[121.19( v 55980'44039961 (55980'44038960,55980'44039961] local-les=55801 n=800 ec=10098 les/c 55801/55806 55800/55800/55800) [17,15] r=0 lpr=55800 mlcod 55980'44039959 active+clean+scrubbing+deep]  done.  pg log is 935009892 bytes