|
awolfegordon@prod-data33:~$ sudo ceph --admin-daemon /var/run/ceph/ceph-osd.751.asok dump_blocked_ops
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "osd_op(client.4105517.0:1 4.bc2dcdb7 rbd_data.3d31f5756d622c.0000000000005b87 [delete] snapc 0=[] ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:22:06.513906",
|
|
"age": 392.882291,
|
|
"duration": 392.882438,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4105517",
|
|
"tid": 1
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:22:06.513906",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.513927",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.513951",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.514130",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"complaint_time": 30.000000,
|
|
"num_blocked_ops": 1
|
|
}
|
|
|
|
awolfegordon@prod-data33:~$ sudo ceph --admin-daemon /var/run/ceph/ceph-osd.751.asok dump_historic_ops
|
|
{
|
|
"num to keep": 20,
|
|
"duration to keep": 600,
|
|
"Ops": [
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473847 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.298369",
|
|
"age": 83.679545,
|
|
"duration": 0.175974,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473847
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.298369",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.298427",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.469592",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.469787",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.470306",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.471094",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.471142",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.471271",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.473873",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.473911",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474132",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474178",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474305",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474343",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473848 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.298487",
|
|
"age": 83.679427,
|
|
"duration": 0.175916,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473848
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.298487",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.298512",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.471176",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.471199",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.471554",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.472227",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.472262",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.472475",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.473913",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474133",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474378",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474392",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474398",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474403",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473849 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.298555",
|
|
"age": 83.679360,
|
|
"duration": 0.181399,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473849
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.298555",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.298588",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.472258",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.472423",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.472923",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.473728",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.473740",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474011",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474361",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474366",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479337",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479488",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479624",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479954",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473850 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.298628",
|
|
"age": 83.679286,
|
|
"duration": 0.181755,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473850
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.298628",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.298720",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474424",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.474624",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.475183",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.477011",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.477048",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.477300",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479357",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479368",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.480081",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.480141",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.480328",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.480383",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473851 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.298736",
|
|
"age": 83.679178,
|
|
"duration": 0.183932,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473851
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.298736",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.298786",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.477184",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.477211",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.477960",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479113",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479124",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479313",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.479361",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.480124",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.482554",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.482587",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.482637",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.482667",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473852 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.298789",
|
|
"age": 83.679125,
|
|
"duration": 0.188309,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473852
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.298789",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.298955",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.480515",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.480842",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.481357",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.482462",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.482473",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.482736",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.485104",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487005",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487072",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487077",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487086",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487098",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473853 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.298959",
|
|
"age": 83.678956,
|
|
"duration": 0.188722,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473853
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.298959",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.299008",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.482877",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.483169",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.483826",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.485053",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.485073",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.485382",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487248",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487260",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487303",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487560",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487671",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487680",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473854 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.299064",
|
|
"age": 83.678850,
|
|
"duration": 0.191079,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473854
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.299064",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.299085",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.485123",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.485199",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.485930",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.486780",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.486829",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487023",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487552",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487693",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.489902",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.489923",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.489982",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.490144",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473855 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.299525",
|
|
"age": 83.678389,
|
|
"duration": 0.194801,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473855
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.299525",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.299547",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487716",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.487912",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.488689",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.489827",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.489865",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.490004",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494176",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494254",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494287",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494312",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494317",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494326",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473856 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300237",
|
|
"age": 83.677677,
|
|
"duration": 0.194110,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473856
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300237",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300275",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.490260",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.490566",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.492007",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.492789",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.492800",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.492927",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494181",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494297",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494304",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494340",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494342",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494348",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473857 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300349",
|
|
"age": 83.677566,
|
|
"duration": 0.197393,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473857
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300349",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300370",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.492819",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.492913",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.493391",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494104",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494161",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494277",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494356",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497351",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497525",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497652",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497730",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497741",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473858 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300378",
|
|
"age": 83.677537,
|
|
"duration": 0.197458,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473858
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300378",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300400",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494360",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.494553",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.495103",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.495875",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.495928",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.496033",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497680",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497772",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497782",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497792",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497803",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497836",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473859 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300402",
|
|
"age": 83.677512,
|
|
"duration": 0.200088,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473859
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300402",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300415",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.495907",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.496003",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.496493",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497275",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497319",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497435",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497695",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497775",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500424",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500460",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500472",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500490",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473860 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300417",
|
|
"age": 83.677497,
|
|
"duration": 0.203468,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473860
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300417",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300444",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.497849",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.498039",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.498641",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.499282",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.499313",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.499428",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500452",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500617",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500640",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503855",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503867",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503885",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473861 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300446",
|
|
"age": 83.677469,
|
|
"duration": 0.203622,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473861
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300446",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300470",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.499316",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.499345",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.499659",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500338",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500361",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500477",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500645",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.502398",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503813",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504050",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504061",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504068",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473862 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300484",
|
|
"age": 83.677430,
|
|
"duration": 0.203609,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473862
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300484",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300524",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500784",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.500960",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.501559",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.502375",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.502384",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.502636",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503846",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503947",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503969",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504085",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504089",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504093",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473863 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300561",
|
|
"age": 83.677353,
|
|
"duration": 0.206144,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473863
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300561",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300632",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.502418",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.502639",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503139",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503746",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503775",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503915",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.503976",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504079",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506627",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506676",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506688",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506705",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473864 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300636",
|
|
"age": 83.677279,
|
|
"duration": 0.206172,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473864
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300636",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300650",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504103",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504124",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.504504",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.505269",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.505314",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.505419",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506668",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506778",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506787",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506791",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506796",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506807",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473865 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300838",
|
|
"age": 83.677076,
|
|
"duration": 0.207550,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473865
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300838",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300895",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.505308",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.505427",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.505850",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506587",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506642",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506825",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508270",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508296",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508305",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508323",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508367",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508388",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4049621.0:361473866 4.1aff6881 rbd_data.38922172dbe9d.0000000000008eff [] snapc 192c=[192c,15bb,15ba] ack+ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:27:18.300899",
|
|
"age": 83.677015,
|
|
"duration": 0.208172,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4049621",
|
|
"tid": 361473866
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:27:18.300899",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.300947",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506851",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.506998",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.507502",
|
|
"event": "waiting for subops from 313,418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508225",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508256",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508462",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508491",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508674",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.508861",
|
|
"event": "sub_op_commit_rec from 418"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.509048",
|
|
"event": "sub_op_commit_rec from 313"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.509062",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:27:18.509071",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
awolfegordon@prod-data33:~$ sudo ceph --admin-daemon /var/run/ceph/ceph-osd.751.asok dump_ops_in_flight
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "osd_op(client.4105517.0:1 4.bc2dcdb7 rbd_data.3d31f5756d622c.0000000000005b87 [delete] snapc 0=[] ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:22:06.513906",
|
|
"age": 399.712375,
|
|
"duration": 399.712404,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4105517",
|
|
"tid": 1
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:22:06.513906",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.513927",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.513951",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.514130",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"num_ops": 1
|
|
}
|
|
|
|
awolfegordon@prod-data33:~$ sudo ceph --admin-daemon /var/run/ceph/ceph-osd.751.asok ops
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "osd_op(client.4105517.0:1 4.bc2dcdb7 rbd_data.3d31f5756d622c.0000000000005b87 [delete] snapc 0=[] ondisk+write+known_if_redirected e235042)",
|
|
"initiated_at": "2017-03-09 21:22:06.513906",
|
|
"age": 402.455955,
|
|
"duration": 402.456232,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4105517",
|
|
"tid": 1
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-03-09 21:22:06.513906",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.513927",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.513951",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-03-09 21:22:06.514130",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"num_ops": 1
|
|
}
|