|
|
|
Last login: Wed Apr 5 21:57:28 2017 from 10.126.86.66
|
|
# ps auxww | grep 397
|
|
root 25397 0.0 0.0 0 0 ? S 2016 223:45 [xfsaild/dm-46]
|
|
root 61397 0.1 0.0 0 0 ? S 14:19 0:01 [kworker/12:2]
|
|
root 184501 0.0 0.0 11752 2160 pts/29 S+ 14:38 0:00 grep --color=auto 397
|
|
ceph 505970 22.5 0.9 9106748 1305040 ? Ssl Apr04 27762:42 /usr/bin/ceph-osd --cluster=ceph -i 397 -f --setuser ceph --setgroup ceph
|
|
root 3972048 0.1 0.0 0 0 ? S 13:37 0:07 [kworker/u289:3]
|
|
ntp 4054458 0.0 0.0 35708 3976 ? Ss Apr19 3:39 /usr/sbin/ntpd -p /var/run/ntpd.pid -g -u 106:113
|
|
# ceph daemon osd.397 ops
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "osd_op(client.4976210.0:180637 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279203)",
|
|
"initiated_at": "2017-06-29 12:35:46.720883",
|
|
"age": 7402.676163,
|
|
"duration": 7402.685720,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4976210",
|
|
"tid": 180637
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 12:35:46.720883",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 12:35:46.721107",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 12:35:46.721128",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 12:35:46.721149",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4981571.0:18791 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279208)",
|
|
"initiated_at": "2017-06-29 13:48:16.277165",
|
|
"age": 3053.119882,
|
|
"duration": 3053.138278,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4981571",
|
|
"tid": 18791
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 13:48:16.277165",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 13:48:16.277573",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 13:48:16.277593",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 13:48:16.278146",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4981346.0:714408 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279200)",
|
|
"initiated_at": "2017-06-29 11:17:44.080284",
|
|
"age": 12085.316763,
|
|
"duration": 12085.335281,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4981346",
|
|
"tid": 714408
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 11:17:44.080284",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 11:17:44.080560",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 11:17:44.080582",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 11:17:44.081240",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4952662.0:1178841 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279187)",
|
|
"initiated_at": "2017-06-29 09:46:50.111424",
|
|
"age": 17539.285623,
|
|
"duration": 17539.304165,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4952662",
|
|
"tid": 1178841
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 09:46:50.111424",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 09:46:50.111695",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 09:46:50.111751",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 09:46:50.112092",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4888992.0:267567696 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279144)",
|
|
"initiated_at": "2017-06-29 02:47:07.669854",
|
|
"age": 42721.727193,
|
|
"duration": 42721.745772,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4888992",
|
|
"tid": 267567696
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 02:47:07.669854",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 02:47:07.670105",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 02:47:07.670126",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 02:47:07.670536",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4976261.0:4069 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279171)",
|
|
"initiated_at": "2017-06-29 07:34:17.457290",
|
|
"age": 25491.939756,
|
|
"duration": 25491.958356,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4976261",
|
|
"tid": 4069
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 07:34:17.457290",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 07:34:17.459186",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 07:34:17.459234",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 07:34:17.459441",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4967951.0:52908 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279163)",
|
|
"initiated_at": "2017-06-29 05:56:29.309705",
|
|
"age": 31360.087341,
|
|
"duration": 31360.105962,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4967951",
|
|
"tid": 52908
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 05:56:29.309705",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 05:56:29.310055",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 05:56:29.310297",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 05:56:29.311467",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4953495.0:809788 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279155)",
|
|
"initiated_at": "2017-06-29 04:39:25.959042",
|
|
"age": 35983.438004,
|
|
"duration": 35983.456662,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4953495",
|
|
"tid": 809788
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 04:39:25.959042",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 04:39:25.960380",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 04:39:25.960421",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 04:39:25.960593",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"num_ops": 8
|
|
}
|
|
|
|
# ceph daemon osd.397 dump_ops_in_flight
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "osd_op(client.4976210.0:180637 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279203)",
|
|
"initiated_at": "2017-06-29 12:35:46.720883",
|
|
"age": 7452.029648,
|
|
"duration": 7452.029766,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4976210",
|
|
"tid": 180637
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 12:35:46.720883",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 12:35:46.721107",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 12:35:46.721128",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 12:35:46.721149",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4981571.0:18791 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279208)",
|
|
"initiated_at": "2017-06-29 13:48:16.277165",
|
|
"age": 3102.473366,
|
|
"duration": 3102.473535,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4981571",
|
|
"tid": 18791
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 13:48:16.277165",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 13:48:16.277573",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 13:48:16.277593",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 13:48:16.278146",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4981346.0:714408 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279200)",
|
|
"initiated_at": "2017-06-29 11:17:44.080284",
|
|
"age": 12134.670247,
|
|
"duration": 12134.670457,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4981346",
|
|
"tid": 714408
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 11:17:44.080284",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 11:17:44.080560",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 11:17:44.080582",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 11:17:44.081240",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4952662.0:1178841 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279187)",
|
|
"initiated_at": "2017-06-29 09:46:50.111424",
|
|
"age": 17588.639107,
|
|
"duration": 17588.639344,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4952662",
|
|
"tid": 1178841
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 09:46:50.111424",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 09:46:50.111695",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 09:46:50.111751",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 09:46:50.112092",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4888992.0:267567696 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279144)",
|
|
"initiated_at": "2017-06-29 02:47:07.669854",
|
|
"age": 42771.080677,
|
|
"duration": 42771.080979,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4888992",
|
|
"tid": 267567696
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 02:47:07.669854",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 02:47:07.670105",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 02:47:07.670126",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 02:47:07.670536",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4976261.0:4069 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279171)",
|
|
"initiated_at": "2017-06-29 07:34:17.457290",
|
|
"age": 25541.293241,
|
|
"duration": 25541.293583,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4976261",
|
|
"tid": 4069
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 07:34:17.457290",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 07:34:17.459186",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 07:34:17.459234",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 07:34:17.459441",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4967951.0:52908 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279163)",
|
|
"initiated_at": "2017-06-29 05:56:29.309705",
|
|
"age": 31409.440826,
|
|
"duration": 31409.441197,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4967951",
|
|
"tid": 52908
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 05:56:29.309705",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 05:56:29.310055",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 05:56:29.310297",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 05:56:29.311467",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4953495.0:809788 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279155)",
|
|
"initiated_at": "2017-06-29 04:39:25.959042",
|
|
"age": 36032.791489,
|
|
"duration": 36032.791883,
|
|
"type_data": [
|
|
"delayed",
|
|
{
|
|
"client": "client.4953495",
|
|
"tid": 809788
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 04:39:25.959042",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 04:39:25.960380",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 04:39:25.960421",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 04:39:25.960593",
|
|
"event": "waiting for scrub"
|
|
}
|
|
]
|
|
]
|
|
}
|
|
],
|
|
"num_ops": 8
|
|
}
|
|
|
|
# ceph daemon osd.397 dump_historic_ops
|
|
{
|
|
"num to keep": 20,
|
|
"duration to keep": 600,
|
|
"Ops": [
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506031 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.948997",
|
|
"age": 169.408361,
|
|
"duration": 0.156108,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506031
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.948997",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949024",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.099462",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.099590",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.100112",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.100843",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.100867",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.101090",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.102150",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103574",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103601",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105093",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105099",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105105",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506032 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949027",
|
|
"age": 169.408331,
|
|
"duration": 0.156341,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506032
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949027",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949044",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.100900",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.101030",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.101405",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.102001",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.102036",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.102212",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103575",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103602",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105061",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105356",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105363",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105368",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506033 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949070",
|
|
"age": 169.408288,
|
|
"duration": 0.159311,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506033
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949070",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949099",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.102189",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.102307",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.102710",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103517",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103547",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103757",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105189",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105328",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105339",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108064",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108298",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108380",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506034 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949107",
|
|
"age": 169.408251,
|
|
"duration": 0.159903,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506034
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949107",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949119",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103628",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.103772",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.104182",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.104837",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.104861",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.104997",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105333",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105346",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105456",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108718",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108865",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109010",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506035 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949169",
|
|
"age": 169.408189,
|
|
"duration": 0.159904,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506035
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949169",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949181",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105462",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105486",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.105681",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.106411",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.106437",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.106560",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108593",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108624",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109044",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109063",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109069",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109073",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506036 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949225",
|
|
"age": 169.408133,
|
|
"duration": 0.160100,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506036
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949225",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949241",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.106496",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.106752",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.107232",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.107890",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.107922",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108107",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108599",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.108629",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109139",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109308",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109318",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109325",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506037 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949275",
|
|
"age": 169.408083,
|
|
"duration": 0.168544,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506037
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949275",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949291",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109347",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.109659",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.110157",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.111587",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.111613",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.111729",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117005",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117035",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117081",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117368",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117681",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117820",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506038 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949398",
|
|
"age": 169.407960,
|
|
"duration": 0.168929,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506038
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949398",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949427",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.111738",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.111945",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.112838",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.113730",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.113758",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.113941",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117018",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117914",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118124",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118212",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118234",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118327",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506039 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949510",
|
|
"age": 169.407848,
|
|
"duration": 0.173132,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506039
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949510",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949584",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.113968",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.114627",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.115383",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.116956",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.116981",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.117124",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118573",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118628",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118635",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122617",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122633",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122642",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506040 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949593",
|
|
"age": 169.407765,
|
|
"duration": 0.175640,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506040
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949593",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949627",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118645",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.118766",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.119198",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.119854",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.119893",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.120035",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122404",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122429",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122501",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125216",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125225",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125233",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506041 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949634",
|
|
"age": 169.407724,
|
|
"duration": 0.175978,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506041
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949634",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949652",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.119911",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.120129",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.120517",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.121162",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.121177",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.121326",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122434",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122610",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125244",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125308",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125500",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125611",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506042 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949690",
|
|
"age": 169.407668,
|
|
"duration": 0.175962,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506042
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949690",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949722",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.121191",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.121293",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.121661",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122362",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122386",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122593",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125187",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125246",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125637",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125645",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125649",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125653",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506043 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949776",
|
|
"age": 169.407582,
|
|
"duration": 0.176188,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506043
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949776",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949826",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122735",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.122974",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.123457",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.124026",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.124035",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.124236",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125193",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125249",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125730",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125875",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125888",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125964",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506044 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949832",
|
|
"age": 169.407526,
|
|
"duration": 0.179841,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506044
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949832",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949847",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.124083",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.124182",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.124580",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125169",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125177",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125316",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125669",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.125854",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.126067",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.129376",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.129447",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.129673",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506045 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949895",
|
|
"age": 169.407463,
|
|
"duration": 0.183653,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506045
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949895",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949913",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.126104",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.126418",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.126690",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.127578",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.127605",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.127752",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.130127",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.130178",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.130203",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133119",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133133",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133548",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506046 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949933",
|
|
"age": 169.407425,
|
|
"duration": 0.183771,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506046
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949933",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949955",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.127621",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.127647",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.127857",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.129257",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.129287",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.129429",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.130185",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.130208",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133589",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133690",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133697",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133704",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.4850621.0:474506047 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)",
|
|
"initiated_at": "2017-06-29 14:37:21.949971",
|
|
"age": 169.407387,
|
|
"duration": 0.183856,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.4850621",
|
|
"tid": 474506047
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:21.949971",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:21.949985",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.130215",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.130859",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.131656",
|
|
"event": "waiting for subops from 69,377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133064",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133115",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133672",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133724",
|
|
"event": "sub_op_commit_rec from 69"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133729",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133804",
|
|
"event": "sub_op_commit_rec from 377"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133809",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133816",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:22.133827",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_repop(client.4687667.0:2079863051 4.8533 4:cca1d83a:::rbd_data.477e0378acd3bb.000000000000e9e6:head v 279218'7289366)",
|
|
"initiated_at": "2017-06-29 14:37:39.188370",
|
|
"age": 152.168988,
|
|
"duration": 0.172779,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:37:39.188370",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.188879",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.188897",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.188994",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.192134",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.192158",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.192304",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.192350",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.361147",
|
|
"event": "sub_op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:37:39.361149",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.2876330.0:27903880541 4.67c36c2a rbd_data.2b56c9548a720e.0000000000001afd [] snapc 0=[] ack+ondisk+write+known_if_redirected e279219)",
|
|
"initiated_at": "2017-06-29 14:39:47.402981",
|
|
"age": 23.954377,
|
|
"duration": 0.125320,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.2876330",
|
|
"tid": 27903880541
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:39:47.402981",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.403113",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.521980",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.522233",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.522580",
|
|
"event": "waiting for subops from 46,122"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.523313",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.523341",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.523517",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.525932",
|
|
"event": "sub_op_commit_rec from 46"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.526490",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528186",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528219",
|
|
"event": "sub_op_commit_rec from 122"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528279",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528301",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"description": "osd_op(client.2876330.0:27903880543 4.67c36c2a rbd_data.2b56c9548a720e.0000000000001afd [] snapc 0=[] ack+ondisk+write+known_if_redirected e279219)",
|
|
"initiated_at": "2017-06-29 14:39:47.403785",
|
|
"age": 23.953573,
|
|
"duration": 0.126216,
|
|
"type_data": [
|
|
"commit sent; apply or cleanup",
|
|
{
|
|
"client": "client.2876330",
|
|
"tid": 27903880543
|
|
},
|
|
[
|
|
{
|
|
"time": "2017-06-29 14:39:47.403785",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.403824",
|
|
"event": "queued_for_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.526536",
|
|
"event": "reached_pg"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.526751",
|
|
"event": "started"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.527289",
|
|
"event": "waiting for subops from 46,122"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528130",
|
|
"event": "commit_queued_for_journal_write"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528172",
|
|
"event": "write_thread_in_journal_buffer"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528442",
|
|
"event": "journaled_completion_queued"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528547",
|
|
"event": "op_commit"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.528631",
|
|
"event": "op_applied"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.529094",
|
|
"event": "sub_op_commit_rec from 46"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.529992",
|
|
"event": "sub_op_commit_rec from 122"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.529998",
|
|
"event": "commit_sent"
|
|
},
|
|
{
|
|
"time": "2017-06-29 14:39:47.530001",
|
|
"event": "done"
|
|
}
|
|
]
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
# restart ceph-osd id=397
|
|
ceph-osd (ceph/397) start/running, process 266709
|
|
|
|
# ceph -v ; dpkg -l | grep ceph
|
|
ceph version 10.2.6 (656b5b63ed7c43bd014bcafd81b001959d5f089f)
|
|
ii ceph 10.2.6-1trusty amd64 distributed storage and file system
|
|
ii ceph-base 10.2.6-1trusty amd64 common ceph daemon libraries and management tools
|
|
ii ceph-common 10.2.6-1trusty amd64 common utilities to mount and interact with a ceph storage cluster
|
|
ii ceph-mds 10.2.6-1trusty amd64 metadata server for the ceph distributed file system
|
|
ii ceph-mon 10.2.6-1trusty amd64 monitor server for the ceph storage system
|
|
ii ceph-osd 10.2.6-1trusty amd64 OSD server for the ceph storage system
|
|
ii libcephfs1 10.2.6-1trusty amd64 Ceph distributed file system client library
|
|
ii python-cephfs 10.2.6-1trusty amd64 Python libraries for the Ceph libcephfs library
|
|
root@prod-data17:/home/adatri# uname -r
|
|
3.19.0-49-generic
|
|
root@prod-data17:/home/adatri#
|