Last login: Wed Apr 5 21:57:28 2017 from 10.126.86.66 # ps auxww | grep 397 root 25397 0.0 0.0 0 0 ? S 2016 223:45 [xfsaild/dm-46] root 61397 0.1 0.0 0 0 ? S 14:19 0:01 [kworker/12:2] root 184501 0.0 0.0 11752 2160 pts/29 S+ 14:38 0:00 grep --color=auto 397 ceph 505970 22.5 0.9 9106748 1305040 ? Ssl Apr04 27762:42 /usr/bin/ceph-osd --cluster=ceph -i 397 -f --setuser ceph --setgroup ceph root 3972048 0.1 0.0 0 0 ? S 13:37 0:07 [kworker/u289:3] ntp 4054458 0.0 0.0 35708 3976 ? Ss Apr19 3:39 /usr/sbin/ntpd -p /var/run/ntpd.pid -g -u 106:113 # ceph daemon osd.397 ops { "ops": [ { "description": "osd_op(client.4976210.0:180637 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279203)", "initiated_at": "2017-06-29 12:35:46.720883", "age": 7402.676163, "duration": 7402.685720, "type_data": [ "delayed", { "client": "client.4976210", "tid": 180637 }, [ { "time": "2017-06-29 12:35:46.720883", "event": "initiated" }, { "time": "2017-06-29 12:35:46.721107", "event": "queued_for_pg" }, { "time": "2017-06-29 12:35:46.721128", "event": "reached_pg" }, { "time": "2017-06-29 12:35:46.721149", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4981571.0:18791 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279208)", "initiated_at": "2017-06-29 13:48:16.277165", "age": 3053.119882, "duration": 3053.138278, "type_data": [ "delayed", { "client": "client.4981571", "tid": 18791 }, [ { "time": "2017-06-29 13:48:16.277165", "event": "initiated" }, { "time": "2017-06-29 13:48:16.277573", "event": "queued_for_pg" }, { "time": "2017-06-29 13:48:16.277593", "event": "reached_pg" }, { "time": "2017-06-29 13:48:16.278146", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4981346.0:714408 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279200)", "initiated_at": "2017-06-29 11:17:44.080284", "age": 12085.316763, "duration": 12085.335281, "type_data": [ "delayed", { "client": "client.4981346", "tid": 714408 }, [ { "time": "2017-06-29 11:17:44.080284", "event": "initiated" }, { "time": "2017-06-29 11:17:44.080560", "event": "queued_for_pg" }, { "time": "2017-06-29 11:17:44.080582", "event": "reached_pg" }, { "time": "2017-06-29 11:17:44.081240", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4952662.0:1178841 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279187)", "initiated_at": "2017-06-29 09:46:50.111424", "age": 17539.285623, "duration": 17539.304165, "type_data": [ "delayed", { "client": "client.4952662", "tid": 1178841 }, [ { "time": "2017-06-29 09:46:50.111424", "event": "initiated" }, { "time": "2017-06-29 09:46:50.111695", "event": "queued_for_pg" }, { "time": "2017-06-29 09:46:50.111751", "event": "reached_pg" }, { "time": "2017-06-29 09:46:50.112092", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4888992.0:267567696 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279144)", "initiated_at": "2017-06-29 02:47:07.669854", "age": 42721.727193, "duration": 42721.745772, "type_data": [ "delayed", { "client": "client.4888992", "tid": 267567696 }, [ { "time": "2017-06-29 02:47:07.669854", "event": "initiated" }, { "time": "2017-06-29 02:47:07.670105", "event": "queued_for_pg" }, { "time": "2017-06-29 02:47:07.670126", "event": "reached_pg" }, { "time": "2017-06-29 02:47:07.670536", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4976261.0:4069 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279171)", "initiated_at": "2017-06-29 07:34:17.457290", "age": 25491.939756, "duration": 25491.958356, "type_data": [ "delayed", { "client": "client.4976261", "tid": 4069 }, [ { "time": "2017-06-29 07:34:17.457290", "event": "initiated" }, { "time": "2017-06-29 07:34:17.459186", "event": "queued_for_pg" }, { "time": "2017-06-29 07:34:17.459234", "event": "reached_pg" }, { "time": "2017-06-29 07:34:17.459441", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4967951.0:52908 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279163)", "initiated_at": "2017-06-29 05:56:29.309705", "age": 31360.087341, "duration": 31360.105962, "type_data": [ "delayed", { "client": "client.4967951", "tid": 52908 }, [ { "time": "2017-06-29 05:56:29.309705", "event": "initiated" }, { "time": "2017-06-29 05:56:29.310055", "event": "queued_for_pg" }, { "time": "2017-06-29 05:56:29.310297", "event": "reached_pg" }, { "time": "2017-06-29 05:56:29.311467", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4953495.0:809788 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279155)", "initiated_at": "2017-06-29 04:39:25.959042", "age": 35983.438004, "duration": 35983.456662, "type_data": [ "delayed", { "client": "client.4953495", "tid": 809788 }, [ { "time": "2017-06-29 04:39:25.959042", "event": "initiated" }, { "time": "2017-06-29 04:39:25.960380", "event": "queued_for_pg" }, { "time": "2017-06-29 04:39:25.960421", "event": "reached_pg" }, { "time": "2017-06-29 04:39:25.960593", "event": "waiting for scrub" } ] ] } ], "num_ops": 8 } # ceph daemon osd.397 dump_ops_in_flight { "ops": [ { "description": "osd_op(client.4976210.0:180637 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279203)", "initiated_at": "2017-06-29 12:35:46.720883", "age": 7452.029648, "duration": 7452.029766, "type_data": [ "delayed", { "client": "client.4976210", "tid": 180637 }, [ { "time": "2017-06-29 12:35:46.720883", "event": "initiated" }, { "time": "2017-06-29 12:35:46.721107", "event": "queued_for_pg" }, { "time": "2017-06-29 12:35:46.721128", "event": "reached_pg" }, { "time": "2017-06-29 12:35:46.721149", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4981571.0:18791 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279208)", "initiated_at": "2017-06-29 13:48:16.277165", "age": 3102.473366, "duration": 3102.473535, "type_data": [ "delayed", { "client": "client.4981571", "tid": 18791 }, [ { "time": "2017-06-29 13:48:16.277165", "event": "initiated" }, { "time": "2017-06-29 13:48:16.277573", "event": "queued_for_pg" }, { "time": "2017-06-29 13:48:16.277593", "event": "reached_pg" }, { "time": "2017-06-29 13:48:16.278146", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4981346.0:714408 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279200)", "initiated_at": "2017-06-29 11:17:44.080284", "age": 12134.670247, "duration": 12134.670457, "type_data": [ "delayed", { "client": "client.4981346", "tid": 714408 }, [ { "time": "2017-06-29 11:17:44.080284", "event": "initiated" }, { "time": "2017-06-29 11:17:44.080560", "event": "queued_for_pg" }, { "time": "2017-06-29 11:17:44.080582", "event": "reached_pg" }, { "time": "2017-06-29 11:17:44.081240", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4952662.0:1178841 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279187)", "initiated_at": "2017-06-29 09:46:50.111424", "age": 17588.639107, "duration": 17588.639344, "type_data": [ "delayed", { "client": "client.4952662", "tid": 1178841 }, [ { "time": "2017-06-29 09:46:50.111424", "event": "initiated" }, { "time": "2017-06-29 09:46:50.111695", "event": "queued_for_pg" }, { "time": "2017-06-29 09:46:50.111751", "event": "reached_pg" }, { "time": "2017-06-29 09:46:50.112092", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4888992.0:267567696 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279144)", "initiated_at": "2017-06-29 02:47:07.669854", "age": 42771.080677, "duration": 42771.080979, "type_data": [ "delayed", { "client": "client.4888992", "tid": 267567696 }, [ { "time": "2017-06-29 02:47:07.669854", "event": "initiated" }, { "time": "2017-06-29 02:47:07.670105", "event": "queued_for_pg" }, { "time": "2017-06-29 02:47:07.670126", "event": "reached_pg" }, { "time": "2017-06-29 02:47:07.670536", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4976261.0:4069 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279171)", "initiated_at": "2017-06-29 07:34:17.457290", "age": 25541.293241, "duration": 25541.293583, "type_data": [ "delayed", { "client": "client.4976261", "tid": 4069 }, [ { "time": "2017-06-29 07:34:17.457290", "event": "initiated" }, { "time": "2017-06-29 07:34:17.459186", "event": "queued_for_pg" }, { "time": "2017-06-29 07:34:17.459234", "event": "reached_pg" }, { "time": "2017-06-29 07:34:17.459441", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4967951.0:52908 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279163)", "initiated_at": "2017-06-29 05:56:29.309705", "age": 31409.440826, "duration": 31409.441197, "type_data": [ "delayed", { "client": "client.4967951", "tid": 52908 }, [ { "time": "2017-06-29 05:56:29.309705", "event": "initiated" }, { "time": "2017-06-29 05:56:29.310055", "event": "queued_for_pg" }, { "time": "2017-06-29 05:56:29.310297", "event": "reached_pg" }, { "time": "2017-06-29 05:56:29.311467", "event": "waiting for scrub" } ] ] }, { "description": "osd_op(client.4953495.0:809788 4.df603eb8 rbd_data.3452e27d9473cb.00000000000003a8 [write 2650112~81920] snapc 0=[] ack+ondisk+write+known_if_redirected e279155)", "initiated_at": "2017-06-29 04:39:25.959042", "age": 36032.791489, "duration": 36032.791883, "type_data": [ "delayed", { "client": "client.4953495", "tid": 809788 }, [ { "time": "2017-06-29 04:39:25.959042", "event": "initiated" }, { "time": "2017-06-29 04:39:25.960380", "event": "queued_for_pg" }, { "time": "2017-06-29 04:39:25.960421", "event": "reached_pg" }, { "time": "2017-06-29 04:39:25.960593", "event": "waiting for scrub" } ] ] } ], "num_ops": 8 } # ceph daemon osd.397 dump_historic_ops { "num to keep": 20, "duration to keep": 600, "Ops": [ { "description": "osd_op(client.4850621.0:474506031 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.948997", "age": 169.408361, "duration": 0.156108, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506031 }, [ { "time": "2017-06-29 14:37:21.948997", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949024", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.099462", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.099590", "event": "started" }, { "time": "2017-06-29 14:37:22.100112", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.100843", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.100867", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.101090", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.102150", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.103574", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.103601", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.105093", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.105099", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.105105", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506032 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949027", "age": 169.408331, "duration": 0.156341, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506032 }, [ { "time": "2017-06-29 14:37:21.949027", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949044", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.100900", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.101030", "event": "started" }, { "time": "2017-06-29 14:37:22.101405", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.102001", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.102036", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.102212", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.103575", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.103602", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.105061", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.105356", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.105363", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.105368", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506033 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949070", "age": 169.408288, "duration": 0.159311, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506033 }, [ { "time": "2017-06-29 14:37:21.949070", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949099", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.102189", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.102307", "event": "started" }, { "time": "2017-06-29 14:37:22.102710", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.103517", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.103547", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.103757", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.105189", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.105328", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.105339", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.108064", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.108298", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.108380", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506034 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949107", "age": 169.408251, "duration": 0.159903, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506034 }, [ { "time": "2017-06-29 14:37:21.949107", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949119", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.103628", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.103772", "event": "started" }, { "time": "2017-06-29 14:37:22.104182", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.104837", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.104861", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.104997", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.105333", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.105346", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.105456", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.108718", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.108865", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.109010", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506035 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949169", "age": 169.408189, "duration": 0.159904, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506035 }, [ { "time": "2017-06-29 14:37:21.949169", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949181", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.105462", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.105486", "event": "started" }, { "time": "2017-06-29 14:37:22.105681", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.106411", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.106437", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.106560", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.108593", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.108624", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.109044", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.109063", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.109069", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.109073", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506036 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949225", "age": 169.408133, "duration": 0.160100, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506036 }, [ { "time": "2017-06-29 14:37:21.949225", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949241", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.106496", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.106752", "event": "started" }, { "time": "2017-06-29 14:37:22.107232", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.107890", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.107922", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.108107", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.108599", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.108629", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.109139", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.109308", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.109318", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.109325", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506037 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949275", "age": 169.408083, "duration": 0.168544, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506037 }, [ { "time": "2017-06-29 14:37:21.949275", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949291", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.109347", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.109659", "event": "started" }, { "time": "2017-06-29 14:37:22.110157", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.111587", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.111613", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.111729", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.117005", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.117035", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.117081", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.117368", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.117681", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.117820", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506038 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949398", "age": 169.407960, "duration": 0.168929, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506038 }, [ { "time": "2017-06-29 14:37:21.949398", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949427", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.111738", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.111945", "event": "started" }, { "time": "2017-06-29 14:37:22.112838", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.113730", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.113758", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.113941", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.117018", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.117914", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.118124", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.118212", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.118234", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.118327", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506039 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949510", "age": 169.407848, "duration": 0.173132, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506039 }, [ { "time": "2017-06-29 14:37:21.949510", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949584", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.113968", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.114627", "event": "started" }, { "time": "2017-06-29 14:37:22.115383", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.116956", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.116981", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.117124", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.118573", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.118628", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.118635", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.122617", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.122633", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.122642", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506040 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949593", "age": 169.407765, "duration": 0.175640, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506040 }, [ { "time": "2017-06-29 14:37:21.949593", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949627", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.118645", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.118766", "event": "started" }, { "time": "2017-06-29 14:37:22.119198", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.119854", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.119893", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.120035", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.122404", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.122429", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.122501", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.125216", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.125225", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.125233", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506041 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949634", "age": 169.407724, "duration": 0.175978, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506041 }, [ { "time": "2017-06-29 14:37:21.949634", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949652", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.119911", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.120129", "event": "started" }, { "time": "2017-06-29 14:37:22.120517", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.121162", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.121177", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.121326", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.122434", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.122610", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.125244", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.125308", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.125500", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.125611", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506042 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949690", "age": 169.407668, "duration": 0.175962, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506042 }, [ { "time": "2017-06-29 14:37:21.949690", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949722", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.121191", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.121293", "event": "started" }, { "time": "2017-06-29 14:37:22.121661", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.122362", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.122386", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.122593", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.125187", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.125246", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.125637", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.125645", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.125649", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.125653", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506043 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949776", "age": 169.407582, "duration": 0.176188, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506043 }, [ { "time": "2017-06-29 14:37:21.949776", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949826", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.122735", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.122974", "event": "started" }, { "time": "2017-06-29 14:37:22.123457", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.124026", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.124035", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.124236", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.125193", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.125249", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.125730", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.125875", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.125888", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.125964", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506044 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949832", "age": 169.407526, "duration": 0.179841, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506044 }, [ { "time": "2017-06-29 14:37:21.949832", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949847", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.124083", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.124182", "event": "started" }, { "time": "2017-06-29 14:37:22.124580", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.125169", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.125177", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.125316", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.125669", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.125854", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.126067", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.129376", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.129447", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.129673", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506045 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949895", "age": 169.407463, "duration": 0.183653, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506045 }, [ { "time": "2017-06-29 14:37:21.949895", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949913", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.126104", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.126418", "event": "started" }, { "time": "2017-06-29 14:37:22.126690", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.127578", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.127605", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.127752", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.130127", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.130178", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.130203", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.133119", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.133133", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.133548", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506046 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949933", "age": 169.407425, "duration": 0.183771, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506046 }, [ { "time": "2017-06-29 14:37:21.949933", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949955", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.127621", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.127647", "event": "started" }, { "time": "2017-06-29 14:37:22.127857", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.129257", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.129287", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.129429", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.130185", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.130208", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.133589", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.133690", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.133697", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.133704", "event": "done" } ] ] }, { "description": "osd_op(client.4850621.0:474506047 4.44543e40 rbd_data.38922b2885e38f.0000000000018be5 [] snapc 0=[] ack+ondisk+write+known_if_redirected e279218)", "initiated_at": "2017-06-29 14:37:21.949971", "age": 169.407387, "duration": 0.183856, "type_data": [ "commit sent; apply or cleanup", { "client": "client.4850621", "tid": 474506047 }, [ { "time": "2017-06-29 14:37:21.949971", "event": "initiated" }, { "time": "2017-06-29 14:37:21.949985", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:22.130215", "event": "reached_pg" }, { "time": "2017-06-29 14:37:22.130859", "event": "started" }, { "time": "2017-06-29 14:37:22.131656", "event": "waiting for subops from 69,377" }, { "time": "2017-06-29 14:37:22.133064", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:22.133115", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:22.133672", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:22.133724", "event": "sub_op_commit_rec from 69" }, { "time": "2017-06-29 14:37:22.133729", "event": "op_commit" }, { "time": "2017-06-29 14:37:22.133804", "event": "sub_op_commit_rec from 377" }, { "time": "2017-06-29 14:37:22.133809", "event": "commit_sent" }, { "time": "2017-06-29 14:37:22.133816", "event": "op_applied" }, { "time": "2017-06-29 14:37:22.133827", "event": "done" } ] ] }, { "description": "osd_repop(client.4687667.0:2079863051 4.8533 4:cca1d83a:::rbd_data.477e0378acd3bb.000000000000e9e6:head v 279218'7289366)", "initiated_at": "2017-06-29 14:37:39.188370", "age": 152.168988, "duration": 0.172779, "type_data": [ "commit sent; apply or cleanup", [ { "time": "2017-06-29 14:37:39.188370", "event": "initiated" }, { "time": "2017-06-29 14:37:39.188879", "event": "queued_for_pg" }, { "time": "2017-06-29 14:37:39.188897", "event": "reached_pg" }, { "time": "2017-06-29 14:37:39.188994", "event": "started" }, { "time": "2017-06-29 14:37:39.192134", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:37:39.192158", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:37:39.192304", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:37:39.192350", "event": "commit_sent" }, { "time": "2017-06-29 14:37:39.361147", "event": "sub_op_applied" }, { "time": "2017-06-29 14:37:39.361149", "event": "done" } ] ] }, { "description": "osd_op(client.2876330.0:27903880541 4.67c36c2a rbd_data.2b56c9548a720e.0000000000001afd [] snapc 0=[] ack+ondisk+write+known_if_redirected e279219)", "initiated_at": "2017-06-29 14:39:47.402981", "age": 23.954377, "duration": 0.125320, "type_data": [ "commit sent; apply or cleanup", { "client": "client.2876330", "tid": 27903880541 }, [ { "time": "2017-06-29 14:39:47.402981", "event": "initiated" }, { "time": "2017-06-29 14:39:47.403113", "event": "queued_for_pg" }, { "time": "2017-06-29 14:39:47.521980", "event": "reached_pg" }, { "time": "2017-06-29 14:39:47.522233", "event": "started" }, { "time": "2017-06-29 14:39:47.522580", "event": "waiting for subops from 46,122" }, { "time": "2017-06-29 14:39:47.523313", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:39:47.523341", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:39:47.523517", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:39:47.525932", "event": "sub_op_commit_rec from 46" }, { "time": "2017-06-29 14:39:47.526490", "event": "op_commit" }, { "time": "2017-06-29 14:39:47.528186", "event": "op_applied" }, { "time": "2017-06-29 14:39:47.528219", "event": "sub_op_commit_rec from 122" }, { "time": "2017-06-29 14:39:47.528279", "event": "commit_sent" }, { "time": "2017-06-29 14:39:47.528301", "event": "done" } ] ] }, { "description": "osd_op(client.2876330.0:27903880543 4.67c36c2a rbd_data.2b56c9548a720e.0000000000001afd [] snapc 0=[] ack+ondisk+write+known_if_redirected e279219)", "initiated_at": "2017-06-29 14:39:47.403785", "age": 23.953573, "duration": 0.126216, "type_data": [ "commit sent; apply or cleanup", { "client": "client.2876330", "tid": 27903880543 }, [ { "time": "2017-06-29 14:39:47.403785", "event": "initiated" }, { "time": "2017-06-29 14:39:47.403824", "event": "queued_for_pg" }, { "time": "2017-06-29 14:39:47.526536", "event": "reached_pg" }, { "time": "2017-06-29 14:39:47.526751", "event": "started" }, { "time": "2017-06-29 14:39:47.527289", "event": "waiting for subops from 46,122" }, { "time": "2017-06-29 14:39:47.528130", "event": "commit_queued_for_journal_write" }, { "time": "2017-06-29 14:39:47.528172", "event": "write_thread_in_journal_buffer" }, { "time": "2017-06-29 14:39:47.528442", "event": "journaled_completion_queued" }, { "time": "2017-06-29 14:39:47.528547", "event": "op_commit" }, { "time": "2017-06-29 14:39:47.528631", "event": "op_applied" }, { "time": "2017-06-29 14:39:47.529094", "event": "sub_op_commit_rec from 46" }, { "time": "2017-06-29 14:39:47.529992", "event": "sub_op_commit_rec from 122" }, { "time": "2017-06-29 14:39:47.529998", "event": "commit_sent" }, { "time": "2017-06-29 14:39:47.530001", "event": "done" } ] ] } ] } # restart ceph-osd id=397 ceph-osd (ceph/397) start/running, process 266709 # ceph -v ; dpkg -l | grep ceph ceph version 10.2.6 (656b5b63ed7c43bd014bcafd81b001959d5f089f) ii ceph 10.2.6-1trusty amd64 distributed storage and file system ii ceph-base 10.2.6-1trusty amd64 common ceph daemon libraries and management tools ii ceph-common 10.2.6-1trusty amd64 common utilities to mount and interact with a ceph storage cluster ii ceph-mds 10.2.6-1trusty amd64 metadata server for the ceph distributed file system ii ceph-mon 10.2.6-1trusty amd64 monitor server for the ceph storage system ii ceph-osd 10.2.6-1trusty amd64 OSD server for the ceph storage system ii libcephfs1 10.2.6-1trusty amd64 Ceph distributed file system client library ii python-cephfs 10.2.6-1trusty amd64 Python libraries for the Ceph libcephfs library root@prod-data17:/home/adatri# uname -r 3.19.0-49-generic root@prod-data17:/home/adatri#