[Wed May 28 17:45:03 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:45:03 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:45:04 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:45:04 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:45:05 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:45:21 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:45:35 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:45:36 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:45:41 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:45:42 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:45:44 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:45:50 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:46:32 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:46:35 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:46:45 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:46:53 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:46:54 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:46:58 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:47:05 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:47:05 2014] libceph: osd4 10.214.132.16:6810 socket closed (con state OPEN) [Wed May 28 17:47:06 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:47:07 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:47:07 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:47:10 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:47:12 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:47:16 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:47:16 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:47:16 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:47:17 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:47:24 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:47:27 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:47:28 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:47:29 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:47:33 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:47:33 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:47:34 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:47:39 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:47:40 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:47:41 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:47:46 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:47:50 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:47:54 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:47:54 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:48:00 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:48:00 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:48:00 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:01 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:48:03 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:48:03 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:05 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:06 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:48:06 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:10 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:10 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:10 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:48:12 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:48:13 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:48:13 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:48:16 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:17 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:48:19 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:19 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:48:19 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:48:19 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:48:19 2014] libceph: osd4 10.214.132.16:6810 socket closed (con state OPEN) [Wed May 28 17:48:20 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:48:20 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:48:21 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:48:21 2014] libceph: osd3 10.214.132.16:6805 socket closed (con state OPEN) [Wed May 28 17:48:22 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:48:27 2014] libceph: osd1 10.214.132.15:6805 socket closed (con state OPEN) [Wed May 28 17:48:29 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:48:29 2014] libceph: osd5 10.214.132.16:6800 socket closed (con state OPEN) [Wed May 28 17:48:29 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:48:31 2014] libceph: osd2 10.214.132.15:6810 socket closed (con state OPEN) [Wed May 28 17:48:31 2014] libceph: osd0 10.214.132.15:6800 socket closed (con state OPEN) [Wed May 28 17:50:56 2014] INFO: task kworker/7:2:902 blocked for more than 120 seconds. [Wed May 28 17:50:56 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:56 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:56 2014] kworker/7:2 D ffffffff818104c0 0 902 2 0x00000000 [Wed May 28 17:50:56 2014] Workqueue: ceph-msgr con_work [libceph] [Wed May 28 17:50:56 2014] ffff880036fe3c28 0000000000000046 0000000000000000 0000000000000000 [Wed May 28 17:50:56 2014] ffff88022165e440 0000000000013100 ffff880036fe3fd8 0000000000013100 [Wed May 28 17:50:56 2014] ffff880223d89910 ffff88022165e440 0000000000000000 ffff8802210397c0 [Wed May 28 17:50:56 2014] Call Trace: [Wed May 28 17:50:56 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:56 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:56 2014] [] __mutex_lock_slowpath+0xd3/0x1c0 [Wed May 28 17:50:56 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:56 2014] [] alloc_msg+0xcf/0x240 [libceph] [Wed May 28 17:50:56 2014] [] con_work+0x19f0/0x20b0 [libceph] [Wed May 28 17:50:56 2014] [] ? put_prev_entity+0x3c/0x320 [Wed May 28 17:50:56 2014] [] ? pick_next_task_fair+0x292/0x470 [Wed May 28 17:50:56 2014] [] process_one_work+0x174/0x4a0 [Wed May 28 17:50:56 2014] [] worker_thread+0x11b/0x370 [Wed May 28 17:50:56 2014] [] ? manage_workers.isra.20+0x2d0/0x2d0 [Wed May 28 17:50:56 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:56 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:56 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:56 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:56 2014] INFO: task kworker/5:2:10903 blocked for more than 120 seconds. [Wed May 28 17:50:56 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:56 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:56 2014] kworker/5:2 D ffff880169cf1910 0 10903 2 0x00000000 [Wed May 28 17:50:57 2014] Workqueue: ceph-msgr con_work [libceph] [Wed May 28 17:50:57 2014] ffff88002fbc5c08 0000000000000046 ffff88002fbc5bd8 ffff880221665600 [Wed May 28 17:50:57 2014] ffff880188274b30 0000000000013100 ffff88002fbc5fd8 0000000000013100 [Wed May 28 17:50:57 2014] ffff880169cf1910 ffff880188274b30 ffffffff818af620 ffff8802210397c0 [Wed May 28 17:50:57 2014] Call Trace: [Wed May 28 17:50:57 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:57 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:57 2014] [] __mutex_lock_slowpath+0xd3/0x1c0 [Wed May 28 17:50:57 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:57 2014] [] osd_reset+0x42/0x200 [libceph] [Wed May 28 17:50:57 2014] [] ? sock_release+0x79/0xa0 [Wed May 28 17:50:57 2014] [] con_work+0x59e/0x20b0 [libceph] [Wed May 28 17:50:57 2014] [] ? put_prev_entity+0x3c/0x320 [Wed May 28 17:50:57 2014] [] ? pick_next_task_fair+0x292/0x470 [Wed May 28 17:50:57 2014] [] process_one_work+0x174/0x4a0 [Wed May 28 17:50:57 2014] [] worker_thread+0x11b/0x370 [Wed May 28 17:50:57 2014] [] ? manage_workers.isra.20+0x2d0/0x2d0 [Wed May 28 17:50:57 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] INFO: task kworker/6:0:10930 blocked for more than 120 seconds. [Wed May 28 17:50:57 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:57 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:57 2014] kworker/6:0 D ffffffff818104c0 0 10930 2 0x00000000 [Wed May 28 17:50:57 2014] Workqueue: ceph-msgr con_work [libceph] [Wed May 28 17:50:57 2014] ffff880151385c08 0000000000000046 ffff880151385c28 ffffffff81597b5c [Wed May 28 17:50:57 2014] ffff88000b25cb30 0000000000013100 ffff880151385fd8 0000000000013100 [Wed May 28 17:50:57 2014] ffff880223d88000 ffff88000b25cb30 ffff880221624b30 ffff8802210397c0 [Wed May 28 17:50:57 2014] Call Trace: [Wed May 28 17:50:57 2014] [] ? sk_reset_timer+0x1c/0x30 [Wed May 28 17:50:57 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:57 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:57 2014] [] __mutex_lock_slowpath+0xd3/0x1c0 [Wed May 28 17:50:57 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:57 2014] [] osd_reset+0x42/0x200 [libceph] [Wed May 28 17:50:57 2014] [] ? sock_release+0x79/0xa0 [Wed May 28 17:50:57 2014] [] con_work+0x59e/0x20b0 [libceph] [Wed May 28 17:50:57 2014] [] ? put_prev_entity+0x3c/0x320 [Wed May 28 17:50:57 2014] [] ? pick_next_task_fair+0x292/0x470 [Wed May 28 17:50:57 2014] [] process_one_work+0x174/0x4a0 [Wed May 28 17:50:57 2014] [] worker_thread+0x11b/0x370 [Wed May 28 17:50:57 2014] [] ? manage_workers.isra.20+0x2d0/0x2d0 [Wed May 28 17:50:57 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] INFO: task kworker/0:0:10954 blocked for more than 120 seconds. [Wed May 28 17:50:57 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:57 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:57 2014] kworker/0:0 D ffffffff818104c0 0 10954 2 0x00000000 [Wed May 28 17:50:57 2014] Workqueue: ceph-msgr con_work [libceph] [Wed May 28 17:50:57 2014] ffff8800b968bc28 0000000000000046 0000000000000000 0000000000000000 [Wed May 28 17:50:57 2014] ffff880223cd4b30 0000000000013100 ffff8800b968bfd8 0000000000013100 [Wed May 28 17:50:57 2014] ffffffff81c144a0 ffff880223cd4b30 0000000000000000 ffff8802210397c0 [Wed May 28 17:50:57 2014] Call Trace: [Wed May 28 17:50:57 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:57 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:57 2014] [] __mutex_lock_slowpath+0xd3/0x1c0 [Wed May 28 17:50:57 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:57 2014] [] alloc_msg+0xcf/0x240 [libceph] [Wed May 28 17:50:57 2014] [] con_work+0x19f0/0x20b0 [libceph] [Wed May 28 17:50:57 2014] [] ? put_prev_entity+0x3c/0x320 [Wed May 28 17:50:57 2014] [] ? pick_next_task_fair+0x292/0x470 [Wed May 28 17:50:57 2014] [] process_one_work+0x174/0x4a0 [Wed May 28 17:50:57 2014] [] worker_thread+0x11b/0x370 [Wed May 28 17:50:57 2014] [] ? manage_workers.isra.20+0x2d0/0x2d0 [Wed May 28 17:50:57 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] INFO: task kworker/4:2:10956 blocked for more than 120 seconds. [Wed May 28 17:50:57 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:57 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:57 2014] kworker/4:2 D ffff8802234d9910 0 10956 2 0x00000000 [Wed May 28 17:50:57 2014] Workqueue: ceph-msgr con_work [libceph] [Wed May 28 17:50:57 2014] ffff8801dd65bc28 0000000000000046 ffff8801dd65bc48 0000000000000000 [Wed May 28 17:50:57 2014] ffff8801994f1910 0000000000013100 ffff8801dd65bfd8 0000000000013100 [Wed May 28 17:50:57 2014] ffff8802234d9910 ffff8801994f1910 ffff880227293c00 ffff8802210397c0 [Wed May 28 17:50:57 2014] Call Trace: [Wed May 28 17:50:57 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:57 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:57 2014] [] __mutex_lock_slowpath+0xd3/0x1c0 [Wed May 28 17:50:57 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:57 2014] [] alloc_msg+0xcf/0x240 [libceph] [Wed May 28 17:50:57 2014] [] con_work+0x19f0/0x20b0 [libceph] [Wed May 28 17:50:57 2014] [] ? put_prev_entity+0x3c/0x320 [Wed May 28 17:50:57 2014] [] ? pick_next_task_fair+0x292/0x470 [Wed May 28 17:50:57 2014] [] process_one_work+0x174/0x4a0 [Wed May 28 17:50:57 2014] [] worker_thread+0x11b/0x370 [Wed May 28 17:50:57 2014] [] ? manage_workers.isra.20+0x2d0/0x2d0 [Wed May 28 17:50:57 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] INFO: task kworker/6:2:10960 blocked for more than 120 seconds. [Wed May 28 17:50:57 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:57 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:57 2014] kworker/6:2 D 0000000000000000 0 10960 2 0x00000000 [Wed May 28 17:50:57 2014] Workqueue: ceph-msgr con_work [libceph] [Wed May 28 17:50:57 2014] ffff880119f83c28 0000000000000046 ffff880119f83c68 0000000000000000 [Wed May 28 17:50:57 2014] ffff8801867fb220 0000000000013100 ffff880119f83fd8 0000000000013100 [Wed May 28 17:50:57 2014] ffff88000b259910 ffff8801867fb220 0000000000000000 ffff8802210397c0 [Wed May 28 17:50:57 2014] Call Trace: [Wed May 28 17:50:57 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:57 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:57 2014] [] __mutex_lock_slowpath+0xd3/0x1c0 [Wed May 28 17:50:57 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:57 2014] [] alloc_msg+0xcf/0x240 [libceph] [Wed May 28 17:50:57 2014] [] con_work+0x19f0/0x20b0 [libceph] [Wed May 28 17:50:57 2014] [] ? ttwu_do_activate.constprop.117+0x66/0x70 [Wed May 28 17:50:57 2014] [] process_one_work+0x174/0x4a0 [Wed May 28 17:50:57 2014] [] worker_thread+0x11b/0x370 [Wed May 28 17:50:57 2014] [] ? manage_workers.isra.20+0x2d0/0x2d0 [Wed May 28 17:50:57 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] INFO: task kworker/3:2:13093 blocked for more than 120 seconds. [Wed May 28 17:50:57 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:57 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:57 2014] kworker/3:2 D ffffffff818104c0 0 13093 2 0x00000000 [Wed May 28 17:50:57 2014] Workqueue: events handle_timeout [libceph] [Wed May 28 17:50:57 2014] ffff88022093bce8 0000000000000046 0000000000013100 0000000000013100 [Wed May 28 17:50:57 2014] ffff88016c209910 0000000000013100 ffff88022093bfd8 0000000000013100 [Wed May 28 17:50:57 2014] ffff880223d73220 ffff88016c209910 0000000000000000 ffff8802210397c0 [Wed May 28 17:50:57 2014] Call Trace: [Wed May 28 17:50:57 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:57 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:57 2014] [] __mutex_lock_slowpath+0xd3/0x1c0 [Wed May 28 17:50:57 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:57 2014] [] handle_timeout+0x65/0x180 [libceph] [Wed May 28 17:50:57 2014] [] process_one_work+0x174/0x4a0 [Wed May 28 17:50:57 2014] [] worker_thread+0x11b/0x370 [Wed May 28 17:50:57 2014] [] ? manage_workers.isra.20+0x2d0/0x2d0 [Wed May 28 17:50:57 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:57 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:57 2014] INFO: task kworker/u64:0:14314 blocked for more than 120 seconds. [Wed May 28 17:50:57 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:57 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:57 2014] kworker/u64:0 D 0000000000000000 0 14314 2 0x00000000 [Wed May 28 17:50:57 2014] Workqueue: writeback bdi_writeback_workfn (flush-251:0) [Wed May 28 17:50:57 2014] ffff8801994ab608 0000000000000046 ffff8801994ab5c8 0000000000000000 [Wed May 28 17:50:57 2014] ffff880036a51910 0000000000013100 ffff8801994abfd8 0000000000013100 [Wed May 28 17:50:57 2014] ffff88016c209910 ffff880036a51910 ffff8801994ab5d8 ffff880227273a10 [Wed May 28 17:50:57 2014] Call Trace: [Wed May 28 17:50:57 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:57 2014] [] io_schedule+0x8f/0xd0 [Wed May 28 17:50:57 2014] [] get_request+0x413/0x800 [Wed May 28 17:50:57 2014] [] ? bio_attempt_back_merge+0x93/0x100 [Wed May 28 17:50:57 2014] [] ? __wake_up_sync+0x20/0x20 [Wed May 28 17:50:57 2014] [] blk_queue_bio+0x95/0x380 [Wed May 28 17:50:57 2014] [] generic_make_request+0xc0/0x100 [Wed May 28 17:50:57 2014] [] submit_bio+0x75/0x140 [Wed May 28 17:50:57 2014] [] _submit_bh+0x136/0x1f0 [Wed May 28 17:50:57 2014] [] submit_bh+0x10/0x20 [Wed May 28 17:50:57 2014] [] __block_write_full_page+0x1a0/0x360 [Wed May 28 17:50:57 2014] [] ? end_buffer_async_read+0x120/0x120 [Wed May 28 17:50:57 2014] [] ? end_buffer_async_read+0x120/0x120 [Wed May 28 17:50:57 2014] [] block_write_full_page_endio+0xb3/0xf0 [Wed May 28 17:50:58 2014] [] block_write_full_page+0x15/0x20 [Wed May 28 17:50:58 2014] [] ext3_ordered_writepage+0x28d/0x2a0 [Wed May 28 17:50:58 2014] [] ? SyS_msync+0x1d0/0x1d0 [Wed May 28 17:50:58 2014] [] __writepage+0x17/0x50 [Wed May 28 17:50:58 2014] [] write_cache_pages+0x20a/0x470 [Wed May 28 17:50:58 2014] [] ? __blk_run_queue+0x37/0x50 [Wed May 28 17:50:58 2014] [] ? set_page_dirty_lock+0x60/0x60 [Wed May 28 17:50:58 2014] [] generic_writepages+0x4a/0x70 [Wed May 28 17:50:58 2014] [] do_writepages+0x35/0x40 [Wed May 28 17:50:58 2014] [] __writeback_single_inode+0x45/0x290 [Wed May 28 17:50:58 2014] [] writeback_sb_inodes+0x216/0x450 [Wed May 28 17:50:58 2014] [] __writeback_inodes_wb+0x9e/0xd0 [Wed May 28 17:50:58 2014] [] wb_writeback+0x27b/0x320 [Wed May 28 17:50:58 2014] [] ? wb_check_old_data_flush+0x4e/0xb0 [Wed May 28 17:50:58 2014] [] bdi_writeback_workfn+0x209/0x460 [Wed May 28 17:50:58 2014] [] process_one_work+0x174/0x4a0 [Wed May 28 17:50:58 2014] [] worker_thread+0x11b/0x370 [Wed May 28 17:50:58 2014] [] ? manage_workers.isra.20+0x2d0/0x2d0 [Wed May 28 17:50:58 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:58 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:58 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:58 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:58 2014] INFO: task kjournald:14978 blocked for more than 120 seconds. [Wed May 28 17:50:58 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:58 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:58 2014] kjournald D 0000000000000002 0 14978 2 0x00000000 [Wed May 28 17:50:58 2014] ffff880214049778 0000000000000046 ffff880214049798 ffff880017a1eea0 [Wed May 28 17:50:58 2014] ffff880221624b30 0000000000013100 ffff880214049fd8 0000000000013100 [Wed May 28 17:50:58 2014] ffff8802217c8000 ffff880221624b30 ffff880188274b30 ffff8800173439b8 [Wed May 28 17:50:58 2014] Call Trace: [Wed May 28 17:50:58 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:58 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:58 2014] [] __mutex_lock_slowpath+0x1b5/0x1c0 [Wed May 28 17:50:58 2014] [] ? ceph_str_hash+0x26/0x80 [libceph] [Wed May 28 17:50:58 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:58 2014] [] ceph_con_send+0x4d/0x150 [libceph] [Wed May 28 17:50:58 2014] [] __send_queued+0x132/0x170 [libceph] [Wed May 28 17:50:58 2014] [] __ceph_osdc_start_request+0x5b/0xd0 [libceph] [Wed May 28 17:50:58 2014] [] ceph_osdc_start_request+0x51/0x80 [libceph] [Wed May 28 17:50:58 2014] [] rbd_obj_request_submit.isra.23+0x10/0x20 [rbd] [Wed May 28 17:50:58 2014] [] rbd_img_obj_request_submit+0x1ae/0x440 [rbd] [Wed May 28 17:50:58 2014] [] rbd_img_request_submit+0x4c/0x60 [rbd] [Wed May 28 17:50:58 2014] [] rbd_request_fn+0x248/0x2a0 [rbd] [Wed May 28 17:50:58 2014] [] __blk_run_queue+0x37/0x50 [Wed May 28 17:50:58 2014] [] queue_unplugged+0x3d/0xc0 [Wed May 28 17:50:58 2014] [] blk_flush_plug_list+0x1fb/0x230 [Wed May 28 17:50:58 2014] [] io_schedule+0x78/0xd0 [Wed May 28 17:50:58 2014] [] get_request+0x413/0x800 [Wed May 28 17:50:58 2014] [] ? __wake_up_sync+0x20/0x20 [Wed May 28 17:50:58 2014] [] blk_queue_bio+0x95/0x380 [Wed May 28 17:50:58 2014] [] generic_make_request+0xc0/0x100 [Wed May 28 17:50:58 2014] [] submit_bio+0x75/0x140 [Wed May 28 17:50:58 2014] [] _submit_bh+0x136/0x1f0 [Wed May 28 17:50:58 2014] [] journal_do_submit_data+0x41/0x50 [Wed May 28 17:50:58 2014] [] journal_commit_transaction+0x1150/0x1350 [Wed May 28 17:50:58 2014] [] ? try_to_del_timer_sync+0x4f/0x70 [Wed May 28 17:50:58 2014] [] kjournald+0xe1/0x260 [Wed May 28 17:50:58 2014] [] ? __wake_up_sync+0x20/0x20 [Wed May 28 17:50:58 2014] [] ? commit_timeout+0x10/0x10 [Wed May 28 17:50:58 2014] [] kthread+0xc9/0xe0 [Wed May 28 17:50:58 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:58 2014] [] ret_from_fork+0x7c/0xb0 [Wed May 28 17:50:58 2014] [] ? flush_kthread_worker+0xb0/0xb0 [Wed May 28 17:50:58 2014] INFO: task ffsb:16690 blocked for more than 120 seconds. [Wed May 28 17:50:58 2014] Tainted: G I E 3.15.0-rc6-ceph-00011-gea590a0 #1 [Wed May 28 17:50:58 2014] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed May 28 17:50:58 2014] ffsb D ffffffff818104c0 0 16690 15017 0x00000000 [Wed May 28 17:50:58 2014] ffff880222f57818 0000000000000082 ffff880222f577c8 ffffffffa06ad258 [Wed May 28 17:50:58 2014] ffff8801176d1910 0000000000013100 ffff880222f57fd8 0000000000013100 [Wed May 28 17:50:58 2014] ffff880223d73220 ffff8801176d1910 ffff880222f57848 ffff8802210397c0 [Wed May 28 17:50:58 2014] Call Trace: [Wed May 28 17:50:58 2014] [] ? ceph_osdc_msg_data_add+0x98/0xa0 [libceph] [Wed May 28 17:50:58 2014] [] schedule+0x29/0x70 [Wed May 28 17:50:58 2014] [] schedule_preempt_disabled+0xe/0x10 [Wed May 28 17:50:58 2014] [] __mutex_lock_slowpath+0xd3/0x1c0 [Wed May 28 17:50:58 2014] [] mutex_lock+0x23/0x37 [Wed May 28 17:50:58 2014] [] ceph_osdc_start_request+0x42/0x80 [libceph] [Wed May 28 17:50:58 2014] [] rbd_obj_request_submit.isra.23+0x10/0x20 [rbd] [Wed May 28 17:50:58 2014] [] rbd_img_obj_request_submit+0x1ae/0x440 [rbd] [Wed May 28 17:50:58 2014] [] rbd_img_request_submit+0x4c/0x60 [rbd] [Wed May 28 17:50:58 2014] [] rbd_request_fn+0x248/0x2a0 [rbd] [Wed May 28 17:50:58 2014] [] __blk_run_queue+0x37/0x50 [Wed May 28 17:50:58 2014] [] blk_queue_bio+0x2eb/0x380 [Wed May 28 17:50:58 2014] [] generic_make_request+0xc0/0x100 [Wed May 28 17:50:58 2014] [] submit_bio+0x75/0x140 [Wed May 28 17:50:58 2014] [] _submit_bh+0x136/0x1f0 [Wed May 28 17:50:58 2014] [] submit_bh+0x10/0x20 [Wed May 28 17:50:58 2014] [] __sync_dirty_buffer+0x77/0xd0 [Wed May 28 17:50:58 2014] [] sync_dirty_buffer+0x13/0x20 [Wed May 28 17:50:58 2014] [] journal_dirty_data+0x1ae/0x220 [Wed May 28 17:50:58 2014] [] ext3_journal_dirty_data+0x20/0x50 [Wed May 28 17:50:58 2014] [] journal_dirty_data_fn+0x25/0x30 [Wed May 28 17:50:58 2014] [] walk_page_buffers+0x85/0xc0 [Wed May 28 17:50:58 2014] [] ? ext3_journal_dirty_data+0x50/0x50 [Wed May 28 17:50:58 2014] [] ext3_ordered_write_end+0xa6/0x1d0 [Wed May 28 17:50:58 2014] [] generic_perform_write+0x111/0x1d0 [Wed May 28 17:50:58 2014] [] ? update_time+0x81/0xc0 [Wed May 28 17:50:58 2014] [] ? mnt_clone_write+0x12/0x30 [Wed May 28 17:50:58 2014] [] __generic_file_aio_write+0x1ce/0x3f0 [Wed May 28 17:50:58 2014] [] ? user_path_at_empty+0x63/0xa0 [Wed May 28 17:50:58 2014] [] generic_file_aio_write+0x5e/0xe0 [Wed May 28 17:50:58 2014] [] do_sync_write+0x5a/0x90 [Wed May 28 17:50:58 2014] [] vfs_write+0xcb/0x1f0 [Wed May 28 17:50:58 2014] [] SyS_write+0x4f/0xb0 [Wed May 28 17:50:58 2014] [] system_call_fastpath+0x1a/0x1f