Ceph

[  362.341358] INFO: task kworker/5:1:245 blocked for more than 120 seconds.
[  362.348271]       Tainted: GF         I  3.12.0-rc2-ceph-00009-gff638b7 #1
[  362.355259] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  362.363216] kworker/5:1     D 0000000000000000     0   245      2 0x00000000
[  362.370515] Workqueue: events do_async_commit [btrfs]
[  362.375686]  ffff88021f81b978 0000000000000046 ffff88021f81bb14 ffff88021f81bb14
[  362.383426]  ffff88021f820000 ffff88021f81bfd8 ffff88021f81bfd8 ffff88021f81bfd8
[  362.391164]  ffff880223d2de00 ffff88021f820000 ffff88021f81b988 ffff8801a96a5e00
[  362.398903] Call Trace:
[  362.401468]  [<ffffffff816c74b9>] schedule+0x29/0x70
[  362.406564]  [<ffffffffa01648fd>] btrfs_tree_lock+0x6d/0x230 [btrfs]
[  362.413034]  [<ffffffff81072530>] ? add_wait_queue+0x60/0x60
[  362.418817]  [<ffffffffa010840b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[  362.425646]  [<ffffffffa010d887>] btrfs_search_slot+0x867/0x930 [btrfs]
[  362.432351]  [<ffffffffa011a562>] ? run_clustered_refs+0x212/0xf10 [btrfs]
[  362.439307]  [<ffffffffa010f038>] btrfs_insert_empty_items+0x78/0xd0 [btrfs]
[  362.446475]  [<ffffffffa012276c>] insert_with_overflow+0x3c/0x110 [btrfs]
[  362.453383]  [<ffffffffa01228ff>] btrfs_insert_dir_item+0xbf/0x200 [btrfs]
[  362.460378]  [<ffffffffa012e36e>] create_pending_snapshot+0x7fe/0x9e0 [btrfs]
[  362.467631]  [<ffffffffa012e5d9>] create_pending_snapshots+0x89/0xa0 [btrfs]
[  362.474800]  [<ffffffffa012fa5c>] btrfs_commit_transaction+0x45c/0xa10 [btrfs]
[  362.482145]  [<ffffffff81072530>] ? add_wait_queue+0x60/0x60
[  362.487921]  [<ffffffffa0130042>] do_async_commit+0x32/0x50 [btrfs]
[  362.494294]  [<ffffffff81069eb4>] process_one_work+0x174/0x490
[  362.500230]  [<ffffffff8106af7b>] worker_thread+0x11b/0x370
[  362.505905]  [<ffffffff8106ae60>] ? manage_workers.isra.20+0x2d0/0x2d0
[  362.512537]  [<ffffffff81071d30>] kthread+0xc0/0xd0
[  362.517518]  [<ffffffff81071c70>] ? flush_kthread_worker+0xb0/0xb0
[  362.523808]  [<ffffffff816d0eec>] ret_from_fork+0x7c/0xb0
[  362.529314]  [<ffffffff81071c70>] ? flush_kthread_worker+0xb0/0xb0
[  362.535614] INFO: task btrfs-transacti:4641 blocked for more than 120 seconds.
[  362.542957]       Tainted: GF         I  3.12.0-rc2-ceph-00009-gff638b7 #1
[  362.549937] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  362.557885] btrfs-transacti D 0000000000000000     0  4641      2 0x00000000
[  362.565141]  ffff88021fa5bd78 0000000000000046 ffff880223d28418 0000000000000001
[  362.572911]  ffff88020191af00 ffff88021fa5bfd8 ffff88021fa5bfd8 ffff88021fa5bfd8
[  362.580683]  ffff880223d28000 ffff88020191af00 ffff88021fa5bd88 ffff8801a978c000
[  362.588457] Call Trace:
[  362.591009]  [<ffffffff816c74b9>] schedule+0x29/0x70
[  362.596095]  [<ffffffffa012e8ff>] wait_current_trans.isra.33+0xbf/0x120 [btrfs]
[  362.603526]  [<ffffffff81072530>] ? add_wait_queue+0x60/0x60
[  362.609363]  [<ffffffffa0130790>] start_transaction+0x340/0x530 [btrfs]
[  362.616137]  [<ffffffff8105b900>] ? call_timer_fn+0x150/0x150
[  362.622054]  [<ffffffffa01309e7>] btrfs_attach_transaction+0x17/0x20 [btrfs]
[  362.629271]  [<ffffffffa0129276>] transaction_kthread+0x86/0x240 [btrfs]
[  362.636146]  [<ffffffffa01291f0>] ? btrfs_destroy_delayed_refs+0x350/0x350 [btrfs]
[  362.643886]  [<ffffffff81071d30>] kthread+0xc0/0xd0
[  362.648930]  [<ffffffff81071c70>] ? flush_kthread_worker+0xb0/0xb0
[  362.655274]  [<ffffffff816d0eec>] ret_from_fork+0x7c/0xb0
[  362.660838]  [<ffffffff81071c70>] ? flush_kthread_worker+0xb0/0xb0
[  362.667207] INFO: task ceph-osd:4911 blocked for more than 120 seconds.
[  362.673988]       Tainted: GF         I  3.12.0-rc2-ceph-00009-gff638b7 #1
[  362.681024] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  362.689030] ceph-osd        D 0000000000000000     0  4911   4806 0x00000000
[  362.696357]  ffff8801a73f5b68 0000000000000086 ffff8801a73f5b08 ffffffff81067414
[  362.704200]  ffff88007a021780 ffff8801a73f5fd8 ffff8801a73f5fd8 ffff8801a73f5fd8
[  362.712045]  ffff88021f820000 ffff88007a021780 ffff8801a73f5b78 ffff8801a978c000
[  362.719880] Call Trace:
[  362.722491]  [<ffffffff81067414>] ? wake_up_worker+0x24/0x30
[  362.728314]  [<ffffffff816c74b9>] schedule+0x29/0x70
[  362.733468]  [<ffffffffa0130b9f>] btrfs_commit_transaction_async+0x14f/0x250 [btrfs]
[  362.741410]  [<ffffffffa01304e6>] ? start_transaction+0x96/0x530 [btrfs]
[  362.748240]  [<ffffffff81072530>] ? add_wait_queue+0x60/0x60
[  362.754084]  [<ffffffffa015f742>] btrfs_mksubvol.isra.60+0x292/0x3f0 [btrfs]
[  362.761321]  [<ffffffffa015f99e>] btrfs_ioctl_snap_create_transid+0xfe/0x190 [btrfs]
[  362.769264]  [<ffffffffa015faef>] ? btrfs_ioctl_snap_create_v2+0x3f/0x140 [btrfs]
[  362.776943]  [<ffffffffa015fbb2>] btrfs_ioctl_snap_create_v2+0x102/0x140 [btrfs]
[  362.784563]  [<ffffffffa0162d8a>] btrfs_ioctl+0xa8a/0x1fe0 [btrfs]
[  362.790911]  [<ffffffff810b5bd9>] ? futex_wake+0x109/0x130
[  362.796572]  [<ffffffff811d5bb2>] ? fsnotify+0x1d2/0x2b0
[  362.802052]  [<ffffffff8119552a>] ? do_sync_write+0x5a/0x90
[  362.807790]  [<ffffffff811a80fb>] do_vfs_ioctl+0x8b/0x4e0
[  362.813353]  [<ffffffff81197df3>] ? __sb_end_write+0x33/0x70
[  362.819177]  [<ffffffff811960f6>] ? vfs_write+0x196/0x200
[  362.824740]  [<ffffffff811a85e1>] SyS_ioctl+0x91/0xb0
[  362.829959]  [<ffffffff81196686>] ? SyS_pwrite64+0x66/0xa0
[  362.835611]  [<ffffffff816d0f96>] system_call_fastpath+0x1a/0x1f
[  362.841755] INFO: task ceph-osd:4912 blocked for more than 120 seconds.
[  362.848541]       Tainted: GF         I  3.12.0-rc2-ceph-00009-gff638b7 #1
[  362.855577] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  362.863587] ceph-osd        D 0000000000000000     0  4912   4806 0x00000000
[  362.870964]  ffff8801a73f76b8 0000000000000086 ffff8801a73f7787 00000019000131c0
[  362.878761]  ffff88007a022f00 ffff8801a73f7fd8 ffff8801a73f7fd8 ffff8801a73f7fd8
[  362.886623]  ffffffff81c114a0 ffff88007a022f00 ffff8801a73f76c8 ffff8801a978c000
[  362.894462] Call Trace:
[  362.896968]  [<ffffffff816c74b9>] schedule+0x29/0x70
[  362.902111]  [<ffffffffa012e8ff>] wait_current_trans.isra.33+0xbf/0x120 [btrfs]
[  362.909608]  [<ffffffff81072530>] ? add_wait_queue+0x60/0x60
[  362.915446]  [<ffffffffa0130790>] start_transaction+0x340/0x530 [btrfs]
[  362.922239]  [<ffffffffa0110d07>] ? block_group_cache_tree_search+0x27/0xe0 [btrfs]
[  362.930089]  [<ffffffffa0130a47>] btrfs_join_transaction+0x17/0x20 [btrfs]
[  362.937152]  [<ffffffffa013ebad>] can_nocow_extent+0x2fd/0x400 [btrfs]
[  362.943858]  [<ffffffffa013eefe>] btrfs_get_blocks_direct+0x24e/0x5f0 [btrfs]
[  362.951166]  [<ffffffff8134ad88>] ? cpumask_next_and+0x38/0x50
[  362.957167]  [<ffffffff811d189c>] do_blockdev_direct_IO+0x101c/0x20e0
[  362.963805]  [<ffffffffa013ecb0>] ? can_nocow_extent+0x400/0x400 [btrfs]
[  362.970688]  [<ffffffffa0134030>] ? record_one_backref+0x420/0x420 [btrfs]
[  362.977697]  [<ffffffff811d29b5>] __blockdev_direct_IO+0x55/0x60
[  362.983889]  [<ffffffffa013ecb0>] ? can_nocow_extent+0x400/0x400 [btrfs]
[  362.990775]  [<ffffffffa0134030>] ? record_one_backref+0x420/0x420 [btrfs]
[  362.997830]  [<ffffffffa0131b48>] btrfs_direct_IO+0x1d8/0x350 [btrfs]
[  363.005589]  [<ffffffffa013ecb0>] ? can_nocow_extent+0x400/0x400 [btrfs]
[  363.012468]  [<ffffffffa0134030>] ? record_one_backref+0x420/0x420 [btrfs]
[  363.019508]  [<ffffffff8112dfe6>] generic_file_direct_write+0xc6/0x180
[  363.026216]  [<ffffffffa0142659>] btrfs_file_aio_write+0x429/0x5c0 [btrfs]
[  363.033260]  [<ffffffff816c878f>] ? _raw_spin_unlock_irqrestore+0xf/0x20
[  363.040137]  [<ffffffff81195498>] do_sync_readv_writev+0x48/0x80
[  363.046309]  [<ffffffff811968c2>] do_readv_writev+0xd2/0x2b0
[  363.052148]  [<ffffffffa0142230>] ? __btrfs_buffered_write+0x490/0x490 [btrfs]
[  363.059550]  [<ffffffff811954d0>] ? do_sync_readv_writev+0x80/0x80
[  363.065894]  [<ffffffff816c5ac6>] ? mutex_lock+0x16/0x40
[  363.071384]  [<ffffffffa014079a>] ? btrfs_file_llseek+0xda/0x370 [btrfs]
[  363.078253]  [<ffffffff81196adc>] vfs_writev+0x3c/0x50
[  363.083558]  [<ffffffff81196c52>] SyS_writev+0x52/0xc0
[  363.088868]  [<ffffffff816d0f96>] system_call_fastpath+0x1a/0x1f