Ceph » Linux kernel client

[ 1199.875490] device fsid 59557a67-f2a6-402d-9a6c-b6ff59811401 devid 1 transid 7 /dev/sdc
[ 1199.889468] btrfs flagging fs with big metadata feature
[ 1530.893444] btrfs: truncated 1 orphans
[ 1564.595119] INFO: task btrfs-transacti:20513 blocked for more than 120 seconds.
[ 1564.602479] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1564.610369] btrfs-transacti D ffff88020bbb3f20     0 20513      2 0x00000000
[ 1564.617492]  ffff880220f7da78 0000000000000046 0000000000000286 ffff88008d1deed8
[ 1564.624979]  ffff88020bbb3f20 ffff880220f7dfd8 ffff880220f7dfd8 ffff880220f7dfd8
[ 1564.632478]  ffff880225d33f20 ffff88020bbb3f20 ffff880220f7da88 ffff88008d1deda0
[ 1564.639980] Call Trace:
[ 1564.642451]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1564.647498]  [<ffffffffa017dcc5>] btrfs_tree_lock+0x75/0x270 [btrfs]
[ 1564.653889]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1564.660212]  [<ffffffffa012170b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[ 1564.666972]  [<ffffffffa0126c5a>] btrfs_search_slot+0x6ca/0x790 [btrfs]
[ 1564.673624]  [<ffffffffa013e3df>] btrfs_lookup_inode+0x2f/0xa0 [btrfs]
[ 1564.680211]  [<ffffffff810b8c05>] ? trace_hardirqs_on_caller+0x105/0x190
[ 1564.686997]  [<ffffffffa0197fec>] __btrfs_update_delayed_inode+0x4c/0x1b0 [btrfs]
[ 1564.694542]  [<ffffffffa019906a>] ? __btrfs_run_delayed_items+0x13a/0x1e0 [btrfs]
[ 1564.702094]  [<ffffffff8134a68d>] ? do_raw_spin_unlock+0x5d/0xb0
[ 1564.708184]  [<ffffffffa01990c2>] __btrfs_run_delayed_items+0x192/0x1e0 [btrfs]
[ 1564.715582]  [<ffffffffa0199123>] btrfs_run_delayed_items+0x13/0x20 [btrfs]
[ 1564.722600]  [<ffffffffa0146e72>] btrfs_flush_all_pending_stuffs+0xa2/0xe0 [btrfs]
[ 1564.730257]  [<ffffffffa01488c1>] btrfs_commit_transaction+0x201/0xa50 [btrfs]
[ 1564.737554]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1564.743866]  [<ffffffffa0141945>] transaction_kthread+0x1b5/0x230 [btrfs]
[ 1564.750728]  [<ffffffffa0141790>] ? btrfs_destroy_delayed_refs.isra.102+0x330/0x330 [btrfs]
[ 1564.759142]  [<ffffffff8107c46a>] kthread+0xea/0xf0
[ 1564.764049]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1564.770280]  [<ffffffff8167b4ac>] ret_from_fork+0x7c/0xb0
[ 1564.775734]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1564.781939] 2 locks held by btrfs-transacti/20513:
[ 1564.786769]  #0:  (&fs_info->transaction_kthread_mutex){+.+...}, at: [<ffffffffa014187f>] transaction_kthread+0xef/0x230 [btrfs]
[ 1564.798502]  #1:  (&delayed_node->mutex){+.+...}, at: [<ffffffffa019906a>] __btrfs_run_delayed_items+0x13a/0x1e0 [btrfs]
[ 1564.809543] INFO: task btrfs-endio-wri:20563 blocked for more than 120 seconds.
[ 1564.816920] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1564.824791] btrfs-endio-wri D ffff880221400000     0 20563      2 0x00000000
[ 1564.831912]  ffff88021e86f928 0000000000000046 0000000000000286 ffff88008d1deed8
[ 1564.839418]  ffff880221400000 ffff88021e86ffd8 ffff88021e86ffd8 ffff88021e86ffd8
[ 1564.846930]  ffff88021c8f3f20 ffff880221400000 ffff88021e86f938 ffff88008d1deda0
[ 1564.854427] Call Trace:
[ 1564.856920]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1564.861943]  [<ffffffffa017dcc5>] btrfs_tree_lock+0x75/0x270 [btrfs]
[ 1564.868351]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1564.874668]  [<ffffffffa012170b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[ 1564.881440]  [<ffffffffa0126c5a>] btrfs_search_slot+0x6ca/0x790 [btrfs]
[ 1564.888133]  [<ffffffffa013c3e8>] btrfs_lookup_file_extent+0x38/0x40 [btrfs]
[ 1564.895229]  [<ffffffffa015c66e>] __btrfs_drop_extents+0x12e/0xaf0 [btrfs]
[ 1564.902168]  [<ffffffff810b36ba>] ? static_obj+0x5a/0x70
[ 1564.907524]  [<ffffffff8117ee28>] ? kmem_cache_alloc+0xc8/0x150
[ 1564.913485]  [<ffffffffa0147b35>] ? join_transaction.isra.30+0x25/0x3d0 [btrfs]
[ 1564.920875]  [<ffffffffa015daee>] btrfs_drop_extents+0x6e/0xa0 [btrfs]
[ 1564.927468]  [<ffffffffa014c22b>] insert_reserved_file_extent.constprop.59+0x7b/0x2c0 [btrfs]
[ 1564.936100]  [<ffffffffa0155a9e>] btrfs_finish_ordered_io+0x51e/0xa10 [btrfs]
[ 1564.943288]  [<ffffffffa0176531>] ? worker_loop+0xa1/0x5a0 [btrfs]
[ 1564.949513]  [<ffffffff816724c0>] ? _raw_spin_unlock_irq+0x30/0x40
[ 1564.955770]  [<ffffffffa0155fa5>] finish_ordered_fn+0x15/0x20 [btrfs]
[ 1564.962270]  [<ffffffffa01765fa>] worker_loop+0x16a/0x5a0 [btrfs]
[ 1564.968450]  [<ffffffffa0176490>] ? btrfs_queue_worker+0x320/0x320 [btrfs]
[ 1564.975386]  [<ffffffff8107c46a>] kthread+0xea/0xf0
[ 1564.980298]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1564.986539]  [<ffffffff8167b4ac>] ret_from_fork+0x7c/0xb0
[ 1564.991972]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1564.998206] 1 lock held by btrfs-endio-wri/20563:
[ 1565.002938]  #0:  (sb_internal#2){.+.+..}, at: [<ffffffffa0149b9d>] start_transaction+0x3dd/0x4f0 [btrfs]
[ 1565.012703] INFO: task flush-btrfs-6:20564 blocked for more than 120 seconds.
[ 1565.019881] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1565.027776] flush-btrfs-6   D ffff880221405eb0     0 20564      2 0x00000000
[ 1565.034890]  ffff880224fb72c8 0000000000000046 0000000000000286 ffff88008d1deed8
[ 1565.042413]  ffff880221405eb0 ffff880224fb7fd8 ffff880224fb7fd8 ffff880224fb7fd8
[ 1565.049946]  ffff880225e9bf20 ffff880221405eb0 ffff880224fb72d8 ffff88008d1deda0
[ 1565.057480] Call Trace:
[ 1565.059960]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1565.064987]  [<ffffffffa017dcc5>] btrfs_tree_lock+0x75/0x270 [btrfs]
[ 1565.071408]  [<ffffffff8117d0af>] ? kmem_cache_free+0xaf/0x1e0
[ 1565.077305]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1565.083623]  [<ffffffffa012170b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[ 1565.090394]  [<ffffffffa0126c5a>] btrfs_search_slot+0x6ca/0x790 [btrfs]
[ 1565.097105]  [<ffffffffa015ea8e>] ? free_extent_map+0x4e/0x90 [btrfs]
[ 1565.103601]  [<ffffffffa013c3e8>] btrfs_lookup_file_extent+0x38/0x40 [btrfs]
[ 1565.110733]  [<ffffffffa015c66e>] __btrfs_drop_extents+0x12e/0xaf0 [btrfs]
[ 1565.118798]  [<ffffffff8117ee28>] ? kmem_cache_alloc+0xc8/0x150
[ 1565.124777]  [<ffffffffa015daee>] btrfs_drop_extents+0x6e/0xa0 [btrfs]
[ 1565.131388]  [<ffffffffa0150811>] cow_file_range_inline+0xf1/0x1e0 [btrfs]
[ 1565.138344]  [<ffffffffa0150c55>] __cow_file_range+0x355/0x4a0 [btrfs]
[ 1565.144928]  [<ffffffffa0151905>] cow_file_range+0x95/0xe0 [btrfs]
[ 1565.151167]  [<ffffffff8134a68d>] ? do_raw_spin_unlock+0x5d/0xb0
[ 1565.157253]  [<ffffffffa0151c9c>] run_delalloc_range+0x34c/0x370 [btrfs]
[ 1565.164013]  [<ffffffffa01674a0>] __extent_writepage+0x5e0/0x770 [btrfs]
[ 1565.170776]  [<ffffffff810b8a46>] ? mark_held_locks+0x86/0x140
[ 1565.176669]  [<ffffffff811329ee>] ? find_get_pages_tag+0x2e/0x1c0
[ 1565.182821]  [<ffffffffa01678b2>] extent_write_cache_pages.isra.28.constprop.42+0x282/0x3e0 [btrfs]
[ 1565.191939]  [<ffffffff811b228b>] ? writeback_sb_inodes+0x12b/0x3e0
[ 1565.198284]  [<ffffffffa0167c5e>] extent_writepages+0x4e/0x70 [btrfs]
[ 1565.204781]  [<ffffffffa014e0d0>] ? can_nocow_odirect+0x2b0/0x2b0 [btrfs]
[ 1565.211632]  [<ffffffff811b2329>] ? writeback_sb_inodes+0x1c9/0x3e0
[ 1565.217969]  [<ffffffffa014b978>] btrfs_writepages+0x28/0x30 [btrfs]
[ 1565.224351]  [<ffffffff8113e0b3>] do_writepages+0x23/0x40
[ 1565.229799]  [<ffffffff811afa9e>] __writeback_single_inode+0x4e/0x2a0
[ 1565.236297]  [<ffffffff811b23e0>] writeback_sb_inodes+0x280/0x3e0
[ 1565.242420]  [<ffffffff811b25de>] __writeback_inodes_wb+0x9e/0xd0
[ 1565.248564]  [<ffffffff811b280b>] wb_writeback+0x1fb/0x310
[ 1565.254085]  [<ffffffff8167250f>] ? _raw_spin_unlock_irqrestore+0x3f/0x80
[ 1565.260934]  [<ffffffff811b29bf>] wb_check_old_data_flush+0x9f/0xb0
[ 1565.267258]  [<ffffffff811b4291>] wb_do_writeback+0xf1/0x1e0
[ 1565.272953]  [<ffffffff811b4413>] bdi_writeback_thread+0x93/0x270
[ 1565.279100]  [<ffffffff811b4380>] ? wb_do_writeback+0x1e0/0x1e0
[ 1565.285049]  [<ffffffff8107c46a>] kthread+0xea/0xf0
[ 1565.289978]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1565.296213]  [<ffffffff8167b4ac>] ret_from_fork+0x7c/0xb0
[ 1565.301646]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1565.307875] 2 locks held by flush-btrfs-6/20564:
[ 1565.312514]  #0:  (&type->s_umount_key#27){+++++.}, at: [<ffffffff81189e74>] grab_super_passive+0x44/0x90
[ 1565.322235]  #1:  (sb_internal#2){.+.+..}, at: [<ffffffffa0149b9d>] start_transaction+0x3dd/0x4f0 [btrfs]
[ 1565.331984] INFO: task ceph-osd:20706 blocked for more than 120 seconds.
[ 1565.338741] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1565.346638] ceph-osd        D ffff880213ee87c8     0 20706  20638 0x00000000
[ 1565.353753]  ffff88021c77bc08 0000000000000046 0000000000000286 ffff880005f27500
[ 1565.361278]  ffff88021e895eb0 ffff88021c77bfd8 ffff88021c77bfd8 ffff88021c77bfd8
[ 1565.368812]  ffff880220539f90 ffff88021e895eb0 ffff88021c77bc18 ffff880005f27490
[ 1565.376330] Call Trace:
[ 1565.378808]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1565.383832]  [<ffffffffa0181305>] log_one_extent.isra.26+0x4f5/0x7a0 [btrfs]
[ 1565.390945]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1565.397295]  [<ffffffffa0184c79>] btrfs_log_inode+0x7a9/0x8d0 [btrfs]
[ 1565.403795]  [<ffffffffa0185ae4>] btrfs_log_inode_parent+0x194/0x460 [btrfs]
[ 1565.410909]  [<ffffffff8119c99c>] ? dget_parent+0x1c/0xd0
[ 1565.416372]  [<ffffffffa0185df6>] btrfs_log_dentry_safe+0x46/0x70 [btrfs]
[ 1565.423205]  [<ffffffffa015a105>] btrfs_sync_file+0x155/0x2c0 [btrfs]
[ 1565.429696]  [<ffffffff811a3900>] ? fget+0xe0/0xe0
[ 1565.434518]  [<ffffffff811a393c>] ? fget_light+0x3c/0x130
[ 1565.439971]  [<ffffffff811b7d5d>] do_fsync+0x5d/0x90
[ 1565.444968]  [<ffffffff813434ee>] ? trace_hardirqs_on_thunk+0x3a/0x3f
[ 1565.451463]  [<ffffffff811b8170>] sys_fsync+0x10/0x20
[ 1565.456555]  [<ffffffff8167b559>] system_call_fastpath+0x16/0x1b
[ 1565.462586] 3 locks held by ceph-osd/20706:
[ 1565.466811]  #0:  (&sb->s_type->i_mutex_key#15){+.+.+.}, at: [<ffffffffa015a06a>] btrfs_sync_file+0xba/0x2c0 [btrfs]
[ 1565.477527]  #1:  (sb_internal#2){.+.+..}, at: [<ffffffffa0149b9d>] start_transaction+0x3dd/0x4f0 [btrfs]
[ 1565.487286]  #2:  (&ei->log_mutex){+.+...}, at: [<ffffffffa01845d7>] btrfs_log_inode+0x107/0x8d0 [btrfs]
[ 1565.496943] INFO: task ceph-osd:20761 blocked for more than 120 seconds.
[ 1565.503677] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1565.511579] ceph-osd        D 00000000ffffffff     0 20761  20638 0x00000000
[ 1565.518704]  ffff880222067dd8 0000000000000046 0000000000000007 0000000000000006
[ 1565.526211]  ffff88018e61bf20 ffff880222067fd8 ffff880222067fd8 ffff880222067fd8
[ 1565.533702]  ffff88018e618000 ffff88018e61bf20 ffffffff8166e8d5 ffff8802238c58c0
[ 1565.541203] Call Trace:
[ 1565.543676]  [<ffffffff8166e8d5>] ? mutex_lock_nested+0x1e5/0x320
[ 1565.549816]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1565.554812]  [<ffffffff8167141e>] schedule_preempt_disabled+0xe/0x10
[ 1565.561209]  [<ffffffff8166e841>] mutex_lock_nested+0x151/0x320
[ 1565.567185]  [<ffffffffa0159c30>] ? btrfs_file_llseek+0x40/0x3c0 [btrfs]
[ 1565.573910]  [<ffffffff81080c53>] ? up_read+0x23/0x40
[ 1565.579041]  [<ffffffffa0159c30>] btrfs_file_llseek+0x40/0x3c0 [btrfs]
[ 1565.585592]  [<ffffffff81186093>] vfs_llseek+0x33/0x40
[ 1565.590818]  [<ffffffff8118663e>] sys_lseek+0x7e/0x90
[ 1565.595966]  [<ffffffff8167b559>] system_call_fastpath+0x16/0x1b
[ 1565.601992] 1 lock held by ceph-osd/20761:
[ 1565.606163]  #0:  (&sb->s_type->i_mutex_key#15){+.+.+.}, at: [<ffffffffa0159c30>] btrfs_file_llseek+0x40/0x3c0 [btrfs]
[ 1565.617017] INFO: task btrfs-endio-wri:20673 blocked for more than 120 seconds.
[ 1565.624363] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1565.632259] btrfs-endio-wri D ffff88020b963f20     0 20673      2 0x00000000
[ 1565.639387]  ffff88011525b928 0000000000000046 0000000000000286 ffff88008d1deed8
[ 1565.646908]  ffff88020b963f20 ffff88011525bfd8 ffff88011525bfd8 ffff88011525bfd8
[ 1565.654406]  ffff88015c170000 ffff88020b963f20 ffff88011525b938 ffff88008d1deda0
[ 1565.661922] Call Trace:
[ 1565.664396]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1565.669434]  [<ffffffffa017dcc5>] btrfs_tree_lock+0x75/0x270 [btrfs]
[ 1565.675843]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1565.682149]  [<ffffffffa012170b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[ 1565.688923]  [<ffffffffa0126c5a>] btrfs_search_slot+0x6ca/0x790 [btrfs]
[ 1565.695594]  [<ffffffffa013c3e8>] btrfs_lookup_file_extent+0x38/0x40 [btrfs]
[ 1565.702723]  [<ffffffffa015c66e>] __btrfs_drop_extents+0x12e/0xaf0 [btrfs]
[ 1565.709650]  [<ffffffff810b36ba>] ? static_obj+0x5a/0x70
[ 1565.714992]  [<ffffffff8117ee28>] ? kmem_cache_alloc+0xc8/0x150
[ 1565.720958]  [<ffffffff813420da>] ? delay_tsc+0x4a/0x80
[ 1565.726239]  [<ffffffffa015daee>] btrfs_drop_extents+0x6e/0xa0 [btrfs]
[ 1565.732807]  [<ffffffffa014c22b>] insert_reserved_file_extent.constprop.59+0x7b/0x2c0 [btrfs]
[ 1565.741402]  [<ffffffffa0155a9e>] btrfs_finish_ordered_io+0x51e/0xa10 [btrfs]
[ 1565.748617]  [<ffffffffa0176531>] ? worker_loop+0xa1/0x5a0 [btrfs]
[ 1565.754832]  [<ffffffff816724c0>] ? _raw_spin_unlock_irq+0x30/0x40
[ 1565.761081]  [<ffffffffa0155fa5>] finish_ordered_fn+0x15/0x20 [btrfs]
[ 1565.767592]  [<ffffffffa01765fa>] worker_loop+0x16a/0x5a0 [btrfs]
[ 1565.773730]  [<ffffffffa0176490>] ? btrfs_queue_worker+0x320/0x320 [btrfs]
[ 1565.780653]  [<ffffffff8107c46a>] kthread+0xea/0xf0
[ 1565.785560]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1565.791794]  [<ffffffff8167b4ac>] ret_from_fork+0x7c/0xb0
[ 1565.797239]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1565.803448] 1 lock held by btrfs-endio-wri/20673:
[ 1565.808192]  #0:  (sb_internal#2){.+.+..}, at: [<ffffffffa0149b9d>] start_transaction+0x3dd/0x4f0 [btrfs]
[ 1565.817931] INFO: task btrfs-endio-wri:22007 blocked for more than 120 seconds.
[ 1565.825281] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1565.833175] btrfs-endio-wri D ffff88020a509f90     0 22007      2 0x00000000
[ 1565.840294]  ffff8800361d3928 0000000000000046 0000000000000286 ffff88008d1deed8
[ 1565.847840]  ffff88020a509f90 ffff8800361d3fd8 ffff8800361d3fd8 ffff8800361d3fd8
[ 1565.855329]  ffff880225d49f90 ffff88020a509f90 ffff8800361d3938 ffff88008d1deda0
[ 1565.862852] Call Trace:
[ 1565.865329]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1565.870377]  [<ffffffffa017dcc5>] btrfs_tree_lock+0x75/0x270 [btrfs]
[ 1565.876789]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1565.883096]  [<ffffffffa012170b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[ 1565.889870]  [<ffffffffa0126c5a>] btrfs_search_slot+0x6ca/0x790 [btrfs]
[ 1565.896555]  [<ffffffffa013c3e8>] btrfs_lookup_file_extent+0x38/0x40 [btrfs]
[ 1565.903663]  [<ffffffffa015c66e>] __btrfs_drop_extents+0x12e/0xaf0 [btrfs]
[ 1565.910594]  [<ffffffff810b36ba>] ? static_obj+0x5a/0x70
[ 1565.915957]  [<ffffffff8117ee28>] ? kmem_cache_alloc+0xc8/0x150
[ 1565.921932]  [<ffffffffa0147b35>] ? join_transaction.isra.30+0x25/0x3d0 [btrfs]
[ 1565.929334]  [<ffffffffa015daee>] btrfs_drop_extents+0x6e/0xa0 [btrfs]
[ 1565.935944]  [<ffffffffa014c22b>] insert_reserved_file_extent.constprop.59+0x7b/0x2c0 [btrfs]
[ 1565.944544]  [<ffffffffa0155a9e>] btrfs_finish_ordered_io+0x51e/0xa10 [btrfs]
[ 1565.951762]  [<ffffffffa0176531>] ? worker_loop+0xa1/0x5a0 [btrfs]
[ 1565.958003]  [<ffffffff816724c0>] ? _raw_spin_unlock_irq+0x30/0x40
[ 1565.964247]  [<ffffffffa0155fa5>] finish_ordered_fn+0x15/0x20 [btrfs]
[ 1565.970764]  [<ffffffffa01765fa>] worker_loop+0x16a/0x5a0 [btrfs]
[ 1565.976903]  [<ffffffff810b8c05>] ? trace_hardirqs_on_caller+0x105/0x190
[ 1565.983661]  [<ffffffffa0176490>] ? btrfs_queue_worker+0x320/0x320 [btrfs]
[ 1565.990599]  [<ffffffff8107c46a>] kthread+0xea/0xf0
[ 1565.995506]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1566.001734]  [<ffffffff8167b4ac>] ret_from_fork+0x7c/0xb0
[ 1566.007182]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1566.013387] 1 lock held by btrfs-endio-wri/22007:
[ 1566.018130]  #0:  (sb_internal#2){.+.+..}, at: [<ffffffffa0149b9d>] start_transaction+0x3dd/0x4f0 [btrfs]
[ 1566.027843] INFO: task btrfs-endio-wri:22009 blocked for more than 120 seconds.
[ 1566.035191] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1566.043081] btrfs-endio-wri D ffff88021bc83f20     0 22009      2 0x00000000
[ 1566.050205]  ffff8800b59f1928 0000000000000046 0000000000000286 ffff88008d1deed8
[ 1566.057736]  ffff88021bc83f20 ffff8800b59f1fd8 ffff8800b59f1fd8 ffff8800b59f1fd8
[ 1566.065242]  ffff880198c45eb0 ffff88021bc83f20 ffff8800b59f1938 ffff88008d1deda0
[ 1566.073923] Call Trace:
[ 1566.076429]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1566.081458]  [<ffffffffa017dcc5>] btrfs_tree_lock+0x75/0x270 [btrfs]
[ 1566.087869]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1566.094188]  [<ffffffffa012170b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[ 1566.100979]  [<ffffffffa0126c5a>] btrfs_search_slot+0x6ca/0x790 [btrfs]
[ 1566.107651]  [<ffffffffa013c3e8>] btrfs_lookup_file_extent+0x38/0x40 [btrfs]
[ 1566.114745]  [<ffffffffa015c66e>] __btrfs_drop_extents+0x12e/0xaf0 [btrfs]
[ 1566.121679]  [<ffffffff810b36ba>] ? static_obj+0x5a/0x70
[ 1566.127037]  [<ffffffff8117ee28>] ? kmem_cache_alloc+0xc8/0x150
[ 1566.133009]  [<ffffffffa0147b35>] ? join_transaction.isra.30+0x25/0x3d0 [btrfs]
[ 1566.140415]  [<ffffffffa015daee>] btrfs_drop_extents+0x6e/0xa0 [btrfs]
[ 1566.147016]  [<ffffffffa014c22b>] insert_reserved_file_extent.constprop.59+0x7b/0x2c0 [btrfs]
[ 1566.155615]  [<ffffffffa0155a9e>] btrfs_finish_ordered_io+0x51e/0xa10 [btrfs]
[ 1566.162821]  [<ffffffffa0176531>] ? worker_loop+0xa1/0x5a0 [btrfs]
[ 1566.169052]  [<ffffffff816724c0>] ? _raw_spin_unlock_irq+0x30/0x40
[ 1566.175286]  [<ffffffffa0155fa5>] finish_ordered_fn+0x15/0x20 [btrfs]
[ 1566.181813]  [<ffffffffa01765fa>] worker_loop+0x16a/0x5a0 [btrfs]
[ 1566.187987]  [<ffffffffa0176490>] ? btrfs_queue_worker+0x320/0x320 [btrfs]
[ 1566.194899]  [<ffffffff8107c46a>] kthread+0xea/0xf0
[ 1566.199839]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1566.206093]  [<ffffffff8167b4ac>] ret_from_fork+0x7c/0xb0
[ 1566.211518]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1566.217737] 1 lock held by btrfs-endio-wri/22009:
[ 1566.222467]  #0:  (sb_internal#2){.+.+..}, at: [<ffffffffa0149b9d>] start_transaction+0x3dd/0x4f0 [btrfs]
[ 1566.232223] INFO: task btrfs-endio-wri:22013 blocked for more than 120 seconds.
[ 1566.239599] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1566.247494] btrfs-endio-wri D ffff88021bc85eb0     0 22013      2 0x00000000
[ 1566.254605]  ffff880162481928 0000000000000046 0000000000000286 ffff88008d1deed8
[ 1566.262135]  ffff88021bc85eb0 ffff880162481fd8 ffff880162481fd8 ffff880162481fd8
[ 1566.269652]  ffff8801f0085eb0 ffff88021bc85eb0 ffff880162481938 ffff88008d1deda0
[ 1566.277175] Call Trace:
[ 1566.279650]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1566.284657]  [<ffffffffa017dcc5>] btrfs_tree_lock+0x75/0x270 [btrfs]
[ 1566.291063]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1566.297395]  [<ffffffffa012170b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[ 1566.304144]  [<ffffffffa0126c5a>] btrfs_search_slot+0x6ca/0x790 [btrfs]
[ 1566.310828]  [<ffffffffa013c3e8>] btrfs_lookup_file_extent+0x38/0x40 [btrfs]
[ 1566.317955]  [<ffffffffa015c66e>] __btrfs_drop_extents+0x12e/0xaf0 [btrfs]
[ 1566.324866]  [<ffffffff810b36ba>] ? static_obj+0x5a/0x70
[ 1566.330237]  [<ffffffff8117ee28>] ? kmem_cache_alloc+0xc8/0x150
[ 1566.336236]  [<ffffffffa0147b35>] ? join_transaction.isra.30+0x25/0x3d0 [btrfs]
[ 1566.343620]  [<ffffffffa015daee>] btrfs_drop_extents+0x6e/0xa0 [btrfs]
[ 1566.350241]  [<ffffffffa014c22b>] insert_reserved_file_extent.constprop.59+0x7b/0x2c0 [btrfs]
[ 1566.358861]  [<ffffffffa0155a9e>] btrfs_finish_ordered_io+0x51e/0xa10 [btrfs]
[ 1566.366053]  [<ffffffffa0176531>] ? worker_loop+0xa1/0x5a0 [btrfs]
[ 1566.372284]  [<ffffffff816724c0>] ? _raw_spin_unlock_irq+0x30/0x40
[ 1566.378533]  [<ffffffffa0155fa5>] finish_ordered_fn+0x15/0x20 [btrfs]
[ 1566.385020]  [<ffffffffa01765fa>] worker_loop+0x16a/0x5a0 [btrfs]
[ 1566.391162]  [<ffffffff810b8c05>] ? trace_hardirqs_on_caller+0x105/0x190
[ 1566.397945]  [<ffffffffa0176490>] ? btrfs_queue_worker+0x320/0x320 [btrfs]
[ 1566.404852]  [<ffffffff8107c46a>] kthread+0xea/0xf0
[ 1566.409776]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1566.415988]  [<ffffffff8167b4ac>] ret_from_fork+0x7c/0xb0
[ 1566.421436]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1566.427668] 1 lock held by btrfs-endio-wri/22013:
[ 1566.432397]  #0:  (sb_internal#2){.+.+..}, at: [<ffffffffa0149b9d>] start_transaction+0x3dd/0x4f0 [btrfs]
[ 1566.442127] INFO: task btrfs-endio-wri:22014 blocked for more than 120 seconds.
[ 1566.449493] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1566.457378] btrfs-endio-wri D ffff88021bc80000     0 22014      2 0x00000000
[ 1566.464475]  ffff8801624a1928 0000000000000046 0000000000000286 ffff88008d1deed8
[ 1566.471972]  ffff88021bc80000 ffff8801624a1fd8 ffff8801624a1fd8 ffff8801624a1fd8
[ 1566.479495]  ffff880188278000 ffff88021bc80000 ffff8801624a1938 ffff88008d1deda0
[ 1566.487014] Call Trace:
[ 1566.489491]  [<ffffffff816710c9>] schedule+0x29/0x70
[ 1566.494516]  [<ffffffffa017dcc5>] btrfs_tree_lock+0x75/0x270 [btrfs]
[ 1566.500932]  [<ffffffff8107cda0>] ? __init_waitqueue_head+0x60/0x60
[ 1566.507264]  [<ffffffffa012170b>] btrfs_lock_root_node+0x3b/0x50 [btrfs]
[ 1566.514016]  [<ffffffffa0126c5a>] btrfs_search_slot+0x6ca/0x790 [btrfs]
[ 1566.520704]  [<ffffffffa013c3e8>] btrfs_lookup_file_extent+0x38/0x40 [btrfs]
[ 1566.527829]  [<ffffffffa015c66e>] __btrfs_drop_extents+0x12e/0xaf0 [btrfs]
[ 1566.534744]  [<ffffffff810b36ba>] ? static_obj+0x5a/0x70
[ 1566.540114]  [<ffffffff8117ee28>] ? kmem_cache_alloc+0xc8/0x150
[ 1566.546092]  [<ffffffffa0147b35>] ? join_transaction.isra.30+0x25/0x3d0 [btrfs]
[ 1566.553487]  [<ffffffffa015daee>] btrfs_drop_extents+0x6e/0xa0 [btrfs]
[ 1566.560088]  [<ffffffffa014c22b>] insert_reserved_file_extent.constprop.59+0x7b/0x2c0 [btrfs]
[ 1566.568700]  [<ffffffffa0155a9e>] btrfs_finish_ordered_io+0x51e/0xa10 [btrfs]
[ 1566.575884]  [<ffffffffa0176531>] ? worker_loop+0xa1/0x5a0 [btrfs]
[ 1566.582120]  [<ffffffff816724c0>] ? _raw_spin_unlock_irq+0x30/0x40
[ 1566.588377]  [<ffffffffa0155fa5>] finish_ordered_fn+0x15/0x20 [btrfs]
[ 1566.594865]  [<ffffffffa01765fa>] worker_loop+0x16a/0x5a0 [btrfs]
[ 1566.601026]  [<ffffffff810b8c05>] ? trace_hardirqs_on_caller+0x105/0x190
[ 1566.607830]  [<ffffffffa0176490>] ? btrfs_queue_worker+0x320/0x320 [btrfs]
[ 1566.614737]  [<ffffffff8107c46a>] kthread+0xea/0xf0
[ 1566.619665]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1566.625874]  [<ffffffff8167b4ac>] ret_from_fork+0x7c/0xb0
[ 1566.631327]  [<ffffffff8107c380>] ? flush_kthread_work+0x1a0/0x1a0
[ 1566.637551] 1 lock held by btrfs-endio-wri/22014:
[ 1566.642276]  #0:  (sb_internal#2){.+.+..}, at: [<ffffffffa0149b9d>] start_transaction+0x3dd/0x4f0 [btrfs]
[ 1566.673279] btrfs: truncated 1 orphans

ubuntu@teuthology:/var/lib/teuthworker/archive/teuthology-2013-06-03_01:00:58-kernel-master-testing-basic/30217$ cat orig.config.yaml 
kernel:
  kdb: true
  sha1: 19bb6a83cb93383b363cc5956e304213f0f1b79f
machine_type: plana
nuke-on-error: true
overrides:
  ceph:
    conf:
      mon:
        debug mon: 20
        debug ms: 20
        debug paxos: 20
      osd:
        filestore flush min: 0
        osd op thread timeout: 60
    fs: btrfs
    log-whitelist:
    - slow request
    sha1: f7c19440290d4b82ced0320d1dfc4676ad5083d2
  install:
    ceph:
      sha1: f7c19440290d4b82ced0320d1dfc4676ad5083d2
  s3tests:
    branch: master
  workunit:
    sha1: f7c19440290d4b82ced0320d1dfc4676ad5083d2
roles:
- - mon.a
  - mon.c
  - osd.0
  - osd.1
  - osd.2
- - mon.b
  - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
tasks:
- chef: null
- clock.check: null
- install: null
- ceph: null
- mon_thrash:
    revive_delay: 20
    thrash_delay: 1
- kclient: null
- workunit:
    clients:
      all:
      - suites/ffsb.sh