Ceph » Linux kernel client

[3485670.784162] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686]

[3485670.788156] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152]

[3485670.788185] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485670.788185] irq event stamp: 0

[3485670.788187] hardirqs last enabled at (0): [< (null)>] (null)

[3485670.788192] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485670.788194] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485670.788195] softirqs last disabled at (0): [< (null)>] (null)

[3485670.788197] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485670.788198] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485670.788203] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1)

[3485670.788204] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000

[3485670.788208] RIP: 0010:[<ffffffff810b2f40>] [<ffffffff810b2f40>] do_raw_spin_lock+0x150/0x160

[3485670.788209] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202

[3485670.788210] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485670.788210] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485670.788211] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000

[3485670.788212] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8

[3485670.788212] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001

[3485670.788213] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000

[3485670.788214] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485670.788215] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0

[3485670.788216] Stack:

[3485670.788217] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485670.788219] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30

[3485670.788220] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf

[3485670.788221] Call Trace:

[3485670.788225] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485670.788236] [<ffffffffa038d3bf>] ? ceph_caps_revoking+0x2f/0x90 [ceph]

[3485670.788243] [<ffffffffa038d3bf>] ceph_caps_revoking+0x2f/0x90 [ceph]

[3485670.788248] [<ffffffffa0388033>] ceph_writepages_start+0x83/0xf90 [ceph]

[3485670.788251] [<ffffffff81275c2e>] ? ext4_writepages+0x4ee/0xd30

[3485670.788257] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485670.788263] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485670.788265] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485670.788267] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485670.788269] [<ffffffff81168713>] do_writepages+0x23/0x40

[3485670.788270] [<ffffffff811f7a85>] __writeback_single_inode+0x45/0x2e0

[3485670.788272] [<ffffffff811f91fd>] writeback_sb_inodes+0x2ad/0x4f0

[3485670.788273] [<ffffffff811f9564>] ? wb_writeback+0x54/0x350

[3485670.788274] [<ffffffff811f94de>] __writeback_inodes_wb+0x9e/0xd0

[3485670.788276] [<ffffffff811f97ab>] wb_writeback+0x29b/0x350

[3485670.788278] [<ffffffff811ea2b2>] ? get_nr_dirty_inodes+0x52/0x80

[3485670.788279] [<ffffffff811f98ff>] wb_check_old_data_flush+0x9f/0xb0

[3485670.788281] [<ffffffff811fbcfc>] bdi_writeback_workfn+0x1cc/0x490

[3485670.788283] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485670.788285] [<ffffffff81076291>] process_one_work+0x1d1/0x570

[3485670.788286] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485670.788288] [<ffffffff810770cc>] worker_thread+0x11c/0x530

[3485670.788289] [<ffffffff81076fb0>] ? create_and_start_worker+0x50/0x50

[3485670.788291] [<ffffffff8107e4b4>] kthread+0xe4/0x100

[3485670.788293] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485670.788295] [<ffffffff81729c2c>] ret_from_fork+0x7c/0xb0

[3485670.788297] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485670.788313] Code: 81 48 89 df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff <0f> b7 03 66 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90

[3485670.804149] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790]

[3485670.804169] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485670.804169] irq event stamp: 0

[3485670.804171] hardirqs last enabled at (0): [< (null)>] (null)

[3485670.804174] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485670.804176] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485670.804177] softirqs last disabled at (0): [< (null)>] (null)

[3485670.804178] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485670.804179] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485670.804180] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000

[3485670.804183] RIP: 0010:[<ffffffff810b2f43>] [<ffffffff810b2f43>] do_raw_spin_lock+0x153/0x160

[3485670.804184] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202

[3485670.804184] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485670.804185] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485670.804186] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000

[3485670.804186] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8

[3485670.804187] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001

[3485670.804188] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000

[3485670.804189] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485670.804190] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485670.804190] Stack:

[3485670.804192] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485670.804194] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485670.804195] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf

[3485670.804195] Call Trace:

[3485670.804198] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485670.804205] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485670.804212] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485670.804217] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485670.804219] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485670.804221] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485670.804223] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485670.804225] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485670.804227] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485670.804228] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485670.804230] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485670.804232] [<ffffffff81006d28>] die+0x58/0x90

[3485670.804233] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485670.804235] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485670.804237] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485670.804243] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485670.804244] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485670.804246] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485670.804248] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485670.804249] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485670.804250] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485670.804252] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485670.804258] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485670.804265] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485670.804267] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485670.804268] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485670.804274] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485670.804276] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485670.804281] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485670.804283] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485670.804289] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485670.804291] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485670.804297] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485670.804298] [<ffffffff817295fb>] ? _raw_spin_unlock+0x2b/0x40

[3485670.804300] [<ffffffff81187454>] ? do_wp_page+0x374/0x740

[3485670.804302] [<ffffffff81189efc>] ? handle_mm_fault+0x86c/0x1060

[3485670.804303] [<ffffffff81046a19>] ? __do_page_fault+0x159/0x580

[3485670.804305] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485670.804307] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485670.804308] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485670.804310] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485670.804312] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485670.804313] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485670.804329] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55

[3485670.816141] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089]

[3485670.816160] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485670.816161] irq event stamp: 0

[3485670.816162] hardirqs last enabled at (0): [< (null)>] (null)

[3485670.816165] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485670.816167] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485670.816168] softirqs last disabled at (0): [< (null)>] (null)

[3485670.816169] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485670.816170] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485670.816171] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000

[3485670.816173] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485670.816174] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202

[3485670.816175] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485670.816176] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485670.816176] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000

[3485670.816177] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8

[3485670.816178] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001

[3485670.816179] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000

[3485670.816180] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485670.816181] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485670.816181] Stack:

[3485670.816183] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690

[3485670.816184] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485670.816186] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf

[3485670.816186] Call Trace:

[3485670.816188] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485670.816195] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485670.816202] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485670.816207] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485670.816209] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485670.816211] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485670.816213] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485670.816214] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485670.816216] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485670.816217] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485670.816219] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485670.816220] [<ffffffff81006d28>] die+0x58/0x90

[3485670.816221] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485670.816223] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485670.816225] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485670.816231] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485670.816232] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485670.816234] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485670.816235] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485670.816237] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485670.816238] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485670.816240] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485670.816246] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485670.816252] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485670.816254] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485670.816256] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485670.816261] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485670.816263] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485670.816268] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485670.816270] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485670.816276] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485670.816278] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485670.816283] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485670.816285] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485670.816286] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485670.816288] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485670.816289] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485670.816291] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485670.816292] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485670.816294] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485670.816295] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485670.816297] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485670.816313] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485672.226042] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485672.289011] irq event stamp: 0

[3485672.292292] hardirqs last enabled at (0): [< (null)>] (null)

[3485672.299974] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485672.308447] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485672.316912] softirqs last disabled at (0): [< (null)>] (null)

[3485672.324605] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485672.333454] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485672.340211] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000

[3485672.347926] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485672.356746] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202

[3485672.362277] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8

[3485672.369649] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001

[3485672.377020] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000

[3485672.384390] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48

[3485672.392897] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001

[3485672.400270] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000

[3485672.408594] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485672.414563] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0

[3485672.421933] Stack:

[3485672.424171] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50

[3485672.432008] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000

[3485672.439883] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee

[3485672.447775] Call Trace:

[3485672.450462] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485672.456206] [<ffffffffa0390bee>] ? ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485672.463076] [<ffffffffa0390bee>] ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485672.469744] [<ffffffff810a3660>] ? __init_waitqueue_head+0x60/0x60

[3485672.476237] [<ffffffffa0384559>] ceph_read_iter+0xc9/0x570 [ceph]

[3485672.482643] [<ffffffff811ccc08>] new_sync_read+0x78/0xb0

[3485672.488277] [<ffffffff811cdcdb>] vfs_read+0xab/0x180

[3485672.493560] [<ffffffff811cdf62>] SyS_read+0x52/0xb0

[3485672.498752] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485672.504979] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485698.769213] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686]

[3485698.773209] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152]

[3485698.773235] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485698.773236] irq event stamp: 0

[3485698.773237] hardirqs last enabled at (0): [< (null)>] (null)

[3485698.773242] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485698.773244] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485698.773245] softirqs last disabled at (0): [< (null)>] (null)

[3485698.773248] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485698.773248] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485698.773253] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1)

[3485698.773254] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000

[3485698.773258] RIP: 0010:[<ffffffff810b2f40>] [<ffffffff810b2f40>] do_raw_spin_lock+0x150/0x160

[3485698.773259] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202

[3485698.773260] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485698.773261] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485698.773261] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000

[3485698.773262] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8

[3485698.773263] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001

[3485698.773263] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000

[3485698.773264] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485698.773265] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0

[3485698.773266] Stack:

[3485698.773268] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485698.773269] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30

[3485698.773271] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf

[3485698.773271] Call Trace:

[3485698.773275] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485698.773286] [<ffffffffa038d3bf>] ? ceph_caps_revoking+0x2f/0x90 [ceph]

[3485698.773292] [<ffffffffa038d3bf>] ceph_caps_revoking+0x2f/0x90 [ceph]

[3485698.773298] [<ffffffffa0388033>] ceph_writepages_start+0x83/0xf90 [ceph]

[3485698.773300] [<ffffffff81275c2e>] ? ext4_writepages+0x4ee/0xd30

[3485698.773307] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485698.773313] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485698.773315] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485698.773316] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485698.773318] [<ffffffff81168713>] do_writepages+0x23/0x40

[3485698.773320] [<ffffffff811f7a85>] __writeback_single_inode+0x45/0x2e0

[3485698.773321] [<ffffffff811f91fd>] writeback_sb_inodes+0x2ad/0x4f0

[3485698.773322] [<ffffffff811f9564>] ? wb_writeback+0x54/0x350

[3485698.773324] [<ffffffff811f94de>] __writeback_inodes_wb+0x9e/0xd0

[3485698.773325] [<ffffffff811f97ab>] wb_writeback+0x29b/0x350

[3485698.773327] [<ffffffff811ea2b2>] ? get_nr_dirty_inodes+0x52/0x80

[3485698.773329] [<ffffffff811f98ff>] wb_check_old_data_flush+0x9f/0xb0

[3485698.773330] [<ffffffff811fbcfc>] bdi_writeback_workfn+0x1cc/0x490

[3485698.773333] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485698.773334] [<ffffffff81076291>] process_one_work+0x1d1/0x570

[3485698.773335] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485698.773337] [<ffffffff810770cc>] worker_thread+0x11c/0x530

[3485698.773339] [<ffffffff81076fb0>] ? create_and_start_worker+0x50/0x50

[3485698.773340] [<ffffffff8107e4b4>] kthread+0xe4/0x100

[3485698.773343] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485698.773344] [<ffffffff81729c2c>] ret_from_fork+0x7c/0xb0

[3485698.773346] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485698.773362] Code: 81 48 89 df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff <0f> b7 03 66 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90

[3485698.789200] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790]

[3485698.789228] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485698.789229] irq event stamp: 0

[3485698.789230] hardirqs last enabled at (0): [< (null)>] (null)

[3485698.789233] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485698.789235] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485698.789236] softirqs last disabled at (0): [< (null)>] (null)

[3485698.789238] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485698.789238] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485698.789239] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000

[3485698.789242] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485698.789243] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202

[3485698.789244] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485698.789244] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485698.789245] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000

[3485698.789246] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8

[3485698.789246] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001

[3485698.789248] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000

[3485698.789248] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485698.789249] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485698.789250] Stack:

[3485698.789252] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485698.789253] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485698.789255] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf

[3485698.789255] Call Trace:

[3485698.789257] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485698.789265] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485698.789272] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485698.789277] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485698.789279] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485698.789281] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485698.789283] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485698.789285] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485698.789287] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485698.789288] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485698.789290] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485698.789291] [<ffffffff81006d28>] die+0x58/0x90

[3485698.789293] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485698.789295] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485698.789296] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485698.789302] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485698.789304] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485698.789305] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485698.789307] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485698.789308] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485698.789310] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485698.789311] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485698.789318] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485698.789324] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485698.789326] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485698.789327] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485698.789333] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485698.789335] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485698.789340] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485698.789342] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485698.789348] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485698.789350] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485698.789356] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485698.789357] [<ffffffff817295fb>] ? _raw_spin_unlock+0x2b/0x40

[3485698.789359] [<ffffffff81187454>] ? do_wp_page+0x374/0x740

[3485698.789360] [<ffffffff81189efc>] ? handle_mm_fault+0x86c/0x1060

[3485698.789362] [<ffffffff81046a19>] ? __do_page_fault+0x159/0x580

[3485698.789364] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485698.789365] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485698.789367] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485698.789368] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485698.789370] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485698.789371] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485698.789387] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485698.801195] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089]

[3485698.801215] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485698.801215] irq event stamp: 0

[3485698.801216] hardirqs last enabled at (0): [< (null)>] (null)

[3485698.801219] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485698.801221] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485698.801222] softirqs last disabled at (0): [< (null)>] (null)

[3485698.801224] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485698.801224] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485698.801225] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000

[3485698.801228] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485698.801229] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202

[3485698.801229] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485698.801230] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485698.801231] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000

[3485698.801231] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8

[3485698.801232] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001

[3485698.801233] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000

[3485698.801234] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485698.801235] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485698.801235] Stack:

[3485698.801237] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690

[3485698.801238] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485698.801240] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf

[3485698.801240] Call Trace:

[3485698.801243] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485698.801250] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485698.801256] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485698.801261] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485698.801263] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485698.801265] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485698.801267] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485698.801269] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485698.801270] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485698.801272] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485698.801273] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485698.801274] [<ffffffff81006d28>] die+0x58/0x90

[3485698.801276] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485698.801277] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485698.801279] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485698.801285] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485698.801287] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485698.801288] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485698.801290] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485698.801291] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485698.801292] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485698.801294] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485698.801300] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485698.801307] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485698.801308] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485698.801310] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485698.801315] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485698.801317] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485698.801323] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485698.801324] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485698.801330] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485698.801332] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485698.801337] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485698.801339] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485698.801341] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485698.801342] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485698.801344] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485698.801345] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485698.801347] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485698.801348] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485698.801350] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485698.801351] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485698.801367] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485700.210998] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485700.274003] irq event stamp: 0

[3485700.277285] hardirqs last enabled at (0): [< (null)>] (null)

[3485700.284971] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485700.293448] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485700.301928] softirqs last disabled at (0): [< (null)>] (null)

[3485700.309619] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485700.318472] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485700.325228] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000

[3485700.332953] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485700.341773] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202

[3485700.347313] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8

[3485700.354684] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001

[3485700.362056] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000

[3485700.369425] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48

[3485700.376798] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001

[3485700.384169] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000

[3485700.392494] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485700.398463] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0

[3485700.405832] Stack:

[3485700.408092] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50

[3485700.415931] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000

[3485700.423771] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee

[3485700.431609] Call Trace:

[3485700.434288] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485700.440009] [<ffffffffa0390bee>] ? ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485700.446850] [<ffffffffa0390bee>] ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485700.454619] [<ffffffff810a3660>] ? __init_waitqueue_head+0x60/0x60

[3485700.461118] [<ffffffffa0384559>] ceph_read_iter+0xc9/0x570 [ceph]

[3485700.467524] [<ffffffff811ccc08>] new_sync_read+0x78/0xb0

[3485700.473150] [<ffffffff811cdcdb>] vfs_read+0xab/0x180

[3485700.478431] [<ffffffff811cdf62>] SyS_read+0x52/0xb0

[3485700.483620] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485700.489846] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485726.754263] BUG: soft lockup - CPU#1 stuck for 23s! [grep:27686]

[3485726.758260] BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u16:1:7152]

[3485726.758288] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485726.758289] irq event stamp: 0

[3485726.758291] hardirqs last enabled at (0): [< (null)>] (null)

[3485726.758296] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485726.758298] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485726.758299] softirqs last disabled at (0): [< (null)>] (null)

[3485726.758301] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485726.758302] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485726.758306] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1)

[3485726.758307] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000

[3485726.758312] RIP: 0010:[<ffffffff810b2f48>] [<ffffffff810b2f48>] do_raw_spin_lock+0x158/0x160

[3485726.758312] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202

[3485726.758313] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485726.758314] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485726.758315] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000

[3485726.758315] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8

[3485726.758316] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001

[3485726.758317] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000

[3485726.758318] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485726.758319] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0

[3485726.758319] Stack:

[3485726.758321] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485726.758323] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30

[3485726.758324] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf

[3485726.758325] Call Trace:

[3485726.758329] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485726.758340] [<ffffffffa038d3bf>] ? ceph_caps_revoking+0x2f/0x90 [ceph]

[3485726.758346] [<ffffffffa038d3bf>] ceph_caps_revoking+0x2f/0x90 [ceph]

[3485726.758352] [<ffffffffa0388033>] ceph_writepages_start+0x83/0xf90 [ceph]

[3485726.758354] [<ffffffff81275c2e>] ? ext4_writepages+0x4ee/0xd30

[3485726.758361] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485726.758367] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485726.758369] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485726.758370] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485726.758373] [<ffffffff81168713>] do_writepages+0x23/0x40

[3485726.758374] [<ffffffff811f7a85>] __writeback_single_inode+0x45/0x2e0

[3485726.758376] [<ffffffff811f91fd>] writeback_sb_inodes+0x2ad/0x4f0

[3485726.758377] [<ffffffff811f9564>] ? wb_writeback+0x54/0x350

[3485726.758379] [<ffffffff811f94de>] __writeback_inodes_wb+0x9e/0xd0

[3485726.758380] [<ffffffff811f97ab>] wb_writeback+0x29b/0x350

[3485726.758382] [<ffffffff811ea2b2>] ? get_nr_dirty_inodes+0x52/0x80

[3485726.758384] [<ffffffff811f98ff>] wb_check_old_data_flush+0x9f/0xb0

[3485726.758385] [<ffffffff811fbcfc>] bdi_writeback_workfn+0x1cc/0x490

[3485726.758387] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485726.758389] [<ffffffff81076291>] process_one_work+0x1d1/0x570

[3485726.758390] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485726.758392] [<ffffffff810770cc>] worker_thread+0x11c/0x530

[3485726.758393] [<ffffffff81076fb0>] ? create_and_start_worker+0x50/0x50

[3485726.758395] [<ffffffff8107e4b4>] kthread+0xe4/0x100

[3485726.758397] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485726.758399] [<ffffffff81729c2c>] ret_from_fork+0x7c/0xb0

[3485726.758401] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485726.758417] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 <f3> 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5

[3485726.774254] BUG: soft lockup - CPU#2 stuck for 23s! [teuthology:19790]

[3485726.774274] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485726.774274] irq event stamp: 0

[3485726.774275] hardirqs last enabled at (0): [< (null)>] (null)

[3485726.774278] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485726.774280] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485726.774281] softirqs last disabled at (0): [< (null)>] (null)

[3485726.774283] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485726.774284] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485726.774284] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000

[3485726.774287] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485726.774288] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202

[3485726.774289] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485726.774289] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485726.774290] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000

[3485726.774291] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8

[3485726.774291] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001

[3485726.774292] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000

[3485726.774293] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485726.774294] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485726.774295] Stack:

[3485726.774296] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485726.774298] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485726.774299] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf

[3485726.774300] Call Trace:

[3485726.774302] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485726.774310] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485726.774316] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485726.774321] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485726.774324] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485726.774325] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485726.774327] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485726.774330] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485726.774331] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485726.774333] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485726.774335] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485726.774336] [<ffffffff81006d28>] die+0x58/0x90

[3485726.774338] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485726.774340] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485726.774341] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485726.774347] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485726.774349] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485726.774350] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485726.774352] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485726.774353] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485726.774354] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485726.774356] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485726.774363] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485726.774369] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485726.774371] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485726.774372] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485726.774378] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485726.774380] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485726.774386] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485726.774387] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485726.774393] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485726.774395] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485726.774401] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485726.774402] [<ffffffff817295fb>] ? _raw_spin_unlock+0x2b/0x40

[3485726.774404] [<ffffffff81187454>] ? do_wp_page+0x374/0x740

[3485726.774406] [<ffffffff81189efc>] ? handle_mm_fault+0x86c/0x1060

[3485726.774407] [<ffffffff81046a19>] ? __do_page_fault+0x159/0x580

[3485726.774409] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485726.774410] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485726.774412] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485726.774414] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485726.774415] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485726.774417] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485726.774433] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485726.786249] BUG: soft lockup - CPU#3 stuck for 23s! [teuthology:20089]

[3485726.786269] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485726.786270] irq event stamp: 0

[3485726.786271] hardirqs last enabled at (0): [< (null)>] (null)

[3485726.786274] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485726.786276] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485726.786276] softirqs last disabled at (0): [< (null)>] (null)

[3485726.786278] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485726.786279] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485726.786280] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000

[3485726.786282] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485726.786283] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202

[3485726.786284] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485726.786285] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485726.786285] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000

[3485726.786286] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8

[3485726.786287] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001

[3485726.786288] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000

[3485726.786289] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485726.786289] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485726.786290] Stack:

[3485726.786292] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690

[3485726.786293] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485726.786295] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf

[3485726.786295] Call Trace:

[3485726.786297] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485726.786304] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485726.786311] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485726.786316] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485726.786318] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485726.786320] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485726.786322] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485726.786324] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485726.786325] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485726.786327] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485726.786328] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485726.786329] [<ffffffff81006d28>] die+0x58/0x90

[3485726.786331] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485726.786332] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485726.786334] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485726.786340] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485726.786342] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485726.786343] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485726.786345] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485726.786346] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485726.786347] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485726.786349] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485726.786355] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485726.786361] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485726.786363] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485726.786365] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485726.786370] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485726.786372] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485726.786377] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485726.786379] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485726.786385] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485726.786387] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485726.786392] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485726.786394] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485726.786395] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485726.786397] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485726.786399] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485726.786400] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485726.786401] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485726.786403] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485726.786405] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485726.786406] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485726.786422] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485728.195927] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485728.258976] irq event stamp: 0

[3485728.262264] hardirqs last enabled at (0): [< (null)>] (null)

[3485728.269961] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485728.278434] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485728.286896] softirqs last disabled at (0): [< (null)>] (null)

[3485728.294584] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485728.303443] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485728.310205] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000

[3485728.317923] RIP: 0010:[<ffffffff810b2f43>] [<ffffffff810b2f43>] do_raw_spin_lock+0x153/0x160

[3485728.326759] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202

[3485728.332295] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8

[3485728.339666] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001

[3485728.347065] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000

[3485728.354433] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48

[3485728.361811] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001

[3485728.369186] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000

[3485728.377511] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485728.383478] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0

[3485728.390861] Stack:

[3485728.393105] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50

[3485728.400938] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000

[3485728.408778] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee

[3485728.416626] Call Trace:

[3485728.419304] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485728.425035] [<ffffffffa0390bee>] ? ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485728.431889] [<ffffffffa0390bee>] ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485728.438563] [<ffffffff810a3660>] ? __init_waitqueue_head+0x60/0x60

[3485728.445062] [<ffffffffa0384559>] ceph_read_iter+0xc9/0x570 [ceph]

[3485728.451488] [<ffffffff811ccc08>] new_sync_read+0x78/0xb0

[3485728.457111] [<ffffffff811cdcdb>] vfs_read+0xab/0x180

[3485728.462394] [<ffffffff811cdf62>] SyS_read+0x52/0xb0

[3485728.467591] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485728.473828] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55

[3485738.671896] INFO: rcu_sched self-detected stall on CPU { 2} (t=7035781 jiffies g=482214633 c=482214632 q=8556771)

[3485738.679894] INFO: rcu_sched detected stalls on CPUs/tasks: { 2} (detected by 3, t=7035782 jiffies, g=482214633, c=482214632, q=8556771)

[3485738.679895] sending NMI to all CPUs:

[3485738.679897] NMI backtrace for cpu 3

[3485738.679900] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485738.679901] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485738.679902] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000

[3485738.679909] RIP: 0010:[<ffffffff813a9cf2>] [<ffffffff813a9cf2>] __const_udelay+0x12/0x30

[3485738.679910] RSP: 0018:ffff88043fcc3ca8 EFLAGS: 00000046

[3485738.679911] RAX: 0000000001062560 RBX: 0000000000002710 RCX: 0000000000000004

[3485738.679912] RDX: 00000000009aa1d0 RSI: 0000000000000100 RDI: 0000000000418958

[3485738.679913] RBP: ffff88043fcc3cc8 R08: 0000000000000001 R09: 0000000000000000

[3485738.679914] R10: 0000000000000001 R11: 0000000000000000 R12: ffffffff81c54f80

[3485738.679915] R13: 0000000000000003 R14: ffffffff81c54f80 R15: ffffffff81d21e80

[3485738.679916] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000

[3485738.679918] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485738.679919] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485738.679920] Stack:

[3485738.679922] ffff88043fcc3cc8 ffffffff81036812 000000001cbe02e8 ffff88043fccd7c0

[3485738.679924] ffff88043fcc3d28 ffffffff810cd6a5 0000000000000002 00000000008290e3

[3485738.679927] 0000000000000082 00000001810aae9d ffff88043fcc3d18 ffff8803c46321f0

[3485738.679927] Call Trace:

[3485738.679930] <IRQ>

[3485738.679934] [<ffffffff81036812>] ? arch_trigger_all_cpu_backtrace+0x92/0x120

[3485738.679938] [<ffffffff810cd6a5>] rcu_check_callbacks+0x685/0x6b0

[3485738.679940] [<ffffffff81067f88>] update_process_times+0x48/0x80

[3485738.679944] [<ffffffff810d8f43>] tick_sched_handle.isra.15+0x33/0x70

[3485738.679946] [<ffffffff810d942c>] tick_sched_timer+0x4c/0x80

[3485738.679948] [<ffffffff81081261>] __run_hrtimer+0x71/0x250

[3485738.679950] [<ffffffff81081a75>] ? hrtimer_interrupt+0x75/0x260

[3485738.679952] [<ffffffff810d93e0>] ? tick_sched_do_timer+0x40/0x40

[3485738.679954] [<ffffffff81081b07>] hrtimer_interrupt+0x107/0x260

[3485738.679958] [<ffffffff810c4c4e>] ? handle_edge_irq+0x1e/0x110

[3485738.679961] [<ffffffff810414b6>] hpet_interrupt_handler+0x16/0x40

[3485738.679963] [<ffffffff810c1e7d>] handle_irq_event_percpu+0x5d/0x200

[3485738.679965] [<ffffffff810c2068>] handle_irq_event+0x48/0x70

[3485738.679967] [<ffffffff810c4ca7>] handle_edge_irq+0x77/0x110

[3485738.679970] [<ffffffff810053c2>] handle_irq+0x22/0x40

[3485738.679974] [<ffffffff8172c88e>] do_IRQ+0x5e/0x110

[3485738.679976] [<ffffffff8172a9af>] common_interrupt+0x6f/0x6f

[3485738.679977] <EOI>

[3485738.679979] [<ffffffff8172aa60>] ? retint_restore_args+0xe/0xe

[3485738.679983] [<ffffffff810b2f48>] ? do_raw_spin_lock+0x158/0x160

[3485738.679985] [<ffffffff810b2f28>] ? do_raw_spin_lock+0x138/0x160

[3485738.679987] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485738.680003] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485738.680011] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485738.680018] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485738.680021] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485738.680023] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485738.680025] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485738.680028] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485738.680030] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485738.680032] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485738.680034] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485738.680036] [<ffffffff81006d28>] die+0x58/0x90

[3485738.680038] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485738.680040] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485738.680042] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485738.680049] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485738.680052] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485738.680053] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485738.680056] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485738.680057] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485738.680059] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485738.680061] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485738.680069] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485738.680076] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485738.680079] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485738.680081] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485738.680088] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485738.680091] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485738.680098] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485738.680100] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485738.680107] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485738.680110] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485738.680116] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485738.680119] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485738.680121] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485738.680123] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485738.680125] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485738.680127] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485738.680129] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485738.680131] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485738.680133] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485738.680135] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485738.680159] Code: 89 e5 ff 15 69 58 8f 00 5d c3 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 55 48 8d 04 bd 00 00 00 00 65 48 8b 14 25 60 30 01 00 <48> 8d 0c 12 48 c1 e2 06 48 89 e5 48 29 ca f7 e2 48 8d 7a 01 ff

[3485738.680161] NMI backtrace for cpu 1

[3485738.680163] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485738.680164] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485738.680165] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000

[3485738.680170] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485738.680171] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202

[3485738.680172] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000057c957c8

[3485738.680173] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001

[3485738.680174] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000

[3485738.680175] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520

[3485738.680176] R13: 0000000097020520 R14: ffff880224754690 R15: ffff88033a8afeb0

[3485738.680177] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000

[3485738.680178] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485738.680179] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0

[3485738.680180] Stack:

[3485738.680183] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50

[3485738.680185] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000

[3485738.680187] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee

[3485738.680187] Call Trace:

[3485738.680190] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485738.680201] [<ffffffffa0390bee>] ? ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485738.680208] [<ffffffffa0390bee>] ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485738.680212] [<ffffffff810a3660>] ? __init_waitqueue_head+0x60/0x60

[3485738.680218] [<ffffffffa0384559>] ceph_read_iter+0xc9/0x570 [ceph]

[3485738.680221] [<ffffffff811ccc08>] new_sync_read+0x78/0xb0

[3485738.680223] [<ffffffff811cdcdb>] vfs_read+0xab/0x180

[3485738.680225] [<ffffffff811cdf62>] SyS_read+0x52/0xb0

[3485738.680228] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485738.680251] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485738.680253] NMI backtrace for cpu 0

[3485738.680256] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485738.680257] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485738.680262] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1)

[3485738.680263] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000

[3485738.680267] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485738.680268] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202

[3485738.680269] RAX: 0000000000008916 RBX: ffff8800049b5830 RCX: 0000000000000004

[3485738.680270] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485738.680271] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000

[3485738.680272] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520

[3485738.680273] R13: 0000000097020520 R14: 0000000000002000 R15: 000000000000001d

[3485738.680275] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000

[3485738.680276] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485738.680277] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0

[3485738.680278] Stack:

[3485738.680280] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485738.680282] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30

[3485738.680285] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf

[3485738.680285] Call Trace:

[3485738.680288] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485738.680297] [<ffffffffa038d3bf>] ? ceph_caps_revoking+0x2f/0x90 [ceph]

[3485738.680305] [<ffffffffa038d3bf>] ceph_caps_revoking+0x2f/0x90 [ceph]

[3485738.680312] [<ffffffffa0388033>] ceph_writepages_start+0x83/0xf90 [ceph]

[3485738.680315] [<ffffffff81275c2e>] ? ext4_writepages+0x4ee/0xd30

[3485738.680323] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485738.680330] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485738.680332] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485738.680334] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485738.680337] [<ffffffff81168713>] do_writepages+0x23/0x40

[3485738.680339] [<ffffffff811f7a85>] __writeback_single_inode+0x45/0x2e0

[3485738.680341] [<ffffffff811f91fd>] writeback_sb_inodes+0x2ad/0x4f0

[3485738.680342] [<ffffffff811f9564>] ? wb_writeback+0x54/0x350

[3485738.680344] [<ffffffff811f94de>] __writeback_inodes_wb+0x9e/0xd0

[3485738.680346] [<ffffffff811f97ab>] wb_writeback+0x29b/0x350

[3485738.680349] [<ffffffff811ea2b2>] ? get_nr_dirty_inodes+0x52/0x80

[3485738.680351] [<ffffffff811f98ff>] wb_check_old_data_flush+0x9f/0xb0

[3485738.680353] [<ffffffff811fbcfc>] bdi_writeback_workfn+0x1cc/0x490

[3485738.680355] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485738.680358] [<ffffffff81076291>] process_one_work+0x1d1/0x570

[3485738.680359] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485738.680362] [<ffffffff810770cc>] worker_thread+0x11c/0x530

[3485738.680364] [<ffffffff81076fb0>] ? create_and_start_worker+0x50/0x50

[3485738.680366] [<ffffffff8107e4b4>] kthread+0xe4/0x100

[3485738.680369] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485738.680371] [<ffffffff81729c2c>] ret_from_fork+0x7c/0xb0

[3485738.680373] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485738.680397] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485738.680398] NMI backtrace for cpu 2

[3485738.680400] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485738.680401] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485738.680402] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000

[3485738.680407] RIP: 0010:[<ffffffff8147a7d8>] [<ffffffff8147a7d8>] io_serial_in+0x18/0x20

[3485738.680408] RSP: 0018:ffff88043fc83a78 EFLAGS: 00000006

[3485738.680409] RAX: 00000000a84e5600 RBX: ffffffff82c09eb0 RCX: 0000000000000000

[3485738.680410] RDX: 00000000000003ed RSI: 00000000000003ed RDI: ffffffff82c09eb0

[3485738.680411] RBP: ffff88043fc83a78 R08: 0000000000000002 R09: 0000000000000000

[3485738.680412] R10: 0000000000000000 R11: 0000000000000001 R12: 00000000000026f6

[3485738.680413] R13: 0000000000000020 R14: ffffffff8147cd70 R15: ffffffff82b607f7

[3485738.680414] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000

[3485738.680415] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485738.680416] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485738.680417] Stack:

[3485738.680419] ffff88043fc83aa8 ffffffff8147cd0b ffffffff813a7184 ffffffff82c09eb0

[3485738.680422] 0000000000000032 ffffffff82b607da ffff88043fc83ac8 ffffffff8147cd95

[3485738.680424] ffffffff82b607da ffffffff82c09eb0 ffff88043fc83b08 ffffffff81475fcd

[3485738.680424] Call Trace:

[3485738.680426] <IRQ>

[3485738.680429] [<ffffffff8147cd0b>] wait_for_xmitr+0x3b/0xa0

[3485738.680431] [<ffffffff813a7184>] ? number.isra.2+0x304/0x330

[3485738.680433] [<ffffffff8147cd95>] serial8250_console_putchar+0x25/0x40

[3485738.680436] [<ffffffff81475fcd>] uart_console_write+0x3d/0x70

[3485738.680438] [<ffffffff8147cf13>] serial8250_console_write+0xc3/0x1a0

[3485738.680441] [<ffffffff810bf2b5>] call_console_drivers.constprop.20+0xa5/0x120

[3485738.680443] [<ffffffff810bf9b7>] console_unlock+0x397/0x480

[3485738.680445] [<ffffffff810bffcc>] vprintk_emit+0x29c/0x600

[3485738.680448] [<ffffffff810a9ab5>] ? cpuacct_account_field+0x5/0xb0

[3485738.680452] [<ffffffff8171b6c2>] printk+0x61/0x63

[3485738.680455] [<ffffffff810cd468>] rcu_check_callbacks+0x448/0x6b0

[3485738.680457] [<ffffffff81067f88>] update_process_times+0x48/0x80

[3485738.680459] [<ffffffff810d8f43>] tick_sched_handle.isra.15+0x33/0x70

[3485738.680461] [<ffffffff810d942c>] tick_sched_timer+0x4c/0x80

[3485738.680463] [<ffffffff81081261>] __run_hrtimer+0x71/0x250

[3485738.680465] [<ffffffff81081a75>] ? hrtimer_interrupt+0x75/0x260

[3485738.680467] [<ffffffff810d93e0>] ? tick_sched_do_timer+0x40/0x40

[3485738.680469] [<ffffffff81081b07>] hrtimer_interrupt+0x107/0x260

[3485738.680471] [<ffffffff810c4c4e>] ? handle_edge_irq+0x1e/0x110

[3485738.680474] [<ffffffff810414b6>] hpet_interrupt_handler+0x16/0x40

[3485738.680476] [<ffffffff810c1e7d>] handle_irq_event_percpu+0x5d/0x200

[3485738.680478] [<ffffffff810c2068>] handle_irq_event+0x48/0x70

[3485738.680480] [<ffffffff810c4ca7>] handle_edge_irq+0x77/0x110

[3485738.680482] [<ffffffff810053c2>] handle_irq+0x22/0x40

[3485738.680485] [<ffffffff8172c88e>] do_IRQ+0x5e/0x110

[3485738.680487] [<ffffffff8172a9af>] common_interrupt+0x6f/0x6f

[3485738.680488] <EOI>

[3485738.680489] [<ffffffff8172aa60>] ? retint_restore_args+0xe/0xe

[3485738.680492] [<ffffffff810b2f4a>] ? do_raw_spin_lock+0x15a/0x160

[3485738.680494] [<ffffffff810b2f28>] ? do_raw_spin_lock+0x138/0x160

[3485738.680496] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485738.680505] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485738.680513] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485738.680519] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485738.680522] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485738.680524] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485738.680526] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485738.680529] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485738.680531] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485738.680532] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485738.680534] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485738.680536] [<ffffffff81006d28>] die+0x58/0x90

[3485738.680538] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485738.680540] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485738.680542] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485738.680550] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485738.680552] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485738.680554] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485738.680556] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485738.680558] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485738.680560] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485738.680562] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485738.680570] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485738.680577] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485738.680579] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485738.680581] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485738.680588] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485738.680591] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485738.680598] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485738.680600] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485738.680607] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485738.680609] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485738.680616] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485738.680618] [<ffffffff817295fb>] ? _raw_spin_unlock+0x2b/0x40

[3485738.680620] [<ffffffff81187454>] ? do_wp_page+0x374/0x740

[3485738.680623] [<ffffffff81189efc>] ? handle_mm_fault+0x86c/0x1060

[3485738.680625] [<ffffffff81046a19>] ? __do_page_fault+0x159/0x580

[3485738.680627] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485738.680629] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485738.680631] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485738.680633] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485738.680635] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485738.680637] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485738.680661] Code: 48 89 e5 d3 e6 48 63 f6 48 03 77 40 8b 06 5d c3 0f 1f 00 66 66 66 66 90 0f b6 8f 91 00 00 00 55 48 89 e5 d3 e6 03 77 38 89 f2 ec <0f> b6 c0 5d c3 0f 1f 00 66 66 66 66 90 0f b6 8f 91 00 00 00 55

[3485738.680662] NMI backtrace for cpu 7

[3485738.680664] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485738.680665] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485738.680666] task: ffff8804296821f0 ti: ffff88042968c000 task.ti: ffff88042968c000

[3485738.680671] RIP: 0010:[<ffffffff8140b56c>] [<ffffffff8140b56c>] intel_idle+0xcc/0x150

[3485738.680672] RSP: 0018:ffff88042968fdf8 EFLAGS: 00000046

[3485738.680673] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001

[3485738.680674] RDX: 0000000000000000 RSI: ffff88042968ffd8 RDI: ffffffff817295bf

[3485738.680675] RBP: ffff88042968fe28 R08: 0000000000000001 R09: 0000000000000000

[3485738.680676] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003

[3485738.680676] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003

[3485738.680678] FS: 0000000000000000(0000) GS:ffff88043fdc0000(0000) knlGS:0000000000000000

[3485738.680679] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485738.680680] CR2: 00007fab827192c0 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485738.680680] Stack:

[3485738.680682] ffffffff815d3fef 00000007ffffffff ffff88043fddb020 000c63f4343bb9c3

[3485738.680684] ffffffff81ca4320 ffffffff81ca4200 ffff88042968fe88 ffffffff815d3fff

[3485738.680686] 0000000000000000 00000000003ca437 0000000000000000 00000000003ca437

[3485738.680687] Call Trace:

[3485738.680691] [<ffffffff815d3fef>] ? cpuidle_enter_state+0x3f/0xe0

[3485738.680693] [<ffffffff815d3fff>] cpuidle_enter_state+0x4f/0xe0

[3485738.680695] [<ffffffff815d4157>] cpuidle_enter+0x17/0x20

[3485738.680697] [<ffffffff810a3d0b>] cpu_startup_entry+0x32b/0x490

[3485738.680699] [<ffffffff810b087d>] ? trace_hardirqs_on+0xd/0x10

[3485738.680701] [<ffffffff81032a38>] start_secondary+0x1d8/0x290

[3485738.680723] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48

[3485738.680724] NMI backtrace for cpu 4

[3485738.680727] CPU: 4 PID: 0 Comm: swapper/4 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485738.680728] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485738.680729] task: ffff88042966a1f0 ti: ffff880429670000 task.ti: ffff880429670000

[3485738.680732] RIP: 0010:[<ffffffff8104080c>] [<ffffffff8104080c>] hpet_msi_next_event+0x3c/0x50

[3485738.680733] RSP: 0018:ffff88043fd03db8 EFLAGS: 00000082

[3485738.680734] RAX: 0000000080ae5de1 RBX: ffff880428ce5c00 RCX: 0000000000000020

[3485738.680735] RDX: 0000000080ae5dcb RSI: ffff880428ce5c00 RDI: 0000000080af3d1f

[3485738.680736] RBP: ffff88043fd03db8 R08: 0000000000000001 R09: 0000000000000000

[3485738.680737] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000

[3485738.680738] R13: 00000000003ced8e R14: ffff88043fd0d4c8 R15: ffff88043fd0d448

[3485738.680740] FS: 0000000000000000(0000) GS:ffff88043fd00000(0000) knlGS:0000000000000000

[3485738.680741] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485738.680742] CR2: 00007f7146cf02c0 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485738.680743] Stack:

[3485738.680745] ffff88043fd03df8 ffffffff810d6d64 0000000000000003 ffff88043fd0d488

[3485738.680747] ffff88043fd0d4c8 ffff88043fd0d360 0000000000000003 ffff88043fd0d488

[3485738.680749] ffff88043fd03e08 ffffffff810d8d14 ffff88043fd03e88 ffffffff81081b4f

[3485738.680750] Call Trace:

[3485738.680751] <IRQ>

[3485738.680753] [<ffffffff810d6d64>] clockevents_program_event+0x74/0x100

[3485738.680755] [<ffffffff810d8d14>] tick_program_event+0x24/0x30

[3485738.680757] [<ffffffff81081b4f>] hrtimer_interrupt+0x14f/0x260

[3485738.680760] [<ffffffff810c4c4e>] ? handle_edge_irq+0x1e/0x110

[3485738.680762] [<ffffffff810414b6>] hpet_interrupt_handler+0x16/0x40

[3485738.680764] [<ffffffff810c1e7d>] handle_irq_event_percpu+0x5d/0x200

[3485738.680766] [<ffffffff810c2068>] handle_irq_event+0x48/0x70

[3485738.680768] [<ffffffff810c4ca7>] handle_edge_irq+0x77/0x110

[3485738.680770] [<ffffffff810053c2>] handle_irq+0x22/0x40

[3485738.680772] [<ffffffff8172c88e>] do_IRQ+0x5e/0x110

[3485738.680774] [<ffffffff8172a9af>] common_interrupt+0x6f/0x6f

[3485738.680776] <EOI>

[3485738.680778] [<ffffffff815d4013>] ? cpuidle_enter_state+0x63/0xe0

[3485738.680780] [<ffffffff815d400f>] ? cpuidle_enter_state+0x5f/0xe0

[3485738.680782] [<ffffffff815d4157>] cpuidle_enter+0x17/0x20

[3485738.680785] [<ffffffff810a3d0b>] cpu_startup_entry+0x32b/0x490

[3485738.680787] [<ffffffff810b087d>] ? trace_hardirqs_on+0xd/0x10

[3485738.680789] [<ffffffff81032a38>] start_secondary+0x1d8/0x290

[3485738.680813] Code: e5 00 48 89 e5 8b 92 f0 00 00 00 c1 e0 05 01 d7 05 08 01 00 00 48 03 05 7b be e5 00 89 38 48 8b 05 72 be e5 00 8b 80 f0 00 00 00 <29> c7 ba 00 00 00 00 b8 c2 ff ff ff 83 ff 7f 0f 4f c2 5d c3 66

[3485738.680814] NMI backtrace for cpu 5

[3485738.680816] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485738.680817] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485738.680818] task: ffff88042966c3e0 ti: ffff880429674000 task.ti: ffff880429674000

[3485738.680822] RIP: 0010:[<ffffffff8140b56c>] [<ffffffff8140b56c>] intel_idle+0xcc/0x150

[3485738.680823] RSP: 0018:ffff880429677df8 EFLAGS: 00000046

[3485738.680824] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001

[3485738.680825] RDX: 0000000000000000 RSI: ffff880429677fd8 RDI: ffffffff817295bf

[3485738.680826] RBP: ffff880429677e28 R08: 0000000000000001 R09: 0000000000000000

[3485738.680827] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003

[3485738.680828] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003

[3485738.680829] FS: 0000000000000000(0000) GS:ffff88043fd40000(0000) knlGS:0000000000000000

[3485738.680830] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485738.680831] CR2: 00007f9f944192c0 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485738.680832] Stack:

[3485738.680834] ffffffff815d3fef 00000005ffffffff ffff88043fd5b020 000c63f433fed48b

[3485738.680836] ffffffff81ca4320 ffffffff81ca4200 ffff880429677e88 ffffffff815d3fff

[3485738.680838] 0000000000000000 0000000000798d3d 0000000000000000 0000000000798d3d

[3485738.680839] Call Trace:

[3485738.680842] [<ffffffff815d3fef>] ? cpuidle_enter_state+0x3f/0xe0

[3485738.680844] [<ffffffff815d3fff>] cpuidle_enter_state+0x4f/0xe0

[3485738.680846] [<ffffffff815d4157>] cpuidle_enter+0x17/0x20

[3485738.680848] [<ffffffff810a3d0b>] cpu_startup_entry+0x32b/0x490

[3485738.680850] [<ffffffff810b087d>] ? trace_hardirqs_on+0xd/0x10

[3485738.680852] [<ffffffff81032a38>] start_secondary+0x1d8/0x290

[3485738.680876] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48

[3485738.680877] NMI backtrace for cpu 6

[3485738.680880] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485738.680880] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485738.680881] task: ffff880429680000 ti: ffff880429688000 task.ti: ffff880429688000

[3485738.680885] RIP: 0010:[<ffffffff8140b56c>] [<ffffffff8140b56c>] intel_idle+0xcc/0x150

[3485738.680885] RSP: 0018:ffff88042968bdf8 EFLAGS: 00000046

[3485738.680886] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

[3485738.680887] RDX: 0000000000000000 RSI: ffff88042968bfd8 RDI: ffffffff817295bf

[3485738.680888] RBP: ffff88042968be28 R08: 0000000000000001 R09: 0000000000000000

[3485738.680888] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004

[3485738.680889] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

[3485738.680890] FS: 0000000000000000(0000) GS:ffff88043fd80000(0000) knlGS:0000000000000000

[3485738.680891] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485738.680892] CR2: 00007f8012d82980 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485738.680893] Stack:

[3485738.680895] ffffffff815d3fef 00000006ffffffff ffff88043fd9b020 000c63f433d9555d

[3485738.680896] ffffffff81ca4378 ffffffff81ca4200 ffff88042968be88 ffffffff815d3fff

[3485738.680898] 0000000000000000 0000000000dc1881 0000000000000000 0000000000dc1881

[3485738.680898] Call Trace:

[3485738.680901] [<ffffffff815d3fef>] ? cpuidle_enter_state+0x3f/0xe0

[3485738.680903] [<ffffffff815d3fff>] cpuidle_enter_state+0x4f/0xe0

[3485738.680905] [<ffffffff815d4157>] cpuidle_enter+0x17/0x20

[3485738.680907] [<ffffffff810a3d0b>] cpu_startup_entry+0x32b/0x490

[3485738.680908] [<ffffffff810b087d>] ? trace_hardirqs_on+0xd/0x10

[3485738.680910] [<ffffffff81032a38>] start_secondary+0x1d8/0x290

[3485738.680929] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48

[3485741.151472] sending NMI to all CPUs:

[3485741.155274] NMI backtrace for cpu 3

[3485741.158991] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485741.168357] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485741.175106] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000

[3485741.182830] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485741.191649] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202

[3485741.197181] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000000000004

[3485741.204554] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485741.211924] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000

[3485741.219297] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520

[3485741.226667] R13: 0000000097020520 R14: 0000000000000002 R15: ffff880425546660

[3485741.234039] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000

[3485741.242366] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485741.248333] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485741.255703] Stack:

[3485741.257940] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690

[3485741.265776] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485741.273623] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf

[3485741.281462] Call Trace:

[3485741.284138] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485741.289862] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485741.296364] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485741.302686] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485741.308832] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485741.313939] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485741.319043] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485741.324664] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485741.329938] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485741.336689] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485741.342231] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485741.347419] [<ffffffff81006d28>] die+0x58/0x90

[3485741.352173] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485741.357363] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485741.364373] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485741.370701] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485741.378421] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485741.385518] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485741.392533] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485741.399454] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485741.405073] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485741.410697] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485741.416063] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485741.423789] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485741.431426] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485741.438434] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485741.445361] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485741.452198] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485741.457831] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485741.465379] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485741.471787] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485741.478280] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485741.484693] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485741.491274] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485741.496902] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485741.502523] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485741.508144] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485741.513765] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485741.519388] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485741.525098] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485741.530457] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485741.535732] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485741.541961] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485741.564420] NMI backtrace for cpu 1

[3485741.568136] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485741.576985] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485741.583739] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000

[3485741.591455] RIP: 0010:[<ffffffff810b2f43>] [<ffffffff810b2f43>] do_raw_spin_lock+0x153/0x160

[3485741.600276] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202

[3485741.605808] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000057c957c8

[3485741.613179] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001

[3485741.620551] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000

[3485741.627921] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520

[3485741.635294] R13: 0000000097020520 R14: ffff880224754690 R15: ffff88033a8afeb0

[3485741.642666] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000

[3485741.650991] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485741.656959] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0

[3485741.664330] Stack:

[3485741.666565] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50

[3485741.674401] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000

[3485741.682241] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee

[3485741.690079] Call Trace:

[3485741.692756] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485741.698475] [<ffffffffa0390bee>] ? ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485741.705319] [<ffffffffa0390bee>] ceph_put_cap_refs+0x3e/0x270 [ceph]

[3485741.711985] [<ffffffff810a3660>] ? __init_waitqueue_head+0x60/0x60

[3485741.718483] [<ffffffffa0384559>] ceph_read_iter+0xc9/0x570 [ceph]

[3485741.724889] [<ffffffff811ccc08>] new_sync_read+0x78/0xb0

[3485741.730511] [<ffffffff811cdcdb>] vfs_read+0xab/0x180

[3485741.735784] [<ffffffff811cdf62>] SyS_read+0x52/0xb0

[3485741.740972] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485741.747200] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55

[3485741.769658] NMI backtrace for cpu 0

[3485741.773375] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485741.782917] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485741.789667] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1)

[3485741.796346] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000

[3485741.804067] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485741.812880] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202

[3485741.818420] RAX: 0000000000008916 RBX: ffff8800049b5830 RCX: 0000000000000004

[3485741.825791] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485741.833163] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000

[3485741.840533] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520

[3485741.847905] R13: 0000000097020520 R14: 0000000000002000 R15: 000000000000001d

[3485741.855278] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000

[3485741.863602] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485741.869570] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0

[3485741.876942] Stack:

[3485741.879178] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485741.887012] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30

[3485741.894852] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf

[3485741.902691] Call Trace:

[3485741.905366] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485741.911087] [<ffffffffa038d3bf>] ? ceph_caps_revoking+0x2f/0x90 [ceph]

[3485741.917930] [<ffffffffa038d3bf>] ceph_caps_revoking+0x2f/0x90 [ceph]

[3485741.924599] [<ffffffffa0388033>] ceph_writepages_start+0x83/0xf90 [ceph]

[3485741.931613] [<ffffffff81275c2e>] ? ext4_writepages+0x4ee/0xd30

[3485741.937766] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485741.944525] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485741.951275] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485741.957769] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485741.964257] [<ffffffff81168713>] do_writepages+0x23/0x40

[3485741.969879] [<ffffffff811f7a85>] __writeback_single_inode+0x45/0x2e0

[3485741.976541] [<ffffffff811f91fd>] writeback_sb_inodes+0x2ad/0x4f0

[3485741.982855] [<ffffffff811f9564>] ? wb_writeback+0x54/0x350

[3485741.988650] [<ffffffff811f94de>] __writeback_inodes_wb+0x9e/0xd0

[3485741.994965] [<ffffffff811f97ab>] wb_writeback+0x29b/0x350

[3485742.000672] [<ffffffff811ea2b2>] ? get_nr_dirty_inodes+0x52/0x80

[3485742.006987] [<ffffffff811f98ff>] wb_check_old_data_flush+0x9f/0xb0

[3485742.013478] [<ffffffff811fbcfc>] bdi_writeback_workfn+0x1cc/0x490

[3485742.019886] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485742.026114] [<ffffffff81076291>] process_one_work+0x1d1/0x570

[3485742.032168] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485742.038397] [<ffffffff810770cc>] worker_thread+0x11c/0x530

[3485742.044191] [<ffffffff81076fb0>] ? create_and_start_worker+0x50/0x50

[3485742.050854] [<ffffffff8107e4b4>] kthread+0xe4/0x100

[3485742.056050] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485742.062625] [<ffffffff81729c2c>] ret_from_fork+0x7c/0xb0

[3485742.068246] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485742.074821] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485742.097269] NMI backtrace for cpu 2

[3485742.100984] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485742.110354] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485742.117108] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000

[3485742.124826] RIP: 0010:[<ffffffff813a9ddf>] [<ffffffff813a9ddf>] delay_tsc+0x4f/0x80

[3485742.132856] RSP: 0018:ffff88043fc83c98 EFLAGS: 00000046

[3485742.138389] RAX: 000000001e474dd6 RBX: 0000000000002710 RCX: 0000000000000002

[3485742.145760] RDX: 00000000001f63d8 RSI: 000000001e474d8d RDI: 000000000026595c

[3485742.153133] RBP: ffff88043fc83c98 R08: 0000000000000002 R09: 0000000000000000

[3485742.160504] R10: 0000000000000001 R11: 0000000000000000 R12: ffffffff81c54f80

[3485742.167876] R13: 0000000000000002 R14: ffffffff81c54f80 R15: ffffffff81d21e80

[3485742.175248] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000

[3485742.183573] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485742.189540] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485742.196909] Stack:

[3485742.199147] ffff88043fc83ca8 ffffffff813a9d0c ffff88043fc83cc8 ffffffff81036812

[3485742.206990] 00000000008290e3 ffff88043fc8d7c0 ffff88043fc83d28 ffffffff810cd472

[3485742.214839] 0000000000000082 00000000008290e3 ffff88043fc83cf8 00000000810aae9d

[3485742.222677] Call Trace:

[3485742.225352] <IRQ>

[3485742.227484] [<ffffffff813a9d0c>] __const_udelay+0x2c/0x30

[3485742.234513] [<ffffffff81036812>] arch_trigger_all_cpu_backtrace+0x92/0x120

[3485742.241701] [<ffffffff810cd472>] rcu_check_callbacks+0x452/0x6b0

[3485742.248021] [<ffffffff81067f88>] update_process_times+0x48/0x80

[3485742.254258] [<ffffffff810d8f43>] tick_sched_handle.isra.15+0x33/0x70

[3485742.260926] [<ffffffff810d942c>] tick_sched_timer+0x4c/0x80

[3485742.266808] [<ffffffff81081261>] __run_hrtimer+0x71/0x250

[3485742.272515] [<ffffffff81081a75>] ? hrtimer_interrupt+0x75/0x260

[3485742.278743] [<ffffffff810d93e0>] ? tick_sched_do_timer+0x40/0x40

[3485742.285058] [<ffffffff81081b07>] hrtimer_interrupt+0x107/0x260

[3485742.291202] [<ffffffff810c4c4e>] ? handle_edge_irq+0x1e/0x110

[3485742.297265] [<ffffffff810414b6>] hpet_interrupt_handler+0x16/0x40

[3485742.303667] [<ffffffff810c1e7d>] handle_irq_event_percpu+0x5d/0x200

[3485742.310248] [<ffffffff810c2068>] handle_irq_event+0x48/0x70

[3485742.316130] [<ffffffff810c4ca7>] handle_edge_irq+0x77/0x110

[3485742.322010] [<ffffffff810053c2>] handle_irq+0x22/0x40

[3485742.327374] [<ffffffff8172c88e>] do_IRQ+0x5e/0x110

[3485742.332472] [<ffffffff8172a9af>] common_interrupt+0x6f/0x6f

[3485742.338354] <EOI>

[3485742.340490] [<ffffffff8172aa60>] ? retint_restore_args+0xe/0xe

[3485742.346940] [<ffffffff810b2f4a>] ? do_raw_spin_lock+0x15a/0x160

[3485742.353171] [<ffffffff810b2f28>] ? do_raw_spin_lock+0x138/0x160

[3485742.359406] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485742.365119] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485742.371616] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485742.377940] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485742.384085] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485742.389183] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485742.394285] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485742.399909] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485742.405183] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485742.410804] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485742.416341] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485742.421529] [<ffffffff81006d28>] die+0x58/0x90

[3485742.426282] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485742.431473] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485742.438485] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485742.444812] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485742.452532] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485742.459628] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485742.466646] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485742.473574] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485742.479200] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485742.484823] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485742.490191] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485742.497919] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485742.505553] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485742.512562] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485742.519487] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485742.526326] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485742.531958] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485742.539509] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485742.545914] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485742.552408] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485742.558822] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485742.565399] [<ffffffff817295fb>] ? _raw_spin_unlock+0x2b/0x40

[3485742.571454] [<ffffffff81187454>] ? do_wp_page+0x374/0x740

[3485742.577161] [<ffffffff81189efc>] ? handle_mm_fault+0x86c/0x1060

[3485742.583390] [<ffffffff81046a19>] ? __do_page_fault+0x159/0x580

[3485742.589532] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485742.595151] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485742.600773] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485742.606483] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485742.611845] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485742.617119] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485742.623348] Code: eb 21 66 2e 0f 1f 84 00 00 00 00 00 f3 90 65 ff 04 25 a0 ba 00 00 65 8b 0c 25 84 b0 00 00 41 39 c8 75 1a 66 66 90 0f ae e8 0f 31 <89> c2 29 f2 65 ff 0c 25 a0 ba 00 00 39 fa 72 d1 5d c3 29 c6 01

[3485742.645807] NMI backtrace for cpu 4

[3485742.649523] CPU: 4 PID: 0 Comm: swapper/4 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485742.658456] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485742.665203] task: ffff88042966a1f0 ti: ffff880429670000 task.ti: ffff880429670000

[3485742.672919] RIP: 0010:[<ffffffff8140b56c>] [<ffffffff8140b56c>] intel_idle+0xcc/0x150

[3485742.681125] RSP: 0018:ffff880429673df8 EFLAGS: 00000046

[3485742.686657] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001

[3485742.694029] RDX: 0000000000000000 RSI: ffff880429673fd8 RDI: ffffffff817295bf

[3485742.701400] RBP: ffff880429673e28 R08: 0000000000000001 R09: 0000000000000000

[3485742.708772] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003

[3485742.716143] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003

[3485742.723516] FS: 0000000000000000(0000) GS:ffff88043fd00000(0000) knlGS:0000000000000000

[3485742.731841] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485742.737808] CR2: 00007fecde790100 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485742.745179] Stack:

[3485742.747417] ffffffff815d3fef 00000004ffffffff ffff88043fd1b020 000c63f4c80d5109

[3485742.755259] ffffffff81ca4320 ffffffff81ca4200 ffff880429673e88 ffffffff815d3fff

[3485742.763098] 0000000000000000 0000000000f3f3ab 0000000000000000 0000000000f3f3ab

[3485742.770939] Call Trace:

[3485742.773615] [<ffffffff815d3fef>] ? cpuidle_enter_state+0x3f/0xe0

[3485742.779935] [<ffffffff815d3fff>] cpuidle_enter_state+0x4f/0xe0

[3485742.786083] [<ffffffff815d4157>] cpuidle_enter+0x17/0x20

[3485742.791706] [<ffffffff810a3d0b>] cpu_startup_entry+0x32b/0x490

[3485742.797846] [<ffffffff810b087d>] ? trace_hardirqs_on+0xd/0x10

[3485742.803901] [<ffffffff81032a38>] start_secondary+0x1d8/0x290

[3485742.809869] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48

[3485742.832328] NMI backtrace for cpu 5

[3485742.836042] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485742.844978] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485742.851726] task: ffff88042966c3e0 ti: ffff880429674000 task.ti: ffff880429674000

[3485742.859442] RIP: 0010:[<ffffffff8140b56c>] [<ffffffff8140b56c>] intel_idle+0xcc/0x150

[3485742.867655] RSP: 0018:ffff880429677df8 EFLAGS: 00000046

[3485742.873188] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001

[3485742.880560] RDX: 0000000000000000 RSI: ffff880429677fd8 RDI: ffffffff817295bf

[3485742.887931] RBP: ffff880429677e28 R08: 0000000000000001 R09: 0000000000000000

[3485742.895302] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003

[3485742.902674] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003

[3485742.910047] FS: 0000000000000000(0000) GS:ffff88043fd40000(0000) knlGS:0000000000000000

[3485742.918372] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485742.924339] CR2: 00007f0b52ac8fe0 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485742.931710] Stack:

[3485742.933947] ffffffff815d3fef 00000005ffffffff ffff88043fd5b020 000c63f4c80d81cb

[3485742.941781] ffffffff81ca4320 ffffffff81ca4200 ffff880429677e88 ffffffff815d3fff

[3485742.949621] 0000000000000000 0000000000335179 0000000000000000 0000000000335179

[3485742.957469] Call Trace:

[3485742.960146] [<ffffffff815d3fef>] ? cpuidle_enter_state+0x3f/0xe0

[3485742.966465] [<ffffffff815d3fff>] cpuidle_enter_state+0x4f/0xe0

[3485742.972615] [<ffffffff815d4157>] cpuidle_enter+0x17/0x20

[3485742.978246] [<ffffffff810a3d0b>] cpu_startup_entry+0x32b/0x490

[3485742.984395] [<ffffffff810b087d>] ? trace_hardirqs_on+0xd/0x10

[3485742.990451] [<ffffffff81032a38>] start_secondary+0x1d8/0x290

[3485742.996426] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48

[3485743.018884] NMI backtrace for cpu 6

[3485743.022654] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485743.031601] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485743.038352] task: ffff880429680000 ti: ffff880429688000 task.ti: ffff880429688000

[3485743.046077] RIP: 0010:[<ffffffff8140b56c>] [<ffffffff8140b56c>] intel_idle+0xcc/0x150

[3485743.054289] RSP: 0018:ffff88042968bdf8 EFLAGS: 00000046

[3485743.059823] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

[3485743.067195] RDX: 0000000000000000 RSI: ffff88042968bfd8 RDI: ffffffff817295bf

[3485743.074565] RBP: ffff88042968be28 R08: 0000000000000001 R09: 0000000000000000

[3485743.081937] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004

[3485743.089309] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

[3485743.096680] FS: 0000000000000000(0000) GS:ffff88043fd80000(0000) knlGS:0000000000000000

[3485743.105007] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485743.110980] CR2: 00007f8012d82980 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485743.118352] Stack:

[3485743.120588] ffffffff815d3fef 00000006ffffffff ffff88043fd9b020 000c63f4c80d9ca1

[3485743.129505] ffffffff81ca4378 ffffffff81ca4200 ffff88042968be88 ffffffff815d3fff

[3485743.137352] 0000000000000000 0000000000f3ac95 0000000000000000 0000000000f3ac95

[3485743.145202] Call Trace:

[3485743.147881] [<ffffffff815d3fef>] ? cpuidle_enter_state+0x3f/0xe0

[3485743.154198] [<ffffffff815d3fff>] cpuidle_enter_state+0x4f/0xe0

[3485743.160339] [<ffffffff815d4157>] cpuidle_enter+0x17/0x20

[3485743.165960] [<ffffffff810a3d0b>] cpu_startup_entry+0x32b/0x490

[3485743.172101] [<ffffffff810b087d>] ? trace_hardirqs_on+0xd/0x10

[3485743.178157] [<ffffffff81032a38>] start_secondary+0x1d8/0x290

[3485743.184124] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48

[3485743.206582] NMI backtrace for cpu 7

[3485743.210300] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485743.219234] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485743.225981] task: ffff8804296821f0 ti: ffff88042968c000 task.ti: ffff88042968c000

[3485743.233698] RIP: 0010:[<ffffffff8140b56c>] [<ffffffff8140b56c>] intel_idle+0xcc/0x150

[3485743.241902] RSP: 0018:ffff88042968fdf8 EFLAGS: 00000046

[3485743.247435] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

[3485743.254806] RDX: 0000000000000000 RSI: ffff88042968ffd8 RDI: ffffffff817295bf

[3485743.262177] RBP: ffff88042968fe28 R08: 0000000000000001 R09: 0000000000000000

[3485743.269547] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004

[3485743.276920] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

[3485743.284292] FS: 0000000000000000(0000) GS:ffff88043fdc0000(0000) knlGS:0000000000000000

[3485743.292625] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485743.298593] CR2: 00000000004238b0 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485743.305964] Stack:

[3485743.308201] ffffffff815d3fef 00000007ffffffff ffff88043fddb020 000c63f4c80d7a9b

[3485743.316035] ffffffff81ca4378 ffffffff81ca4200 ffff88042968fe88 ffffffff815d3fff

[3485743.323885] 0000000000000000 0000000000b6c420 0000000000000000 0000000000b6c420

[3485743.331722] Call Trace:

[3485743.334402] [<ffffffff815d3fef>] ? cpuidle_enter_state+0x3f/0xe0

[3485743.340721] [<ffffffff815d3fff>] cpuidle_enter_state+0x4f/0xe0

[3485743.346869] [<ffffffff815d4157>] cpuidle_enter+0x17/0x20

[3485743.352490] [<ffffffff810a3d0b>] cpu_startup_entry+0x32b/0x490

[3485743.358633] [<ffffffff810b087d>] ? trace_hardirqs_on+0xd/0x10

[3485743.364687] [<ffffffff81032a38>] start_secondary+0x1d8/0x290

[3485743.370654] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48

[3485766.732909] BUG: soft lockup - CPU#1 stuck for 23s! [grep:27686]

[3485766.736905] BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u16:1:7152]

[3485766.736936] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485766.736936] irq event stamp: 0

[3485766.736938] hardirqs last enabled at (0): [< (null)>] (null)

[3485766.736943] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485766.736945] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485766.736946] softirqs last disabled at (0): [< (null)>] (null)

[3485766.736948] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485766.736949] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485766.736954] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1)

[3485766.736955] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000

[3485766.736959] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485766.736960] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202

[3485766.736961] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485766.736961] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485766.736962] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000

[3485766.736963] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8

[3485766.736963] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001

[3485766.736964] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000

[3485766.736965] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

[3485766.736966] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0

[3485766.736967] Stack:

[3485766.736969] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485766.736970] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30

[3485766.736972] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf

[3485766.736972] Call Trace:

[3485766.736976] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485766.736987] [<ffffffffa038d3bf>] ? ceph_caps_revoking+0x2f/0x90 [ceph]

[3485766.736993] [<ffffffffa038d3bf>] ceph_caps_revoking+0x2f/0x90 [ceph]

[3485766.736999] [<ffffffffa0388033>] ceph_writepages_start+0x83/0xf90 [ceph]

[3485766.737001] [<ffffffff81275c2e>] ? ext4_writepages+0x4ee/0xd30

[3485766.737008] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485766.737014] [<ffffffffa0390546>] ? ceph_write_inode+0x86/0x200 [ceph]

[3485766.737016] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485766.737017] [<ffffffff811f9089>] ? writeback_sb_inodes+0x139/0x4f0

[3485766.737020] [<ffffffff81168713>] do_writepages+0x23/0x40

[3485766.737021] [<ffffffff811f7a85>] __writeback_single_inode+0x45/0x2e0

[3485766.737022] [<ffffffff811f91fd>] writeback_sb_inodes+0x2ad/0x4f0

[3485766.737024] [<ffffffff811f9564>] ? wb_writeback+0x54/0x350

[3485766.737025] [<ffffffff811f94de>] __writeback_inodes_wb+0x9e/0xd0

[3485766.737026] [<ffffffff811f97ab>] wb_writeback+0x29b/0x350

[3485766.737029] [<ffffffff811ea2b2>] ? get_nr_dirty_inodes+0x52/0x80

[3485766.737030] [<ffffffff811f98ff>] wb_check_old_data_flush+0x9f/0xb0

[3485766.737032] [<ffffffff811fbcfc>] bdi_writeback_workfn+0x1cc/0x490

[3485766.737034] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485766.737036] [<ffffffff81076291>] process_one_work+0x1d1/0x570

[3485766.737037] [<ffffffff8107622f>] ? process_one_work+0x16f/0x570

[3485766.737039] [<ffffffff810770cc>] worker_thread+0x11c/0x530

[3485766.737040] [<ffffffff81076fb0>] ? create_and_start_worker+0x50/0x50

[3485766.737043] [<ffffffff8107e4b4>] kthread+0xe4/0x100

[3485766.737045] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485766.737047] [<ffffffff81729c2c>] ret_from_fork+0x7c/0xb0

[3485766.737049] [<ffffffff8107e3d0>] ? flush_kthread_worker+0x130/0x130

[3485766.737065] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485766.752895] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790]

[3485766.752915] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485766.752915] irq event stamp: 0

[3485766.752916] hardirqs last enabled at (0): [< (null)>] (null)

[3485766.752919] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485766.752921] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485766.752922] softirqs last disabled at (0): [< (null)>] (null)

[3485766.752924] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485766.752925] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485766.752925] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000

[3485766.752928] RIP: 0010:[<ffffffff810b2f4a>] [<ffffffff810b2f4a>] do_raw_spin_lock+0x15a/0x160

[3485766.752929] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202

[3485766.752930] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485766.752931] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485766.752931] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000

[3485766.752932] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8

[3485766.752933] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001

[3485766.752934] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000

[3485766.752934] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485766.752935] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485766.752936] Stack:

[3485766.752937] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830

[3485766.752939] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485766.752940] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf

[3485766.752941] Call Trace:

[3485766.752943] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485766.752951] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485766.752957] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485766.752962] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485766.752965] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485766.752966] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485766.752968] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485766.752970] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485766.752972] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485766.752974] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485766.752976] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485766.752977] [<ffffffff81006d28>] die+0x58/0x90

[3485766.752978] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485766.752980] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485766.752982] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485766.752988] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485766.752989] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485766.752991] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485766.752992] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485766.752994] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485766.752995] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485766.752997] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485766.753003] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485766.753009] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485766.753011] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485766.753013] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485766.753018] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485766.753020] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485766.753026] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485766.753028] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485766.753033] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485766.753035] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485766.753041] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485766.753042] [<ffffffff817295fb>] ? _raw_spin_unlock+0x2b/0x40

[3485766.753044] [<ffffffff81187454>] ? do_wp_page+0x374/0x740

[3485766.753046] [<ffffffff81189efc>] ? handle_mm_fault+0x86c/0x1060

[3485766.753048] [<ffffffff81046a19>] ? __do_page_fault+0x159/0x580

[3485766.753049] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485766.753051] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485766.753052] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485766.753054] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485766.753056] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485766.753057] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485766.753073] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 <eb> f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0

[3485766.764890] BUG: soft lockup - CPU#3 stuck for 23s! [teuthology:20089]

[3485766.764909] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485766.764909] irq event stamp: 0

[3485766.764911] hardirqs last enabled at (0): [< (null)>] (null)

[3485766.764913] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485766.764915] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485766.764916] softirqs last disabled at (0): [< (null)>] (null)

[3485766.764918] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485766.764918] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010

[3485766.764919] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000

[3485766.764922] RIP: 0010:[<ffffffff810b2f48>] [<ffffffff810b2f48>] do_raw_spin_lock+0x158/0x160

[3485766.764923] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202

[3485766.764923] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004

[3485766.764924] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20

[3485766.764925] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000

[3485766.764925] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8

[3485766.764926] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001

[3485766.764927] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000

[3485766.764928] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033

[3485766.764929] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0

[3485766.764929] Stack:

[3485766.764931] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690

[3485766.764932] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00

[3485766.764934] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf

[3485766.764934] Call Trace:

[3485766.764936] [<ffffffff81728e31>] _raw_spin_lock+0x41/0x50

[3485766.764943] [<ffffffffa03925cf>] ? ceph_put_fmode+0x2f/0xd0 [ceph]

[3485766.764950] [<ffffffffa03925cf>] ceph_put_fmode+0x2f/0xd0 [ceph]

[3485766.764955] [<ffffffffa03859ee>] ceph_release+0x2e/0xc0 [ceph]

[3485766.764957] [<ffffffff811cf028>] __fput+0xc8/0x210

[3485766.764959] [<ffffffff811cf1be>] ____fput+0xe/0x10

[3485766.764961] [<ffffffff8107aa44>] task_work_run+0xb4/0xe0

[3485766.764962] [<ffffffff8105b5d8>] do_exit+0x2e8/0xae0

[3485766.764964] [<ffffffff810c11cd>] ? kmsg_dump+0x10d/0x120

[3485766.764965] [<ffffffff810c10e4>] ? kmsg_dump+0x24/0x120

[3485766.764967] [<ffffffff81006a85>] oops_end+0xa5/0xf0

[3485766.764968] [<ffffffff81006d28>] die+0x58/0x90

[3485766.764969] [<ffffffff8100336d>] do_trap+0xcd/0x160

[3485766.764971] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485766.764973] [<ffffffff810039c1>] do_error_trap.part.16+0xa1/0xe0

[3485766.764979] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485766.764980] [<ffffffff81082b2b>] ? __atomic_notifier_call_chain+0x8b/0xb0

[3485766.764982] [<ffffffff81082aa5>] ? __atomic_notifier_call_chain+0x5/0xb0

[3485766.764983] [<ffffffff81082b66>] ? atomic_notifier_call_chain+0x16/0x20

[3485766.764985] [<ffffffff81003a64>] do_error_trap+0x64/0x80

[3485766.764986] [<ffffffff81003fb0>] do_invalid_op+0x20/0x30

[3485766.764988] [<ffffffff8172b808>] invalid_op+0x18/0x20

[3485766.764994] [<ffffffffa0390f40>] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph]

[3485766.765000] [<ffffffffa0390e59>] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph]

[3485766.765002] [<ffffffff811688a3>] ? test_clear_page_writeback+0x143/0x260

[3485766.765004] [<ffffffff81168798>] ? test_clear_page_writeback+0x38/0x260

[3485766.765009] [<ffffffffa0389854>] writepage_nounlock+0x2a4/0x450 [ceph]

[3485766.765011] [<ffffffff81193d60>] ? SyS_msync+0x230/0x230

[3485766.765016] [<ffffffffa0389b50>] ceph_update_writeable_page+0x150/0x480 [ceph]

[3485766.765018] [<ffffffff81166536>] ? wait_for_stable_page+0x16/0x50

[3485766.765024] [<ffffffffa0389eec>] ceph_write_begin+0x6c/0xc0 [ceph]

[3485766.765026] [<ffffffff8115a9e6>] generic_perform_write+0xc6/0x1e0

[3485766.765031] [<ffffffffa03856b0>] ceph_write_iter+0x890/0xa30 [ceph]

[3485766.765033] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485766.765034] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485766.765036] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485766.765037] [<ffffffff811cdc13>] ? vfs_write+0x1d3/0x1f0

[3485766.765039] [<ffffffff81186526>] ? might_fault+0x66/0xc0

[3485766.765040] [<ffffffff811ccdcb>] new_sync_write+0x7b/0xb0

[3485766.765042] [<ffffffff811cdb07>] vfs_write+0xc7/0x1f0

[3485766.765043] [<ffffffff811ce012>] SyS_write+0x52/0xb0

[3485766.765045] [<ffffffff81729cd6>] system_call_fastpath+0x1a/0x1f

[3485766.765061] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 <f3> 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5

[3485768.174611] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E)

[3485768.237664] irq event stamp: 0

[3485768.240943] hardirqs last enabled at (0): [< (null)>] (null)

[3485768.248629] hardirqs last disabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485768.257094] softirqs last enabled at (0): [<ffffffff8105626d>] copy_process+0x78d/0x22f0

[3485768.265557] softirqs last disabled at (0): [< (null)>] (null)

[3485768.273242] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1

[3485768.282087] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010