[3485670.784162] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3485670.788156] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3485670.788185] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485670.788185] irq event stamp: 0 [3485670.788187] hardirqs last enabled at (0): [< (null)>] (null) [3485670.788192] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485670.788194] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485670.788195] softirqs last disabled at (0): [< (null)>] (null) [3485670.788197] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485670.788198] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485670.788203] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485670.788204] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485670.788208] RIP: 0010:[] [] do_raw_spin_lock+0x150/0x160 [3485670.788209] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485670.788210] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485670.788210] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485670.788211] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485670.788212] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485670.788212] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485670.788213] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485670.788214] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485670.788215] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485670.788216] Stack: [3485670.788217] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485670.788219] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485670.788220] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485670.788221] Call Trace: [3485670.788225] [] _raw_spin_lock+0x41/0x50 [3485670.788236] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485670.788243] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485670.788248] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485670.788251] [] ? ext4_writepages+0x4ee/0xd30 [3485670.788257] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485670.788263] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485670.788265] [] ? writeback_sb_inodes+0x139/0x4f0 [3485670.788267] [] ? writeback_sb_inodes+0x139/0x4f0 [3485670.788269] [] do_writepages+0x23/0x40 [3485670.788270] [] __writeback_single_inode+0x45/0x2e0 [3485670.788272] [] writeback_sb_inodes+0x2ad/0x4f0 [3485670.788273] [] ? wb_writeback+0x54/0x350 [3485670.788274] [] __writeback_inodes_wb+0x9e/0xd0 [3485670.788276] [] wb_writeback+0x29b/0x350 [3485670.788278] [] ? get_nr_dirty_inodes+0x52/0x80 [3485670.788279] [] wb_check_old_data_flush+0x9f/0xb0 [3485670.788281] [] bdi_writeback_workfn+0x1cc/0x490 [3485670.788283] [] ? process_one_work+0x16f/0x570 [3485670.788285] [] process_one_work+0x1d1/0x570 [3485670.788286] [] ? process_one_work+0x16f/0x570 [3485670.788288] [] worker_thread+0x11c/0x530 [3485670.788289] [] ? create_and_start_worker+0x50/0x50 [3485670.788291] [] kthread+0xe4/0x100 [3485670.788293] [] ? flush_kthread_worker+0x130/0x130 [3485670.788295] [] ret_from_fork+0x7c/0xb0 [3485670.788297] [] ? flush_kthread_worker+0x130/0x130 [3485670.788313] Code: 81 48 89 df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff <0f> b7 03 66 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 [3485670.804149] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3485670.804169] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485670.804169] irq event stamp: 0 [3485670.804171] hardirqs last enabled at (0): [< (null)>] (null) [3485670.804174] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485670.804176] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485670.804177] softirqs last disabled at (0): [< (null)>] (null) [3485670.804178] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485670.804179] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485670.804180] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485670.804183] RIP: 0010:[] [] do_raw_spin_lock+0x153/0x160 [3485670.804184] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485670.804184] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485670.804185] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485670.804186] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485670.804186] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485670.804187] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485670.804188] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485670.804189] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485670.804190] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485670.804190] Stack: [3485670.804192] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485670.804194] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485670.804195] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485670.804195] Call Trace: [3485670.804198] [] _raw_spin_lock+0x41/0x50 [3485670.804205] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485670.804212] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485670.804217] [] ceph_release+0x2e/0xc0 [ceph] [3485670.804219] [] __fput+0xc8/0x210 [3485670.804221] [] ____fput+0xe/0x10 [3485670.804223] [] task_work_run+0xb4/0xe0 [3485670.804225] [] do_exit+0x2e8/0xae0 [3485670.804227] [] ? kmsg_dump+0x10d/0x120 [3485670.804228] [] ? kmsg_dump+0x24/0x120 [3485670.804230] [] oops_end+0xa5/0xf0 [3485670.804232] [] die+0x58/0x90 [3485670.804233] [] do_trap+0xcd/0x160 [3485670.804235] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485670.804237] [] do_error_trap.part.16+0xa1/0xe0 [3485670.804243] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485670.804244] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485670.804246] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485670.804248] [] ? atomic_notifier_call_chain+0x16/0x20 [3485670.804249] [] do_error_trap+0x64/0x80 [3485670.804250] [] do_invalid_op+0x20/0x30 [3485670.804252] [] invalid_op+0x18/0x20 [3485670.804258] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485670.804265] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485670.804267] [] ? test_clear_page_writeback+0x143/0x260 [3485670.804268] [] ? test_clear_page_writeback+0x38/0x260 [3485670.804274] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485670.804276] [] ? SyS_msync+0x230/0x230 [3485670.804281] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485670.804283] [] ? wait_for_stable_page+0x16/0x50 [3485670.804289] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485670.804291] [] generic_perform_write+0xc6/0x1e0 [3485670.804297] [] ceph_write_iter+0x890/0xa30 [ceph] [3485670.804298] [] ? _raw_spin_unlock+0x2b/0x40 [3485670.804300] [] ? do_wp_page+0x374/0x740 [3485670.804302] [] ? handle_mm_fault+0x86c/0x1060 [3485670.804303] [] ? __do_page_fault+0x159/0x580 [3485670.804305] [] ? vfs_write+0x1d3/0x1f0 [3485670.804307] [] ? vfs_write+0x1d3/0x1f0 [3485670.804308] [] new_sync_write+0x7b/0xb0 [3485670.804310] [] vfs_write+0xc7/0x1f0 [3485670.804312] [] SyS_write+0x52/0xb0 [3485670.804313] [] system_call_fastpath+0x1a/0x1f [3485670.804329] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 [3485670.816141] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485670.816160] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485670.816161] irq event stamp: 0 [3485670.816162] hardirqs last enabled at (0): [< (null)>] (null) [3485670.816165] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485670.816167] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485670.816168] softirqs last disabled at (0): [< (null)>] (null) [3485670.816169] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485670.816170] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485670.816171] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485670.816173] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485670.816174] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485670.816175] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485670.816176] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485670.816176] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485670.816177] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485670.816178] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485670.816179] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485670.816180] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485670.816181] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485670.816181] Stack: [3485670.816183] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485670.816184] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485670.816186] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485670.816186] Call Trace: [3485670.816188] [] _raw_spin_lock+0x41/0x50 [3485670.816195] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485670.816202] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485670.816207] [] ceph_release+0x2e/0xc0 [ceph] [3485670.816209] [] __fput+0xc8/0x210 [3485670.816211] [] ____fput+0xe/0x10 [3485670.816213] [] task_work_run+0xb4/0xe0 [3485670.816214] [] do_exit+0x2e8/0xae0 [3485670.816216] [] ? kmsg_dump+0x10d/0x120 [3485670.816217] [] ? kmsg_dump+0x24/0x120 [3485670.816219] [] oops_end+0xa5/0xf0 [3485670.816220] [] die+0x58/0x90 [3485670.816221] [] do_trap+0xcd/0x160 [3485670.816223] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485670.816225] [] do_error_trap.part.16+0xa1/0xe0 [3485670.816231] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485670.816232] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485670.816234] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485670.816235] [] ? atomic_notifier_call_chain+0x16/0x20 [3485670.816237] [] do_error_trap+0x64/0x80 [3485670.816238] [] do_invalid_op+0x20/0x30 [3485670.816240] [] invalid_op+0x18/0x20 [3485670.816246] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485670.816252] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485670.816254] [] ? test_clear_page_writeback+0x143/0x260 [3485670.816256] [] ? test_clear_page_writeback+0x38/0x260 [3485670.816261] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485670.816263] [] ? SyS_msync+0x230/0x230 [3485670.816268] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485670.816270] [] ? wait_for_stable_page+0x16/0x50 [3485670.816276] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485670.816278] [] generic_perform_write+0xc6/0x1e0 [3485670.816283] [] ceph_write_iter+0x890/0xa30 [ceph] [3485670.816285] [] ? might_fault+0x66/0xc0 [3485670.816286] [] ? might_fault+0x66/0xc0 [3485670.816288] [] ? vfs_write+0x1d3/0x1f0 [3485670.816289] [] ? vfs_write+0x1d3/0x1f0 [3485670.816291] [] ? might_fault+0x66/0xc0 [3485670.816292] [] new_sync_write+0x7b/0xb0 [3485670.816294] [] vfs_write+0xc7/0x1f0 [3485670.816295] [] SyS_write+0x52/0xb0 [3485670.816297] [] system_call_fastpath+0x1a/0x1f [3485670.816313] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485672.226042] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485672.289011] irq event stamp: 0 [3485672.292292] hardirqs last enabled at (0): [< (null)>] (null) [3485672.299974] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485672.308447] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485672.316912] softirqs last disabled at (0): [< (null)>] (null) [3485672.324605] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485672.333454] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485672.340211] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485672.347926] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485672.356746] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485672.362277] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485672.369649] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485672.377020] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485672.384390] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485672.392897] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485672.400270] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485672.408594] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485672.414563] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485672.421933] Stack: [3485672.424171] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485672.432008] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485672.439883] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485672.447775] Call Trace: [3485672.450462] [] _raw_spin_lock+0x41/0x50 [3485672.456206] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485672.463076] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485672.469744] [] ? __init_waitqueue_head+0x60/0x60 [3485672.476237] [] ceph_read_iter+0xc9/0x570 [ceph] [3485672.482643] [] new_sync_read+0x78/0xb0 [3485672.488277] [] vfs_read+0xab/0x180 [3485672.493560] [] SyS_read+0x52/0xb0 [3485672.498752] [] system_call_fastpath+0x1a/0x1f [3485672.504979] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485698.769213] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3485698.773209] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3485698.773235] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485698.773236] irq event stamp: 0 [3485698.773237] hardirqs last enabled at (0): [< (null)>] (null) [3485698.773242] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485698.773244] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485698.773245] softirqs last disabled at (0): [< (null)>] (null) [3485698.773248] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485698.773248] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485698.773253] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485698.773254] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485698.773258] RIP: 0010:[] [] do_raw_spin_lock+0x150/0x160 [3485698.773259] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485698.773260] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485698.773261] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485698.773261] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485698.773262] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485698.773263] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485698.773263] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485698.773264] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485698.773265] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485698.773266] Stack: [3485698.773268] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485698.773269] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485698.773271] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485698.773271] Call Trace: [3485698.773275] [] _raw_spin_lock+0x41/0x50 [3485698.773286] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485698.773292] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485698.773298] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485698.773300] [] ? ext4_writepages+0x4ee/0xd30 [3485698.773307] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485698.773313] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485698.773315] [] ? writeback_sb_inodes+0x139/0x4f0 [3485698.773316] [] ? writeback_sb_inodes+0x139/0x4f0 [3485698.773318] [] do_writepages+0x23/0x40 [3485698.773320] [] __writeback_single_inode+0x45/0x2e0 [3485698.773321] [] writeback_sb_inodes+0x2ad/0x4f0 [3485698.773322] [] ? wb_writeback+0x54/0x350 [3485698.773324] [] __writeback_inodes_wb+0x9e/0xd0 [3485698.773325] [] wb_writeback+0x29b/0x350 [3485698.773327] [] ? get_nr_dirty_inodes+0x52/0x80 [3485698.773329] [] wb_check_old_data_flush+0x9f/0xb0 [3485698.773330] [] bdi_writeback_workfn+0x1cc/0x490 [3485698.773333] [] ? process_one_work+0x16f/0x570 [3485698.773334] [] process_one_work+0x1d1/0x570 [3485698.773335] [] ? process_one_work+0x16f/0x570 [3485698.773337] [] worker_thread+0x11c/0x530 [3485698.773339] [] ? create_and_start_worker+0x50/0x50 [3485698.773340] [] kthread+0xe4/0x100 [3485698.773343] [] ? flush_kthread_worker+0x130/0x130 [3485698.773344] [] ret_from_fork+0x7c/0xb0 [3485698.773346] [] ? flush_kthread_worker+0x130/0x130 [3485698.773362] Code: 81 48 89 df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff <0f> b7 03 66 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 [3485698.789200] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3485698.789228] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485698.789229] irq event stamp: 0 [3485698.789230] hardirqs last enabled at (0): [< (null)>] (null) [3485698.789233] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485698.789235] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485698.789236] softirqs last disabled at (0): [< (null)>] (null) [3485698.789238] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485698.789238] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485698.789239] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485698.789242] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485698.789243] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485698.789244] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485698.789244] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485698.789245] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485698.789246] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485698.789246] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485698.789248] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485698.789248] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485698.789249] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485698.789250] Stack: [3485698.789252] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485698.789253] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485698.789255] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485698.789255] Call Trace: [3485698.789257] [] _raw_spin_lock+0x41/0x50 [3485698.789265] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485698.789272] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485698.789277] [] ceph_release+0x2e/0xc0 [ceph] [3485698.789279] [] __fput+0xc8/0x210 [3485698.789281] [] ____fput+0xe/0x10 [3485698.789283] [] task_work_run+0xb4/0xe0 [3485698.789285] [] do_exit+0x2e8/0xae0 [3485698.789287] [] ? kmsg_dump+0x10d/0x120 [3485698.789288] [] ? kmsg_dump+0x24/0x120 [3485698.789290] [] oops_end+0xa5/0xf0 [3485698.789291] [] die+0x58/0x90 [3485698.789293] [] do_trap+0xcd/0x160 [3485698.789295] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485698.789296] [] do_error_trap.part.16+0xa1/0xe0 [3485698.789302] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485698.789304] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485698.789305] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485698.789307] [] ? atomic_notifier_call_chain+0x16/0x20 [3485698.789308] [] do_error_trap+0x64/0x80 [3485698.789310] [] do_invalid_op+0x20/0x30 [3485698.789311] [] invalid_op+0x18/0x20 [3485698.789318] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485698.789324] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485698.789326] [] ? test_clear_page_writeback+0x143/0x260 [3485698.789327] [] ? test_clear_page_writeback+0x38/0x260 [3485698.789333] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485698.789335] [] ? SyS_msync+0x230/0x230 [3485698.789340] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485698.789342] [] ? wait_for_stable_page+0x16/0x50 [3485698.789348] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485698.789350] [] generic_perform_write+0xc6/0x1e0 [3485698.789356] [] ceph_write_iter+0x890/0xa30 [ceph] [3485698.789357] [] ? _raw_spin_unlock+0x2b/0x40 [3485698.789359] [] ? do_wp_page+0x374/0x740 [3485698.789360] [] ? handle_mm_fault+0x86c/0x1060 [3485698.789362] [] ? __do_page_fault+0x159/0x580 [3485698.789364] [] ? vfs_write+0x1d3/0x1f0 [3485698.789365] [] ? vfs_write+0x1d3/0x1f0 [3485698.789367] [] new_sync_write+0x7b/0xb0 [3485698.789368] [] vfs_write+0xc7/0x1f0 [3485698.789370] [] SyS_write+0x52/0xb0 [3485698.789371] [] system_call_fastpath+0x1a/0x1f [3485698.789387] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485698.801195] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485698.801215] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485698.801215] irq event stamp: 0 [3485698.801216] hardirqs last enabled at (0): [< (null)>] (null) [3485698.801219] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485698.801221] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485698.801222] softirqs last disabled at (0): [< (null)>] (null) [3485698.801224] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485698.801224] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485698.801225] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485698.801228] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485698.801229] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485698.801229] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485698.801230] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485698.801231] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485698.801231] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485698.801232] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485698.801233] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485698.801234] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485698.801235] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485698.801235] Stack: [3485698.801237] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485698.801238] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485698.801240] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485698.801240] Call Trace: [3485698.801243] [] _raw_spin_lock+0x41/0x50 [3485698.801250] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485698.801256] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485698.801261] [] ceph_release+0x2e/0xc0 [ceph] [3485698.801263] [] __fput+0xc8/0x210 [3485698.801265] [] ____fput+0xe/0x10 [3485698.801267] [] task_work_run+0xb4/0xe0 [3485698.801269] [] do_exit+0x2e8/0xae0 [3485698.801270] [] ? kmsg_dump+0x10d/0x120 [3485698.801272] [] ? kmsg_dump+0x24/0x120 [3485698.801273] [] oops_end+0xa5/0xf0 [3485698.801274] [] die+0x58/0x90 [3485698.801276] [] do_trap+0xcd/0x160 [3485698.801277] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485698.801279] [] do_error_trap.part.16+0xa1/0xe0 [3485698.801285] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485698.801287] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485698.801288] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485698.801290] [] ? atomic_notifier_call_chain+0x16/0x20 [3485698.801291] [] do_error_trap+0x64/0x80 [3485698.801292] [] do_invalid_op+0x20/0x30 [3485698.801294] [] invalid_op+0x18/0x20 [3485698.801300] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485698.801307] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485698.801308] [] ? test_clear_page_writeback+0x143/0x260 [3485698.801310] [] ? test_clear_page_writeback+0x38/0x260 [3485698.801315] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485698.801317] [] ? SyS_msync+0x230/0x230 [3485698.801323] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485698.801324] [] ? wait_for_stable_page+0x16/0x50 [3485698.801330] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485698.801332] [] generic_perform_write+0xc6/0x1e0 [3485698.801337] [] ceph_write_iter+0x890/0xa30 [ceph] [3485698.801339] [] ? might_fault+0x66/0xc0 [3485698.801341] [] ? might_fault+0x66/0xc0 [3485698.801342] [] ? vfs_write+0x1d3/0x1f0 [3485698.801344] [] ? vfs_write+0x1d3/0x1f0 [3485698.801345] [] ? might_fault+0x66/0xc0 [3485698.801347] [] new_sync_write+0x7b/0xb0 [3485698.801348] [] vfs_write+0xc7/0x1f0 [3485698.801350] [] SyS_write+0x52/0xb0 [3485698.801351] [] system_call_fastpath+0x1a/0x1f [3485698.801367] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485700.210998] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485700.274003] irq event stamp: 0 [3485700.277285] hardirqs last enabled at (0): [< (null)>] (null) [3485700.284971] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485700.293448] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485700.301928] softirqs last disabled at (0): [< (null)>] (null) [3485700.309619] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485700.318472] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485700.325228] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485700.332953] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485700.341773] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485700.347313] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485700.354684] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485700.362056] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485700.369425] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485700.376798] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485700.384169] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485700.392494] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485700.398463] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485700.405832] Stack: [3485700.408092] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485700.415931] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485700.423771] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485700.431609] Call Trace: [3485700.434288] [] _raw_spin_lock+0x41/0x50 [3485700.440009] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485700.446850] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485700.454619] [] ? __init_waitqueue_head+0x60/0x60 [3485700.461118] [] ceph_read_iter+0xc9/0x570 [ceph] [3485700.467524] [] new_sync_read+0x78/0xb0 [3485700.473150] [] vfs_read+0xab/0x180 [3485700.478431] [] SyS_read+0x52/0xb0 [3485700.483620] [] system_call_fastpath+0x1a/0x1f [3485700.489846] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485726.754263] BUG: soft lockup - CPU#1 stuck for 23s! [grep:27686] [3485726.758260] BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u16:1:7152] [3485726.758288] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485726.758289] irq event stamp: 0 [3485726.758291] hardirqs last enabled at (0): [< (null)>] (null) [3485726.758296] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485726.758298] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485726.758299] softirqs last disabled at (0): [< (null)>] (null) [3485726.758301] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485726.758302] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485726.758306] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485726.758307] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485726.758312] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3485726.758312] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485726.758313] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485726.758314] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485726.758315] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485726.758315] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485726.758316] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485726.758317] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485726.758318] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485726.758319] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485726.758319] Stack: [3485726.758321] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485726.758323] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485726.758324] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485726.758325] Call Trace: [3485726.758329] [] _raw_spin_lock+0x41/0x50 [3485726.758340] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485726.758346] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485726.758352] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485726.758354] [] ? ext4_writepages+0x4ee/0xd30 [3485726.758361] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485726.758367] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485726.758369] [] ? writeback_sb_inodes+0x139/0x4f0 [3485726.758370] [] ? writeback_sb_inodes+0x139/0x4f0 [3485726.758373] [] do_writepages+0x23/0x40 [3485726.758374] [] __writeback_single_inode+0x45/0x2e0 [3485726.758376] [] writeback_sb_inodes+0x2ad/0x4f0 [3485726.758377] [] ? wb_writeback+0x54/0x350 [3485726.758379] [] __writeback_inodes_wb+0x9e/0xd0 [3485726.758380] [] wb_writeback+0x29b/0x350 [3485726.758382] [] ? get_nr_dirty_inodes+0x52/0x80 [3485726.758384] [] wb_check_old_data_flush+0x9f/0xb0 [3485726.758385] [] bdi_writeback_workfn+0x1cc/0x490 [3485726.758387] [] ? process_one_work+0x16f/0x570 [3485726.758389] [] process_one_work+0x1d1/0x570 [3485726.758390] [] ? process_one_work+0x16f/0x570 [3485726.758392] [] worker_thread+0x11c/0x530 [3485726.758393] [] ? create_and_start_worker+0x50/0x50 [3485726.758395] [] kthread+0xe4/0x100 [3485726.758397] [] ? flush_kthread_worker+0x130/0x130 [3485726.758399] [] ret_from_fork+0x7c/0xb0 [3485726.758401] [] ? flush_kthread_worker+0x130/0x130 [3485726.758417] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3485726.774254] BUG: soft lockup - CPU#2 stuck for 23s! [teuthology:19790] [3485726.774274] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485726.774274] irq event stamp: 0 [3485726.774275] hardirqs last enabled at (0): [< (null)>] (null) [3485726.774278] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485726.774280] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485726.774281] softirqs last disabled at (0): [< (null)>] (null) [3485726.774283] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485726.774284] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485726.774284] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485726.774287] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485726.774288] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485726.774289] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485726.774289] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485726.774290] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485726.774291] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485726.774291] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485726.774292] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485726.774293] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485726.774294] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485726.774295] Stack: [3485726.774296] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485726.774298] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485726.774299] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485726.774300] Call Trace: [3485726.774302] [] _raw_spin_lock+0x41/0x50 [3485726.774310] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485726.774316] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485726.774321] [] ceph_release+0x2e/0xc0 [ceph] [3485726.774324] [] __fput+0xc8/0x210 [3485726.774325] [] ____fput+0xe/0x10 [3485726.774327] [] task_work_run+0xb4/0xe0 [3485726.774330] [] do_exit+0x2e8/0xae0 [3485726.774331] [] ? kmsg_dump+0x10d/0x120 [3485726.774333] [] ? kmsg_dump+0x24/0x120 [3485726.774335] [] oops_end+0xa5/0xf0 [3485726.774336] [] die+0x58/0x90 [3485726.774338] [] do_trap+0xcd/0x160 [3485726.774340] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485726.774341] [] do_error_trap.part.16+0xa1/0xe0 [3485726.774347] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485726.774349] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485726.774350] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485726.774352] [] ? atomic_notifier_call_chain+0x16/0x20 [3485726.774353] [] do_error_trap+0x64/0x80 [3485726.774354] [] do_invalid_op+0x20/0x30 [3485726.774356] [] invalid_op+0x18/0x20 [3485726.774363] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485726.774369] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485726.774371] [] ? test_clear_page_writeback+0x143/0x260 [3485726.774372] [] ? test_clear_page_writeback+0x38/0x260 [3485726.774378] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485726.774380] [] ? SyS_msync+0x230/0x230 [3485726.774386] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485726.774387] [] ? wait_for_stable_page+0x16/0x50 [3485726.774393] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485726.774395] [] generic_perform_write+0xc6/0x1e0 [3485726.774401] [] ceph_write_iter+0x890/0xa30 [ceph] [3485726.774402] [] ? _raw_spin_unlock+0x2b/0x40 [3485726.774404] [] ? do_wp_page+0x374/0x740 [3485726.774406] [] ? handle_mm_fault+0x86c/0x1060 [3485726.774407] [] ? __do_page_fault+0x159/0x580 [3485726.774409] [] ? vfs_write+0x1d3/0x1f0 [3485726.774410] [] ? vfs_write+0x1d3/0x1f0 [3485726.774412] [] new_sync_write+0x7b/0xb0 [3485726.774414] [] vfs_write+0xc7/0x1f0 [3485726.774415] [] SyS_write+0x52/0xb0 [3485726.774417] [] system_call_fastpath+0x1a/0x1f [3485726.774433] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485726.786249] BUG: soft lockup - CPU#3 stuck for 23s! [teuthology:20089] [3485726.786269] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485726.786270] irq event stamp: 0 [3485726.786271] hardirqs last enabled at (0): [< (null)>] (null) [3485726.786274] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485726.786276] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485726.786276] softirqs last disabled at (0): [< (null)>] (null) [3485726.786278] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485726.786279] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485726.786280] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485726.786282] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485726.786283] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485726.786284] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485726.786285] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485726.786285] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485726.786286] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485726.786287] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485726.786288] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485726.786289] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485726.786289] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485726.786290] Stack: [3485726.786292] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485726.786293] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485726.786295] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485726.786295] Call Trace: [3485726.786297] [] _raw_spin_lock+0x41/0x50 [3485726.786304] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485726.786311] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485726.786316] [] ceph_release+0x2e/0xc0 [ceph] [3485726.786318] [] __fput+0xc8/0x210 [3485726.786320] [] ____fput+0xe/0x10 [3485726.786322] [] task_work_run+0xb4/0xe0 [3485726.786324] [] do_exit+0x2e8/0xae0 [3485726.786325] [] ? kmsg_dump+0x10d/0x120 [3485726.786327] [] ? kmsg_dump+0x24/0x120 [3485726.786328] [] oops_end+0xa5/0xf0 [3485726.786329] [] die+0x58/0x90 [3485726.786331] [] do_trap+0xcd/0x160 [3485726.786332] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485726.786334] [] do_error_trap.part.16+0xa1/0xe0 [3485726.786340] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485726.786342] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485726.786343] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485726.786345] [] ? atomic_notifier_call_chain+0x16/0x20 [3485726.786346] [] do_error_trap+0x64/0x80 [3485726.786347] [] do_invalid_op+0x20/0x30 [3485726.786349] [] invalid_op+0x18/0x20 [3485726.786355] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485726.786361] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485726.786363] [] ? test_clear_page_writeback+0x143/0x260 [3485726.786365] [] ? test_clear_page_writeback+0x38/0x260 [3485726.786370] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485726.786372] [] ? SyS_msync+0x230/0x230 [3485726.786377] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485726.786379] [] ? wait_for_stable_page+0x16/0x50 [3485726.786385] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485726.786387] [] generic_perform_write+0xc6/0x1e0 [3485726.786392] [] ceph_write_iter+0x890/0xa30 [ceph] [3485726.786394] [] ? might_fault+0x66/0xc0 [3485726.786395] [] ? might_fault+0x66/0xc0 [3485726.786397] [] ? vfs_write+0x1d3/0x1f0 [3485726.786399] [] ? vfs_write+0x1d3/0x1f0 [3485726.786400] [] ? might_fault+0x66/0xc0 [3485726.786401] [] new_sync_write+0x7b/0xb0 [3485726.786403] [] vfs_write+0xc7/0x1f0 [3485726.786405] [] SyS_write+0x52/0xb0 [3485726.786406] [] system_call_fastpath+0x1a/0x1f [3485726.786422] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485728.195927] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485728.258976] irq event stamp: 0 [3485728.262264] hardirqs last enabled at (0): [< (null)>] (null) [3485728.269961] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485728.278434] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485728.286896] softirqs last disabled at (0): [< (null)>] (null) [3485728.294584] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485728.303443] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485728.310205] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485728.317923] RIP: 0010:[] [] do_raw_spin_lock+0x153/0x160 [3485728.326759] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485728.332295] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485728.339666] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485728.347065] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485728.354433] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485728.361811] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485728.369186] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485728.377511] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485728.383478] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485728.390861] Stack: [3485728.393105] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485728.400938] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485728.408778] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485728.416626] Call Trace: [3485728.419304] [] _raw_spin_lock+0x41/0x50 [3485728.425035] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485728.431889] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485728.438563] [] ? __init_waitqueue_head+0x60/0x60 [3485728.445062] [] ceph_read_iter+0xc9/0x570 [ceph] [3485728.451488] [] new_sync_read+0x78/0xb0 [3485728.457111] [] vfs_read+0xab/0x180 [3485728.462394] [] SyS_read+0x52/0xb0 [3485728.467591] [] system_call_fastpath+0x1a/0x1f [3485728.473828] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 [3485738.671896] INFO: rcu_sched self-detected stall on CPU { 2} (t=7035781 jiffies g=482214633 c=482214632 q=8556771) [3485738.679894] INFO: rcu_sched detected stalls on CPUs/tasks: { 2} (detected by 3, t=7035782 jiffies, g=482214633, c=482214632, q=8556771) [3485738.679895] sending NMI to all CPUs: [3485738.679897] NMI backtrace for cpu 3 [3485738.679900] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485738.679901] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485738.679902] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485738.679909] RIP: 0010:[] [] __const_udelay+0x12/0x30 [3485738.679910] RSP: 0018:ffff88043fcc3ca8 EFLAGS: 00000046 [3485738.679911] RAX: 0000000001062560 RBX: 0000000000002710 RCX: 0000000000000004 [3485738.679912] RDX: 00000000009aa1d0 RSI: 0000000000000100 RDI: 0000000000418958 [3485738.679913] RBP: ffff88043fcc3cc8 R08: 0000000000000001 R09: 0000000000000000 [3485738.679914] R10: 0000000000000001 R11: 0000000000000000 R12: ffffffff81c54f80 [3485738.679915] R13: 0000000000000003 R14: ffffffff81c54f80 R15: ffffffff81d21e80 [3485738.679916] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485738.679918] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485738.679919] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485738.679920] Stack: [3485738.679922] ffff88043fcc3cc8 ffffffff81036812 000000001cbe02e8 ffff88043fccd7c0 [3485738.679924] ffff88043fcc3d28 ffffffff810cd6a5 0000000000000002 00000000008290e3 [3485738.679927] 0000000000000082 00000001810aae9d ffff88043fcc3d18 ffff8803c46321f0 [3485738.679927] Call Trace: [3485738.679930] [3485738.679934] [] ? arch_trigger_all_cpu_backtrace+0x92/0x120 [3485738.679938] [] rcu_check_callbacks+0x685/0x6b0 [3485738.679940] [] update_process_times+0x48/0x80 [3485738.679944] [] tick_sched_handle.isra.15+0x33/0x70 [3485738.679946] [] tick_sched_timer+0x4c/0x80 [3485738.679948] [] __run_hrtimer+0x71/0x250 [3485738.679950] [] ? hrtimer_interrupt+0x75/0x260 [3485738.679952] [] ? tick_sched_do_timer+0x40/0x40 [3485738.679954] [] hrtimer_interrupt+0x107/0x260 [3485738.679958] [] ? handle_edge_irq+0x1e/0x110 [3485738.679961] [] hpet_interrupt_handler+0x16/0x40 [3485738.679963] [] handle_irq_event_percpu+0x5d/0x200 [3485738.679965] [] handle_irq_event+0x48/0x70 [3485738.679967] [] handle_edge_irq+0x77/0x110 [3485738.679970] [] handle_irq+0x22/0x40 [3485738.679974] [] do_IRQ+0x5e/0x110 [3485738.679976] [] common_interrupt+0x6f/0x6f [3485738.679977] [3485738.679979] [] ? retint_restore_args+0xe/0xe [3485738.679983] [] ? do_raw_spin_lock+0x158/0x160 [3485738.679985] [] ? do_raw_spin_lock+0x138/0x160 [3485738.679987] [] _raw_spin_lock+0x41/0x50 [3485738.680003] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485738.680011] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485738.680018] [] ceph_release+0x2e/0xc0 [ceph] [3485738.680021] [] __fput+0xc8/0x210 [3485738.680023] [] ____fput+0xe/0x10 [3485738.680025] [] task_work_run+0xb4/0xe0 [3485738.680028] [] do_exit+0x2e8/0xae0 [3485738.680030] [] ? kmsg_dump+0x10d/0x120 [3485738.680032] [] ? kmsg_dump+0x24/0x120 [3485738.680034] [] oops_end+0xa5/0xf0 [3485738.680036] [] die+0x58/0x90 [3485738.680038] [] do_trap+0xcd/0x160 [3485738.680040] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485738.680042] [] do_error_trap.part.16+0xa1/0xe0 [3485738.680049] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485738.680052] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485738.680053] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485738.680056] [] ? atomic_notifier_call_chain+0x16/0x20 [3485738.680057] [] do_error_trap+0x64/0x80 [3485738.680059] [] do_invalid_op+0x20/0x30 [3485738.680061] [] invalid_op+0x18/0x20 [3485738.680069] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485738.680076] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485738.680079] [] ? test_clear_page_writeback+0x143/0x260 [3485738.680081] [] ? test_clear_page_writeback+0x38/0x260 [3485738.680088] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485738.680091] [] ? SyS_msync+0x230/0x230 [3485738.680098] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485738.680100] [] ? wait_for_stable_page+0x16/0x50 [3485738.680107] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485738.680110] [] generic_perform_write+0xc6/0x1e0 [3485738.680116] [] ceph_write_iter+0x890/0xa30 [ceph] [3485738.680119] [] ? might_fault+0x66/0xc0 [3485738.680121] [] ? might_fault+0x66/0xc0 [3485738.680123] [] ? vfs_write+0x1d3/0x1f0 [3485738.680125] [] ? vfs_write+0x1d3/0x1f0 [3485738.680127] [] ? might_fault+0x66/0xc0 [3485738.680129] [] new_sync_write+0x7b/0xb0 [3485738.680131] [] vfs_write+0xc7/0x1f0 [3485738.680133] [] SyS_write+0x52/0xb0 [3485738.680135] [] system_call_fastpath+0x1a/0x1f [3485738.680159] Code: 89 e5 ff 15 69 58 8f 00 5d c3 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 55 48 8d 04 bd 00 00 00 00 65 48 8b 14 25 60 30 01 00 <48> 8d 0c 12 48 c1 e2 06 48 89 e5 48 29 ca f7 e2 48 8d 7a 01 ff [3485738.680161] NMI backtrace for cpu 1 [3485738.680163] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485738.680164] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485738.680165] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485738.680170] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485738.680171] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485738.680172] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000057c957c8 [3485738.680173] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485738.680174] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485738.680175] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485738.680176] R13: 0000000097020520 R14: ffff880224754690 R15: ffff88033a8afeb0 [3485738.680177] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485738.680178] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485738.680179] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485738.680180] Stack: [3485738.680183] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485738.680185] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485738.680187] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485738.680187] Call Trace: [3485738.680190] [] _raw_spin_lock+0x41/0x50 [3485738.680201] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485738.680208] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485738.680212] [] ? __init_waitqueue_head+0x60/0x60 [3485738.680218] [] ceph_read_iter+0xc9/0x570 [ceph] [3485738.680221] [] new_sync_read+0x78/0xb0 [3485738.680223] [] vfs_read+0xab/0x180 [3485738.680225] [] SyS_read+0x52/0xb0 [3485738.680228] [] system_call_fastpath+0x1a/0x1f [3485738.680251] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485738.680253] NMI backtrace for cpu 0 [3485738.680256] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485738.680257] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485738.680262] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485738.680263] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485738.680267] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485738.680268] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485738.680269] RAX: 0000000000008916 RBX: ffff8800049b5830 RCX: 0000000000000004 [3485738.680270] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485738.680271] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485738.680272] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485738.680273] R13: 0000000097020520 R14: 0000000000002000 R15: 000000000000001d [3485738.680275] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485738.680276] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485738.680277] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485738.680278] Stack: [3485738.680280] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485738.680282] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485738.680285] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485738.680285] Call Trace: [3485738.680288] [] _raw_spin_lock+0x41/0x50 [3485738.680297] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485738.680305] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485738.680312] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485738.680315] [] ? ext4_writepages+0x4ee/0xd30 [3485738.680323] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485738.680330] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485738.680332] [] ? writeback_sb_inodes+0x139/0x4f0 [3485738.680334] [] ? writeback_sb_inodes+0x139/0x4f0 [3485738.680337] [] do_writepages+0x23/0x40 [3485738.680339] [] __writeback_single_inode+0x45/0x2e0 [3485738.680341] [] writeback_sb_inodes+0x2ad/0x4f0 [3485738.680342] [] ? wb_writeback+0x54/0x350 [3485738.680344] [] __writeback_inodes_wb+0x9e/0xd0 [3485738.680346] [] wb_writeback+0x29b/0x350 [3485738.680349] [] ? get_nr_dirty_inodes+0x52/0x80 [3485738.680351] [] wb_check_old_data_flush+0x9f/0xb0 [3485738.680353] [] bdi_writeback_workfn+0x1cc/0x490 [3485738.680355] [] ? process_one_work+0x16f/0x570 [3485738.680358] [] process_one_work+0x1d1/0x570 [3485738.680359] [] ? process_one_work+0x16f/0x570 [3485738.680362] [] worker_thread+0x11c/0x530 [3485738.680364] [] ? create_and_start_worker+0x50/0x50 [3485738.680366] [] kthread+0xe4/0x100 [3485738.680369] [] ? flush_kthread_worker+0x130/0x130 [3485738.680371] [] ret_from_fork+0x7c/0xb0 [3485738.680373] [] ? flush_kthread_worker+0x130/0x130 [3485738.680397] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485738.680398] NMI backtrace for cpu 2 [3485738.680400] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485738.680401] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485738.680402] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485738.680407] RIP: 0010:[] [] io_serial_in+0x18/0x20 [3485738.680408] RSP: 0018:ffff88043fc83a78 EFLAGS: 00000006 [3485738.680409] RAX: 00000000a84e5600 RBX: ffffffff82c09eb0 RCX: 0000000000000000 [3485738.680410] RDX: 00000000000003ed RSI: 00000000000003ed RDI: ffffffff82c09eb0 [3485738.680411] RBP: ffff88043fc83a78 R08: 0000000000000002 R09: 0000000000000000 [3485738.680412] R10: 0000000000000000 R11: 0000000000000001 R12: 00000000000026f6 [3485738.680413] R13: 0000000000000020 R14: ffffffff8147cd70 R15: ffffffff82b607f7 [3485738.680414] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485738.680415] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485738.680416] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485738.680417] Stack: [3485738.680419] ffff88043fc83aa8 ffffffff8147cd0b ffffffff813a7184 ffffffff82c09eb0 [3485738.680422] 0000000000000032 ffffffff82b607da ffff88043fc83ac8 ffffffff8147cd95 [3485738.680424] ffffffff82b607da ffffffff82c09eb0 ffff88043fc83b08 ffffffff81475fcd [3485738.680424] Call Trace: [3485738.680426] [3485738.680429] [] wait_for_xmitr+0x3b/0xa0 [3485738.680431] [] ? number.isra.2+0x304/0x330 [3485738.680433] [] serial8250_console_putchar+0x25/0x40 [3485738.680436] [] uart_console_write+0x3d/0x70 [3485738.680438] [] serial8250_console_write+0xc3/0x1a0 [3485738.680441] [] call_console_drivers.constprop.20+0xa5/0x120 [3485738.680443] [] console_unlock+0x397/0x480 [3485738.680445] [] vprintk_emit+0x29c/0x600 [3485738.680448] [] ? cpuacct_account_field+0x5/0xb0 [3485738.680452] [] printk+0x61/0x63 [3485738.680455] [] rcu_check_callbacks+0x448/0x6b0 [3485738.680457] [] update_process_times+0x48/0x80 [3485738.680459] [] tick_sched_handle.isra.15+0x33/0x70 [3485738.680461] [] tick_sched_timer+0x4c/0x80 [3485738.680463] [] __run_hrtimer+0x71/0x250 [3485738.680465] [] ? hrtimer_interrupt+0x75/0x260 [3485738.680467] [] ? tick_sched_do_timer+0x40/0x40 [3485738.680469] [] hrtimer_interrupt+0x107/0x260 [3485738.680471] [] ? handle_edge_irq+0x1e/0x110 [3485738.680474] [] hpet_interrupt_handler+0x16/0x40 [3485738.680476] [] handle_irq_event_percpu+0x5d/0x200 [3485738.680478] [] handle_irq_event+0x48/0x70 [3485738.680480] [] handle_edge_irq+0x77/0x110 [3485738.680482] [] handle_irq+0x22/0x40 [3485738.680485] [] do_IRQ+0x5e/0x110 [3485738.680487] [] common_interrupt+0x6f/0x6f [3485738.680488] [3485738.680489] [] ? retint_restore_args+0xe/0xe [3485738.680492] [] ? do_raw_spin_lock+0x15a/0x160 [3485738.680494] [] ? do_raw_spin_lock+0x138/0x160 [3485738.680496] [] _raw_spin_lock+0x41/0x50 [3485738.680505] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485738.680513] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485738.680519] [] ceph_release+0x2e/0xc0 [ceph] [3485738.680522] [] __fput+0xc8/0x210 [3485738.680524] [] ____fput+0xe/0x10 [3485738.680526] [] task_work_run+0xb4/0xe0 [3485738.680529] [] do_exit+0x2e8/0xae0 [3485738.680531] [] ? kmsg_dump+0x10d/0x120 [3485738.680532] [] ? kmsg_dump+0x24/0x120 [3485738.680534] [] oops_end+0xa5/0xf0 [3485738.680536] [] die+0x58/0x90 [3485738.680538] [] do_trap+0xcd/0x160 [3485738.680540] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485738.680542] [] do_error_trap.part.16+0xa1/0xe0 [3485738.680550] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485738.680552] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485738.680554] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485738.680556] [] ? atomic_notifier_call_chain+0x16/0x20 [3485738.680558] [] do_error_trap+0x64/0x80 [3485738.680560] [] do_invalid_op+0x20/0x30 [3485738.680562] [] invalid_op+0x18/0x20 [3485738.680570] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485738.680577] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485738.680579] [] ? test_clear_page_writeback+0x143/0x260 [3485738.680581] [] ? test_clear_page_writeback+0x38/0x260 [3485738.680588] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485738.680591] [] ? SyS_msync+0x230/0x230 [3485738.680598] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485738.680600] [] ? wait_for_stable_page+0x16/0x50 [3485738.680607] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485738.680609] [] generic_perform_write+0xc6/0x1e0 [3485738.680616] [] ceph_write_iter+0x890/0xa30 [ceph] [3485738.680618] [] ? _raw_spin_unlock+0x2b/0x40 [3485738.680620] [] ? do_wp_page+0x374/0x740 [3485738.680623] [] ? handle_mm_fault+0x86c/0x1060 [3485738.680625] [] ? __do_page_fault+0x159/0x580 [3485738.680627] [] ? vfs_write+0x1d3/0x1f0 [3485738.680629] [] ? vfs_write+0x1d3/0x1f0 [3485738.680631] [] new_sync_write+0x7b/0xb0 [3485738.680633] [] vfs_write+0xc7/0x1f0 [3485738.680635] [] SyS_write+0x52/0xb0 [3485738.680637] [] system_call_fastpath+0x1a/0x1f [3485738.680661] Code: 48 89 e5 d3 e6 48 63 f6 48 03 77 40 8b 06 5d c3 0f 1f 00 66 66 66 66 90 0f b6 8f 91 00 00 00 55 48 89 e5 d3 e6 03 77 38 89 f2 ec <0f> b6 c0 5d c3 0f 1f 00 66 66 66 66 90 0f b6 8f 91 00 00 00 55 [3485738.680662] NMI backtrace for cpu 7 [3485738.680664] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485738.680665] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485738.680666] task: ffff8804296821f0 ti: ffff88042968c000 task.ti: ffff88042968c000 [3485738.680671] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485738.680672] RSP: 0018:ffff88042968fdf8 EFLAGS: 00000046 [3485738.680673] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 [3485738.680674] RDX: 0000000000000000 RSI: ffff88042968ffd8 RDI: ffffffff817295bf [3485738.680675] RBP: ffff88042968fe28 R08: 0000000000000001 R09: 0000000000000000 [3485738.680676] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003 [3485738.680676] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 [3485738.680678] FS: 0000000000000000(0000) GS:ffff88043fdc0000(0000) knlGS:0000000000000000 [3485738.680679] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485738.680680] CR2: 00007fab827192c0 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485738.680680] Stack: [3485738.680682] ffffffff815d3fef 00000007ffffffff ffff88043fddb020 000c63f4343bb9c3 [3485738.680684] ffffffff81ca4320 ffffffff81ca4200 ffff88042968fe88 ffffffff815d3fff [3485738.680686] 0000000000000000 00000000003ca437 0000000000000000 00000000003ca437 [3485738.680687] Call Trace: [3485738.680691] [] ? cpuidle_enter_state+0x3f/0xe0 [3485738.680693] [] cpuidle_enter_state+0x4f/0xe0 [3485738.680695] [] cpuidle_enter+0x17/0x20 [3485738.680697] [] cpu_startup_entry+0x32b/0x490 [3485738.680699] [] ? trace_hardirqs_on+0xd/0x10 [3485738.680701] [] start_secondary+0x1d8/0x290 [3485738.680723] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485738.680724] NMI backtrace for cpu 4 [3485738.680727] CPU: 4 PID: 0 Comm: swapper/4 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485738.680728] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485738.680729] task: ffff88042966a1f0 ti: ffff880429670000 task.ti: ffff880429670000 [3485738.680732] RIP: 0010:[] [] hpet_msi_next_event+0x3c/0x50 [3485738.680733] RSP: 0018:ffff88043fd03db8 EFLAGS: 00000082 [3485738.680734] RAX: 0000000080ae5de1 RBX: ffff880428ce5c00 RCX: 0000000000000020 [3485738.680735] RDX: 0000000080ae5dcb RSI: ffff880428ce5c00 RDI: 0000000080af3d1f [3485738.680736] RBP: ffff88043fd03db8 R08: 0000000000000001 R09: 0000000000000000 [3485738.680737] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 [3485738.680738] R13: 00000000003ced8e R14: ffff88043fd0d4c8 R15: ffff88043fd0d448 [3485738.680740] FS: 0000000000000000(0000) GS:ffff88043fd00000(0000) knlGS:0000000000000000 [3485738.680741] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485738.680742] CR2: 00007f7146cf02c0 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485738.680743] Stack: [3485738.680745] ffff88043fd03df8 ffffffff810d6d64 0000000000000003 ffff88043fd0d488 [3485738.680747] ffff88043fd0d4c8 ffff88043fd0d360 0000000000000003 ffff88043fd0d488 [3485738.680749] ffff88043fd03e08 ffffffff810d8d14 ffff88043fd03e88 ffffffff81081b4f [3485738.680750] Call Trace: [3485738.680751] [3485738.680753] [] clockevents_program_event+0x74/0x100 [3485738.680755] [] tick_program_event+0x24/0x30 [3485738.680757] [] hrtimer_interrupt+0x14f/0x260 [3485738.680760] [] ? handle_edge_irq+0x1e/0x110 [3485738.680762] [] hpet_interrupt_handler+0x16/0x40 [3485738.680764] [] handle_irq_event_percpu+0x5d/0x200 [3485738.680766] [] handle_irq_event+0x48/0x70 [3485738.680768] [] handle_edge_irq+0x77/0x110 [3485738.680770] [] handle_irq+0x22/0x40 [3485738.680772] [] do_IRQ+0x5e/0x110 [3485738.680774] [] common_interrupt+0x6f/0x6f [3485738.680776] [3485738.680778] [] ? cpuidle_enter_state+0x63/0xe0 [3485738.680780] [] ? cpuidle_enter_state+0x5f/0xe0 [3485738.680782] [] cpuidle_enter+0x17/0x20 [3485738.680785] [] cpu_startup_entry+0x32b/0x490 [3485738.680787] [] ? trace_hardirqs_on+0xd/0x10 [3485738.680789] [] start_secondary+0x1d8/0x290 [3485738.680813] Code: e5 00 48 89 e5 8b 92 f0 00 00 00 c1 e0 05 01 d7 05 08 01 00 00 48 03 05 7b be e5 00 89 38 48 8b 05 72 be e5 00 8b 80 f0 00 00 00 <29> c7 ba 00 00 00 00 b8 c2 ff ff ff 83 ff 7f 0f 4f c2 5d c3 66 [3485738.680814] NMI backtrace for cpu 5 [3485738.680816] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485738.680817] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485738.680818] task: ffff88042966c3e0 ti: ffff880429674000 task.ti: ffff880429674000 [3485738.680822] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485738.680823] RSP: 0018:ffff880429677df8 EFLAGS: 00000046 [3485738.680824] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 [3485738.680825] RDX: 0000000000000000 RSI: ffff880429677fd8 RDI: ffffffff817295bf [3485738.680826] RBP: ffff880429677e28 R08: 0000000000000001 R09: 0000000000000000 [3485738.680827] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003 [3485738.680828] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 [3485738.680829] FS: 0000000000000000(0000) GS:ffff88043fd40000(0000) knlGS:0000000000000000 [3485738.680830] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485738.680831] CR2: 00007f9f944192c0 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485738.680832] Stack: [3485738.680834] ffffffff815d3fef 00000005ffffffff ffff88043fd5b020 000c63f433fed48b [3485738.680836] ffffffff81ca4320 ffffffff81ca4200 ffff880429677e88 ffffffff815d3fff [3485738.680838] 0000000000000000 0000000000798d3d 0000000000000000 0000000000798d3d [3485738.680839] Call Trace: [3485738.680842] [] ? cpuidle_enter_state+0x3f/0xe0 [3485738.680844] [] cpuidle_enter_state+0x4f/0xe0 [3485738.680846] [] cpuidle_enter+0x17/0x20 [3485738.680848] [] cpu_startup_entry+0x32b/0x490 [3485738.680850] [] ? trace_hardirqs_on+0xd/0x10 [3485738.680852] [] start_secondary+0x1d8/0x290 [3485738.680876] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485738.680877] NMI backtrace for cpu 6 [3485738.680880] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485738.680880] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485738.680881] task: ffff880429680000 ti: ffff880429688000 task.ti: ffff880429688000 [3485738.680885] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485738.680885] RSP: 0018:ffff88042968bdf8 EFLAGS: 00000046 [3485738.680886] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 [3485738.680887] RDX: 0000000000000000 RSI: ffff88042968bfd8 RDI: ffffffff817295bf [3485738.680888] RBP: ffff88042968be28 R08: 0000000000000001 R09: 0000000000000000 [3485738.680888] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004 [3485738.680889] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 [3485738.680890] FS: 0000000000000000(0000) GS:ffff88043fd80000(0000) knlGS:0000000000000000 [3485738.680891] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485738.680892] CR2: 00007f8012d82980 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485738.680893] Stack: [3485738.680895] ffffffff815d3fef 00000006ffffffff ffff88043fd9b020 000c63f433d9555d [3485738.680896] ffffffff81ca4378 ffffffff81ca4200 ffff88042968be88 ffffffff815d3fff [3485738.680898] 0000000000000000 0000000000dc1881 0000000000000000 0000000000dc1881 [3485738.680898] Call Trace: [3485738.680901] [] ? cpuidle_enter_state+0x3f/0xe0 [3485738.680903] [] cpuidle_enter_state+0x4f/0xe0 [3485738.680905] [] cpuidle_enter+0x17/0x20 [3485738.680907] [] cpu_startup_entry+0x32b/0x490 [3485738.680908] [] ? trace_hardirqs_on+0xd/0x10 [3485738.680910] [] start_secondary+0x1d8/0x290 [3485738.680929] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485741.151472] sending NMI to all CPUs: [3485741.155274] NMI backtrace for cpu 3 [3485741.158991] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485741.168357] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485741.175106] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485741.182830] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485741.191649] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485741.197181] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000000000004 [3485741.204554] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485741.211924] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485741.219297] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485741.226667] R13: 0000000097020520 R14: 0000000000000002 R15: ffff880425546660 [3485741.234039] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485741.242366] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485741.248333] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485741.255703] Stack: [3485741.257940] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485741.265776] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485741.273623] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485741.281462] Call Trace: [3485741.284138] [] _raw_spin_lock+0x41/0x50 [3485741.289862] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485741.296364] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485741.302686] [] ceph_release+0x2e/0xc0 [ceph] [3485741.308832] [] __fput+0xc8/0x210 [3485741.313939] [] ____fput+0xe/0x10 [3485741.319043] [] task_work_run+0xb4/0xe0 [3485741.324664] [] do_exit+0x2e8/0xae0 [3485741.329938] [] ? kmsg_dump+0x10d/0x120 [3485741.336689] [] ? kmsg_dump+0x24/0x120 [3485741.342231] [] oops_end+0xa5/0xf0 [3485741.347419] [] die+0x58/0x90 [3485741.352173] [] do_trap+0xcd/0x160 [3485741.357363] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485741.364373] [] do_error_trap.part.16+0xa1/0xe0 [3485741.370701] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485741.378421] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485741.385518] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485741.392533] [] ? atomic_notifier_call_chain+0x16/0x20 [3485741.399454] [] do_error_trap+0x64/0x80 [3485741.405073] [] do_invalid_op+0x20/0x30 [3485741.410697] [] invalid_op+0x18/0x20 [3485741.416063] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485741.423789] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485741.431426] [] ? test_clear_page_writeback+0x143/0x260 [3485741.438434] [] ? test_clear_page_writeback+0x38/0x260 [3485741.445361] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485741.452198] [] ? SyS_msync+0x230/0x230 [3485741.457831] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485741.465379] [] ? wait_for_stable_page+0x16/0x50 [3485741.471787] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485741.478280] [] generic_perform_write+0xc6/0x1e0 [3485741.484693] [] ceph_write_iter+0x890/0xa30 [ceph] [3485741.491274] [] ? might_fault+0x66/0xc0 [3485741.496902] [] ? might_fault+0x66/0xc0 [3485741.502523] [] ? vfs_write+0x1d3/0x1f0 [3485741.508144] [] ? vfs_write+0x1d3/0x1f0 [3485741.513765] [] ? might_fault+0x66/0xc0 [3485741.519388] [] new_sync_write+0x7b/0xb0 [3485741.525098] [] vfs_write+0xc7/0x1f0 [3485741.530457] [] SyS_write+0x52/0xb0 [3485741.535732] [] system_call_fastpath+0x1a/0x1f [3485741.541961] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485741.564420] NMI backtrace for cpu 1 [3485741.568136] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485741.576985] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485741.583739] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485741.591455] RIP: 0010:[] [] do_raw_spin_lock+0x153/0x160 [3485741.600276] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485741.605808] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000057c957c8 [3485741.613179] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485741.620551] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485741.627921] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485741.635294] R13: 0000000097020520 R14: ffff880224754690 R15: ffff88033a8afeb0 [3485741.642666] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485741.650991] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485741.656959] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485741.664330] Stack: [3485741.666565] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485741.674401] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485741.682241] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485741.690079] Call Trace: [3485741.692756] [] _raw_spin_lock+0x41/0x50 [3485741.698475] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485741.705319] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485741.711985] [] ? __init_waitqueue_head+0x60/0x60 [3485741.718483] [] ceph_read_iter+0xc9/0x570 [ceph] [3485741.724889] [] new_sync_read+0x78/0xb0 [3485741.730511] [] vfs_read+0xab/0x180 [3485741.735784] [] SyS_read+0x52/0xb0 [3485741.740972] [] system_call_fastpath+0x1a/0x1f [3485741.747200] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 [3485741.769658] NMI backtrace for cpu 0 [3485741.773375] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485741.782917] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485741.789667] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485741.796346] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485741.804067] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485741.812880] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485741.818420] RAX: 0000000000008916 RBX: ffff8800049b5830 RCX: 0000000000000004 [3485741.825791] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485741.833163] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485741.840533] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485741.847905] R13: 0000000097020520 R14: 0000000000002000 R15: 000000000000001d [3485741.855278] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485741.863602] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485741.869570] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485741.876942] Stack: [3485741.879178] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485741.887012] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485741.894852] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485741.902691] Call Trace: [3485741.905366] [] _raw_spin_lock+0x41/0x50 [3485741.911087] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485741.917930] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485741.924599] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485741.931613] [] ? ext4_writepages+0x4ee/0xd30 [3485741.937766] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485741.944525] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485741.951275] [] ? writeback_sb_inodes+0x139/0x4f0 [3485741.957769] [] ? writeback_sb_inodes+0x139/0x4f0 [3485741.964257] [] do_writepages+0x23/0x40 [3485741.969879] [] __writeback_single_inode+0x45/0x2e0 [3485741.976541] [] writeback_sb_inodes+0x2ad/0x4f0 [3485741.982855] [] ? wb_writeback+0x54/0x350 [3485741.988650] [] __writeback_inodes_wb+0x9e/0xd0 [3485741.994965] [] wb_writeback+0x29b/0x350 [3485742.000672] [] ? get_nr_dirty_inodes+0x52/0x80 [3485742.006987] [] wb_check_old_data_flush+0x9f/0xb0 [3485742.013478] [] bdi_writeback_workfn+0x1cc/0x490 [3485742.019886] [] ? process_one_work+0x16f/0x570 [3485742.026114] [] process_one_work+0x1d1/0x570 [3485742.032168] [] ? process_one_work+0x16f/0x570 [3485742.038397] [] worker_thread+0x11c/0x530 [3485742.044191] [] ? create_and_start_worker+0x50/0x50 [3485742.050854] [] kthread+0xe4/0x100 [3485742.056050] [] ? flush_kthread_worker+0x130/0x130 [3485742.062625] [] ret_from_fork+0x7c/0xb0 [3485742.068246] [] ? flush_kthread_worker+0x130/0x130 [3485742.074821] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485742.097269] NMI backtrace for cpu 2 [3485742.100984] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485742.110354] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485742.117108] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485742.124826] RIP: 0010:[] [] delay_tsc+0x4f/0x80 [3485742.132856] RSP: 0018:ffff88043fc83c98 EFLAGS: 00000046 [3485742.138389] RAX: 000000001e474dd6 RBX: 0000000000002710 RCX: 0000000000000002 [3485742.145760] RDX: 00000000001f63d8 RSI: 000000001e474d8d RDI: 000000000026595c [3485742.153133] RBP: ffff88043fc83c98 R08: 0000000000000002 R09: 0000000000000000 [3485742.160504] R10: 0000000000000001 R11: 0000000000000000 R12: ffffffff81c54f80 [3485742.167876] R13: 0000000000000002 R14: ffffffff81c54f80 R15: ffffffff81d21e80 [3485742.175248] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485742.183573] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485742.189540] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485742.196909] Stack: [3485742.199147] ffff88043fc83ca8 ffffffff813a9d0c ffff88043fc83cc8 ffffffff81036812 [3485742.206990] 00000000008290e3 ffff88043fc8d7c0 ffff88043fc83d28 ffffffff810cd472 [3485742.214839] 0000000000000082 00000000008290e3 ffff88043fc83cf8 00000000810aae9d [3485742.222677] Call Trace: [3485742.225352] [3485742.227484] [] __const_udelay+0x2c/0x30 [3485742.234513] [] arch_trigger_all_cpu_backtrace+0x92/0x120 [3485742.241701] [] rcu_check_callbacks+0x452/0x6b0 [3485742.248021] [] update_process_times+0x48/0x80 [3485742.254258] [] tick_sched_handle.isra.15+0x33/0x70 [3485742.260926] [] tick_sched_timer+0x4c/0x80 [3485742.266808] [] __run_hrtimer+0x71/0x250 [3485742.272515] [] ? hrtimer_interrupt+0x75/0x260 [3485742.278743] [] ? tick_sched_do_timer+0x40/0x40 [3485742.285058] [] hrtimer_interrupt+0x107/0x260 [3485742.291202] [] ? handle_edge_irq+0x1e/0x110 [3485742.297265] [] hpet_interrupt_handler+0x16/0x40 [3485742.303667] [] handle_irq_event_percpu+0x5d/0x200 [3485742.310248] [] handle_irq_event+0x48/0x70 [3485742.316130] [] handle_edge_irq+0x77/0x110 [3485742.322010] [] handle_irq+0x22/0x40 [3485742.327374] [] do_IRQ+0x5e/0x110 [3485742.332472] [] common_interrupt+0x6f/0x6f [3485742.338354] [3485742.340490] [] ? retint_restore_args+0xe/0xe [3485742.346940] [] ? do_raw_spin_lock+0x15a/0x160 [3485742.353171] [] ? do_raw_spin_lock+0x138/0x160 [3485742.359406] [] _raw_spin_lock+0x41/0x50 [3485742.365119] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485742.371616] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485742.377940] [] ceph_release+0x2e/0xc0 [ceph] [3485742.384085] [] __fput+0xc8/0x210 [3485742.389183] [] ____fput+0xe/0x10 [3485742.394285] [] task_work_run+0xb4/0xe0 [3485742.399909] [] do_exit+0x2e8/0xae0 [3485742.405183] [] ? kmsg_dump+0x10d/0x120 [3485742.410804] [] ? kmsg_dump+0x24/0x120 [3485742.416341] [] oops_end+0xa5/0xf0 [3485742.421529] [] die+0x58/0x90 [3485742.426282] [] do_trap+0xcd/0x160 [3485742.431473] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485742.438485] [] do_error_trap.part.16+0xa1/0xe0 [3485742.444812] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485742.452532] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485742.459628] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485742.466646] [] ? atomic_notifier_call_chain+0x16/0x20 [3485742.473574] [] do_error_trap+0x64/0x80 [3485742.479200] [] do_invalid_op+0x20/0x30 [3485742.484823] [] invalid_op+0x18/0x20 [3485742.490191] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485742.497919] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485742.505553] [] ? test_clear_page_writeback+0x143/0x260 [3485742.512562] [] ? test_clear_page_writeback+0x38/0x260 [3485742.519487] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485742.526326] [] ? SyS_msync+0x230/0x230 [3485742.531958] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485742.539509] [] ? wait_for_stable_page+0x16/0x50 [3485742.545914] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485742.552408] [] generic_perform_write+0xc6/0x1e0 [3485742.558822] [] ceph_write_iter+0x890/0xa30 [ceph] [3485742.565399] [] ? _raw_spin_unlock+0x2b/0x40 [3485742.571454] [] ? do_wp_page+0x374/0x740 [3485742.577161] [] ? handle_mm_fault+0x86c/0x1060 [3485742.583390] [] ? __do_page_fault+0x159/0x580 [3485742.589532] [] ? vfs_write+0x1d3/0x1f0 [3485742.595151] [] ? vfs_write+0x1d3/0x1f0 [3485742.600773] [] new_sync_write+0x7b/0xb0 [3485742.606483] [] vfs_write+0xc7/0x1f0 [3485742.611845] [] SyS_write+0x52/0xb0 [3485742.617119] [] system_call_fastpath+0x1a/0x1f [3485742.623348] Code: eb 21 66 2e 0f 1f 84 00 00 00 00 00 f3 90 65 ff 04 25 a0 ba 00 00 65 8b 0c 25 84 b0 00 00 41 39 c8 75 1a 66 66 90 0f ae e8 0f 31 <89> c2 29 f2 65 ff 0c 25 a0 ba 00 00 39 fa 72 d1 5d c3 29 c6 01 [3485742.645807] NMI backtrace for cpu 4 [3485742.649523] CPU: 4 PID: 0 Comm: swapper/4 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485742.658456] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485742.665203] task: ffff88042966a1f0 ti: ffff880429670000 task.ti: ffff880429670000 [3485742.672919] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485742.681125] RSP: 0018:ffff880429673df8 EFLAGS: 00000046 [3485742.686657] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 [3485742.694029] RDX: 0000000000000000 RSI: ffff880429673fd8 RDI: ffffffff817295bf [3485742.701400] RBP: ffff880429673e28 R08: 0000000000000001 R09: 0000000000000000 [3485742.708772] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003 [3485742.716143] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 [3485742.723516] FS: 0000000000000000(0000) GS:ffff88043fd00000(0000) knlGS:0000000000000000 [3485742.731841] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485742.737808] CR2: 00007fecde790100 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485742.745179] Stack: [3485742.747417] ffffffff815d3fef 00000004ffffffff ffff88043fd1b020 000c63f4c80d5109 [3485742.755259] ffffffff81ca4320 ffffffff81ca4200 ffff880429673e88 ffffffff815d3fff [3485742.763098] 0000000000000000 0000000000f3f3ab 0000000000000000 0000000000f3f3ab [3485742.770939] Call Trace: [3485742.773615] [] ? cpuidle_enter_state+0x3f/0xe0 [3485742.779935] [] cpuidle_enter_state+0x4f/0xe0 [3485742.786083] [] cpuidle_enter+0x17/0x20 [3485742.791706] [] cpu_startup_entry+0x32b/0x490 [3485742.797846] [] ? trace_hardirqs_on+0xd/0x10 [3485742.803901] [] start_secondary+0x1d8/0x290 [3485742.809869] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485742.832328] NMI backtrace for cpu 5 [3485742.836042] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485742.844978] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485742.851726] task: ffff88042966c3e0 ti: ffff880429674000 task.ti: ffff880429674000 [3485742.859442] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485742.867655] RSP: 0018:ffff880429677df8 EFLAGS: 00000046 [3485742.873188] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 [3485742.880560] RDX: 0000000000000000 RSI: ffff880429677fd8 RDI: ffffffff817295bf [3485742.887931] RBP: ffff880429677e28 R08: 0000000000000001 R09: 0000000000000000 [3485742.895302] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003 [3485742.902674] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 [3485742.910047] FS: 0000000000000000(0000) GS:ffff88043fd40000(0000) knlGS:0000000000000000 [3485742.918372] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485742.924339] CR2: 00007f0b52ac8fe0 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485742.931710] Stack: [3485742.933947] ffffffff815d3fef 00000005ffffffff ffff88043fd5b020 000c63f4c80d81cb [3485742.941781] ffffffff81ca4320 ffffffff81ca4200 ffff880429677e88 ffffffff815d3fff [3485742.949621] 0000000000000000 0000000000335179 0000000000000000 0000000000335179 [3485742.957469] Call Trace: [3485742.960146] [] ? cpuidle_enter_state+0x3f/0xe0 [3485742.966465] [] cpuidle_enter_state+0x4f/0xe0 [3485742.972615] [] cpuidle_enter+0x17/0x20 [3485742.978246] [] cpu_startup_entry+0x32b/0x490 [3485742.984395] [] ? trace_hardirqs_on+0xd/0x10 [3485742.990451] [] start_secondary+0x1d8/0x290 [3485742.996426] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485743.018884] NMI backtrace for cpu 6 [3485743.022654] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485743.031601] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485743.038352] task: ffff880429680000 ti: ffff880429688000 task.ti: ffff880429688000 [3485743.046077] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485743.054289] RSP: 0018:ffff88042968bdf8 EFLAGS: 00000046 [3485743.059823] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 [3485743.067195] RDX: 0000000000000000 RSI: ffff88042968bfd8 RDI: ffffffff817295bf [3485743.074565] RBP: ffff88042968be28 R08: 0000000000000001 R09: 0000000000000000 [3485743.081937] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004 [3485743.089309] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 [3485743.096680] FS: 0000000000000000(0000) GS:ffff88043fd80000(0000) knlGS:0000000000000000 [3485743.105007] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485743.110980] CR2: 00007f8012d82980 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485743.118352] Stack: [3485743.120588] ffffffff815d3fef 00000006ffffffff ffff88043fd9b020 000c63f4c80d9ca1 [3485743.129505] ffffffff81ca4378 ffffffff81ca4200 ffff88042968be88 ffffffff815d3fff [3485743.137352] 0000000000000000 0000000000f3ac95 0000000000000000 0000000000f3ac95 [3485743.145202] Call Trace: [3485743.147881] [] ? cpuidle_enter_state+0x3f/0xe0 [3485743.154198] [] cpuidle_enter_state+0x4f/0xe0 [3485743.160339] [] cpuidle_enter+0x17/0x20 [3485743.165960] [] cpu_startup_entry+0x32b/0x490 [3485743.172101] [] ? trace_hardirqs_on+0xd/0x10 [3485743.178157] [] start_secondary+0x1d8/0x290 [3485743.184124] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485743.206582] NMI backtrace for cpu 7 [3485743.210300] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485743.219234] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485743.225981] task: ffff8804296821f0 ti: ffff88042968c000 task.ti: ffff88042968c000 [3485743.233698] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485743.241902] RSP: 0018:ffff88042968fdf8 EFLAGS: 00000046 [3485743.247435] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 [3485743.254806] RDX: 0000000000000000 RSI: ffff88042968ffd8 RDI: ffffffff817295bf [3485743.262177] RBP: ffff88042968fe28 R08: 0000000000000001 R09: 0000000000000000 [3485743.269547] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004 [3485743.276920] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 [3485743.284292] FS: 0000000000000000(0000) GS:ffff88043fdc0000(0000) knlGS:0000000000000000 [3485743.292625] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485743.298593] CR2: 00000000004238b0 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485743.305964] Stack: [3485743.308201] ffffffff815d3fef 00000007ffffffff ffff88043fddb020 000c63f4c80d7a9b [3485743.316035] ffffffff81ca4378 ffffffff81ca4200 ffff88042968fe88 ffffffff815d3fff [3485743.323885] 0000000000000000 0000000000b6c420 0000000000000000 0000000000b6c420 [3485743.331722] Call Trace: [3485743.334402] [] ? cpuidle_enter_state+0x3f/0xe0 [3485743.340721] [] cpuidle_enter_state+0x4f/0xe0 [3485743.346869] [] cpuidle_enter+0x17/0x20 [3485743.352490] [] cpu_startup_entry+0x32b/0x490 [3485743.358633] [] ? trace_hardirqs_on+0xd/0x10 [3485743.364687] [] start_secondary+0x1d8/0x290 [3485743.370654] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485766.732909] BUG: soft lockup - CPU#1 stuck for 23s! [grep:27686] [3485766.736905] BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u16:1:7152] [3485766.736936] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485766.736936] irq event stamp: 0 [3485766.736938] hardirqs last enabled at (0): [< (null)>] (null) [3485766.736943] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485766.736945] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485766.736946] softirqs last disabled at (0): [< (null)>] (null) [3485766.736948] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485766.736949] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485766.736954] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485766.736955] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485766.736959] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485766.736960] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485766.736961] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485766.736961] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485766.736962] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485766.736963] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485766.736963] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485766.736964] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485766.736965] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485766.736966] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485766.736967] Stack: [3485766.736969] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485766.736970] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485766.736972] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485766.736972] Call Trace: [3485766.736976] [] _raw_spin_lock+0x41/0x50 [3485766.736987] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485766.736993] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485766.736999] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485766.737001] [] ? ext4_writepages+0x4ee/0xd30 [3485766.737008] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485766.737014] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485766.737016] [] ? writeback_sb_inodes+0x139/0x4f0 [3485766.737017] [] ? writeback_sb_inodes+0x139/0x4f0 [3485766.737020] [] do_writepages+0x23/0x40 [3485766.737021] [] __writeback_single_inode+0x45/0x2e0 [3485766.737022] [] writeback_sb_inodes+0x2ad/0x4f0 [3485766.737024] [] ? wb_writeback+0x54/0x350 [3485766.737025] [] __writeback_inodes_wb+0x9e/0xd0 [3485766.737026] [] wb_writeback+0x29b/0x350 [3485766.737029] [] ? get_nr_dirty_inodes+0x52/0x80 [3485766.737030] [] wb_check_old_data_flush+0x9f/0xb0 [3485766.737032] [] bdi_writeback_workfn+0x1cc/0x490 [3485766.737034] [] ? process_one_work+0x16f/0x570 [3485766.737036] [] process_one_work+0x1d1/0x570 [3485766.737037] [] ? process_one_work+0x16f/0x570 [3485766.737039] [] worker_thread+0x11c/0x530 [3485766.737040] [] ? create_and_start_worker+0x50/0x50 [3485766.737043] [] kthread+0xe4/0x100 [3485766.737045] [] ? flush_kthread_worker+0x130/0x130 [3485766.737047] [] ret_from_fork+0x7c/0xb0 [3485766.737049] [] ? flush_kthread_worker+0x130/0x130 [3485766.737065] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485766.752895] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3485766.752915] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485766.752915] irq event stamp: 0 [3485766.752916] hardirqs last enabled at (0): [< (null)>] (null) [3485766.752919] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485766.752921] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485766.752922] softirqs last disabled at (0): [< (null)>] (null) [3485766.752924] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485766.752925] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485766.752925] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485766.752928] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485766.752929] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485766.752930] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485766.752931] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485766.752931] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485766.752932] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485766.752933] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485766.752934] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485766.752934] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485766.752935] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485766.752936] Stack: [3485766.752937] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485766.752939] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485766.752940] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485766.752941] Call Trace: [3485766.752943] [] _raw_spin_lock+0x41/0x50 [3485766.752951] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485766.752957] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485766.752962] [] ceph_release+0x2e/0xc0 [ceph] [3485766.752965] [] __fput+0xc8/0x210 [3485766.752966] [] ____fput+0xe/0x10 [3485766.752968] [] task_work_run+0xb4/0xe0 [3485766.752970] [] do_exit+0x2e8/0xae0 [3485766.752972] [] ? kmsg_dump+0x10d/0x120 [3485766.752974] [] ? kmsg_dump+0x24/0x120 [3485766.752976] [] oops_end+0xa5/0xf0 [3485766.752977] [] die+0x58/0x90 [3485766.752978] [] do_trap+0xcd/0x160 [3485766.752980] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485766.752982] [] do_error_trap.part.16+0xa1/0xe0 [3485766.752988] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485766.752989] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485766.752991] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485766.752992] [] ? atomic_notifier_call_chain+0x16/0x20 [3485766.752994] [] do_error_trap+0x64/0x80 [3485766.752995] [] do_invalid_op+0x20/0x30 [3485766.752997] [] invalid_op+0x18/0x20 [3485766.753003] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485766.753009] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485766.753011] [] ? test_clear_page_writeback+0x143/0x260 [3485766.753013] [] ? test_clear_page_writeback+0x38/0x260 [3485766.753018] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485766.753020] [] ? SyS_msync+0x230/0x230 [3485766.753026] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485766.753028] [] ? wait_for_stable_page+0x16/0x50 [3485766.753033] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485766.753035] [] generic_perform_write+0xc6/0x1e0 [3485766.753041] [] ceph_write_iter+0x890/0xa30 [ceph] [3485766.753042] [] ? _raw_spin_unlock+0x2b/0x40 [3485766.753044] [] ? do_wp_page+0x374/0x740 [3485766.753046] [] ? handle_mm_fault+0x86c/0x1060 [3485766.753048] [] ? __do_page_fault+0x159/0x580 [3485766.753049] [] ? vfs_write+0x1d3/0x1f0 [3485766.753051] [] ? vfs_write+0x1d3/0x1f0 [3485766.753052] [] new_sync_write+0x7b/0xb0 [3485766.753054] [] vfs_write+0xc7/0x1f0 [3485766.753056] [] SyS_write+0x52/0xb0 [3485766.753057] [] system_call_fastpath+0x1a/0x1f [3485766.753073] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485766.764890] BUG: soft lockup - CPU#3 stuck for 23s! [teuthology:20089] [3485766.764909] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485766.764909] irq event stamp: 0 [3485766.764911] hardirqs last enabled at (0): [< (null)>] (null) [3485766.764913] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485766.764915] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485766.764916] softirqs last disabled at (0): [< (null)>] (null) [3485766.764918] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485766.764918] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485766.764919] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485766.764922] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3485766.764923] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485766.764923] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485766.764924] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485766.764925] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485766.764925] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485766.764926] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485766.764927] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485766.764928] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485766.764929] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485766.764929] Stack: [3485766.764931] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485766.764932] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485766.764934] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485766.764934] Call Trace: [3485766.764936] [] _raw_spin_lock+0x41/0x50 [3485766.764943] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485766.764950] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485766.764955] [] ceph_release+0x2e/0xc0 [ceph] [3485766.764957] [] __fput+0xc8/0x210 [3485766.764959] [] ____fput+0xe/0x10 [3485766.764961] [] task_work_run+0xb4/0xe0 [3485766.764962] [] do_exit+0x2e8/0xae0 [3485766.764964] [] ? kmsg_dump+0x10d/0x120 [3485766.764965] [] ? kmsg_dump+0x24/0x120 [3485766.764967] [] oops_end+0xa5/0xf0 [3485766.764968] [] die+0x58/0x90 [3485766.764969] [] do_trap+0xcd/0x160 [3485766.764971] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485766.764973] [] do_error_trap.part.16+0xa1/0xe0 [3485766.764979] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485766.764980] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485766.764982] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485766.764983] [] ? atomic_notifier_call_chain+0x16/0x20 [3485766.764985] [] do_error_trap+0x64/0x80 [3485766.764986] [] do_invalid_op+0x20/0x30 [3485766.764988] [] invalid_op+0x18/0x20 [3485766.764994] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485766.765000] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485766.765002] [] ? test_clear_page_writeback+0x143/0x260 [3485766.765004] [] ? test_clear_page_writeback+0x38/0x260 [3485766.765009] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485766.765011] [] ? SyS_msync+0x230/0x230 [3485766.765016] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485766.765018] [] ? wait_for_stable_page+0x16/0x50 [3485766.765024] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485766.765026] [] generic_perform_write+0xc6/0x1e0 [3485766.765031] [] ceph_write_iter+0x890/0xa30 [ceph] [3485766.765033] [] ? might_fault+0x66/0xc0 [3485766.765034] [] ? might_fault+0x66/0xc0 [3485766.765036] [] ? vfs_write+0x1d3/0x1f0 [3485766.765037] [] ? vfs_write+0x1d3/0x1f0 [3485766.765039] [] ? might_fault+0x66/0xc0 [3485766.765040] [] new_sync_write+0x7b/0xb0 [3485766.765042] [] vfs_write+0xc7/0x1f0 [3485766.765043] [] SyS_write+0x52/0xb0 [3485766.765045] [] system_call_fastpath+0x1a/0x1f [3485766.765061] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3485768.174611] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485768.237664] irq event stamp: 0 [3485768.240943] hardirqs last enabled at (0): [< (null)>] (null) [3485768.248629] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485768.257094] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485768.265557] softirqs last disabled at (0): [< (null)>] (null) [3485768.273242] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485768.282087] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485768.288840] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485768.297687] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485768.306504] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485768.312037] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485768.319406] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485768.326779] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485768.334150] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485768.341522] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485768.348894] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485768.357219] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485768.363187] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485768.370556] Stack: [3485768.372794] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485768.380630] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485768.388467] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485768.396299] Call Trace: [3485768.398976] [] _raw_spin_lock+0x41/0x50 [3485768.404700] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485768.411546] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485768.418211] [] ? __init_waitqueue_head+0x60/0x60 [3485768.424703] [] ceph_read_iter+0xc9/0x570 [ceph] [3485768.431110] [] new_sync_read+0x78/0xb0 [3485768.436730] [] vfs_read+0xab/0x180 [3485768.442008] [] SyS_read+0x52/0xb0 [3485768.447203] [] system_call_fastpath+0x1a/0x1f [3485768.453430] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485794.717960] BUG: soft lockup - CPU#1 stuck for 23s! [grep:27686] [3485794.721956] BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u16:1:7152] [3485794.721988] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485794.721989] irq event stamp: 0 [3485794.721990] hardirqs last enabled at (0): [< (null)>] (null) [3485794.721996] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485794.721997] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485794.721998] softirqs last disabled at (0): [< (null)>] (null) [3485794.722001] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485794.722001] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485794.722006] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485794.722007] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485794.722012] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485794.722013] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485794.722014] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485794.722014] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485794.722015] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485794.722016] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485794.722016] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485794.722017] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485794.722018] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485794.722019] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485794.722019] Stack: [3485794.722021] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485794.722023] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485794.722024] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485794.722025] Call Trace: [3485794.722029] [] _raw_spin_lock+0x41/0x50 [3485794.722040] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485794.722046] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485794.722052] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485794.722055] [] ? ext4_writepages+0x4ee/0xd30 [3485794.722061] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485794.722067] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485794.722069] [] ? writeback_sb_inodes+0x139/0x4f0 [3485794.722071] [] ? writeback_sb_inodes+0x139/0x4f0 [3485794.722073] [] do_writepages+0x23/0x40 [3485794.722074] [] __writeback_single_inode+0x45/0x2e0 [3485794.722076] [] writeback_sb_inodes+0x2ad/0x4f0 [3485794.722077] [] ? wb_writeback+0x54/0x350 [3485794.722079] [] __writeback_inodes_wb+0x9e/0xd0 [3485794.722080] [] wb_writeback+0x29b/0x350 [3485794.722082] [] ? get_nr_dirty_inodes+0x52/0x80 [3485794.722084] [] wb_check_old_data_flush+0x9f/0xb0 [3485794.722085] [] bdi_writeback_workfn+0x1cc/0x490 [3485794.722087] [] ? process_one_work+0x16f/0x570 [3485794.722089] [] process_one_work+0x1d1/0x570 [3485794.722090] [] ? process_one_work+0x16f/0x570 [3485794.722092] [] worker_thread+0x11c/0x530 [3485794.722093] [] ? create_and_start_worker+0x50/0x50 [3485794.722095] [] kthread+0xe4/0x100 [3485794.722097] [] ? flush_kthread_worker+0x130/0x130 [3485794.722099] [] ret_from_fork+0x7c/0xb0 [3485794.722101] [] ? flush_kthread_worker+0x130/0x130 [3485794.722117] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485794.737947] BUG: soft lockup - CPU#2 stuck for 23s! [teuthology:19790] [3485794.737967] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485794.737968] irq event stamp: 0 [3485794.737969] hardirqs last enabled at (0): [< (null)>] (null) [3485794.737972] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485794.737974] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485794.737975] softirqs last disabled at (0): [< (null)>] (null) [3485794.737976] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485794.737977] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485794.737978] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485794.737981] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485794.737982] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485794.737982] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485794.737983] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485794.737984] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485794.737985] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485794.737985] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485794.737986] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485794.737987] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485794.737988] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485794.737988] Stack: [3485794.737990] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485794.737992] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485794.737993] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485794.737994] Call Trace: [3485794.737996] [] _raw_spin_lock+0x41/0x50 [3485794.738003] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485794.738010] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485794.738015] [] ceph_release+0x2e/0xc0 [ceph] [3485794.738018] [] __fput+0xc8/0x210 [3485794.738020] [] ____fput+0xe/0x10 [3485794.738022] [] task_work_run+0xb4/0xe0 [3485794.738024] [] do_exit+0x2e8/0xae0 [3485794.738026] [] ? kmsg_dump+0x10d/0x120 [3485794.738027] [] ? kmsg_dump+0x24/0x120 [3485794.738029] [] oops_end+0xa5/0xf0 [3485794.738030] [] die+0x58/0x90 [3485794.738032] [] do_trap+0xcd/0x160 [3485794.738034] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485794.738035] [] do_error_trap.part.16+0xa1/0xe0 [3485794.738041] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485794.738043] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485794.738044] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485794.738046] [] ? atomic_notifier_call_chain+0x16/0x20 [3485794.738047] [] do_error_trap+0x64/0x80 [3485794.738049] [] do_invalid_op+0x20/0x30 [3485794.738050] [] invalid_op+0x18/0x20 [3485794.738057] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485794.738063] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485794.738065] [] ? test_clear_page_writeback+0x143/0x260 [3485794.738066] [] ? test_clear_page_writeback+0x38/0x260 [3485794.738072] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485794.738074] [] ? SyS_msync+0x230/0x230 [3485794.738079] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485794.738081] [] ? wait_for_stable_page+0x16/0x50 [3485794.738087] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485794.738089] [] generic_perform_write+0xc6/0x1e0 [3485794.738095] [] ceph_write_iter+0x890/0xa30 [ceph] [3485794.738096] [] ? _raw_spin_unlock+0x2b/0x40 [3485794.738098] [] ? do_wp_page+0x374/0x740 [3485794.738100] [] ? handle_mm_fault+0x86c/0x1060 [3485794.738101] [] ? __do_page_fault+0x159/0x580 [3485794.738103] [] ? vfs_write+0x1d3/0x1f0 [3485794.738104] [] ? vfs_write+0x1d3/0x1f0 [3485794.738106] [] new_sync_write+0x7b/0xb0 [3485794.738108] [] vfs_write+0xc7/0x1f0 [3485794.738109] [] SyS_write+0x52/0xb0 [3485794.738111] [] system_call_fastpath+0x1a/0x1f [3485794.738127] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485794.749940] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485794.749959] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485794.749960] irq event stamp: 0 [3485794.749961] hardirqs last enabled at (0): [< (null)>] (null) [3485794.749964] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485794.749966] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485794.749966] softirqs last disabled at (0): [< (null)>] (null) [3485794.749968] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485794.749969] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485794.749970] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485794.749972] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3485794.749973] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485794.749974] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485794.749974] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485794.749975] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485794.749976] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485794.749976] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485794.749977] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485794.749978] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485794.749979] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485794.749979] Stack: [3485794.749981] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485794.749983] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485794.749984] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485794.749984] Call Trace: [3485794.749987] [] _raw_spin_lock+0x41/0x50 [3485794.749994] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485794.750000] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485794.750006] [] ceph_release+0x2e/0xc0 [ceph] [3485794.750007] [] __fput+0xc8/0x210 [3485794.750009] [] ____fput+0xe/0x10 [3485794.750011] [] task_work_run+0xb4/0xe0 [3485794.750013] [] do_exit+0x2e8/0xae0 [3485794.750014] [] ? kmsg_dump+0x10d/0x120 [3485794.750016] [] ? kmsg_dump+0x24/0x120 [3485794.750017] [] oops_end+0xa5/0xf0 [3485794.750019] [] die+0x58/0x90 [3485794.750020] [] do_trap+0xcd/0x160 [3485794.750022] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485794.750023] [] do_error_trap.part.16+0xa1/0xe0 [3485794.750029] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485794.750031] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485794.750032] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485794.750034] [] ? atomic_notifier_call_chain+0x16/0x20 [3485794.750035] [] do_error_trap+0x64/0x80 [3485794.750036] [] do_invalid_op+0x20/0x30 [3485794.750038] [] invalid_op+0x18/0x20 [3485794.750044] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485794.750051] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485794.750052] [] ? test_clear_page_writeback+0x143/0x260 [3485794.750054] [] ? test_clear_page_writeback+0x38/0x260 [3485794.750060] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485794.750061] [] ? SyS_msync+0x230/0x230 [3485794.750067] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485794.750068] [] ? wait_for_stable_page+0x16/0x50 [3485794.750074] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485794.750076] [] generic_perform_write+0xc6/0x1e0 [3485794.750082] [] ceph_write_iter+0x890/0xa30 [ceph] [3485794.750083] [] ? might_fault+0x66/0xc0 [3485794.750085] [] ? might_fault+0x66/0xc0 [3485794.750086] [] ? vfs_write+0x1d3/0x1f0 [3485794.750088] [] ? vfs_write+0x1d3/0x1f0 [3485794.750089] [] ? might_fault+0x66/0xc0 [3485794.750091] [] new_sync_write+0x7b/0xb0 [3485794.750093] [] vfs_write+0xc7/0x1f0 [3485794.750094] [] SyS_write+0x52/0xb0 [3485794.750096] [] system_call_fastpath+0x1a/0x1f [3485794.750112] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3485796.159562] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485796.222567] irq event stamp: 0 [3485796.225847] hardirqs last enabled at (0): [< (null)>] (null) [3485796.233552] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485796.242035] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485796.250509] softirqs last disabled at (0): [< (null)>] (null) [3485796.258213] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485796.267067] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485796.273831] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485796.281557] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485796.290382] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485796.295922] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485796.303303] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485796.310685] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485796.318065] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485796.325446] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485796.332826] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485796.341159] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485796.347133] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485796.355615] Stack: [3485796.357858] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485796.365698] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485796.373547] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485796.381386] Call Trace: [3485796.384064] [] _raw_spin_lock+0x41/0x50 [3485796.389785] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485796.396623] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485796.403290] [] ? __init_waitqueue_head+0x60/0x60 [3485796.409790] [] ceph_read_iter+0xc9/0x570 [ceph] [3485796.416194] [] new_sync_read+0x78/0xb0 [3485796.421814] [] vfs_read+0xab/0x180 [3485796.427088] [] SyS_read+0x52/0xb0 [3485796.432278] [] system_call_fastpath+0x1a/0x1f [3485796.438508] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485822.703011] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3485822.707007] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3485822.707035] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485822.707036] irq event stamp: 0 [3485822.707037] hardirqs last enabled at (0): [< (null)>] (null) [3485822.707042] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485822.707044] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485822.707045] softirqs last disabled at (0): [< (null)>] (null) [3485822.707047] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485822.707048] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485822.707053] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485822.707054] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485822.707058] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485822.707059] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485822.707060] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485822.707060] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485822.707061] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485822.707062] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485822.707062] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485822.707063] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485822.707064] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485822.707065] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485822.707066] Stack: [3485822.707068] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485822.707069] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485822.707071] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485822.707071] Call Trace: [3485822.707075] [] _raw_spin_lock+0x41/0x50 [3485822.707085] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485822.707092] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485822.707097] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485822.707099] [] ? ext4_writepages+0x4ee/0xd30 [3485822.707106] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485822.707112] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485822.707114] [] ? writeback_sb_inodes+0x139/0x4f0 [3485822.707115] [] ? writeback_sb_inodes+0x139/0x4f0 [3485822.707118] [] do_writepages+0x23/0x40 [3485822.707119] [] __writeback_single_inode+0x45/0x2e0 [3485822.707120] [] writeback_sb_inodes+0x2ad/0x4f0 [3485822.707122] [] ? wb_writeback+0x54/0x350 [3485822.707123] [] __writeback_inodes_wb+0x9e/0xd0 [3485822.707125] [] wb_writeback+0x29b/0x350 [3485822.707127] [] ? get_nr_dirty_inodes+0x52/0x80 [3485822.707128] [] wb_check_old_data_flush+0x9f/0xb0 [3485822.707130] [] bdi_writeback_workfn+0x1cc/0x490 [3485822.707132] [] ? process_one_work+0x16f/0x570 [3485822.707133] [] process_one_work+0x1d1/0x570 [3485822.707135] [] ? process_one_work+0x16f/0x570 [3485822.707136] [] worker_thread+0x11c/0x530 [3485822.707138] [] ? create_and_start_worker+0x50/0x50 [3485822.707140] [] kthread+0xe4/0x100 [3485822.707142] [] ? flush_kthread_worker+0x130/0x130 [3485822.707144] [] ret_from_fork+0x7c/0xb0 [3485822.707145] [] ? flush_kthread_worker+0x130/0x130 [3485822.707162] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485822.723001] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3485822.723028] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485822.723029] irq event stamp: 0 [3485822.723030] hardirqs last enabled at (0): [< (null)>] (null) [3485822.723033] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485822.723035] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485822.723036] softirqs last disabled at (0): [< (null)>] (null) [3485822.723038] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485822.723038] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485822.723039] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485822.723042] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485822.723043] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485822.723043] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485822.723044] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485822.723045] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485822.723045] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485822.723046] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485822.723047] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485822.723048] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485822.723049] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485822.723049] Stack: [3485822.723051] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485822.723052] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485822.723054] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485822.723054] Call Trace: [3485822.723057] [] _raw_spin_lock+0x41/0x50 [3485822.723064] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485822.723071] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485822.723076] [] ceph_release+0x2e/0xc0 [ceph] [3485822.723078] [] __fput+0xc8/0x210 [3485822.723080] [] ____fput+0xe/0x10 [3485822.723082] [] task_work_run+0xb4/0xe0 [3485822.723084] [] do_exit+0x2e8/0xae0 [3485822.723086] [] ? kmsg_dump+0x10d/0x120 [3485822.723087] [] ? kmsg_dump+0x24/0x120 [3485822.723089] [] oops_end+0xa5/0xf0 [3485822.723091] [] die+0x58/0x90 [3485822.723092] [] do_trap+0xcd/0x160 [3485822.723094] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485822.723095] [] do_error_trap.part.16+0xa1/0xe0 [3485822.723102] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485822.723103] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485822.723105] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485822.723106] [] ? atomic_notifier_call_chain+0x16/0x20 [3485822.723108] [] do_error_trap+0x64/0x80 [3485822.723109] [] do_invalid_op+0x20/0x30 [3485822.723111] [] invalid_op+0x18/0x20 [3485822.723117] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485822.723123] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485822.723126] [] ? test_clear_page_writeback+0x143/0x260 [3485822.723127] [] ? test_clear_page_writeback+0x38/0x260 [3485822.723133] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485822.723135] [] ? SyS_msync+0x230/0x230 [3485822.723140] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485822.723142] [] ? wait_for_stable_page+0x16/0x50 [3485822.723148] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485822.723151] [] generic_perform_write+0xc6/0x1e0 [3485822.723156] [] ceph_write_iter+0x890/0xa30 [ceph] [3485822.723158] [] ? _raw_spin_unlock+0x2b/0x40 [3485822.723160] [] ? do_wp_page+0x374/0x740 [3485822.723162] [] ? handle_mm_fault+0x86c/0x1060 [3485822.723164] [] ? __do_page_fault+0x159/0x580 [3485822.723165] [] ? vfs_write+0x1d3/0x1f0 [3485822.723167] [] ? vfs_write+0x1d3/0x1f0 [3485822.723168] [] new_sync_write+0x7b/0xb0 [3485822.723170] [] vfs_write+0xc7/0x1f0 [3485822.723171] [] SyS_write+0x52/0xb0 [3485822.723173] [] system_call_fastpath+0x1a/0x1f [3485822.723189] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485822.734992] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485822.735011] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485822.735011] irq event stamp: 0 [3485822.735012] hardirqs last enabled at (0): [< (null)>] (null) [3485822.735015] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485822.735017] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485822.735018] softirqs last disabled at (0): [< (null)>] (null) [3485822.735020] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485822.735020] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485822.735021] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485822.735024] RIP: 0010:[] [] do_raw_spin_lock+0x153/0x160 [3485822.735025] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485822.735025] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485822.735026] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485822.735027] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485822.735027] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485822.735028] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485822.735029] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485822.735030] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485822.735031] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485822.735031] Stack: [3485822.735033] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485822.735034] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485822.735036] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485822.735036] Call Trace: [3485822.735038] [] _raw_spin_lock+0x41/0x50 [3485822.735045] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485822.735052] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485822.735057] [] ceph_release+0x2e/0xc0 [ceph] [3485822.735059] [] __fput+0xc8/0x210 [3485822.735061] [] ____fput+0xe/0x10 [3485822.735063] [] task_work_run+0xb4/0xe0 [3485822.735065] [] do_exit+0x2e8/0xae0 [3485822.735067] [] ? kmsg_dump+0x10d/0x120 [3485822.735068] [] ? kmsg_dump+0x24/0x120 [3485822.735070] [] oops_end+0xa5/0xf0 [3485822.735071] [] die+0x58/0x90 [3485822.735072] [] do_trap+0xcd/0x160 [3485822.735074] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485822.735075] [] do_error_trap.part.16+0xa1/0xe0 [3485822.735082] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485822.735083] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485822.735085] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485822.735086] [] ? atomic_notifier_call_chain+0x16/0x20 [3485822.735087] [] do_error_trap+0x64/0x80 [3485822.735089] [] do_invalid_op+0x20/0x30 [3485822.735090] [] invalid_op+0x18/0x20 [3485822.735097] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485822.735103] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485822.735105] [] ? test_clear_page_writeback+0x143/0x260 [3485822.735106] [] ? test_clear_page_writeback+0x38/0x260 [3485822.735112] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485822.735113] [] ? SyS_msync+0x230/0x230 [3485822.735119] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485822.735121] [] ? wait_for_stable_page+0x16/0x50 [3485822.735126] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485822.735128] [] generic_perform_write+0xc6/0x1e0 [3485822.735134] [] ceph_write_iter+0x890/0xa30 [ceph] [3485822.735136] [] ? might_fault+0x66/0xc0 [3485822.735137] [] ? might_fault+0x66/0xc0 [3485822.735139] [] ? vfs_write+0x1d3/0x1f0 [3485822.735140] [] ? vfs_write+0x1d3/0x1f0 [3485822.735141] [] ? might_fault+0x66/0xc0 [3485822.735143] [] new_sync_write+0x7b/0xb0 [3485822.735145] [] vfs_write+0xc7/0x1f0 [3485822.735146] [] SyS_write+0x52/0xb0 [3485822.735148] [] system_call_fastpath+0x1a/0x1f [3485822.735164] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 [3485824.144794] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485824.207894] irq event stamp: 0 [3485824.211174] hardirqs last enabled at (0): [< (null)>] (null) [3485824.218859] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485824.227335] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485824.235796] softirqs last disabled at (0): [< (null)>] (null) [3485824.243483] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485824.252325] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485824.259074] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485824.266789] RIP: 0010:[] [] do_raw_spin_lock+0x153/0x160 [3485824.275601] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485824.281133] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485824.288505] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485824.295875] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485824.303246] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485824.310619] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485824.317990] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485824.326318] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485824.332284] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485824.339655] Stack: [3485824.341891] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485824.349728] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485824.357567] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485824.365405] Call Trace: [3485824.368082] [] _raw_spin_lock+0x41/0x50 [3485824.373812] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485824.380652] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485824.387320] [] ? __init_waitqueue_head+0x60/0x60 [3485824.393816] [] ceph_read_iter+0xc9/0x570 [ceph] [3485824.400231] [] new_sync_read+0x78/0xb0 [3485824.405853] [] vfs_read+0xab/0x180 [3485824.411132] [] SyS_read+0x52/0xb0 [3485824.416326] [] system_call_fastpath+0x1a/0x1f [3485824.422554] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 [3485850.688062] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3485850.692058] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3485850.692087] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485850.692087] irq event stamp: 0 [3485850.692089] hardirqs last enabled at (0): [< (null)>] (null) [3485850.692094] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485850.692096] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485850.692097] softirqs last disabled at (0): [< (null)>] (null) [3485850.692099] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485850.692100] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485850.692104] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485850.692105] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485850.692110] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485850.692111] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485850.692112] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485850.692112] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485850.692113] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485850.692114] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485850.692114] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485850.692115] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485850.692116] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485850.692117] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485850.692118] Stack: [3485850.692119] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485850.692121] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485850.692122] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485850.692123] Call Trace: [3485850.692127] [] _raw_spin_lock+0x41/0x50 [3485850.692138] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485850.692144] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485850.692150] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485850.692152] [] ? ext4_writepages+0x4ee/0xd30 [3485850.692159] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485850.692165] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485850.692167] [] ? writeback_sb_inodes+0x139/0x4f0 [3485850.692168] [] ? writeback_sb_inodes+0x139/0x4f0 [3485850.692170] [] do_writepages+0x23/0x40 [3485850.692172] [] __writeback_single_inode+0x45/0x2e0 [3485850.692173] [] writeback_sb_inodes+0x2ad/0x4f0 [3485850.692174] [] ? wb_writeback+0x54/0x350 [3485850.692176] [] __writeback_inodes_wb+0x9e/0xd0 [3485850.692177] [] wb_writeback+0x29b/0x350 [3485850.692179] [] ? get_nr_dirty_inodes+0x52/0x80 [3485850.692181] [] wb_check_old_data_flush+0x9f/0xb0 [3485850.692182] [] bdi_writeback_workfn+0x1cc/0x490 [3485850.692184] [] ? process_one_work+0x16f/0x570 [3485850.692186] [] process_one_work+0x1d1/0x570 [3485850.692187] [] ? process_one_work+0x16f/0x570 [3485850.692189] [] worker_thread+0x11c/0x530 [3485850.692191] [] ? create_and_start_worker+0x50/0x50 [3485850.692192] [] kthread+0xe4/0x100 [3485850.692195] [] ? flush_kthread_worker+0x130/0x130 [3485850.692196] [] ret_from_fork+0x7c/0xb0 [3485850.692198] [] ? flush_kthread_worker+0x130/0x130 [3485850.692215] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485850.708049] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3485850.708077] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485850.708078] irq event stamp: 0 [3485850.708079] hardirqs last enabled at (0): [< (null)>] (null) [3485850.708082] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485850.708084] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485850.708085] softirqs last disabled at (0): [< (null)>] (null) [3485850.708086] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485850.708087] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485850.708088] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485850.708091] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3485850.708092] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485850.708092] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485850.708093] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485850.708094] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485850.708094] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485850.708095] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485850.708096] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485850.708097] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485850.708098] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485850.708098] Stack: [3485850.708100] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485850.708101] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485850.708103] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485850.708103] Call Trace: [3485850.708106] [] _raw_spin_lock+0x41/0x50 [3485850.708113] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485850.708120] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485850.708125] [] ceph_release+0x2e/0xc0 [ceph] [3485850.708128] [] __fput+0xc8/0x210 [3485850.708129] [] ____fput+0xe/0x10 [3485850.708131] [] task_work_run+0xb4/0xe0 [3485850.708133] [] do_exit+0x2e8/0xae0 [3485850.708135] [] ? kmsg_dump+0x10d/0x120 [3485850.708137] [] ? kmsg_dump+0x24/0x120 [3485850.708139] [] oops_end+0xa5/0xf0 [3485850.708140] [] die+0x58/0x90 [3485850.708141] [] do_trap+0xcd/0x160 [3485850.708143] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485850.708145] [] do_error_trap.part.16+0xa1/0xe0 [3485850.708151] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485850.708153] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485850.708154] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485850.708156] [] ? atomic_notifier_call_chain+0x16/0x20 [3485850.708157] [] do_error_trap+0x64/0x80 [3485850.708158] [] do_invalid_op+0x20/0x30 [3485850.708160] [] invalid_op+0x18/0x20 [3485850.708167] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485850.708173] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485850.708175] [] ? test_clear_page_writeback+0x143/0x260 [3485850.708176] [] ? test_clear_page_writeback+0x38/0x260 [3485850.708182] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485850.708184] [] ? SyS_msync+0x230/0x230 [3485850.708189] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485850.708191] [] ? wait_for_stable_page+0x16/0x50 [3485850.708197] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485850.708199] [] generic_perform_write+0xc6/0x1e0 [3485850.708204] [] ceph_write_iter+0x890/0xa30 [ceph] [3485850.708206] [] ? _raw_spin_unlock+0x2b/0x40 [3485850.708208] [] ? do_wp_page+0x374/0x740 [3485850.708210] [] ? handle_mm_fault+0x86c/0x1060 [3485850.708211] [] ? __do_page_fault+0x159/0x580 [3485850.708213] [] ? vfs_write+0x1d3/0x1f0 [3485850.708214] [] ? vfs_write+0x1d3/0x1f0 [3485850.708216] [] new_sync_write+0x7b/0xb0 [3485850.708217] [] vfs_write+0xc7/0x1f0 [3485850.708219] [] SyS_write+0x52/0xb0 [3485850.708220] [] system_call_fastpath+0x1a/0x1f [3485850.708237] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3485850.720047] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485850.720066] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485850.720067] irq event stamp: 0 [3485850.720068] hardirqs last enabled at (0): [< (null)>] (null) [3485850.720070] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485850.720072] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485850.720073] softirqs last disabled at (0): [< (null)>] (null) [3485850.720075] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485850.720076] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485850.720076] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485850.720079] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3485850.720080] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485850.720081] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485850.720081] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485850.720082] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485850.720083] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485850.720083] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485850.720084] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485850.720085] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485850.720086] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485850.720086] Stack: [3485850.720088] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485850.720090] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485850.720091] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485850.720092] Call Trace: [3485850.720094] [] _raw_spin_lock+0x41/0x50 [3485850.720101] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485850.720107] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485850.720113] [] ceph_release+0x2e/0xc0 [ceph] [3485850.720114] [] __fput+0xc8/0x210 [3485850.720116] [] ____fput+0xe/0x10 [3485850.720118] [] task_work_run+0xb4/0xe0 [3485850.720120] [] do_exit+0x2e8/0xae0 [3485850.720121] [] ? kmsg_dump+0x10d/0x120 [3485850.720123] [] ? kmsg_dump+0x24/0x120 [3485850.720124] [] oops_end+0xa5/0xf0 [3485850.720126] [] die+0x58/0x90 [3485850.720127] [] do_trap+0xcd/0x160 [3485850.720129] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485850.720130] [] do_error_trap.part.16+0xa1/0xe0 [3485850.720136] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485850.720138] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485850.720139] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485850.720141] [] ? atomic_notifier_call_chain+0x16/0x20 [3485850.720142] [] do_error_trap+0x64/0x80 [3485850.720143] [] do_invalid_op+0x20/0x30 [3485850.720145] [] invalid_op+0x18/0x20 [3485850.720151] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485850.720158] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485850.720159] [] ? test_clear_page_writeback+0x143/0x260 [3485850.720161] [] ? test_clear_page_writeback+0x38/0x260 [3485850.720167] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485850.720168] [] ? SyS_msync+0x230/0x230 [3485850.720174] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485850.720175] [] ? wait_for_stable_page+0x16/0x50 [3485850.720181] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485850.720183] [] generic_perform_write+0xc6/0x1e0 [3485850.720188] [] ceph_write_iter+0x890/0xa30 [ceph] [3485850.720190] [] ? might_fault+0x66/0xc0 [3485850.720192] [] ? might_fault+0x66/0xc0 [3485850.720193] [] ? vfs_write+0x1d3/0x1f0 [3485850.720195] [] ? vfs_write+0x1d3/0x1f0 [3485850.720196] [] ? might_fault+0x66/0xc0 [3485850.720198] [] new_sync_write+0x7b/0xb0 [3485850.720199] [] vfs_write+0xc7/0x1f0 [3485850.720201] [] SyS_write+0x52/0xb0 [3485850.720202] [] system_call_fastpath+0x1a/0x1f [3485850.720218] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3485852.129924] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485852.192972] irq event stamp: 0 [3485852.196250] hardirqs last enabled at (0): [< (null)>] (null) [3485852.203932] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485852.212395] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485852.220858] softirqs last disabled at (0): [< (null)>] (null) [3485852.228543] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485852.237385] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485852.244132] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485852.251849] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3485852.260665] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485852.266204] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485852.273574] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485852.280945] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485852.288315] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485852.295690] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485852.303059] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485852.311423] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485852.317397] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485852.324786] Stack: [3485852.327021] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485852.334857] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485852.342703] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485852.350542] Call Trace: [3485852.353221] [] _raw_spin_lock+0x41/0x50 [3485852.358941] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485852.365780] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485852.372452] [] ? __init_waitqueue_head+0x60/0x60 [3485852.378955] [] ceph_read_iter+0xc9/0x570 [ceph] [3485852.385365] [] new_sync_read+0x78/0xb0 [3485852.390990] [] vfs_read+0xab/0x180 [3485852.396267] [] SyS_read+0x52/0xb0 [3485852.401454] [] system_call_fastpath+0x1a/0x1f [3485852.407683] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3485878.673115] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3485878.677110] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3485878.677140] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485878.677141] irq event stamp: 0 [3485878.677142] hardirqs last enabled at (0): [< (null)>] (null) [3485878.677147] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485878.677149] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485878.677150] softirqs last disabled at (0): [< (null)>] (null) [3485878.677153] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485878.677153] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485878.677158] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485878.677159] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485878.677163] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485878.677164] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485878.677165] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485878.677166] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485878.677166] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485878.677167] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485878.677168] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485878.677169] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485878.677170] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485878.677170] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485878.677171] Stack: [3485878.677173] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485878.677175] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485878.677176] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485878.677177] Call Trace: [3485878.677181] [] _raw_spin_lock+0x41/0x50 [3485878.677193] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485878.677199] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485878.677205] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485878.677207] [] ? ext4_writepages+0x4ee/0xd30 [3485878.677213] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485878.677220] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485878.677222] [] ? writeback_sb_inodes+0x139/0x4f0 [3485878.677223] [] ? writeback_sb_inodes+0x139/0x4f0 [3485878.677226] [] do_writepages+0x23/0x40 [3485878.677227] [] __writeback_single_inode+0x45/0x2e0 [3485878.677228] [] writeback_sb_inodes+0x2ad/0x4f0 [3485878.677230] [] ? wb_writeback+0x54/0x350 [3485878.677231] [] __writeback_inodes_wb+0x9e/0xd0 [3485878.677233] [] wb_writeback+0x29b/0x350 [3485878.677235] [] ? get_nr_dirty_inodes+0x52/0x80 [3485878.677236] [] wb_check_old_data_flush+0x9f/0xb0 [3485878.677238] [] bdi_writeback_workfn+0x1cc/0x490 [3485878.677240] [] ? process_one_work+0x16f/0x570 [3485878.677241] [] process_one_work+0x1d1/0x570 [3485878.677243] [] ? process_one_work+0x16f/0x570 [3485878.677244] [] worker_thread+0x11c/0x530 [3485878.677246] [] ? create_and_start_worker+0x50/0x50 [3485878.677248] [] kthread+0xe4/0x100 [3485878.677250] [] ? flush_kthread_worker+0x130/0x130 [3485878.677252] [] ret_from_fork+0x7c/0xb0 [3485878.677253] [] ? flush_kthread_worker+0x130/0x130 [3485878.677270] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485878.693100] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3485878.693125] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485878.693126] irq event stamp: 0 [3485878.693127] hardirqs last enabled at (0): [< (null)>] (null) [3485878.693130] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485878.693132] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485878.693133] softirqs last disabled at (0): [< (null)>] (null) [3485878.693135] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485878.693136] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485878.693136] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485878.693139] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3485878.693140] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485878.693141] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485878.693142] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485878.693142] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485878.693143] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485878.693144] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485878.693145] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485878.693146] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485878.693146] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485878.693147] Stack: [3485878.693149] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485878.693150] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485878.693151] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485878.693152] Call Trace: [3485878.693154] [] _raw_spin_lock+0x41/0x50 [3485878.693162] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485878.693168] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485878.693173] [] ceph_release+0x2e/0xc0 [ceph] [3485878.693176] [] __fput+0xc8/0x210 [3485878.693178] [] ____fput+0xe/0x10 [3485878.693180] [] task_work_run+0xb4/0xe0 [3485878.693182] [] do_exit+0x2e8/0xae0 [3485878.693184] [] ? kmsg_dump+0x10d/0x120 [3485878.693185] [] ? kmsg_dump+0x24/0x120 [3485878.693188] [] oops_end+0xa5/0xf0 [3485878.693189] [] die+0x58/0x90 [3485878.693190] [] do_trap+0xcd/0x160 [3485878.693192] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485878.693193] [] do_error_trap.part.16+0xa1/0xe0 [3485878.693200] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485878.693201] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485878.693203] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485878.693204] [] ? atomic_notifier_call_chain+0x16/0x20 [3485878.693206] [] do_error_trap+0x64/0x80 [3485878.693207] [] do_invalid_op+0x20/0x30 [3485878.693209] [] invalid_op+0x18/0x20 [3485878.693215] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485878.693221] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485878.693223] [] ? test_clear_page_writeback+0x143/0x260 [3485878.693225] [] ? test_clear_page_writeback+0x38/0x260 [3485878.693231] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485878.693233] [] ? SyS_msync+0x230/0x230 [3485878.693238] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485878.693240] [] ? wait_for_stable_page+0x16/0x50 [3485878.693246] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485878.693248] [] generic_perform_write+0xc6/0x1e0 [3485878.693254] [] ceph_write_iter+0x890/0xa30 [ceph] [3485878.693255] [] ? _raw_spin_unlock+0x2b/0x40 [3485878.693258] [] ? do_wp_page+0x374/0x740 [3485878.693260] [] ? handle_mm_fault+0x86c/0x1060 [3485878.693262] [] ? __do_page_fault+0x159/0x580 [3485878.693264] [] ? vfs_write+0x1d3/0x1f0 [3485878.693265] [] ? vfs_write+0x1d3/0x1f0 [3485878.693267] [] new_sync_write+0x7b/0xb0 [3485878.693269] [] vfs_write+0xc7/0x1f0 [3485878.693270] [] SyS_write+0x52/0xb0 [3485878.693272] [] system_call_fastpath+0x1a/0x1f [3485878.693288] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3485878.705100] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485878.705120] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485878.705120] irq event stamp: 0 [3485878.705121] hardirqs last enabled at (0): [< (null)>] (null) [3485878.705124] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485878.705126] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485878.705127] softirqs last disabled at (0): [< (null)>] (null) [3485878.705129] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485878.705129] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485878.705130] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485878.705133] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485878.705134] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485878.705134] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485878.705135] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485878.705136] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485878.705136] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485878.705137] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485878.705138] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485878.705139] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485878.705140] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485878.705140] Stack: [3485878.705142] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485878.705143] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485878.705145] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485878.705145] Call Trace: [3485878.705148] [] _raw_spin_lock+0x41/0x50 [3485878.705155] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485878.705161] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485878.705166] [] ceph_release+0x2e/0xc0 [ceph] [3485878.705168] [] __fput+0xc8/0x210 [3485878.705170] [] ____fput+0xe/0x10 [3485878.705172] [] task_work_run+0xb4/0xe0 [3485878.705174] [] do_exit+0x2e8/0xae0 [3485878.705175] [] ? kmsg_dump+0x10d/0x120 [3485878.705177] [] ? kmsg_dump+0x24/0x120 [3485878.705178] [] oops_end+0xa5/0xf0 [3485878.705180] [] die+0x58/0x90 [3485878.705181] [] do_trap+0xcd/0x160 [3485878.705182] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485878.705184] [] do_error_trap.part.16+0xa1/0xe0 [3485878.705190] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485878.705192] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485878.705193] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485878.705195] [] ? atomic_notifier_call_chain+0x16/0x20 [3485878.705196] [] do_error_trap+0x64/0x80 [3485878.705197] [] do_invalid_op+0x20/0x30 [3485878.705199] [] invalid_op+0x18/0x20 [3485878.705205] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485878.705211] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485878.705213] [] ? test_clear_page_writeback+0x143/0x260 [3485878.705215] [] ? test_clear_page_writeback+0x38/0x260 [3485878.705220] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485878.705222] [] ? SyS_msync+0x230/0x230 [3485878.705228] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485878.705229] [] ? wait_for_stable_page+0x16/0x50 [3485878.705235] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485878.705237] [] generic_perform_write+0xc6/0x1e0 [3485878.705242] [] ceph_write_iter+0x890/0xa30 [ceph] [3485878.705244] [] ? might_fault+0x66/0xc0 [3485878.705246] [] ? might_fault+0x66/0xc0 [3485878.705247] [] ? vfs_write+0x1d3/0x1f0 [3485878.705249] [] ? vfs_write+0x1d3/0x1f0 [3485878.705250] [] ? might_fault+0x66/0xc0 [3485878.705252] [] new_sync_write+0x7b/0xb0 [3485878.705253] [] vfs_write+0xc7/0x1f0 [3485878.705255] [] SyS_write+0x52/0xb0 [3485878.705256] [] system_call_fastpath+0x1a/0x1f [3485878.705272] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485880.115926] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485880.178969] irq event stamp: 0 [3485880.182249] hardirqs last enabled at (0): [< (null)>] (null) [3485880.189935] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485880.198406] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485880.206871] softirqs last disabled at (0): [< (null)>] (null) [3485880.214559] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485880.223411] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485880.230167] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485880.237889] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3485880.246703] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485880.252241] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485880.259616] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485880.266989] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485880.274368] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485880.281748] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485880.289120] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485880.297452] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485880.303419] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485880.310788] Stack: [3485880.313025] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485880.320860] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485880.328699] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485880.336539] Call Trace: [3485880.339219] [] _raw_spin_lock+0x41/0x50 [3485880.344938] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485880.351776] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485880.358443] [] ? __init_waitqueue_head+0x60/0x60 [3485880.364933] [] ceph_read_iter+0xc9/0x570 [ceph] [3485880.371339] [] new_sync_read+0x78/0xb0 [3485880.376959] [] vfs_read+0xab/0x180 [3485880.382232] [] SyS_read+0x52/0xb0 [3485880.387424] [] system_call_fastpath+0x1a/0x1f [3485880.393652] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3485906.658163] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3485906.662160] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3485906.662186] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485906.662186] irq event stamp: 0 [3485906.662188] hardirqs last enabled at (0): [< (null)>] (null) [3485906.662193] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485906.662195] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485906.662196] softirqs last disabled at (0): [< (null)>] (null) [3485906.662198] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485906.662199] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485906.662203] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485906.662204] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485906.662209] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485906.662210] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485906.662210] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485906.662211] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485906.662212] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485906.662212] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485906.662213] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485906.662214] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485906.662215] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485906.662216] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485906.662216] Stack: [3485906.662218] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485906.662220] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485906.662221] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485906.662222] Call Trace: [3485906.662226] [] _raw_spin_lock+0x41/0x50 [3485906.662236] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485906.662242] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485906.662248] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485906.662250] [] ? ext4_writepages+0x4ee/0xd30 [3485906.662257] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485906.662263] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485906.662265] [] ? writeback_sb_inodes+0x139/0x4f0 [3485906.662266] [] ? writeback_sb_inodes+0x139/0x4f0 [3485906.662269] [] do_writepages+0x23/0x40 [3485906.662270] [] __writeback_single_inode+0x45/0x2e0 [3485906.662271] [] writeback_sb_inodes+0x2ad/0x4f0 [3485906.662273] [] ? wb_writeback+0x54/0x350 [3485906.662274] [] __writeback_inodes_wb+0x9e/0xd0 [3485906.662276] [] wb_writeback+0x29b/0x350 [3485906.662278] [] ? get_nr_dirty_inodes+0x52/0x80 [3485906.662279] [] wb_check_old_data_flush+0x9f/0xb0 [3485906.662281] [] bdi_writeback_workfn+0x1cc/0x490 [3485906.662283] [] ? process_one_work+0x16f/0x570 [3485906.662284] [] process_one_work+0x1d1/0x570 [3485906.662286] [] ? process_one_work+0x16f/0x570 [3485906.662287] [] worker_thread+0x11c/0x530 [3485906.662289] [] ? create_and_start_worker+0x50/0x50 [3485906.662291] [] kthread+0xe4/0x100 [3485906.662293] [] ? flush_kthread_worker+0x130/0x130 [3485906.662294] [] ret_from_fork+0x7c/0xb0 [3485906.662296] [] ? flush_kthread_worker+0x130/0x130 [3485906.662312] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485906.678153] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3485906.678181] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485906.678182] irq event stamp: 0 [3485906.678183] hardirqs last enabled at (0): [< (null)>] (null) [3485906.678186] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485906.678188] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485906.678189] softirqs last disabled at (0): [< (null)>] (null) [3485906.678190] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485906.678191] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485906.678192] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485906.678195] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485906.678196] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485906.678197] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485906.678197] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485906.678198] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485906.678198] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485906.678199] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485906.678200] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485906.678201] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485906.678202] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485906.678202] Stack: [3485906.678204] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485906.678206] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485906.678207] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485906.678208] Call Trace: [3485906.678210] [] _raw_spin_lock+0x41/0x50 [3485906.678218] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485906.678224] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485906.678229] [] ceph_release+0x2e/0xc0 [ceph] [3485906.678232] [] __fput+0xc8/0x210 [3485906.678233] [] ____fput+0xe/0x10 [3485906.678235] [] task_work_run+0xb4/0xe0 [3485906.678237] [] do_exit+0x2e8/0xae0 [3485906.678239] [] ? kmsg_dump+0x10d/0x120 [3485906.678241] [] ? kmsg_dump+0x24/0x120 [3485906.678243] [] oops_end+0xa5/0xf0 [3485906.678244] [] die+0x58/0x90 [3485906.678245] [] do_trap+0xcd/0x160 [3485906.678247] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485906.678249] [] do_error_trap.part.16+0xa1/0xe0 [3485906.678255] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485906.678256] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485906.678258] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485906.678259] [] ? atomic_notifier_call_chain+0x16/0x20 [3485906.678261] [] do_error_trap+0x64/0x80 [3485906.678262] [] do_invalid_op+0x20/0x30 [3485906.678264] [] invalid_op+0x18/0x20 [3485906.678270] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485906.678276] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485906.678278] [] ? test_clear_page_writeback+0x143/0x260 [3485906.678280] [] ? test_clear_page_writeback+0x38/0x260 [3485906.678285] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485906.678287] [] ? SyS_msync+0x230/0x230 [3485906.678293] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485906.678295] [] ? wait_for_stable_page+0x16/0x50 [3485906.678300] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485906.678302] [] generic_perform_write+0xc6/0x1e0 [3485906.678308] [] ceph_write_iter+0x890/0xa30 [ceph] [3485906.678309] [] ? _raw_spin_unlock+0x2b/0x40 [3485906.678311] [] ? do_wp_page+0x374/0x740 [3485906.678313] [] ? handle_mm_fault+0x86c/0x1060 [3485906.678314] [] ? __do_page_fault+0x159/0x580 [3485906.678316] [] ? vfs_write+0x1d3/0x1f0 [3485906.678318] [] ? vfs_write+0x1d3/0x1f0 [3485906.678319] [] new_sync_write+0x7b/0xb0 [3485906.678321] [] vfs_write+0xc7/0x1f0 [3485906.678322] [] SyS_write+0x52/0xb0 [3485906.678324] [] system_call_fastpath+0x1a/0x1f [3485906.678340] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485906.690145] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485906.690165] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485906.690165] irq event stamp: 0 [3485906.690166] hardirqs last enabled at (0): [< (null)>] (null) [3485906.690169] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485906.690171] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485906.690172] softirqs last disabled at (0): [< (null)>] (null) [3485906.690174] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485906.690174] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485906.690175] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485906.690178] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485906.690179] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485906.690179] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485906.690180] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485906.690181] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485906.690181] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485906.690182] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485906.690183] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485906.690184] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485906.690185] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485906.690185] Stack: [3485906.690187] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485906.690188] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485906.690190] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485906.690190] Call Trace: [3485906.690192] [] _raw_spin_lock+0x41/0x50 [3485906.690199] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485906.690206] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485906.690211] [] ceph_release+0x2e/0xc0 [ceph] [3485906.690213] [] __fput+0xc8/0x210 [3485906.690215] [] ____fput+0xe/0x10 [3485906.690217] [] task_work_run+0xb4/0xe0 [3485906.690219] [] do_exit+0x2e8/0xae0 [3485906.690220] [] ? kmsg_dump+0x10d/0x120 [3485906.690222] [] ? kmsg_dump+0x24/0x120 [3485906.690223] [] oops_end+0xa5/0xf0 [3485906.690225] [] die+0x58/0x90 [3485906.690226] [] do_trap+0xcd/0x160 [3485906.690228] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485906.690229] [] do_error_trap.part.16+0xa1/0xe0 [3485906.690235] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485906.690237] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485906.690238] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485906.690240] [] ? atomic_notifier_call_chain+0x16/0x20 [3485906.690241] [] do_error_trap+0x64/0x80 [3485906.690243] [] do_invalid_op+0x20/0x30 [3485906.690244] [] invalid_op+0x18/0x20 [3485906.690251] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485906.690257] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485906.690259] [] ? test_clear_page_writeback+0x143/0x260 [3485906.690260] [] ? test_clear_page_writeback+0x38/0x260 [3485906.690266] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485906.690268] [] ? SyS_msync+0x230/0x230 [3485906.690273] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485906.690275] [] ? wait_for_stable_page+0x16/0x50 [3485906.690280] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485906.690282] [] generic_perform_write+0xc6/0x1e0 [3485906.690288] [] ceph_write_iter+0x890/0xa30 [ceph] [3485906.690290] [] ? might_fault+0x66/0xc0 [3485906.690291] [] ? might_fault+0x66/0xc0 [3485906.690293] [] ? vfs_write+0x1d3/0x1f0 [3485906.690294] [] ? vfs_write+0x1d3/0x1f0 [3485906.690296] [] ? might_fault+0x66/0xc0 [3485906.690297] [] new_sync_write+0x7b/0xb0 [3485906.690299] [] vfs_write+0xc7/0x1f0 [3485906.690300] [] SyS_write+0x52/0xb0 [3485906.690302] [] system_call_fastpath+0x1a/0x1f [3485906.690318] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485908.100977] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485908.163957] irq event stamp: 0 [3485908.167239] hardirqs last enabled at (0): [< (null)>] (null) [3485908.174927] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485908.183400] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485908.191861] softirqs last disabled at (0): [< (null)>] (null) [3485908.199545] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485908.208394] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485908.215150] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485908.222874] RIP: 0010:[] [] do_raw_spin_lock+0x153/0x160 [3485908.231702] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485908.237243] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485908.244612] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485908.251985] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485908.259359] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485908.266738] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485908.274117] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485908.282451] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485908.288420] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485908.295801] Stack: [3485908.298056] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485908.305894] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485908.313733] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485908.321573] Call Trace: [3485908.324250] [] _raw_spin_lock+0x41/0x50 [3485908.329971] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485908.336811] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485908.343477] [] ? __init_waitqueue_head+0x60/0x60 [3485908.349970] [] ceph_read_iter+0xc9/0x570 [ceph] [3485908.356374] [] new_sync_read+0x78/0xb0 [3485908.362001] [] vfs_read+0xab/0x180 [3485908.367276] [] SyS_read+0x52/0xb0 [3485908.372466] [] system_call_fastpath+0x1a/0x1f [3485908.378699] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 [3485918.599784] INFO: rcu_sched self-detected stall on CPU[3485918.603781] INFO: rcu_sched detected stalls on CPUs/tasks: { 2} (detected by 3, t=7080787 jiffies, g=482214633, c=482214632, q=8583795) [3485918.603785] sending NMI to all CPUs: [3485918.603788] NMI backtrace for cpu 1 [3485918.603792] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485918.603793] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485918.603794] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485918.603796] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485918.603803] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485918.603804] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000057c957c8 [3485918.603805] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485918.603806] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485918.603807] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485918.603808] R13: 0000000097020520 R14: ffff880224754690 R15: ffff88033a8afeb0 [3485918.603809] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485918.603810] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485918.603812] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485918.603813] Stack: [3485918.603813] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485918.603816] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485918.603818] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485918.603821] Call Trace: [3485918.603822] [] _raw_spin_lock+0x41/0x50 [3485918.603827] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485918.603840] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485918.603848] [] ? __init_waitqueue_head+0x60/0x60 [3485918.603851] [] ceph_read_iter+0xc9/0x570 [ceph] [3485918.603858] [] new_sync_read+0x78/0xb0 [3485918.603861] [] vfs_read+0xab/0x180 [3485918.603864] [] SyS_read+0x52/0xb0 [3485918.603866] [] system_call_fastpath+0x1a/0x1f [3485918.603869] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485918.603898] NMI backtrace for cpu 3 [3485918.603900] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485918.603901] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485918.603902] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485918.603903] RIP: 0010:[] [] __const_udelay+0x22/0x30 [3485918.603908] RSP: 0018:ffff88043fcc3ca8 EFLAGS: 00000807 [3485918.603909] RAX: 0000000033700400 RBX: 0000000000002710 RCX: 00000000013543a0 [3485918.603910] RDX: 000000000026595b RSI: 0000000000000100 RDI: 0000000000418958 [3485918.603911] RBP: ffff88043fcc3ca8 R08: 0000000000000001 R09: 0000000000000000 [3485918.603912] R10: 0000000000000001 R11: 0000000000000000 R12: ffffffff81c54f80 [3485918.603913] R13: 0000000000000003 R14: ffffffff81c54f80 R15: ffffffff81d21e80 [3485918.603915] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485918.603916] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485918.603917] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485918.603918] Stack: [3485918.603918] ffff88043fcc3cc8 ffffffff81036812 000000001cbe02e8 ffff88043fccd7c0 [3485918.603921] ffff88043fcc3d28 ffffffff810cd6a5 0000000000000002 000000000082fa73 [3485918.603923] 0000000000000082 00000001810aae9d ffff88043fcc3d18 ffff8803c46321f0 [3485918.603926] Call Trace: [3485918.603927] [] arch_trigger_all_cpu_backtrace+0x92/0x120 [3485918.603931] [] rcu_check_callbacks+0x685/0x6b0 [3485918.603935] [] update_process_times+0x48/0x80 [3485918.603938] [] tick_sched_handle.isra.15+0x33/0x70 [3485918.603941] [] tick_sched_timer+0x4c/0x80 [3485918.603943] [] __run_hrtimer+0x71/0x250 [3485918.603946] [] ? hrtimer_interrupt+0x75/0x260 [3485918.603948] [] ? tick_sched_do_timer+0x40/0x40 [3485918.603950] [] hrtimer_interrupt+0x107/0x260 [3485918.603952] [] ? handle_edge_irq+0x1e/0x110 [3485918.603955] [] hpet_interrupt_handler+0x16/0x40 [3485918.603958] [] handle_irq_event_percpu+0x5d/0x200 [3485918.603961] [] handle_irq_event+0x48/0x70 [3485918.603963] [] handle_edge_irq+0x77/0x110 [3485918.603965] [] handle_irq+0x22/0x40 [3485918.603968] [] do_IRQ+0x5e/0x110 [3485918.603972] [] common_interrupt+0x6f/0x6f [3485918.603974] [] ? retint_restore_args+0xe/0xe [3485918.603977] [] ? do_raw_spin_lock+0x15a/0x160 [3485918.603980] [] ? do_raw_spin_lock+0x138/0x160 [3485918.603982] [] _raw_spin_lock+0x41/0x50 [3485918.603985] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485918.603994] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485918.604002] [] ceph_release+0x2e/0xc0 [ceph] [3485918.604008] [] __fput+0xc8/0x210 [3485918.604011] [] ____fput+0xe/0x10 [3485918.604013] [] task_work_run+0xb4/0xe0 [3485918.604016] [] do_exit+0x2e8/0xae0 [3485918.604019] [] ? kmsg_dump+0x10d/0x120 [3485918.604021] [] ? kmsg_dump+0x24/0x120 [3485918.604023] [] oops_end+0xa5/0xf0 [3485918.604025] [] die+0x58/0x90 [3485918.604027] [] do_trap+0xcd/0x160 [3485918.604029] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485918.604031] [] do_error_trap.part.16+0xa1/0xe0 [3485918.604033] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485918.604041] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485918.604043] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485918.604045] [] ? atomic_notifier_call_chain+0x16/0x20 [3485918.604047] [] do_error_trap+0x64/0x80 [3485918.604049] [] do_invalid_op+0x20/0x30 [3485918.604051] [] invalid_op+0x18/0x20 [3485918.604053] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485918.604061] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485918.604068] [] ? test_clear_page_writeback+0x143/0x260 [3485918.604071] [] ? test_clear_page_writeback+0x38/0x260 [3485918.604074] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485918.604080] [] ? SyS_msync+0x230/0x230 [3485918.604083] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485918.604090] [] ? wait_for_stable_page+0x16/0x50 [3485918.604092] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485918.604099] [] generic_perform_write+0xc6/0x1e0 [3485918.604103] [] ceph_write_iter+0x890/0xa30 [ceph] [3485918.604109] [] ? might_fault+0x66/0xc0 [3485918.604111] [] ? might_fault+0x66/0xc0 [3485918.604113] [] ? vfs_write+0x1d3/0x1f0 [3485918.604116] [] ? vfs_write+0x1d3/0x1f0 [3485918.604118] [] ? might_fault+0x66/0xc0 [3485918.604120] [] new_sync_write+0x7b/0xb0 [3485918.604122] [] vfs_write+0xc7/0x1f0 [3485918.604124] [] SyS_write+0x52/0xb0 [3485918.604126] [] system_call_fastpath+0x1a/0x1f [3485918.604128] Code: 2e 0f 1f 84 00 00 00 00 00 55 48 8d 04 bd 00 00 00 00 65 48 8b 14 25 60 30 01 00 48 8d 0c 12 48 c1 e2 06 48 89 e5 48 29 ca f7 e2 <48> 8d 7a 01 ff 15 2c 58 8f 00 5d c3 66 90 66 66 66 66 90 55 65 [3485918.604158] NMI backtrace for cpu 5 [3485918.604160] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485918.604161] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485918.604162] task: ffff88042966c3e0 ti: ffff880429674000 task.ti: ffff880429674000 [3485918.604163] RIP: 0010:[] [] rebalance_domains+0x1a9/0x2c0 [3485918.604166] RSP: 0018:ffff88043fd43e28 EFLAGS: 00000246 [3485918.604167] RAX: 0000000133f7ae6e RBX: ffff88042914c600 RCX: 0000000000000000 [3485918.604168] RDX: ffff88042914c600 RSI: 0000000000000001 RDI: 0000000000000002 [3485918.604169] RBP: ffff88043fd43e98 R08: 0000000000000000 R09: 0000000000000000 [3485918.604170] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000c81 [3485918.604171] R13: 0000000000000000 R14: 0000000133f7e906 R15: 0000000000000000 [3485918.604172] FS: 0000000000000000(0000) GS:ffff88043fd40000(0000) knlGS:0000000000000000 [3485918.604173] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485918.604174] CR2: 00000000004ad420 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485918.604175] Stack: [3485918.604176] ffffffff8109d335 ffff88043fd4d360 ffff88043fd43eb8 0000000500000000 [3485918.604178] ffff88043fd53b40 0000000000000000 0000000000000000 0000000100000003 [3485918.604181] ffffffff810d9cf4 ffffffff81c0a0b8 ffff88043fd53b40 0000000000013b40 [3485918.604183] Call Trace: [3485918.604184] [] ? rebalance_domains+0x5/0x2c0 [3485918.604186] [] ? tick_irq_enter+0x44/0xe0 [3485918.604189] [] run_rebalance_domains+0x4c/0x1b0 [3485918.604191] [] __do_softirq+0x10f/0x350 [3485918.604194] [] irq_exit+0x96/0xc0 [3485918.604196] [] scheduler_ipi+0xa0/0x180 [3485918.604199] [] smp_reschedule_interrupt+0x2a/0x30 [3485918.604201] [] reschedule_interrupt+0x6f/0x80 [3485918.604204] [] ? cpuidle_enter_state+0x63/0xe0 [3485918.604208] [] ? cpuidle_enter_state+0x5f/0xe0 [3485918.604211] [] cpuidle_enter+0x17/0x20 [3485918.604213] [] cpu_startup_entry+0x32b/0x490 [3485918.604215] [] ? trace_hardirqs_on+0xd/0x10 [3485918.604217] [] start_secondary+0x1d8/0x290 [3485918.604220] Code: f0 ff ff 85 c0 74 13 8b 7d ac e8 b3 0a ff ff 85 c0 0f 94 c0 0f b6 c0 89 45 b8 44 8b 45 b8 48 8b 05 2d fb c8 00 8b 7b 60 45 85 c0 <48> 89 43 58 8b 43 28 74 04 48 0f af f8 e8 65 f8 fb ff ba 01 00 [3485918.604250] NMI backtrace for cpu 0 [3485918.604253] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485918.604253] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485918.604258] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485918.604260] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485918.604261] RIP: 0010:[] [] run_timer_softirq+0x5/0x320 [3485918.604265] RSP: 0018:ffff88043fc03ef0 EFLAGS: 00000202 [3485918.604266] RAX: 0000000000000002 RBX: ffffffff81c0a088 RCX: 0000000000000001 [3485918.604267] RDX: ffff8801018ebfd8 RSI: 0000000000000000 RDI: ffffffff81c0a088 [3485918.604268] RBP: ffff88043fc03f58 R08: 0000000000000001 R09: 0000000000000000 [3485918.604269] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001 [3485918.604270] R13: 0000000000000101 R14: 0000000000000002 R15: 0000000000000002 [3485918.604271] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485918.604272] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485918.604273] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485918.604274] Stack: [3485918.604275] ffffffff8105dfbf 0100000004a08060 ffff8801018ebfd8 0000000133f7ae6f [3485918.604277] ffffffff0000000a ffff8804290112a8 0000000000000001 ffff88043fc03f48 [3485918.604280] 0000000000000000 0000000000000041 0000000000000028 0000000000000041 [3485918.604282] Call Trace: [3485918.604283] [] ? __do_softirq+0x10f/0x350 [3485918.604286] [] irq_exit+0x96/0xc0 [3485918.604288] [] do_IRQ+0x67/0x110 [3485918.604291] [] common_interrupt+0x6f/0x6f [3485918.604293] [] ? retint_restore_args+0xe/0xe [3485918.604296] [] ? do_raw_spin_lock+0x158/0x160 [3485918.604299] [] ? do_raw_spin_lock+0x138/0x160 [3485918.604301] [] _raw_spin_lock+0x41/0x50 [3485918.604304] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485918.604312] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485918.604320] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485918.604327] [] ? ext4_writepages+0x4ee/0xd30 [3485918.604330] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485918.604338] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485918.604346] [] ? writeback_sb_inodes+0x139/0x4f0 [3485918.604348] [] ? writeback_sb_inodes+0x139/0x4f0 [3485918.604350] [] do_writepages+0x23/0x40 [3485918.604352] [] __writeback_single_inode+0x45/0x2e0 [3485918.604355] [] writeback_sb_inodes+0x2ad/0x4f0 [3485918.604357] [] ? wb_writeback+0x54/0x350 [3485918.604359] [] __writeback_inodes_wb+0x9e/0xd0 [3485918.604361] [] wb_writeback+0x29b/0x350 [3485918.604363] [] ? get_nr_dirty_inodes+0x52/0x80 [3485918.604365] [] wb_check_old_data_flush+0x9f/0xb0 [3485918.604368] [] bdi_writeback_workfn+0x1cc/0x490 [3485918.604370] [] ? process_one_work+0x16f/0x570 [3485918.604372] [] process_one_work+0x1d1/0x570 [3485918.604374] [] ? process_one_work+0x16f/0x570 [3485918.604376] [] worker_thread+0x11c/0x530 [3485918.604379] [] ? create_and_start_worker+0x50/0x50 [3485918.604381] [] kthread+0xe4/0x100 [3485918.604384] [] ? flush_kthread_worker+0x130/0x130 [3485918.604386] [] ret_from_fork+0x7c/0xb0 [3485918.604388] [] ? flush_kthread_worker+0x130/0x130 [3485918.604391] Code: 24 08 00 00 00 00 89 14 24 48 89 c2 e8 f5 ae 0e 00 48 8b 5d e0 4c 8b 65 e8 4c 8b 6d f0 4c 8b 75 f8 c9 c3 0f 1f 00 66 66 66 66 90 <55> 48 89 e5 41 57 41 56 41 55 41 54 53 48 83 ec 58 65 48 8b 1c [3485918.604421] NMI backtrace for cpu 2 [3485918.604423] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485918.604424] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485918.604425] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485918.604426] RIP: 0010:[] [] io_serial_in+0x18/0x20 [3485918.604431] RSP: 0018:ffff88043fc83a78 EFLAGS: 00000006 [3485918.604432] RAX: 00000000d6b06a00 RBX: ffffffff82c09eb0 RCX: 0000000000000000 [3485918.604433] RDX: 00000000000003ed RSI: 00000000000003ed RDI: ffffffff82c09eb0 [3485918.604434] RBP: ffff88043fc83a78 R08: 0000000000000002 R09: 0000000000000000 [3485918.604435] R10: 0000000000000000 R11: 0000000000000001 R12: 000000000000270b [3485918.604436] R13: 0000000000000020 R14: ffffffff8147cd70 R15: ffffffff82b607fb [3485918.604437] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485918.604438] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485918.604439] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485918.604440] Stack: [3485918.604441] ffff88043fc83aa8 ffffffff8147cd0b ffffffff81a755a9 ffffffff82c09eb0 [3485918.604443] 0000000000000073 ffffffff82b607f0 ffff88043fc83ac8 ffffffff8147cd95 [3485918.604446] ffffffff82b607f0 ffffffff82c09eb0 ffff88043fc83b08 ffffffff81475fcd [3485918.604448] Call Trace: [3485918.604449] [] wait_for_xmitr+0x3b/0xa0 [3485918.604453] [] serial8250_console_putchar+0x25/0x40 [3485918.604455] [] uart_console_write+0x3d/0x70 [3485918.604458] [] serial8250_console_write+0xc3/0x1a0 [3485918.604460] [] call_console_drivers.constprop.20+0xa5/0x120 [3485918.604463] [] console_unlock+0x397/0x480 [3485918.604465] [] vprintk_emit+0x29c/0x600 [3485918.604467] [] ? cpuacct_account_field+0x5/0xb0 [3485918.604470] [] printk+0x61/0x63 [3485918.604474] [] rcu_check_callbacks+0x39e/0x6b0 [3485918.604477] [] update_process_times+0x48/0x80 [3485918.604479] [] tick_sched_handle.isra.15+0x33/0x70 [3485918.604481] [] tick_sched_timer+0x4c/0x80 [3485918.604483] [] __run_hrtimer+0x71/0x250 [3485918.604486] [] ? hrtimer_interrupt+0x75/0x260 [3485918.604487] [] ? tick_sched_do_timer+0x40/0x40 [3485918.604490] [] hrtimer_interrupt+0x107/0x260 [3485918.604492] [] ? handle_edge_irq+0x1e/0x110 [3485918.604494] [] hpet_interrupt_handler+0x16/0x40 [3485918.604497] [] handle_irq_event_percpu+0x5d/0x200 [3485918.604499] [] handle_irq_event+0x48/0x70 [3485918.604501] [] handle_edge_irq+0x77/0x110 [3485918.604503] [] handle_irq+0x22/0x40 [3485918.604505] [] do_IRQ+0x5e/0x110 [3485918.604507] [] common_interrupt+0x6f/0x6f [3485918.604510] [] ? retint_restore_args+0xe/0xe [3485918.604512] [] ? do_raw_spin_lock+0x15a/0x160 [3485918.604515] [] ? do_raw_spin_lock+0x138/0x160 [3485918.604517] [] _raw_spin_lock+0x41/0x50 [3485918.604520] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485918.604528] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485918.604536] [] ceph_release+0x2e/0xc0 [ceph] [3485918.604542] [] __fput+0xc8/0x210 [3485918.604545] [] ____fput+0xe/0x10 [3485918.604547] [] task_work_run+0xb4/0xe0 [3485918.604549] [] do_exit+0x2e8/0xae0 [3485918.604552] [] ? kmsg_dump+0x10d/0x120 [3485918.604554] [] ? kmsg_dump+0x24/0x120 [3485918.604556] [] oops_end+0xa5/0xf0 [3485918.604558] [] die+0x58/0x90 [3485918.604560] [] do_trap+0xcd/0x160 [3485918.604561] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485918.604564] [] do_error_trap.part.16+0xa1/0xe0 [3485918.604566] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485918.604574] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485918.604576] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485918.604578] [] ? atomic_notifier_call_chain+0x16/0x20 [3485918.604580] [] do_error_trap+0x64/0x80 [3485918.604582] [] do_invalid_op+0x20/0x30 [3485918.604584] [] invalid_op+0x18/0x20 [3485918.604586] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485918.604594] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485918.604601] [] ? test_clear_page_writeback+0x143/0x260 [3485918.604603] [] ? test_clear_page_writeback+0x38/0x260 [3485918.604606] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485918.604613] [] ? SyS_msync+0x230/0x230 [3485918.604615] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485918.604622] [] ? wait_for_stable_page+0x16/0x50 [3485918.604624] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485918.604631] [] generic_perform_write+0xc6/0x1e0 [3485918.604634] [] ceph_write_iter+0x890/0xa30 [ceph] [3485918.604640] [] ? _raw_spin_unlock+0x2b/0x40 [3485918.604642] [] ? do_wp_page+0x374/0x740 [3485918.604645] [] ? handle_mm_fault+0x86c/0x1060 [3485918.604647] [] ? __do_page_fault+0x159/0x580 [3485918.604649] [] ? vfs_write+0x1d3/0x1f0 [3485918.604651] [] ? vfs_write+0x1d3/0x1f0 [3485918.604653] [] new_sync_write+0x7b/0xb0 [3485918.604656] [] vfs_write+0xc7/0x1f0 [3485918.604658] [] SyS_write+0x52/0xb0 [3485918.604660] [] system_call_fastpath+0x1a/0x1f [3485918.604662] Code: 48 89 e5 d3 e6 48 63 f6 48 03 77 40 8b 06 5d c3 0f 1f 00 66 66 66 66 90 0f b6 8f 91 00 00 00 55 48 89 e5 d3 e6 03 77 38 89 f2 ec <0f> b6 c0 5d c3 0f 1f 00 66 66 66 66 90 0f b6 8f 91 00 00 00 55 [3485918.604692] NMI backtrace for cpu 7 [3485918.604694] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485918.604694] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485918.604696] task: ffff8804296821f0 ti: ffff88042968c000 task.ti: ffff88042968c000 [3485918.604697] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485918.604702] RSP: 0000:ffff88042968fdf8 EFLAGS: 00000046 [3485918.604702] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 [3485918.604703] RDX: 0000000000000000 RSI: ffff88042968ffd8 RDI: ffffffff817295bf [3485918.604704] RBP: ffff88042968fe28 R08: 0000000000000001 R09: 0000000000000000 [3485918.604705] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004 [3485918.604706] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 [3485918.604707] FS: 0000000000000000(0000) GS:ffff88043fdc0000(0000) knlGS:0000000000000000 [3485918.604708] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485918.604709] CR2: 00007f01affe35e0 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485918.604710] Stack: [3485918.604710] ffffffff815d3fef 00000007ffffffff ffff88043fddb020 000c641e1d8be3bb [3485918.604713] ffffffff81ca4378 ffffffff81ca4200 ffff88042968fe88 ffffffff815d3fff [3485918.604715] 0000000000000000 00000000016dc502 0000000000000000 00000000016dc502 [3485918.604717] Call Trace: [3485918.604718] [] ? cpuidle_enter_state+0x3f/0xe0 [3485918.604721] [] cpuidle_enter_state+0x4f/0xe0 [3485918.604723] [] cpuidle_enter+0x17/0x20 [3485918.604725] [] cpu_startup_entry+0x32b/0x490 [3485918.604727] [] ? trace_hardirqs_on+0xd/0x10 [3485918.604729] [] start_secondary+0x1d8/0x290 [3485918.604731] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485918.604759] NMI backtrace for cpu 4 [3485918.604761] CPU: 4 PID: 0 Comm: swapper/4 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485918.604762] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485918.604763] task: ffff88042966a1f0 ti: ffff880429670000 task.ti: ffff880429670000 [3485918.604764] RIP: 0010:[] [] hpet_msi_next_event+0x3c/0x50 [3485918.604767] RSP: 0018:ffff88043fd03db8 EFLAGS: 00000082 [3485918.604768] RAX: 000000001a5048a2 RBX: ffff880428ce5c00 RCX: 0000000000000020 [3485918.604769] RDX: 000000001a504884 RSI: ffff880428ce5c00 RDI: 000000001a5127f0 [3485918.604770] RBP: ffff88043fd03db8 R08: 0000000000000001 R09: 0000000000000000 [3485918.604771] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 [3485918.604772] R13: 00000000003cf440 R14: ffff88043fd0d4c8 R15: ffff88043fd0d448 [3485918.604774] FS: 0000000000000000(0000) GS:ffff88043fd00000(0000) knlGS:0000000000000000 [3485918.604775] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485918.604776] CR2: 00007f2cd15220f8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485918.604776] Stack: [3485918.604777] ffff88043fd03df8 ffffffff810d6d64 0000000000000003 ffff88043fd0d488 [3485918.604779] ffff88043fd0d4c8 ffff88043fd0d360 0000000000000003 ffff88043fd0d488 [3485918.604782] ffff88043fd03e08 ffffffff810d8d14 ffff88043fd03e88 ffffffff81081b4f [3485918.604784] Call Trace: [3485918.604785] [] clockevents_program_event+0x74/0x100 [3485918.604788] [] tick_program_event+0x24/0x30 [3485918.604790] [] hrtimer_interrupt+0x14f/0x260 [3485918.604792] [] ? rcu_eqs_enter_common.isra.58+0x16d/0x280 [3485918.604795] [] ? handle_edge_irq+0x1e/0x110 [3485918.604798] [] hpet_interrupt_handler+0x16/0x40 [3485918.604800] [] handle_irq_event_percpu+0x5d/0x200 [3485918.604802] [] handle_irq_event+0x48/0x70 [3485918.604804] [] handle_edge_irq+0x77/0x110 [3485918.604806] [] handle_irq+0x22/0x40 [3485918.604808] [] do_IRQ+0x5e/0x110 [3485918.604811] [] common_interrupt+0x6f/0x6f [3485918.604813] [] ? cpuidle_enter_state+0x63/0xe0 [3485918.604816] [] ? cpuidle_enter_state+0x5f/0xe0 [3485918.604818] [] cpuidle_enter+0x17/0x20 [3485918.604820] [] cpu_startup_entry+0x32b/0x490 [3485918.604823] [] ? trace_hardirqs_on+0xd/0x10 [3485918.604825] [] start_secondary+0x1d8/0x290 [3485918.604827] Code: e5 00 48 89 e5 8b 92 f0 00 00 00 c1 e0 05 01 d7 05 08 01 00 00 48 03 05 7b be e5 00 89 38 48 8b 05 72 be e5 00 8b 80 f0 00 00 00 <29> c7 ba 00 00 00 00 b8 c2 ff ff ff 83 ff 7f 0f 4f c2 5d c3 66 [3485918.604857] NMI backtrace for cpu 6 [3485918.604859] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485918.604860] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485918.604861] task: ffff880429680000 ti: ffff880429688000 task.ti: ffff880429688000 [3485918.604862] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485918.604865] RSP: 0000:ffff88042968bdf8 EFLAGS: 00000046 [3485918.604866] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 [3485918.604867] RDX: 0000000000000000 RSI: ffff88042968bfd8 RDI: ffffffff817295bf [3485918.604868] RBP: ffff88042968be28 R08: 0000000000000001 R09: 0000000000000000 [3485918.604869] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004 [3485918.604869] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 [3485918.604870] FS: 0000000000000000(0000) GS:ffff88043fd80000(0000) knlGS:0000000000000000 [3485918.604871] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485918.604872] CR2: 000000000040f3b3 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485918.604873] Stack: [3485918.604873] ffffffff815d3fef 00000006ffffffff ffff88043fd9b020 000c641e1e430293 [3485918.604876] ffffffff81ca4378 ffffffff81ca4200 ffff88042968be88 ffffffff815d3fff [3485918.604877] 0000000000000000 0000000001aacec4 0000000000000000 0000000001aacec4 [3485918.604879] Call Trace: [3485918.604880] [] ? cpuidle_enter_state+0x3f/0xe0 [3485918.604882] [] cpuidle_enter_state+0x4f/0xe0 [3485918.604884] [] cpuidle_enter+0x17/0x20 [3485918.604886] [] cpu_startup_entry+0x32b/0x490 [3485918.604888] [] ? trace_hardirqs_on+0xd/0x10 [3485918.604890] [] start_secondary+0x1d8/0x290 [3485918.604892] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485921.200261] { 2} (t=7081437 jiffies g=482214633 c=482214632 q=8584053) [3485921.207587] sending NMI to all CPUs: [3485921.211391] NMI backtrace for cpu 3 [3485921.215109] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485921.224477] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485921.231231] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485921.238946] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485921.247769] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485921.253309] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000000000004 [3485921.260680] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485921.268051] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485921.275421] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485921.282794] R13: 0000000097020520 R14: 0000000000000002 R15: ffff880425546660 [3485921.290166] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485921.298493] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485921.304459] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485921.311829] Stack: [3485921.314066] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485921.321910] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485921.329749] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485921.337597] Call Trace: [3485921.340274] [] _raw_spin_lock+0x41/0x50 [3485921.345999] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485921.352498] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485921.358819] [] ceph_release+0x2e/0xc0 [ceph] [3485921.364967] [] __fput+0xc8/0x210 [3485921.370073] [] ____fput+0xe/0x10 [3485921.375178] [] task_work_run+0xb4/0xe0 [3485921.380799] [] do_exit+0x2e8/0xae0 [3485921.386075] [] ? kmsg_dump+0x10d/0x120 [3485921.391696] [] ? kmsg_dump+0x24/0x120 [3485921.397230] [] oops_end+0xa5/0xf0 [3485921.402418] [] die+0x58/0x90 [3485921.407173] [] do_trap+0xcd/0x160 [3485921.412362] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485921.419374] [] do_error_trap.part.16+0xa1/0xe0 [3485921.425702] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485921.433420] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485921.440518] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485921.447534] [] ? atomic_notifier_call_chain+0x16/0x20 [3485921.454454] [] do_error_trap+0x64/0x80 [3485921.460075] [] do_invalid_op+0x20/0x30 [3485921.465696] [] invalid_op+0x18/0x20 [3485921.471064] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485921.478789] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485921.486427] [] ? test_clear_page_writeback+0x143/0x260 [3485921.493445] [] ? test_clear_page_writeback+0x38/0x260 [3485921.500377] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485921.507217] [] ? SyS_msync+0x230/0x230 [3485921.512848] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485921.520398] [] ? wait_for_stable_page+0x16/0x50 [3485921.526806] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485921.533299] [] generic_perform_write+0xc6/0x1e0 [3485921.539711] [] ceph_write_iter+0x890/0xa30 [ceph] [3485921.546293] [] ? might_fault+0x66/0xc0 [3485921.551918] [] ? might_fault+0x66/0xc0 [3485921.557539] [] ? vfs_write+0x1d3/0x1f0 [3485921.564284] [] ? vfs_write+0x1d3/0x1f0 [3485921.569910] [] ? might_fault+0x66/0xc0 [3485921.575531] [] new_sync_write+0x7b/0xb0 [3485921.581239] [] vfs_write+0xc7/0x1f0 [3485921.586602] [] SyS_write+0x52/0xb0 [3485921.591875] [] system_call_fastpath+0x1a/0x1f [3485921.598105] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485921.620563] NMI backtrace for cpu 7 [3485921.624278] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485921.633215] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485921.639971] task: ffff8804296821f0 ti: ffff88042968c000 task.ti: ffff88042968c000 [3485921.647696] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485921.655908] RSP: 0018:ffff88042968fdf8 EFLAGS: 00000046 [3485921.661441] RAX: 0000000000000000 RBX: 0000000000000002 RCX: 0000000000000001 [3485921.668812] RDX: 0000000000000000 RSI: ffff88042968ffd8 RDI: 0000000000000007 [3485921.676183] RBP: ffff88042968fe28 R08: 0000000000000001 R09: 0000000000000000 [3485921.683554] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001 [3485921.690927] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000001 [3485921.698298] FS: 0000000000000000(0000) GS:ffff88043fdc0000(0000) knlGS:0000000000000000 [3485921.706624] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485921.712591] CR2: 00007f63bc91a5c0 CR3: 00000004277cf000 CR4: 00000000000007e0 [3485921.719961] Stack: [3485921.722198] ffffffff815d3fef 00000007ffffffff ffff88043fddb020 000c641eb978c990 [3485921.730033] ffffffff81ca4270 ffffffff81ca4200 ffff88042968fe88 ffffffff815d3fff [3485921.737873] 0000000000000000 0000000000f3ca1c 0000000000000000 0000000000f3ca1c [3485921.745721] Call Trace: [3485921.748398] [] ? cpuidle_enter_state+0x3f/0xe0 [3485921.754716] [] cpuidle_enter_state+0x4f/0xe0 [3485921.760859] [] cpuidle_enter+0x17/0x20 [3485921.766481] [] cpu_startup_entry+0x32b/0x490 [3485921.772621] [] ? trace_hardirqs_on+0xd/0x10 [3485921.778676] [] start_secondary+0x1d8/0x290 [3485921.784643] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485921.807101] NMI backtrace for cpu 1 [3485921.810818] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485921.819665] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485921.826414] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485921.834130] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485921.842941] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485921.848473] RAX: 000000000000e223 RBX: ffff880224754690 RCX: 0000000057c957c8 [3485921.855845] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485921.863217] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485921.870588] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485921.877960] R13: 0000000097020520 R14: ffff880224754690 R15: ffff88033a8afeb0 [3485921.885331] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485921.893656] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485921.899624] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485921.906994] Stack: [3485921.909231] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485921.917074] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485921.924923] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485921.932761] Call Trace: [3485921.935437] [] _raw_spin_lock+0x41/0x50 [3485921.941159] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485921.948001] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485921.954665] [] ? __init_waitqueue_head+0x60/0x60 [3485921.961156] [] ceph_read_iter+0xc9/0x570 [ceph] [3485921.967564] [] new_sync_read+0x78/0xb0 [3485921.973192] [] vfs_read+0xab/0x180 [3485921.978465] [] SyS_read+0x52/0xb0 [3485921.983654] [] system_call_fastpath+0x1a/0x1f [3485921.989884] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485922.012343] NMI backtrace for cpu 0 [3485922.016057] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485922.025601] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485922.032358] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485922.039039] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485922.046759] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485922.055571] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485922.061111] RAX: 0000000000008916 RBX: ffff8800049b5830 RCX: 0000000000000004 [3485922.068481] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485922.075853] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485922.083226] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000097020520 [3485922.090597] R13: 0000000097020520 R14: 0000000000002000 R15: 000000000000001d [3485922.097968] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485922.106293] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485922.112261] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485922.119632] Stack: [3485922.121869] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485922.129704] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485922.137542] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485922.145391] Call Trace: [3485922.148066] [] _raw_spin_lock+0x41/0x50 [3485922.153785] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485922.160630] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485922.167299] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485922.174313] [] ? ext4_writepages+0x4ee/0xd30 [3485922.180466] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485922.187223] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485922.193974] [] ? writeback_sb_inodes+0x139/0x4f0 [3485922.200469] [] ? writeback_sb_inodes+0x139/0x4f0 [3485922.206957] [] do_writepages+0x23/0x40 [3485922.212579] [] __writeback_single_inode+0x45/0x2e0 [3485922.219240] [] writeback_sb_inodes+0x2ad/0x4f0 [3485922.225555] [] ? wb_writeback+0x54/0x350 [3485922.231349] [] __writeback_inodes_wb+0x9e/0xd0 [3485922.237664] [] wb_writeback+0x29b/0x350 [3485922.243373] [] ? get_nr_dirty_inodes+0x52/0x80 [3485922.249687] [] wb_check_old_data_flush+0x9f/0xb0 [3485922.256177] [] bdi_writeback_workfn+0x1cc/0x490 [3485922.262587] [] ? process_one_work+0x16f/0x570 [3485922.268822] [] process_one_work+0x1d1/0x570 [3485922.274876] [] ? process_one_work+0x16f/0x570 [3485922.281105] [] worker_thread+0x11c/0x530 [3485922.286899] [] ? create_and_start_worker+0x50/0x50 [3485922.293563] [] kthread+0xe4/0x100 [3485922.298758] [] ? flush_kthread_worker+0x130/0x130 [3485922.305334] [] ret_from_fork+0x7c/0xb0 [3485922.310953] [] ? flush_kthread_worker+0x130/0x130 [3485922.317527] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485922.339977] NMI backtrace for cpu 2 [3485922.343692] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485922.353061] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485922.359808] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485922.367525] RIP: 0010:[] [] delay_tsc+0x4d/0x80 [3485922.375563] RSP: 0018:ffff88043fc83c98 EFLAGS: 00000046 [3485922.381098] RAX: 0000000060a35502 RBX: 0000000000002710 RCX: 0000000000000002 [3485922.388468] RDX: 0000000000000045 RSI: 0000000060a354bd RDI: 000000000026595c [3485922.395840] RBP: ffff88043fc83c98 R08: 0000000000000002 R09: 0000000000000000 [3485922.403212] R10: 0000000000000001 R11: 0000000000000000 R12: ffffffff81c54f80 [3485922.410582] R13: 0000000000000002 R14: ffffffff81c54f80 R15: ffffffff81d21e80 [3485922.417956] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485922.426282] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485922.432249] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485922.440825] Stack: [3485922.443060] ffff88043fc83ca8 ffffffff813a9d0c ffff88043fc83cc8 ffffffff81036812 [3485922.450894] 000000000082fb75 ffff88043fc8d7c0 ffff88043fc83d28 ffffffff810cd472 [3485922.458733] 0000000000000082 000000000082fb75 ffff88043fc83cf8 00000000810aae9d [3485922.466581] Call Trace: [3485922.469256] [3485922.471388] [] __const_udelay+0x2c/0x30 [3485922.477402] [] arch_trigger_all_cpu_backtrace+0x92/0x120 [3485922.484590] [] rcu_check_callbacks+0x452/0x6b0 [3485922.490911] [] update_process_times+0x48/0x80 [3485922.497147] [] tick_sched_handle.isra.15+0x33/0x70 [3485922.503807] [] tick_sched_timer+0x4c/0x80 [3485922.509688] [] __run_hrtimer+0x71/0x250 [3485922.515395] [] ? hrtimer_interrupt+0x75/0x260 [3485922.521625] [] ? tick_sched_do_timer+0x40/0x40 [3485922.527942] [] hrtimer_interrupt+0x107/0x260 [3485922.534092] [] ? handle_edge_irq+0x1e/0x110 [3485922.540155] [] hpet_interrupt_handler+0x16/0x40 [3485922.546565] [] handle_irq_event_percpu+0x5d/0x200 [3485922.553140] [] handle_irq_event+0x48/0x70 [3485922.559027] [] handle_edge_irq+0x77/0x110 [3485922.564910] [] handle_irq+0x22/0x40 [3485922.570272] [] do_IRQ+0x5e/0x110 [3485922.575371] [] common_interrupt+0x6f/0x6f [3485922.581254] [3485922.583390] [] ? retint_restore_args+0xe/0xe [3485922.589837] [] ? do_raw_spin_lock+0x15a/0x160 [3485922.596067] [] ? do_raw_spin_lock+0x138/0x160 [3485922.602296] [] _raw_spin_lock+0x41/0x50 [3485922.608010] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485922.614506] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485922.620830] [] ceph_release+0x2e/0xc0 [ceph] [3485922.626975] [] __fput+0xc8/0x210 [3485922.632073] [] ____fput+0xe/0x10 [3485922.637175] [] task_work_run+0xb4/0xe0 [3485922.642800] [] do_exit+0x2e8/0xae0 [3485922.648073] [] ? kmsg_dump+0x10d/0x120 [3485922.653696] [] ? kmsg_dump+0x24/0x120 [3485922.659230] [] oops_end+0xa5/0xf0 [3485922.664418] [] die+0x58/0x90 [3485922.669173] [] do_trap+0xcd/0x160 [3485922.674363] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485922.681375] [] do_error_trap.part.16+0xa1/0xe0 [3485922.687702] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485922.695422] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485922.702518] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485922.709534] [] ? atomic_notifier_call_chain+0x16/0x20 [3485922.716464] [] do_error_trap+0x64/0x80 [3485922.722091] [] do_invalid_op+0x20/0x30 [3485922.727713] [] invalid_op+0x18/0x20 [3485922.733079] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485922.740807] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485922.748442] [] ? test_clear_page_writeback+0x143/0x260 [3485922.755451] [] ? test_clear_page_writeback+0x38/0x260 [3485922.762377] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485922.769215] [] ? SyS_msync+0x230/0x230 [3485922.774841] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485922.782390] [] ? wait_for_stable_page+0x16/0x50 [3485922.788796] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485922.795289] [] generic_perform_write+0xc6/0x1e0 [3485922.801702] [] ceph_write_iter+0x890/0xa30 [ceph] [3485922.808282] [] ? _raw_spin_unlock+0x2b/0x40 [3485922.814345] [] ? do_wp_page+0x374/0x740 [3485922.820053] [] ? handle_mm_fault+0x86c/0x1060 [3485922.826280] [] ? __do_page_fault+0x159/0x580 [3485922.832422] [] ? vfs_write+0x1d3/0x1f0 [3485922.838043] [] ? vfs_write+0x1d3/0x1f0 [3485922.843665] [] new_sync_write+0x7b/0xb0 [3485922.849374] [] vfs_write+0xc7/0x1f0 [3485922.854733] [] SyS_write+0x52/0xb0 [3485922.860009] [] system_call_fastpath+0x1a/0x1f [3485922.866236] Code: 89 c6 eb 21 66 2e 0f 1f 84 00 00 00 00 00 f3 90 65 ff 04 25 a0 ba 00 00 65 8b 0c 25 84 b0 00 00 41 39 c8 75 1a 66 66 90 0f ae e8 <0f> 31 89 c2 29 f2 65 ff 0c 25 a0 ba 00 00 39 fa 72 d1 5d c3 29 [3485922.888695] NMI backtrace for cpu 6 [3485922.892411] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485922.901347] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485922.908094] task: ffff880429680000 ti: ffff880429688000 task.ti: ffff880429688000 [3485922.915809] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485922.924014] RSP: 0000:ffff88042968bdf8 EFLAGS: 00000046 [3485922.929547] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 [3485922.936919] RDX: 0000000000000000 RSI: ffff88042968bfd8 RDI: ffffffff817295bf [3485922.944290] RBP: ffff88042968be28 R08: 0000000000000001 R09: 0000000000000000 [3485922.951663] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003 [3485922.959034] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 [3485922.966406] FS: 0000000000000000(0000) GS:ffff88043fd80000(0000) knlGS:0000000000000000 [3485922.974731] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485922.980698] CR2: 000000000040a240 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485922.988067] Stack: [3485922.990305] ffffffff815d3fef 00000006ffffffff ffff88043fd9b020 000c641eb93bf307 [3485922.998140] ffffffff81ca4320 ffffffff81ca4200 ffff88042968be88 ffffffff815d3fff [3485923.005987] 0000000000000000 000000000130a3f5 0000000000000000 000000000130a3f5 [3485923.013836] Call Trace: [3485923.016512] [] ? cpuidle_enter_state+0x3f/0xe0 [3485923.022833] [] cpuidle_enter_state+0x4f/0xe0 [3485923.028975] [] cpuidle_enter+0x17/0x20 [3485923.034594] [] cpu_startup_entry+0x32b/0x490 [3485923.040737] [] ? trace_hardirqs_on+0xd/0x10 [3485923.046792] [] start_secondary+0x1d8/0x290 [3485923.052759] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485923.075216] NMI backtrace for cpu 4 [3485923.078934] CPU: 4 PID: 0 Comm: swapper/4 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485923.087868] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485923.094617] task: ffff88042966a1f0 ti: ffff880429670000 task.ti: ffff880429670000 [3485923.102340] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485923.110545] RSP: 0018:ffff880429673df8 EFLAGS: 00000046 [3485923.116077] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 [3485923.123449] RDX: 0000000000000000 RSI: ffff880429673fd8 RDI: ffffffff817295bf [3485923.130820] RBP: ffff880429673e28 R08: 0000000000000001 R09: 0000000000000000 [3485923.138192] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003 [3485923.145563] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 [3485923.152936] FS: 0000000000000000(0000) GS:ffff88043fd00000(0000) knlGS:0000000000000000 [3485923.161262] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485923.167228] CR2: 00007f145a0d00f8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485923.174598] Stack: [3485923.176835] ffffffff815d3fef 00000004ffffffff ffff88043fd1b020 000c641eb9f2cdfb [3485923.184679] ffffffff81ca4320 ffffffff81ca4200 ffff880429673e88 ffffffff815d3fff [3485923.192528] 0000000000000000 00000000003cba26 0000000000000000 00000000003cba26 [3485923.200366] Call Trace: [3485923.203042] [] ? cpuidle_enter_state+0x3f/0xe0 [3485923.209364] [] cpuidle_enter_state+0x4f/0xe0 [3485923.215514] [] cpuidle_enter+0x17/0x20 [3485923.221141] [] cpu_startup_entry+0x32b/0x490 [3485923.227284] [] ? trace_hardirqs_on+0xd/0x10 [3485923.233339] [] start_secondary+0x1d8/0x290 [3485923.239305] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485923.261765] NMI backtrace for cpu 5 [3485923.265487] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485923.274423] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485923.281171] task: ffff88042966c3e0 ti: ffff880429674000 task.ti: ffff880429674000 [3485923.288888] RIP: 0010:[] [] intel_idle+0xcc/0x150 [3485923.297101] RSP: 0018:ffff880429677df8 EFLAGS: 00000046 [3485923.302634] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 [3485923.310004] RDX: 0000000000000000 RSI: ffff880429677fd8 RDI: ffffffff817295bf [3485923.317375] RBP: ffff880429677e28 R08: 0000000000000001 R09: 0000000000000000 [3485923.324747] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000003 [3485923.333248] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 [3485923.340618] FS: 0000000000000000(0000) GS:ffff88043fd40000(0000) knlGS:0000000000000000 [3485923.348944] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485923.354910] CR2: 00007f40a6b665c0 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485923.362280] Stack: [3485923.364517] ffffffff815d3fef 00000005ffffffff ffff88043fd5b020 000c641eb9f2b8af [3485923.372362] ffffffff81ca4320 ffffffff81ca4200 ffff880429677e88 ffffffff815d3fff [3485923.380201] 0000000000000000 000000000079da60 0000000000000000 000000000079da60 [3485923.388040] Call Trace: [3485923.390720] [] ? cpuidle_enter_state+0x3f/0xe0 [3485923.397037] [] cpuidle_enter_state+0x4f/0xe0 [3485923.403185] [] cpuidle_enter+0x17/0x20 [3485923.408808] [] cpu_startup_entry+0x32b/0x490 [3485923.414950] [] ? trace_hardirqs_on+0xd/0x10 [3485923.421003] [] start_secondary+0x1d8/0x290 [3485923.426971] Code: 31 d2 65 48 8b 34 25 c8 ba 00 00 48 89 d1 48 8d 86 38 c0 ff ff 0f 01 c8 48 8b 86 38 c0 ff ff a8 08 75 08 b1 01 4c 89 e8 0f 01 c9 <65> 48 8b 04 25 c8 ba 00 00 f0 80 a0 3a c0 ff ff df 0f ae f0 48 [3485946.636811] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3485946.640805] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3485946.640834] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485946.640834] irq event stamp: 0 [3485946.640836] hardirqs last enabled at (0): [< (null)>] (null) [3485946.640841] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485946.640843] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485946.640844] softirqs last disabled at (0): [< (null)>] (null) [3485946.640846] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485946.640847] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485946.640852] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485946.640853] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485946.640857] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485946.640858] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485946.640859] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485946.640859] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485946.640860] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485946.640861] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485946.640861] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485946.640862] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485946.640863] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485946.640864] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485946.640865] Stack: [3485946.640867] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485946.640868] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485946.640870] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485946.640870] Call Trace: [3485946.640875] [] _raw_spin_lock+0x41/0x50 [3485946.640885] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485946.640892] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485946.640897] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485946.640900] [] ? ext4_writepages+0x4ee/0xd30 [3485946.640906] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485946.640912] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485946.640914] [] ? writeback_sb_inodes+0x139/0x4f0 [3485946.640916] [] ? writeback_sb_inodes+0x139/0x4f0 [3485946.640918] [] do_writepages+0x23/0x40 [3485946.640919] [] __writeback_single_inode+0x45/0x2e0 [3485946.640921] [] writeback_sb_inodes+0x2ad/0x4f0 [3485946.640922] [] ? wb_writeback+0x54/0x350 [3485946.640923] [] __writeback_inodes_wb+0x9e/0xd0 [3485946.640925] [] wb_writeback+0x29b/0x350 [3485946.640927] [] ? get_nr_dirty_inodes+0x52/0x80 [3485946.640928] [] wb_check_old_data_flush+0x9f/0xb0 [3485946.640930] [] bdi_writeback_workfn+0x1cc/0x490 [3485946.640932] [] ? process_one_work+0x16f/0x570 [3485946.640933] [] process_one_work+0x1d1/0x570 [3485946.640935] [] ? process_one_work+0x16f/0x570 [3485946.640936] [] worker_thread+0x11c/0x530 [3485946.640938] [] ? create_and_start_worker+0x50/0x50 [3485946.640940] [] kthread+0xe4/0x100 [3485946.640942] [] ? flush_kthread_worker+0x130/0x130 [3485946.640944] [] ret_from_fork+0x7c/0xb0 [3485946.640946] [] ? flush_kthread_worker+0x130/0x130 [3485946.640962] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485946.656798] BUG: soft lockup - CPU#2 stuck for 21s! [teuthology:19790] [3485946.656819] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485946.656819] irq event stamp: 0 [3485946.656820] hardirqs last enabled at (0): [< (null)>] (null) [3485946.656824] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485946.656826] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485946.656826] softirqs last disabled at (0): [< (null)>] (null) [3485946.656828] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485946.656829] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485946.656830] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485946.656833] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485946.656834] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485946.656834] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485946.656835] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485946.656836] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485946.656836] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485946.656837] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485946.656838] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485946.656839] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485946.656840] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485946.656840] Stack: [3485946.656842] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485946.656843] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485946.656845] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485946.656845] Call Trace: [3485946.656848] [] _raw_spin_lock+0x41/0x50 [3485946.656855] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485946.656862] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485946.656867] [] ceph_release+0x2e/0xc0 [ceph] [3485946.656870] [] __fput+0xc8/0x210 [3485946.656871] [] ____fput+0xe/0x10 [3485946.656874] [] task_work_run+0xb4/0xe0 [3485946.656876] [] do_exit+0x2e8/0xae0 [3485946.656878] [] ? kmsg_dump+0x10d/0x120 [3485946.656879] [] ? kmsg_dump+0x24/0x120 [3485946.656881] [] oops_end+0xa5/0xf0 [3485946.656883] [] die+0x58/0x90 [3485946.656884] [] do_trap+0xcd/0x160 [3485946.656886] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485946.656887] [] do_error_trap.part.16+0xa1/0xe0 [3485946.656894] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485946.656895] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485946.656897] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485946.656898] [] ? atomic_notifier_call_chain+0x16/0x20 [3485946.656900] [] do_error_trap+0x64/0x80 [3485946.656901] [] do_invalid_op+0x20/0x30 [3485946.656902] [] invalid_op+0x18/0x20 [3485946.656909] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485946.656915] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485946.656917] [] ? test_clear_page_writeback+0x143/0x260 [3485946.656919] [] ? test_clear_page_writeback+0x38/0x260 [3485946.656924] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485946.656926] [] ? SyS_msync+0x230/0x230 [3485946.656932] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485946.656933] [] ? wait_for_stable_page+0x16/0x50 [3485946.656939] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485946.656941] [] generic_perform_write+0xc6/0x1e0 [3485946.656947] [] ceph_write_iter+0x890/0xa30 [ceph] [3485946.656948] [] ? _raw_spin_unlock+0x2b/0x40 [3485946.656950] [] ? do_wp_page+0x374/0x740 [3485946.656952] [] ? handle_mm_fault+0x86c/0x1060 [3485946.656954] [] ? __do_page_fault+0x159/0x580 [3485946.656955] [] ? vfs_write+0x1d3/0x1f0 [3485946.656957] [] ? vfs_write+0x1d3/0x1f0 [3485946.656958] [] new_sync_write+0x7b/0xb0 [3485946.656960] [] vfs_write+0xc7/0x1f0 [3485946.656962] [] SyS_write+0x52/0xb0 [3485946.656963] [] system_call_fastpath+0x1a/0x1f [3485946.656979] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485946.668791] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485946.668810] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485946.668810] irq event stamp: 0 [3485946.668812] hardirqs last enabled at (0): [< (null)>] (null) [3485946.668814] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485946.668816] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485946.668817] softirqs last disabled at (0): [< (null)>] (null) [3485946.668819] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485946.668820] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485946.668820] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485946.668823] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485946.668824] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485946.668825] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485946.668825] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485946.668826] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485946.668827] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485946.668827] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485946.668828] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485946.668829] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485946.668830] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485946.668830] Stack: [3485946.668832] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485946.668834] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485946.668835] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485946.668835] Call Trace: [3485946.668838] [] _raw_spin_lock+0x41/0x50 [3485946.668845] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485946.668851] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485946.668856] [] ceph_release+0x2e/0xc0 [ceph] [3485946.668859] [] __fput+0xc8/0x210 [3485946.668860] [] ____fput+0xe/0x10 [3485946.668862] [] task_work_run+0xb4/0xe0 [3485946.668864] [] do_exit+0x2e8/0xae0 [3485946.668866] [] ? kmsg_dump+0x10d/0x120 [3485946.668867] [] ? kmsg_dump+0x24/0x120 [3485946.668868] [] oops_end+0xa5/0xf0 [3485946.668870] [] die+0x58/0x90 [3485946.668871] [] do_trap+0xcd/0x160 [3485946.668873] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485946.668874] [] do_error_trap.part.16+0xa1/0xe0 [3485946.668880] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485946.668882] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485946.668883] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485946.668885] [] ? atomic_notifier_call_chain+0x16/0x20 [3485946.668886] [] do_error_trap+0x64/0x80 [3485946.668888] [] do_invalid_op+0x20/0x30 [3485946.668889] [] invalid_op+0x18/0x20 [3485946.668896] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485946.668902] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485946.668904] [] ? test_clear_page_writeback+0x143/0x260 [3485946.668905] [] ? test_clear_page_writeback+0x38/0x260 [3485946.668911] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485946.668912] [] ? SyS_msync+0x230/0x230 [3485946.668918] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485946.668920] [] ? wait_for_stable_page+0x16/0x50 [3485946.668925] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485946.668928] [] generic_perform_write+0xc6/0x1e0 [3485946.668933] [] ceph_write_iter+0x890/0xa30 [ceph] [3485946.668935] [] ? might_fault+0x66/0xc0 [3485946.668937] [] ? might_fault+0x66/0xc0 [3485946.668938] [] ? vfs_write+0x1d3/0x1f0 [3485946.668940] [] ? vfs_write+0x1d3/0x1f0 [3485946.668941] [] ? might_fault+0x66/0xc0 [3485946.668943] [] new_sync_write+0x7b/0xb0 [3485946.668944] [] vfs_write+0xc7/0x1f0 [3485946.668946] [] SyS_write+0x52/0xb0 [3485946.668947] [] system_call_fastpath+0x1a/0x1f [3485946.668963] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485948.078607] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485948.141604] irq event stamp: 0 [3485948.144883] hardirqs last enabled at (0): [< (null)>] (null) [3485948.152565] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485948.161039] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485948.169511] softirqs last disabled at (0): [< (null)>] (null) [3485948.177201] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485948.186045] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485948.192794] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485948.200518] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485948.209350] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485948.214897] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485948.222277] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485948.229647] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485948.237031] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485948.244417] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485948.251799] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485948.260131] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485948.266103] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485948.273480] Stack: [3485948.275726] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485948.283584] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485948.291442] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485948.299279] Call Trace: [3485948.301958] [] _raw_spin_lock+0x41/0x50 [3485948.307680] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485948.314530] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485948.321198] [] ? __init_waitqueue_head+0x60/0x60 [3485948.327697] [] ceph_read_iter+0xc9/0x570 [ceph] [3485948.335214] [] new_sync_read+0x78/0xb0 [3485948.340842] [] vfs_read+0xab/0x180 [3485948.346124] [] SyS_read+0x52/0xb0 [3485948.351320] [] system_call_fastpath+0x1a/0x1f [3485948.357557] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485974.621863] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3485974.625858] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3485974.625890] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485974.625890] irq event stamp: 0 [3485974.625892] hardirqs last enabled at (0): [< (null)>] (null) [3485974.625897] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485974.625899] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485974.625899] softirqs last disabled at (0): [< (null)>] (null) [3485974.625902] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485974.625903] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485974.625907] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3485974.625908] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3485974.625913] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485974.625913] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3485974.625914] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485974.625915] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485974.625916] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3485974.625916] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3485974.625917] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3485974.625918] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3485974.625919] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3485974.625919] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3485974.625920] Stack: [3485974.625922] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485974.625924] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3485974.625925] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3485974.625926] Call Trace: [3485974.625930] [] _raw_spin_lock+0x41/0x50 [3485974.625941] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3485974.625948] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3485974.625953] [] ceph_writepages_start+0x83/0xf90 [ceph] [3485974.625956] [] ? ext4_writepages+0x4ee/0xd30 [3485974.625962] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485974.625968] [] ? ceph_write_inode+0x86/0x200 [ceph] [3485974.625970] [] ? writeback_sb_inodes+0x139/0x4f0 [3485974.625972] [] ? writeback_sb_inodes+0x139/0x4f0 [3485974.625974] [] do_writepages+0x23/0x40 [3485974.625975] [] __writeback_single_inode+0x45/0x2e0 [3485974.625977] [] writeback_sb_inodes+0x2ad/0x4f0 [3485974.625978] [] ? wb_writeback+0x54/0x350 [3485974.625980] [] __writeback_inodes_wb+0x9e/0xd0 [3485974.625981] [] wb_writeback+0x29b/0x350 [3485974.625983] [] ? get_nr_dirty_inodes+0x52/0x80 [3485974.625985] [] wb_check_old_data_flush+0x9f/0xb0 [3485974.625986] [] bdi_writeback_workfn+0x1cc/0x490 [3485974.625988] [] ? process_one_work+0x16f/0x570 [3485974.625990] [] process_one_work+0x1d1/0x570 [3485974.625991] [] ? process_one_work+0x16f/0x570 [3485974.625993] [] worker_thread+0x11c/0x530 [3485974.625994] [] ? create_and_start_worker+0x50/0x50 [3485974.625996] [] kthread+0xe4/0x100 [3485974.625998] [] ? flush_kthread_worker+0x130/0x130 [3485974.626000] [] ret_from_fork+0x7c/0xb0 [3485974.626002] [] ? flush_kthread_worker+0x130/0x130 [3485974.626018] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485974.641849] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3485974.641869] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485974.641870] irq event stamp: 0 [3485974.641871] hardirqs last enabled at (0): [< (null)>] (null) [3485974.641874] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485974.641876] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485974.641877] softirqs last disabled at (0): [< (null)>] (null) [3485974.641879] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485974.641880] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485974.641880] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3485974.641883] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485974.641884] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3485974.641885] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485974.641886] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485974.641886] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3485974.641887] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3485974.641888] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3485974.641889] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3485974.641889] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485974.641890] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485974.641891] Stack: [3485974.641892] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3485974.641894] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485974.641895] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3485974.641896] Call Trace: [3485974.641898] [] _raw_spin_lock+0x41/0x50 [3485974.641906] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485974.641912] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485974.641917] [] ceph_release+0x2e/0xc0 [ceph] [3485974.641920] [] __fput+0xc8/0x210 [3485974.641922] [] ____fput+0xe/0x10 [3485974.641924] [] task_work_run+0xb4/0xe0 [3485974.641927] [] do_exit+0x2e8/0xae0 [3485974.641930] [] ? kmsg_dump+0x10d/0x120 [3485974.641931] [] ? kmsg_dump+0x24/0x120 [3485974.641933] [] oops_end+0xa5/0xf0 [3485974.641934] [] die+0x58/0x90 [3485974.641936] [] do_trap+0xcd/0x160 [3485974.641938] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485974.641939] [] do_error_trap.part.16+0xa1/0xe0 [3485974.641946] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485974.641947] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485974.641948] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485974.641950] [] ? atomic_notifier_call_chain+0x16/0x20 [3485974.641951] [] do_error_trap+0x64/0x80 [3485974.641953] [] do_invalid_op+0x20/0x30 [3485974.641954] [] invalid_op+0x18/0x20 [3485974.641961] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485974.641967] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485974.641969] [] ? test_clear_page_writeback+0x143/0x260 [3485974.641970] [] ? test_clear_page_writeback+0x38/0x260 [3485974.641976] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485974.641978] [] ? SyS_msync+0x230/0x230 [3485974.641984] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485974.641985] [] ? wait_for_stable_page+0x16/0x50 [3485974.641991] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485974.641993] [] generic_perform_write+0xc6/0x1e0 [3485974.641999] [] ceph_write_iter+0x890/0xa30 [ceph] [3485974.642000] [] ? _raw_spin_unlock+0x2b/0x40 [3485974.642002] [] ? do_wp_page+0x374/0x740 [3485974.642004] [] ? handle_mm_fault+0x86c/0x1060 [3485974.642005] [] ? __do_page_fault+0x159/0x580 [3485974.642007] [] ? vfs_write+0x1d3/0x1f0 [3485974.642008] [] ? vfs_write+0x1d3/0x1f0 [3485974.642010] [] new_sync_write+0x7b/0xb0 [3485974.642012] [] vfs_write+0xc7/0x1f0 [3485974.642013] [] SyS_write+0x52/0xb0 [3485974.642015] [] system_call_fastpath+0x1a/0x1f [3485974.642031] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485974.653842] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3485974.653862] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485974.653862] irq event stamp: 0 [3485974.653863] hardirqs last enabled at (0): [< (null)>] (null) [3485974.653866] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485974.653868] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485974.653869] softirqs last disabled at (0): [< (null)>] (null) [3485974.653870] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485974.653871] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485974.653872] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3485974.653875] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3485974.653875] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3485974.653876] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3485974.653877] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3485974.653878] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3485974.653878] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3485974.653879] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3485974.653880] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3485974.653881] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485974.653882] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3485974.653882] Stack: [3485974.653884] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3485974.653885] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3485974.653887] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3485974.653887] Call Trace: [3485974.653889] [] _raw_spin_lock+0x41/0x50 [3485974.653896] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3485974.653903] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3485974.653908] [] ceph_release+0x2e/0xc0 [ceph] [3485974.653910] [] __fput+0xc8/0x210 [3485974.653912] [] ____fput+0xe/0x10 [3485974.653914] [] task_work_run+0xb4/0xe0 [3485974.653916] [] do_exit+0x2e8/0xae0 [3485974.653918] [] ? kmsg_dump+0x10d/0x120 [3485974.653919] [] ? kmsg_dump+0x24/0x120 [3485974.653921] [] oops_end+0xa5/0xf0 [3485974.653922] [] die+0x58/0x90 [3485974.653923] [] do_trap+0xcd/0x160 [3485974.653925] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485974.653927] [] do_error_trap.part.16+0xa1/0xe0 [3485974.653933] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485974.653934] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3485974.653936] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3485974.653937] [] ? atomic_notifier_call_chain+0x16/0x20 [3485974.653939] [] do_error_trap+0x64/0x80 [3485974.653940] [] do_invalid_op+0x20/0x30 [3485974.653942] [] invalid_op+0x18/0x20 [3485974.653948] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3485974.653954] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3485974.653956] [] ? test_clear_page_writeback+0x143/0x260 [3485974.653958] [] ? test_clear_page_writeback+0x38/0x260 [3485974.653963] [] writepage_nounlock+0x2a4/0x450 [ceph] [3485974.653965] [] ? SyS_msync+0x230/0x230 [3485974.653970] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3485974.653972] [] ? wait_for_stable_page+0x16/0x50 [3485974.653978] [] ceph_write_begin+0x6c/0xc0 [ceph] [3485974.653980] [] generic_perform_write+0xc6/0x1e0 [3485974.653985] [] ceph_write_iter+0x890/0xa30 [ceph] [3485974.653987] [] ? might_fault+0x66/0xc0 [3485974.653988] [] ? might_fault+0x66/0xc0 [3485974.653990] [] ? vfs_write+0x1d3/0x1f0 [3485974.653991] [] ? vfs_write+0x1d3/0x1f0 [3485974.653993] [] ? might_fault+0x66/0xc0 [3485974.653994] [] new_sync_write+0x7b/0xb0 [3485974.653996] [] vfs_write+0xc7/0x1f0 [3485974.653998] [] SyS_write+0x52/0xb0 [3485974.653999] [] system_call_fastpath+0x1a/0x1f [3485974.654015] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3485976.063518] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3485976.126490] irq event stamp: 0 [3485976.129768] hardirqs last enabled at (0): [< (null)>] (null) [3485976.137451] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3485976.145923] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3485976.154385] softirqs last disabled at (0): [< (null)>] (null) [3485976.162072] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3485976.170924] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3485976.177682] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3485976.185405] RIP: 0010:[] [] do_raw_spin_lock+0x150/0x160 [3485976.194214] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3485976.199747] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3485976.207118] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3485976.214490] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3485976.221862] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3485976.229233] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3485976.236605] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3485976.244929] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3485976.250897] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3485976.258268] Stack: [3485976.260505] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3485976.268349] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3485976.276224] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3485976.284065] Call Trace: [3485976.286744] [] _raw_spin_lock+0x41/0x50 [3485976.292463] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3485976.299310] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3485976.305976] [] ? __init_waitqueue_head+0x60/0x60 [3485976.312473] [] ceph_read_iter+0xc9/0x570 [ceph] [3485976.318883] [] new_sync_read+0x78/0xb0 [3485976.324510] [] vfs_read+0xab/0x180 [3485976.329783] [] SyS_read+0x52/0xb0 [3485976.334972] [] system_call_fastpath+0x1a/0x1f [3485976.341199] Code: 81 48 89 df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff <0f> b7 03 66 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 [3486002.606913] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3486002.610908] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3486002.610934] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486002.610935] irq event stamp: 0 [3486002.610936] hardirqs last enabled at (0): [< (null)>] (null) [3486002.610941] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486002.610943] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486002.610944] softirqs last disabled at (0): [< (null)>] (null) [3486002.610946] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486002.610947] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486002.610951] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3486002.610952] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3486002.610957] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486002.610958] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3486002.610958] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486002.610959] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486002.610960] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3486002.610960] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3486002.610961] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3486002.610962] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3486002.610963] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3486002.610963] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3486002.610964] Stack: [3486002.610966] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3486002.610968] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3486002.610969] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3486002.610970] Call Trace: [3486002.610974] [] _raw_spin_lock+0x41/0x50 [3486002.610984] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3486002.610991] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3486002.610996] [] ceph_writepages_start+0x83/0xf90 [ceph] [3486002.610999] [] ? ext4_writepages+0x4ee/0xd30 [3486002.611005] [] ? ceph_write_inode+0x86/0x200 [ceph] [3486002.611011] [] ? ceph_write_inode+0x86/0x200 [ceph] [3486002.611013] [] ? writeback_sb_inodes+0x139/0x4f0 [3486002.611015] [] ? writeback_sb_inodes+0x139/0x4f0 [3486002.611017] [] do_writepages+0x23/0x40 [3486002.611018] [] __writeback_single_inode+0x45/0x2e0 [3486002.611020] [] writeback_sb_inodes+0x2ad/0x4f0 [3486002.611021] [] ? wb_writeback+0x54/0x350 [3486002.611023] [] __writeback_inodes_wb+0x9e/0xd0 [3486002.611024] [] wb_writeback+0x29b/0x350 [3486002.611027] [] ? get_nr_dirty_inodes+0x52/0x80 [3486002.611028] [] wb_check_old_data_flush+0x9f/0xb0 [3486002.611030] [] bdi_writeback_workfn+0x1cc/0x490 [3486002.611032] [] ? process_one_work+0x16f/0x570 [3486002.611033] [] process_one_work+0x1d1/0x570 [3486002.611035] [] ? process_one_work+0x16f/0x570 [3486002.611037] [] worker_thread+0x11c/0x530 [3486002.611038] [] ? create_and_start_worker+0x50/0x50 [3486002.611041] [] kthread+0xe4/0x100 [3486002.611043] [] ? flush_kthread_worker+0x130/0x130 [3486002.611045] [] ret_from_fork+0x7c/0xb0 [3486002.611047] [] ? flush_kthread_worker+0x130/0x130 [3486002.611063] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3486002.626901] BUG: soft lockup - CPU#2 stuck for 22s! [teuthology:19790] [3486002.626929] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486002.626929] irq event stamp: 0 [3486002.626930] hardirqs last enabled at (0): [< (null)>] (null) [3486002.626934] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486002.626935] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486002.626936] softirqs last disabled at (0): [< (null)>] (null) [3486002.626938] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486002.626939] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486002.626940] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3486002.626943] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486002.626944] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3486002.626944] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486002.626945] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486002.626946] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3486002.626946] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3486002.626947] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3486002.626948] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3486002.626949] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486002.626949] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3486002.626950] Stack: [3486002.626952] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3486002.626953] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3486002.626955] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3486002.626955] Call Trace: [3486002.626958] [] _raw_spin_lock+0x41/0x50 [3486002.626965] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3486002.626972] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3486002.626977] [] ceph_release+0x2e/0xc0 [ceph] [3486002.626980] [] __fput+0xc8/0x210 [3486002.626982] [] ____fput+0xe/0x10 [3486002.626984] [] task_work_run+0xb4/0xe0 [3486002.626986] [] do_exit+0x2e8/0xae0 [3486002.626988] [] ? kmsg_dump+0x10d/0x120 [3486002.626990] [] ? kmsg_dump+0x24/0x120 [3486002.626992] [] oops_end+0xa5/0xf0 [3486002.626993] [] die+0x58/0x90 [3486002.626994] [] do_trap+0xcd/0x160 [3486002.626997] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486002.626998] [] do_error_trap.part.16+0xa1/0xe0 [3486002.627004] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486002.627006] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3486002.627007] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486002.627009] [] ? atomic_notifier_call_chain+0x16/0x20 [3486002.627010] [] do_error_trap+0x64/0x80 [3486002.627012] [] do_invalid_op+0x20/0x30 [3486002.627013] [] invalid_op+0x18/0x20 [3486002.627020] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486002.627026] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3486002.627028] [] ? test_clear_page_writeback+0x143/0x260 [3486002.627029] [] ? test_clear_page_writeback+0x38/0x260 [3486002.627035] [] writepage_nounlock+0x2a4/0x450 [ceph] [3486002.627037] [] ? SyS_msync+0x230/0x230 [3486002.627043] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3486002.627044] [] ? wait_for_stable_page+0x16/0x50 [3486002.627050] [] ceph_write_begin+0x6c/0xc0 [ceph] [3486002.627052] [] generic_perform_write+0xc6/0x1e0 [3486002.627058] [] ceph_write_iter+0x890/0xa30 [ceph] [3486002.627059] [] ? _raw_spin_unlock+0x2b/0x40 [3486002.627061] [] ? do_wp_page+0x374/0x740 [3486002.627063] [] ? handle_mm_fault+0x86c/0x1060 [3486002.627064] [] ? __do_page_fault+0x159/0x580 [3486002.627066] [] ? vfs_write+0x1d3/0x1f0 [3486002.627067] [] ? vfs_write+0x1d3/0x1f0 [3486002.627069] [] new_sync_write+0x7b/0xb0 [3486002.627071] [] vfs_write+0xc7/0x1f0 [3486002.627072] [] SyS_write+0x52/0xb0 [3486002.627074] [] system_call_fastpath+0x1a/0x1f [3486002.627090] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3486002.638894] BUG: soft lockup - CPU#3 stuck for 22s! [teuthology:20089] [3486002.638913] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486002.638913] irq event stamp: 0 [3486002.638914] hardirqs last enabled at (0): [< (null)>] (null) [3486002.638917] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486002.638919] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486002.638920] softirqs last disabled at (0): [< (null)>] (null) [3486002.638922] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486002.638922] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486002.638923] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3486002.638926] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486002.638927] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3486002.638928] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486002.638928] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486002.638929] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3486002.638930] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3486002.638930] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3486002.638931] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3486002.638932] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486002.638933] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3486002.638933] Stack: [3486002.638935] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3486002.638937] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3486002.638938] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3486002.638938] Call Trace: [3486002.638941] [] _raw_spin_lock+0x41/0x50 [3486002.638948] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3486002.638954] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3486002.638959] [] ceph_release+0x2e/0xc0 [ceph] [3486002.638961] [] __fput+0xc8/0x210 [3486002.638963] [] ____fput+0xe/0x10 [3486002.638965] [] task_work_run+0xb4/0xe0 [3486002.638967] [] do_exit+0x2e8/0xae0 [3486002.638969] [] ? kmsg_dump+0x10d/0x120 [3486002.638970] [] ? kmsg_dump+0x24/0x120 [3486002.638971] [] oops_end+0xa5/0xf0 [3486002.638973] [] die+0x58/0x90 [3486002.638974] [] do_trap+0xcd/0x160 [3486002.638976] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486002.638977] [] do_error_trap.part.16+0xa1/0xe0 [3486002.638983] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486002.638985] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3486002.638986] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486002.638988] [] ? atomic_notifier_call_chain+0x16/0x20 [3486002.638989] [] do_error_trap+0x64/0x80 [3486002.638991] [] do_invalid_op+0x20/0x30 [3486002.638992] [] invalid_op+0x18/0x20 [3486002.638999] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486002.639005] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3486002.639007] [] ? test_clear_page_writeback+0x143/0x260 [3486002.639008] [] ? test_clear_page_writeback+0x38/0x260 [3486002.639014] [] writepage_nounlock+0x2a4/0x450 [ceph] [3486002.639015] [] ? SyS_msync+0x230/0x230 [3486002.639021] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3486002.639023] [] ? wait_for_stable_page+0x16/0x50 [3486002.639028] [] ceph_write_begin+0x6c/0xc0 [ceph] [3486002.639030] [] generic_perform_write+0xc6/0x1e0 [3486002.639036] [] ceph_write_iter+0x890/0xa30 [ceph] [3486002.639037] [] ? might_fault+0x66/0xc0 [3486002.639039] [] ? might_fault+0x66/0xc0 [3486002.639040] [] ? vfs_write+0x1d3/0x1f0 [3486002.639042] [] ? vfs_write+0x1d3/0x1f0 [3486002.639043] [] ? might_fault+0x66/0xc0 [3486002.639045] [] new_sync_write+0x7b/0xb0 [3486002.639046] [] vfs_write+0xc7/0x1f0 [3486002.639048] [] SyS_write+0x52/0xb0 [3486002.639050] [] system_call_fastpath+0x1a/0x1f [3486002.639066] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3486004.049776] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486004.112735] irq event stamp: 0 [3486004.116015] hardirqs last enabled at (0): [< (null)>] (null) [3486004.123696] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486004.132159] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486004.140635] softirqs last disabled at (0): [< (null)>] (null) [3486004.148328] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486004.157180] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486004.163938] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3486004.171658] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486004.180470] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3486004.186001] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3486004.193371] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3486004.200743] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3486004.208116] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3486004.215488] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3486004.222859] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3486004.231184] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486004.237152] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3486004.244523] Stack: [3486004.246761] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3486004.254603] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3486004.262442] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3486004.270282] Call Trace: [3486004.272959] [] _raw_spin_lock+0x41/0x50 [3486004.278679] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3486004.285520] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3486004.292186] [] ? __init_waitqueue_head+0x60/0x60 [3486004.298677] [] ceph_read_iter+0xc9/0x570 [ceph] [3486004.305085] [] new_sync_read+0x78/0xb0 [3486004.310711] [] vfs_read+0xab/0x180 [3486004.315985] [] SyS_read+0x52/0xb0 [3486004.321174] [] system_call_fastpath+0x1a/0x1f [3486004.327402] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3486030.591964] BUG: soft lockup - CPU#1 stuck for 22s! [grep:27686] [3486030.595959] BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u16:1:7152] [3486030.595988] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486030.595988] irq event stamp: 0 [3486030.595990] hardirqs last enabled at (0): [< (null)>] (null) [3486030.595995] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486030.595997] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486030.595998] softirqs last disabled at (0): [< (null)>] (null) [3486030.596000] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486030.596001] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486030.596006] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3486030.596006] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3486030.596011] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486030.596012] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3486030.596012] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486030.596013] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486030.596014] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3486030.596014] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3486030.596015] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3486030.596016] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3486030.596017] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3486030.596018] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3486030.596018] Stack: [3486030.596020] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3486030.596022] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3486030.596023] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3486030.596024] Call Trace: [3486030.596028] [] _raw_spin_lock+0x41/0x50 [3486030.596039] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3486030.596045] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3486030.596051] [] ceph_writepages_start+0x83/0xf90 [ceph] [3486030.596053] [] ? ext4_writepages+0x4ee/0xd30 [3486030.596060] [] ? ceph_write_inode+0x86/0x200 [ceph] [3486030.596066] [] ? ceph_write_inode+0x86/0x200 [ceph] [3486030.596068] [] ? writeback_sb_inodes+0x139/0x4f0 [3486030.596069] [] ? writeback_sb_inodes+0x139/0x4f0 [3486030.596071] [] do_writepages+0x23/0x40 [3486030.596073] [] __writeback_single_inode+0x45/0x2e0 [3486030.596074] [] writeback_sb_inodes+0x2ad/0x4f0 [3486030.596076] [] ? wb_writeback+0x54/0x350 [3486030.596077] [] __writeback_inodes_wb+0x9e/0xd0 [3486030.596078] [] wb_writeback+0x29b/0x350 [3486030.596081] [] ? get_nr_dirty_inodes+0x52/0x80 [3486030.596082] [] wb_check_old_data_flush+0x9f/0xb0 [3486030.596084] [] bdi_writeback_workfn+0x1cc/0x490 [3486030.596086] [] ? process_one_work+0x16f/0x570 [3486030.596087] [] process_one_work+0x1d1/0x570 [3486030.596088] [] ? process_one_work+0x16f/0x570 [3486030.596090] [] worker_thread+0x11c/0x530 [3486030.596092] [] ? create_and_start_worker+0x50/0x50 [3486030.596094] [] kthread+0xe4/0x100 [3486030.596096] [] ? flush_kthread_worker+0x130/0x130 [3486030.596097] [] ret_from_fork+0x7c/0xb0 [3486030.596099] [] ? flush_kthread_worker+0x130/0x130 [3486030.596116] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3486030.611952] BUG: soft lockup - CPU#2 stuck for 23s! [teuthology:19790] [3486030.611980] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486030.611980] irq event stamp: 0 [3486030.611981] hardirqs last enabled at (0): [< (null)>] (null) [3486030.611985] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486030.611986] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486030.611987] softirqs last disabled at (0): [< (null)>] (null) [3486030.611989] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486030.611990] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486030.611991] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3486030.611994] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3486030.611994] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3486030.611995] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486030.611996] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486030.611997] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3486030.611997] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3486030.611998] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3486030.611999] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3486030.612000] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486030.612001] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3486030.612001] Stack: [3486030.612003] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3486030.612005] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3486030.612006] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3486030.612006] Call Trace: [3486030.612009] [] _raw_spin_lock+0x41/0x50 [3486030.612016] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3486030.612023] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3486030.612028] [] ceph_release+0x2e/0xc0 [ceph] [3486030.612031] [] __fput+0xc8/0x210 [3486030.612033] [] ____fput+0xe/0x10 [3486030.612035] [] task_work_run+0xb4/0xe0 [3486030.612037] [] do_exit+0x2e8/0xae0 [3486030.612039] [] ? kmsg_dump+0x10d/0x120 [3486030.612040] [] ? kmsg_dump+0x24/0x120 [3486030.612042] [] oops_end+0xa5/0xf0 [3486030.612043] [] die+0x58/0x90 [3486030.612045] [] do_trap+0xcd/0x160 [3486030.612047] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486030.612048] [] do_error_trap.part.16+0xa1/0xe0 [3486030.612055] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486030.612056] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3486030.612057] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486030.612059] [] ? atomic_notifier_call_chain+0x16/0x20 [3486030.612060] [] do_error_trap+0x64/0x80 [3486030.612062] [] do_invalid_op+0x20/0x30 [3486030.612064] [] invalid_op+0x18/0x20 [3486030.612070] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486030.612076] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3486030.612078] [] ? test_clear_page_writeback+0x143/0x260 [3486030.612080] [] ? test_clear_page_writeback+0x38/0x260 [3486030.612086] [] writepage_nounlock+0x2a4/0x450 [ceph] [3486030.612088] [] ? SyS_msync+0x230/0x230 [3486030.612093] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3486030.612095] [] ? wait_for_stable_page+0x16/0x50 [3486030.612100] [] ceph_write_begin+0x6c/0xc0 [ceph] [3486030.612104] [] generic_perform_write+0xc6/0x1e0 [3486030.612109] [] ceph_write_iter+0x890/0xa30 [ceph] [3486030.612111] [] ? _raw_spin_unlock+0x2b/0x40 [3486030.612113] [] ? do_wp_page+0x374/0x740 [3486030.612115] [] ? handle_mm_fault+0x86c/0x1060 [3486030.612116] [] ? __do_page_fault+0x159/0x580 [3486030.612118] [] ? vfs_write+0x1d3/0x1f0 [3486030.612120] [] ? vfs_write+0x1d3/0x1f0 [3486030.612121] [] new_sync_write+0x7b/0xb0 [3486030.612123] [] vfs_write+0xc7/0x1f0 [3486030.612125] [] SyS_write+0x52/0xb0 [3486030.612126] [] system_call_fastpath+0x1a/0x1f [3486030.612142] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3486030.623944] BUG: soft lockup - CPU#3 stuck for 23s! [teuthology:20089] [3486030.623963] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486030.623964] irq event stamp: 0 [3486030.623965] hardirqs last enabled at (0): [< (null)>] (null) [3486030.623968] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486030.623970] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486030.623971] softirqs last disabled at (0): [< (null)>] (null) [3486030.623972] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486030.623973] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486030.623974] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3486030.623976] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486030.623977] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3486030.623978] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486030.623979] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486030.623980] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3486030.623980] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3486030.623981] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3486030.623982] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3486030.623983] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486030.623984] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3486030.623984] Stack: [3486030.623986] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3486030.623988] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3486030.623989] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3486030.623990] Call Trace: [3486030.623992] [] _raw_spin_lock+0x41/0x50 [3486030.623999] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3486030.624006] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3486030.624011] [] ceph_release+0x2e/0xc0 [ceph] [3486030.624013] [] __fput+0xc8/0x210 [3486030.624015] [] ____fput+0xe/0x10 [3486030.624017] [] task_work_run+0xb4/0xe0 [3486030.624018] [] do_exit+0x2e8/0xae0 [3486030.624020] [] ? kmsg_dump+0x10d/0x120 [3486030.624022] [] ? kmsg_dump+0x24/0x120 [3486030.624023] [] oops_end+0xa5/0xf0 [3486030.624024] [] die+0x58/0x90 [3486030.624026] [] do_trap+0xcd/0x160 [3486030.624027] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486030.624029] [] do_error_trap.part.16+0xa1/0xe0 [3486030.624035] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486030.624037] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3486030.624038] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486030.624040] [] ? atomic_notifier_call_chain+0x16/0x20 [3486030.624041] [] do_error_trap+0x64/0x80 [3486030.624042] [] do_invalid_op+0x20/0x30 [3486030.624044] [] invalid_op+0x18/0x20 [3486030.624050] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486030.624056] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3486030.624058] [] ? test_clear_page_writeback+0x143/0x260 [3486030.624060] [] ? test_clear_page_writeback+0x38/0x260 [3486030.624065] [] writepage_nounlock+0x2a4/0x450 [ceph] [3486030.624067] [] ? SyS_msync+0x230/0x230 [3486030.624073] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3486030.624074] [] ? wait_for_stable_page+0x16/0x50 [3486030.624080] [] ceph_write_begin+0x6c/0xc0 [ceph] [3486030.624082] [] generic_perform_write+0xc6/0x1e0 [3486030.624087] [] ceph_write_iter+0x890/0xa30 [ceph] [3486030.624089] [] ? might_fault+0x66/0xc0 [3486030.624091] [] ? might_fault+0x66/0xc0 [3486030.624092] [] ? vfs_write+0x1d3/0x1f0 [3486030.624094] [] ? vfs_write+0x1d3/0x1f0 [3486030.624095] [] ? might_fault+0x66/0xc0 [3486030.624097] [] new_sync_write+0x7b/0xb0 [3486030.624098] [] vfs_write+0xc7/0x1f0 [3486030.624100] [] SyS_write+0x52/0xb0 [3486030.624101] [] system_call_fastpath+0x1a/0x1f [3486030.624118] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3486032.034798] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486032.097856] irq event stamp: 0 [3486032.101135] hardirqs last enabled at (0): [< (null)>] (null) [3486032.108814] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486032.117281] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486032.125742] softirqs last disabled at (0): [< (null)>] (null) [3486032.133427] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486032.142270] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486032.149018] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3486032.156733] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486032.165547] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3486032.171079] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3486032.178451] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3486032.185820] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3486032.193191] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3486032.200565] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3486032.207935] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3486032.216262] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486032.222229] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3486032.229598] Stack: [3486032.231836] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3486032.239671] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3486032.247512] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3486032.255350] Call Trace: [3486032.258026] [] _raw_spin_lock+0x41/0x50 [3486032.263749] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3486032.270589] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3486032.277254] [] ? __init_waitqueue_head+0x60/0x60 [3486032.283748] [] ceph_read_iter+0xc9/0x570 [ceph] [3486032.290159] [] new_sync_read+0x78/0xb0 [3486032.295788] [] vfs_read+0xab/0x180 [3486032.301065] [] SyS_read+0x52/0xb0 [3486032.306251] [] system_call_fastpath+0x1a/0x1f [3486032.312478] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3486058.577018] BUG: soft lockup - CPU#1 stuck for 23s! [grep:27686] [3486058.581011] BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u16:1:7152] [3486058.581042] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486058.581043] irq event stamp: 0 [3486058.581045] hardirqs last enabled at (0): [< (null)>] (null) [3486058.581050] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486058.581052] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486058.581052] softirqs last disabled at (0): [< (null)>] (null) [3486058.581055] CPU: 0 PID: 7152 Comm: kworker/u16:1 Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486058.581056] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486058.581060] Workqueue: writeback bdi_writeback_workfn (flush-ceph-1) [3486058.581061] task: ffff8802796ec3e0 ti: ffff8801018e8000 task.ti: ffff8801018e8000 [3486058.581066] RIP: 0010:[] [] do_raw_spin_lock+0x153/0x160 [3486058.581066] RSP: 0018:ffff8801018eb778 EFLAGS: 00000202 [3486058.581067] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486058.581068] RDX: 0000000000008918 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486058.581069] RBP: ffff8801018eb798 R08: 0000000000000000 R09: 0000000000000000 [3486058.581069] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801018eb6e8 [3486058.581070] R13: ffff88043fc13b40 R14: ffff8801018e8000 R15: 0000000000000001 [3486058.581071] FS: 0000000000000000(0000) GS:ffff88043fc00000(0000) knlGS:0000000000000000 [3486058.581072] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [3486058.581073] CR2: 00007f57d7d74000 CR3: 0000000001c11000 CR4: 00000000000007f0 [3486058.581073] Stack: [3486058.581075] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3486058.581077] ffff8801018eb7c8 ffffffff81728e31 ffffffffa038d3bf ffff880003d18b30 [3486058.581078] ffff8801018eb7f8 0000000000000016 ffff8801018eb7f8 ffffffffa038d3bf [3486058.581079] Call Trace: [3486058.581083] [] _raw_spin_lock+0x41/0x50 [3486058.581093] [] ? ceph_caps_revoking+0x2f/0x90 [ceph] [3486058.581100] [] ceph_caps_revoking+0x2f/0x90 [ceph] [3486058.581105] [] ceph_writepages_start+0x83/0xf90 [ceph] [3486058.581108] [] ? ext4_writepages+0x4ee/0xd30 [3486058.581114] [] ? ceph_write_inode+0x86/0x200 [ceph] [3486058.581121] [] ? ceph_write_inode+0x86/0x200 [ceph] [3486058.581122] [] ? writeback_sb_inodes+0x139/0x4f0 [3486058.581124] [] ? writeback_sb_inodes+0x139/0x4f0 [3486058.581126] [] do_writepages+0x23/0x40 [3486058.581128] [] __writeback_single_inode+0x45/0x2e0 [3486058.581129] [] writeback_sb_inodes+0x2ad/0x4f0 [3486058.581130] [] ? wb_writeback+0x54/0x350 [3486058.581132] [] __writeback_inodes_wb+0x9e/0xd0 [3486058.581133] [] wb_writeback+0x29b/0x350 [3486058.581136] [] ? get_nr_dirty_inodes+0x52/0x80 [3486058.581137] [] wb_check_old_data_flush+0x9f/0xb0 [3486058.581139] [] bdi_writeback_workfn+0x1cc/0x490 [3486058.581141] [] ? process_one_work+0x16f/0x570 [3486058.581143] [] process_one_work+0x1d1/0x570 [3486058.581144] [] ? process_one_work+0x16f/0x570 [3486058.581146] [] worker_thread+0x11c/0x530 [3486058.581147] [] ? create_and_start_worker+0x50/0x50 [3486058.581150] [] kthread+0xe4/0x100 [3486058.581152] [] ? flush_kthread_worker+0x130/0x130 [3486058.581154] [] ret_from_fork+0x7c/0xb0 [3486058.581156] [] ? flush_kthread_worker+0x130/0x130 [3486058.581172] Code: df e8 85 85 66 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 <66> 39 c2 74 f3 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 [3486058.597004] BUG: soft lockup - CPU#2 stuck for 23s! [teuthology:19790] [3486058.597034] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486058.597034] irq event stamp: 0 [3486058.597036] hardirqs last enabled at (0): [< (null)>] (null) [3486058.597040] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486058.597043] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486058.597044] softirqs last disabled at (0): [< (null)>] (null) [3486058.597046] CPU: 2 PID: 19790 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486058.597047] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486058.597048] task: ffff8803e1f621f0 ti: ffff8803e4ae4000 task.ti: ffff8803e4ae4000 [3486058.597052] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486058.597053] RSP: 0018:ffff8803e4ae7578 EFLAGS: 00000202 [3486058.597054] RAX: 0000000000008916 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486058.597055] RDX: 0000000000008917 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486058.597056] RBP: ffff8803e4ae7598 R08: 0000000000000002 R09: 0000000000000000 [3486058.597057] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8803e4ae74e8 [3486058.597058] R13: ffff88043fc93b40 R14: ffff8803e4ae4000 R15: 0000000000000001 [3486058.597060] FS: 0000000000000000(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000 [3486058.597061] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486058.597062] CR2: 00000000032fd000 CR3: 0000000001c11000 CR4: 00000000000007e0 [3486058.597063] Stack: [3486058.597065] ffff8800049b5820 ffff8800049b5830 ffff8800049b5820 ffff8800049b5830 [3486058.597067] ffff8803e4ae75c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3486058.597069] 0000000000000000 ffff88003b3bf7c0 ffff8803e4ae75f8 ffffffffa03925cf [3486058.597070] Call Trace: [3486058.597074] [] _raw_spin_lock+0x41/0x50 [3486058.597084] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3486058.597092] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3486058.597100] [] ceph_release+0x2e/0xc0 [ceph] [3486058.597103] [] __fput+0xc8/0x210 [3486058.597105] [] ____fput+0xe/0x10 [3486058.597108] [] task_work_run+0xb4/0xe0 [3486058.597111] [] do_exit+0x2e8/0xae0 [3486058.597114] [] ? kmsg_dump+0x10d/0x120 [3486058.597116] [] ? kmsg_dump+0x24/0x120 [3486058.597119] [] oops_end+0xa5/0xf0 [3486058.597120] [] die+0x58/0x90 [3486058.597122] [] do_trap+0xcd/0x160 [3486058.597125] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486058.597127] [] do_error_trap.part.16+0xa1/0xe0 [3486058.597134] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486058.597137] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3486058.597138] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486058.597140] [] ? atomic_notifier_call_chain+0x16/0x20 [3486058.597142] [] do_error_trap+0x64/0x80 [3486058.597144] [] do_invalid_op+0x20/0x30 [3486058.597146] [] invalid_op+0x18/0x20 [3486058.597154] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486058.597161] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3486058.597164] [] ? test_clear_page_writeback+0x143/0x260 [3486058.597166] [] ? test_clear_page_writeback+0x38/0x260 [3486058.597173] [] writepage_nounlock+0x2a4/0x450 [ceph] [3486058.597175] [] ? SyS_msync+0x230/0x230 [3486058.597182] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3486058.597184] [] ? wait_for_stable_page+0x16/0x50 [3486058.597191] [] ceph_write_begin+0x6c/0xc0 [ceph] [3486058.597194] [] generic_perform_write+0xc6/0x1e0 [3486058.597201] [] ceph_write_iter+0x890/0xa30 [ceph] [3486058.597203] [] ? _raw_spin_unlock+0x2b/0x40 [3486058.597206] [] ? do_wp_page+0x374/0x740 [3486058.597208] [] ? handle_mm_fault+0x86c/0x1060 [3486058.597210] [] ? __do_page_fault+0x159/0x580 [3486058.597213] [] ? vfs_write+0x1d3/0x1f0 [3486058.597215] [] ? vfs_write+0x1d3/0x1f0 [3486058.597217] [] new_sync_write+0x7b/0xb0 [3486058.597219] [] vfs_write+0xc7/0x1f0 [3486058.597221] [] SyS_write+0x52/0xb0 [3486058.597223] [] system_call_fastpath+0x1a/0x1f [3486058.597246] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 [3486058.608999] BUG: soft lockup - CPU#3 stuck for 23s! [teuthology:20089] [3486058.609029] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486058.609029] irq event stamp: 0 [3486058.609031] hardirqs last enabled at (0): [< (null)>] (null) [3486058.609035] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486058.609037] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486058.609039] softirqs last disabled at (0): [< (null)>] (null) [3486058.609041] CPU: 3 PID: 20089 Comm: teuthology Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486058.609042] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486058.609043] task: ffff8803c46321f0 ti: ffff88036eb30000 task.ti: ffff88036eb30000 [3486058.609047] RIP: 0010:[] [] do_raw_spin_lock+0x158/0x160 [3486058.609048] RSP: 0018:ffff88036eb33578 EFLAGS: 00000202 [3486058.609049] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000000000004 [3486058.609050] RDX: 000000000000e224 RSI: 0000000000000100 RDI: ffffffff81d21a20 [3486058.609051] RBP: ffff88036eb33598 R08: 0000000000000003 R09: 0000000000000000 [3486058.609052] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88036eb334e8 [3486058.609053] R13: ffff88043fcd3b40 R14: ffff88036eb30000 R15: 0000000000000001 [3486058.609054] FS: 0000000000000000(0000) GS:ffff88043fcc0000(0000) knlGS:0000000000000000 [3486058.609055] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486058.609056] CR2: 00007f22056c49b8 CR3: 0000000001c11000 CR4: 00000000000007e0 [3486058.609057] Stack: [3486058.609060] ffff880224754680 ffff880224754690 ffff880224754680 ffff880224754690 [3486058.609062] ffff88036eb335c8 ffffffff81728e31 ffffffffa03925cf ffff88042465ea00 [3486058.609064] 0000000000000000 ffff88003b3bf6c0 ffff88036eb335f8 ffffffffa03925cf [3486058.609065] Call Trace: [3486058.609068] [] _raw_spin_lock+0x41/0x50 [3486058.609079] [] ? ceph_put_fmode+0x2f/0xd0 [ceph] [3486058.609087] [] ceph_put_fmode+0x2f/0xd0 [ceph] [3486058.609094] [] ceph_release+0x2e/0xc0 [ceph] [3486058.609097] [] __fput+0xc8/0x210 [3486058.609099] [] ____fput+0xe/0x10 [3486058.609101] [] task_work_run+0xb4/0xe0 [3486058.609104] [] do_exit+0x2e8/0xae0 [3486058.609107] [] ? kmsg_dump+0x10d/0x120 [3486058.609108] [] ? kmsg_dump+0x24/0x120 [3486058.609111] [] oops_end+0xa5/0xf0 [3486058.609113] [] die+0x58/0x90 [3486058.609115] [] do_trap+0xcd/0x160 [3486058.609117] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486058.609119] [] do_error_trap.part.16+0xa1/0xe0 [3486058.609127] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486058.609130] [] ? __atomic_notifier_call_chain+0x8b/0xb0 [3486058.609131] [] ? __atomic_notifier_call_chain+0x5/0xb0 [3486058.609134] [] ? atomic_notifier_call_chain+0x16/0x20 [3486058.609136] [] do_error_trap+0x64/0x80 [3486058.609138] [] do_invalid_op+0x20/0x30 [3486058.609140] [] invalid_op+0x18/0x20 [3486058.609149] [] ? ceph_put_wrbuffer_cap_refs+0x120/0x360 [ceph] [3486058.609157] [] ? ceph_put_wrbuffer_cap_refs+0x39/0x360 [ceph] [3486058.609159] [] ? test_clear_page_writeback+0x143/0x260 [3486058.609161] [] ? test_clear_page_writeback+0x38/0x260 [3486058.609169] [] writepage_nounlock+0x2a4/0x450 [ceph] [3486058.609171] [] ? SyS_msync+0x230/0x230 [3486058.609178] [] ceph_update_writeable_page+0x150/0x480 [ceph] [3486058.609180] [] ? wait_for_stable_page+0x16/0x50 [3486058.609187] [] ceph_write_begin+0x6c/0xc0 [ceph] [3486058.609190] [] generic_perform_write+0xc6/0x1e0 [3486058.609198] [] ceph_write_iter+0x890/0xa30 [ceph] [3486058.609200] [] ? might_fault+0x66/0xc0 [3486058.609202] [] ? might_fault+0x66/0xc0 [3486058.609205] [] ? vfs_write+0x1d3/0x1f0 [3486058.609207] [] ? vfs_write+0x1d3/0x1f0 [3486058.609209] [] ? might_fault+0x66/0xc0 [3486058.609211] [] new_sync_write+0x7b/0xb0 [3486058.609213] [] vfs_write+0xc7/0x1f0 [3486058.609215] [] SyS_write+0x52/0xb0 [3486058.609217] [] system_call_fastpath+0x1a/0x1f [3486058.609240] Code: 00 bf 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 90 eb f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 [3486060.019945] Modules linked in: ipmi_devintf(E) ipmi_si(E) ipmi_msghandler(E) ip6table_filter(E) ip6_tables(E) ebtable_nat(E) ebtables(E) ipt_MASQUERADE(E) iptable_nat(E) nf_nat_ipv4(E) nf_nat(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) xt_state(E) nf_conntrack(E) ipt_REJECT(E) xt_CHECKSUM(E) iptable_mangle(E) xt_tcpudp(E) iptable_filter(E) ip_tables(E) x_tables(E) bridge(E) stp(E) llc(E) ceph(E) gpio_ich(E) i7core_edac(E) libceph(E) edac_core(E) psmouse(E) serio_raw(E) joydev(E) fscache(E) lpc_ich(E) tpm_infineon(E) tpm_tis(E) xfs(E) lp(E) parport(E) hid_generic(E) usbhid(E) hid(E) btrfs(E) e1000e(E) ptp(E) raid6_pq(E) ahci(E) libahci(E) pps_core(E) arcmsr(E) xor(E) libcrc32c(E) [3486060.082985] irq event stamp: 0 [3486060.086266] hardirqs last enabled at (0): [< (null)>] (null) [3486060.093956] hardirqs last disabled at (0): [] copy_process+0x78d/0x22f0 [3486060.102427] softirqs last enabled at (0): [] copy_process+0x78d/0x22f0 [3486060.110900] softirqs last disabled at (0): [< (null)>] (null) [3486060.118590] CPU: 1 PID: 27686 Comm: grep Tainted: G D E 3.16.3-ceph-00306-g76c7fd1 #1 [3486060.127439] Hardware name: Supermicro X8SIL/X8SIL, BIOS 1.1 05/27/2010 [3486060.134192] task: ffff8804275043e0 ti: ffff88033a8ac000 task.ti: ffff88033a8ac000 [3486060.141916] RIP: 0010:[] [] do_raw_spin_lock+0x15a/0x160 [3486060.150735] RSP: 0018:ffff88033a8afcd8 EFLAGS: 00000202 [3486060.156267] RAX: 000000000000e223 RBX: ffffffff8172aa60 RCX: 0000000057c957c8 [3486060.163638] RDX: 000000000000e225 RSI: 0000000000000001 RDI: 0000000000000001 [3486060.171009] RBP: ffff88033a8afcf8 R08: 0000000000000001 R09: 0000000000000000 [3486060.178382] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88033a8afc48 [3486060.185754] R13: ffff88043fc53b40 R14: ffff88033a8ac000 R15: 0000000000000001 [3486060.193124] FS: 00007f0bcbcfd700(0000) GS:ffff88043fc40000(0000) knlGS:0000000000000000 [3486060.201451] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [3486060.207420] CR2: 00000000022f5198 CR3: 000000020b984000 CR4: 00000000000007e0 [3486060.214790] Stack: [3486060.217028] 0000000000000c00 ffff880224754690 0000000000000c00 ffff88033a8aff50 [3486060.224887] ffff88033a8afd28 ffffffff81728e31 ffffffffa0390bee 0000000000001000 [3486060.232738] 0000000000008000 ffff880224754680 ffff88033a8afd88 ffffffffa0390bee [3486060.240584] Call Trace: [3486060.243261] [] _raw_spin_lock+0x41/0x50 [3486060.248983] [] ? ceph_put_cap_refs+0x3e/0x270 [ceph] [3486060.255829] [] ceph_put_cap_refs+0x3e/0x270 [ceph] [3486060.262498] [] ? __init_waitqueue_head+0x60/0x60 [3486060.269002] [] ceph_read_iter+0xc9/0x570 [ceph] [3486060.275416] [] new_sync_read+0x78/0xb0 [3486060.281043] [] vfs_read+0xab/0x180 [3486060.286326] [] SyS_read+0x52/0xb0 [3486060.291524] [] system_call_fastpath+0x1a/0x1f [3486060.297759] Code: 01 00 00 00 e8 58 38 f8 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 75 05 e9 70 ff ff ff 0f b7 03 66 39 c2 74 f3 f3 90 f4 0f 1f 40 00 66 66 66 66 90 8b 17 55 31 c9 48 89 e5 89 d0 ~. [terminated ipmitool] silver:~ john$