Bug #107
lockup on __cap_is_valid (via aio_write) vs __ceph_remove_cap
Description
on master. rsync workload.
[168476.538425] BUG: soft lockup - CPU#0 stuck for 61s! [kswapd0:318]
[168476.538430] Modules linked in: aes_x86_64 aes_generic ceph fan ac battery psmouse ohci_hcd ide_pci_generic ehci_hcd thermal button processor
[168476.538430] irq event stamp: 1474370159
[168476.538430] hardirqs last enabled at (1474370159): [<ffffffff81072e13>] __call_rcu+0x119/0x125
[168476.538430] hardirqs last disabled at (1474370158): [<ffffffff81072d22>] __call_rcu+0x28/0x125
[168476.538430] softirqs last enabled at (1474370062): [<ffffffff8103ab8f>] __do_softirq+0x147/0x156
[168476.538430] softirqs last disabled at (1474370049): [<ffffffff8100388c>] call_softirq+0x1c/0x34
[168476.538430] CPU 0
[168476.538430] Modules linked in: aes_x86_64 aes_generic ceph fan ac battery psmouse ohci_hcd ide_pci_generic ehci_hcd thermal button processor
[168476.538430]
[168476.538430] Pid: 318, comm: kswapd0 Tainted: G D 2.6.34-rc3 #26 H8SSL/H8SSL
[168476.538430] RIP: 0010:[<ffffffff81228220>] [<ffffffff81228220>] debug_smp_processor_id+0x0/0xd8
[168476.538430] RSP: 0018:ffff8800f8363a08 EFLAGS: 00000202
[168476.538430] RAX: ffff8800f8363fd8 RBX: ffff8800f8363a40 RCX: 0000000000007601
[168476.538430] RDX: 0000000000005f00 RSI: 0000000000000002 RDI: 0000000000000001
[168476.538430] RBP: ffffffff8100334e R08: ffff77adf83638f0 R09: 0000000000000000
[168476.538430] R10: ffffffff8104ea18 R11: ffffffff8104e457 R12: ffff8800f8362000
[168476.538430] R13: ffff8800f8363fd8 R14: 0000000000000002 R15: ffffffff8103f845
[168476.538430] FS: 00007fa089b8d6e0(0000) GS:ffff880002600000(0000) knlGS:0000000000000000
[168476.538430] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[168476.538430] CR2: 00007fffab7cdff8 CR3: 00000000f4de8000 CR4: 00000000000006f0
[168476.538430] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[168476.538430] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[168476.538430] Process kswapd0 (pid: 318, threadinfo ffff8800f8362000, task ffff8800f83602c0)
[168476.538430] Stack:
[168476.538430] ffffffff812241ef 0000000000000002 ffff8800f7de1908 000000007787870a
[168476.538430] <0> ffff8800f83602c0 0000000000000000 ffff8800f8360660 ffff8800f8363a50
[168476.538430] <0> ffffffff81224115 ffff8800f8363a90 ffffffff81227fec 0000000082cf4342
[168476.538430] Call Trace:
[168476.538430] [<ffffffff812241ef>] ? delay_tsc+0x23/0xc4
[168476.538430] [<ffffffff81224115>] ? __delay+0xa/0xc
[168476.538430] [<ffffffff81227fec>] ? do_raw_spin_lock+0xd6/0x135
[168476.538430] [<ffffffff814278c3>] ? _raw_spin_lock+0x5c/0x6f
[168476.538430] [<ffffffffa008330a>] ? __ceph_remove_cap+0x8b/0x220 [ceph]
[168476.538430] [<ffffffffa008330a>] ? __ceph_remove_cap+0x8b/0x220 [ceph]
[168476.538430] [<ffffffffa008409f>] ? ceph_queue_caps_release+0x19c/0x263 [ceph]
[168476.538430] [<ffffffffa007634c>] ? ceph_destroy_inode+0x3c/0x1b8 [ceph]
[168476.538430] [<ffffffff810c1f7e>] ? destroy_inode+0x2a/0x3f
[168476.538430] [<ffffffff810c20f5>] ? generic_drop_inode+0x52/0x56
[168476.538430] [<ffffffff810c16fb>] ? iput+0x61/0x65
[168476.538430] [<ffffffff810bee8c>] ? dentry_iput+0xcc/0xe4
[168476.538430] [<ffffffff810beec8>] ? d_kill+0x24/0x45
[168476.538430] [<ffffffff810bf115>] ? __shrink_dcache_sb+0x22c/0x2e4
[168476.538430] [<ffffffff810bf2b9>] ? shrink_dcache_memory+0xdd/0x18b
[168476.538430] [<ffffffff810bf2ca>] ? shrink_dcache_memory+0xee/0x18b
[168476.538430] [<ffffffff81089818>] ? shrink_slab+0xdb/0x15e
[168476.538430] [<ffffffff8108a3c9>] ? kswapd+0x49d/0x682
[168476.538430] [<ffffffff81087a10>] ? isolate_pages_global+0x0/0x209
[168476.538430] [<ffffffff8104a8b0>] ? autoremove_wake_function+0x0/0x38
[168476.538430] [<ffffffff814283cc>] ? _raw_spin_unlock_irqrestore+0x63/0x68
[168476.538430] [<ffffffff81089f2c>] ? kswapd+0x0/0x682
[168476.538430] [<ffffffff8104a57e>] ? kthread+0x7d/0x85
[168476.538430] [<ffffffff81003794>] ? kernel_thread_helper+0x4/0x10
[168476.538430] [<ffffffff81428700>] ? restore_args+0x0/0x30
[168476.538430] [<ffffffff8104a501>] ? kthread+0x0/0x85
[168476.538430] [<ffffffff81003790>] ? kernel_thread_helper+0x0/0x10
[168476.538430] Code: 89 d0 48 c1 e8 04 48 01 d0 48 ba 0f 0f 0f 0f 0f 0f 0f 0f 48 21 d0 48 ba 01 01 01 01 01 01 01 01 48 0f af c2 48 c1 e8 38 c3 90 90 <55> 48 89 e5 41 56 65 44 8b 34 25 e8 d1 00 00 41 55 65 4c 8b 2c
[168476.538430] Call Trace:
[168476.538430] [<ffffffff812241ef>] ? delay_tsc+0x23/0xc4
[168476.538430] [<ffffffff81224115>] ? __delay+0xa/0xc
[168476.538430] [<ffffffff81227fec>] ? do_raw_spin_lock+0xd6/0x135
[168476.538430] [<ffffffff814278c3>] ? _raw_spin_lock+0x5c/0x6f
[168476.538430] [<ffffffffa008330a>] ? __ceph_remove_cap+0x8b/0x220 [ceph]
[168476.538430] [<ffffffffa008330a>] ? __ceph_remove_cap+0x8b/0x220 [ceph]
[168476.538430] [<ffffffffa008409f>] ? ceph_queue_caps_release+0x19c/0x263 [ceph]
[168476.538430] [<ffffffffa007634c>] ? ceph_destroy_inode+0x3c/0x1b8 [ceph]
[168476.538430] [<ffffffff810c1f7e>] ? destroy_inode+0x2a/0x3f
[168476.538430] [<ffffffff810c20f5>] ? generic_drop_inode+0x52/0x56
[168476.538430] [<ffffffff810c16fb>] ? iput+0x61/0x65
[168476.538430] [<ffffffff810bee8c>] ? dentry_iput+0xcc/0xe4
[168476.538430] [<ffffffff810beec8>] ? d_kill+0x24/0x45
[168476.538430] [<ffffffff810bf115>] ? __shrink_dcache_sb+0x22c/0x2e4
[168476.538430] [<ffffffff810bf2b9>] ? shrink_dcache_memory+0xdd/0x18b
[168476.538430] [<ffffffff810bf2ca>] ? shrink_dcache_memory+0xee/0x18b
[168476.538430] [<ffffffff81089818>] ? shrink_slab+0xdb/0x15e
[168476.538430] [<ffffffff8108a3c9>] ? kswapd+0x49d/0x682
[168476.538430] [<ffffffff81087a10>] ? isolate_pages_global+0x0/0x209
[168476.538430] [<ffffffff8104a8b0>] ? autoremove_wake_function+0x0/0x38
[168476.538430] [<ffffffff814283cc>] ? _raw_spin_unlock_irqrestore+0x63/0x68
[168476.538430] [<ffffffff81089f2c>] ? kswapd+0x0/0x682
[168476.538430] [<ffffffff8104a57e>] ? kthread+0x7d/0x85
[168476.538430] [<ffffffff81003794>] ? kernel_thread_helper+0x4/0x10
[168476.538430] [<ffffffff81428700>] ? restore_args+0x0/0x30
[168476.538430] [<ffffffff8104a501>] ? kthread+0x0/0x85
[168476.538430] [<ffffffff81003790>] ? kernel_thread_helper+0x0/0x10
[168476.538489] BUG: soft lockup - CPU#1 stuck for 61s! [rsync:3249]
[168476.538489] Modules linked in: aes_x86_64 aes_generic ceph fan ac battery psmouse ohci_hcd ide_pci_generic ehci_hcd thermal button processor
[168476.538489] irq event stamp: 209302094
[168476.538489] hardirqs last enabled at (209302093): [<ffffffff81425688>] preempt_schedule_irq+0x38/0x5b
[168476.538489] hardirqs last disabled at (209302094): [<ffffffff8142796f>] _raw_spin_lock_irq+0x14/0x77
[168476.538489] softirqs last enabled at (209302092): [<ffffffff8103ab8f>] __do_softirq+0x147/0x156
[168476.538489] softirqs last disabled at (209302031): [<ffffffff8100388c>] call_softirq+0x1c/0x34
[168476.538489] CPU 1
[168476.538489] Modules linked in: aes_x86_64 aes_generic ceph fan ac battery psmouse ohci_hcd ide_pci_generic ehci_hcd thermal button processor
[168476.538489]
[168476.538489] Pid: 3249, comm: rsync Tainted: G D 2.6.34-rc3 #26 H8SSL/H8SSL
[168476.538489] RIP: 0010:[<ffffffff812282ea>] [<ffffffff812282ea>] debug_smp_processor_id+0xca/0xd8
[168476.538489] RSP: 0018:ffff8800f4db7918 EFLAGS: 00000206
[168476.538489] RAX: ffff8800f4db7fd8 RBX: ffff8800f4db7938 RCX: 0000000001062560
[168476.538489] RDX: 0000000000005f00 RSI: 0000000000000002 RDI: 0000000000000001
[168476.538489] RBP: ffffffff8100334e R08: ffffba9ef4db7828 R09: 0000000000000000
[168476.538489] R10: ffffffff8104ea18 R11: ffffffff8104e457 R12: 0000000000000001
[168476.538489] R13: 0000000000000000 R14: ffffffff81427645 R15: ffff8800f4db7978
[168476.538489] FS: 00007fa089b8d6e0(0000) GS:ffff880002800000(0000) knlGS:0000000000000000
[168476.538489] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[168476.538489] CR2: 00000000000008a8 CR3: 00000000f4de8000 CR4: 00000000000006e0
[168476.538489] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[168476.538489] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[168476.538489] Process rsync (pid: 3249, threadinfo ffff8800f4db6000, task ffff880087ffe240)
[168476.538489] Stack:
[168476.538489] ffff8800f7de1908 0000000000000001 ffff880087ffe240 0000000000000000
[168476.538489] <0> ffff8800f4db7978 ffffffff812241ef 0000000000005f00 ffff8800f7de1908
[168476.538489] <0> 000000002a8536f6 ffff880087ffe240 0000000000000000 ffff880087ffe5e0
[168476.538489] Call Trace:
[168476.538489] [<ffffffff812241ef>] ? delay_tsc+0x23/0xc4
[168476.538489] [<ffffffff81224115>] ? __delay+0xa/0xc
[168476.538489] [<ffffffff81227fec>] ? do_raw_spin_lock+0xd6/0x135
[168476.538489] [<ffffffff814278c3>] ? _raw_spin_lock+0x5c/0x6f
[168476.538489] [<ffffffffa007fd1d>] ? __cap_is_valid+0x24/0xda [ceph]
[168476.538489] [<ffffffffa007fd1d>] ? __cap_is_valid+0x24/0xda [ceph]
[168476.538489] [<ffffffffa008207a>] ? ceph_check_caps+0x81/0xb66 [ceph]
[168476.538489] [<ffffffffa007ffa5>] ? __ceph_caps_issued+0x5c/0xee [ceph]
[168476.538489] [<ffffffffa0082153>] ? ceph_check_caps+0x15a/0xb66 [ceph]
[168476.538489] [<ffffffff81057502>] ? print_lock_contention_bug+0x1b/0xe1
[168476.538489] [<ffffffffa008493e>] ? ceph_put_cap_refs+0x35/0x278 [ceph]
[168476.538489] [<ffffffffa0084ae6>] ? ceph_put_cap_refs+0x1dd/0x278 [ceph]
[168476.538489] [<ffffffffa007b2cd>] ? ceph_aio_write+0x8cb/0xb19 [ceph]
[168476.538489] [<ffffffff81094ffe>] ? might_fault+0x63/0xb3
[168476.538489] [<ffffffff8138793e>] ? sock_aio_read+0x10a/0x11e
[168476.538489] [<ffffffff810af4d1>] ? do_sync_write+0xcb/0x112
[168476.538489] [<ffffffff81095047>] ? might_fault+0xac/0xb3
[168476.538489] [<ffffffff810bd37c>] ? poll_select_copy_remaining+0xdd/0x103
[168476.538489] [<ffffffff810afc49>] ? vfs_write+0xae/0x14a
[168476.538489] [<ffffffff810b0155>] ? sys_write+0x47/0x6f
[168476.538489] [<ffffffff810029eb>] ? system_call_fastpath+0x16/0x1b
[168476.538489] Code: 03 00 00 48 c7 c7 89 18 59 81 31 c0 ff ce e8 54 e3 e0 ff 48 8b 75 08 48 c7 c7 cd 18 59 81 e8 21 d9 e3 ff e8 2e ea dd ff ff 4b 1c <5b> 41 5c 41 5d 44 89 f0 41 5e c9 c3 90 90 55 48 89 e5 53 48 89
[168476.538489] Call Trace:
[168476.538489] [<ffffffff812241ef>] ? delay_tsc+0x23/0xc4
[168476.538489] [<ffffffff81224115>] ? __delay+0xa/0xc
[168476.538489] [<ffffffff81227fec>] ? do_raw_spin_lock+0xd6/0x135
[168476.538489] [<ffffffff814278c3>] ? _raw_spin_lock+0x5c/0x6f
[168476.538489] [<ffffffffa007fd1d>] ? __cap_is_valid+0x24/0xda [ceph]
[168476.538489] [<ffffffffa007fd1d>] ? __cap_is_valid+0x24/0xda [ceph]
[168476.538489] [<ffffffffa008207a>] ? ceph_check_caps+0x81/0xb66 [ceph]
[168476.538489] [<ffffffffa007ffa5>] ? __ceph_caps_issued+0x5c/0xee [ceph]
[168476.538489] [<ffffffffa0082153>] ? ceph_check_caps+0x15a/0xb66 [ceph]
[168476.538489] [<ffffffff81057502>] ? print_lock_contention_bug+0x1b/0xe1
[168476.538489] [<ffffffffa008493e>] ? ceph_put_cap_refs+0x35/0x278 [ceph]
[168476.538489] [<ffffffffa0084ae6>] ? ceph_put_cap_refs+0x1dd/0x278 [ceph]
[168476.538489] [<ffffffffa007b2cd>] ? ceph_aio_write+0x8cb/0xb19 [ceph]
[168476.538489] [<ffffffff81094ffe>] ? might_fault+0x63/0xb3
[168476.538489] [<ffffffff8138793e>] ? sock_aio_read+0x10a/0x11e
[168476.538489] [<ffffffff810af4d1>] ? do_sync_write+0xcb/0x112
[168476.538489] [<ffffffff81095047>] ? might_fault+0xac/0xb3
[168476.538489] [<ffffffff810bd37c>] ? poll_select_copy_remaining+0xdd/0x103
[168476.538489] [<ffffffff810afc49>] ? vfs_write+0xae/0x14a
[168476.538489] [<ffffffff810b0155>] ? sys_write+0x47/0x6f
[168476.538489] [<ffffffff810029eb>] ? system_call_fastpath+0x16/0x1b
History
#1 Updated by Sage Weil almost 14 years ago
hmm, s_cap_lock usage sites look okay... don't think it's a leaked spinlock
#2 Updated by Sage Weil almost 14 years ago
again:
[77092.515725] BUG: soft lockup - CPU#0 stuck for 61s! [kswapd0:318] [77092.515729] Modules linked in: aes_x86_64 aes_generic ceph [77092.515729] irq event stamp: 580263187 [77092.515729] hardirqs last enabled at (580263187): [<ffffffff81072fef>] __call_rcu+0x119/0x125 [77092.515729] hardirqs last disabled at (580263186): [<ffffffff81072efe>] __call_rcu+0x28/0x125 [77092.515729] softirqs last enabled at (580262350): [<ffffffff8103ac63>] __do_softirq+0x147/0x156 [77092.515729] softirqs last disabled at (580262337): [<ffffffff810038cc>] call_softirq+0x1c/0x34 [77092.515729] CPU 0 [77092.515729] Modules linked in: aes_x86_64 aes_generic ceph [77092.515729] [77092.515729] Pid: 318, comm: kswapd0 Tainted: G D 2.6.34-rc6 #27 H8SSL/H8SSL [77092.515729] RIP: 0010:[<ffffffff81224945>] [<ffffffff81224945>] delay_tsc+0x29/0xc4 [77092.515729] RSP: 0018:ffff8800f8363a10 EFLAGS: 00000202 [77092.515729] RAX: 0000000000000000 RBX: ffff8800f8363a40 RCX: 0000000001062560 [77092.515729] RDX: 0000000000007000 RSI: 0000000000000002 RDI: 0000000000000001 [77092.515729] RBP: ffffffff8100338e R08: ffffb265f83638ef R09: 0000000000000000 [77092.515729] R10: ffffffff8104eb10 R11: ffffffff8104e54f R12: ffffffff81429040 [77092.515729] R13: 0000000000000001 R14: ffff8800027ce890 R15: ffff8800f8360380 [77092.515729] FS: 00007f7f9ee146e0(0000) GS:ffff880002600000(0000) knlGS:0000000000000000 [77092.515729] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [77092.515729] CR2: 00007f7d3e5c3000 CR3: 00000000f654c000 CR4: 00000000000006f0 [77092.515729] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [77092.515729] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [77092.515729] Process kswapd0 (pid: 318, threadinfo ffff8800f8362000, task ffff8800f8360380) [77092.515729] Stack: [77092.515729] 0000000001062560 ffff8800f7dc3470 000000006e77db4a ffff8800f8360380 [77092.515729] <0> 0000000000000000 ffff8800f8360720 ffff8800f8363a50 ffffffff81224865 [77092.515729] <0> ffff8800f8363a90 ffffffff81228784 0000000082cf3b72 ffff8800f7dc3488 [77092.515729] Call Trace: [77092.515729] [<ffffffff81224865>] ? __delay+0xa/0xc [77092.515729] [<ffffffff81228784>] ? do_raw_spin_lock+0xd6/0x135 [77092.515729] [<ffffffff8142821b>] ? _raw_spin_lock+0x5c/0x6f [77092.515729] [<ffffffffa001330a>] ? __ceph_remove_cap+0x8b/0x220 [ceph] [77092.515729] [<ffffffffa001330a>] ? __ceph_remove_cap+0x8b/0x220 [ceph] [77092.515729] [<ffffffffa001409f>] ? ceph_queue_caps_release+0x19c/0x263 [ceph] [77092.515729] [<ffffffffa000634c>] ? ceph_destroy_inode+0x3c/0x1b8 [ceph] [77092.515729] [<ffffffff810c23a2>] ? destroy_inode+0x2a/0x3f [77092.515729] [<ffffffff810c2519>] ? generic_drop_inode+0x52/0x56 [77092.515729] [<ffffffff810c1b1f>] ? iput+0x61/0x65 [77092.515729] [<ffffffff810bf2af>] ? dentry_iput+0xcc/0xe4 [77092.515729] [<ffffffff810bf2eb>] ? d_kill+0x24/0x45 [77092.515729] [<ffffffff810bf538>] ? __shrink_dcache_sb+0x22c/0x2e4 [77092.515729] [<ffffffff810bf6dc>] ? shrink_dcache_memory+0xdd/0x18b [77092.515729] [<ffffffff810bf6ed>] ? shrink_dcache_memory+0xee/0x18b [77092.515729] [<ffffffff81089a6a>] ? shrink_slab+0xdb/0x15e [77092.515729] [<ffffffff8108a61b>] ? kswapd+0x49d/0x682 [77092.515729] [<ffffffff81087c78>] ? isolate_pages_global+0x0/0x209 [77092.515729] [<ffffffff8104a9a8>] ? autoremove_wake_function+0x0/0x38 [77092.515729] [<ffffffff81428d24>] ? _raw_spin_unlock_irqrestore+0x63/0x68 [77092.515729] [<ffffffff8108a17e>] ? kswapd+0x0/0x682 [77092.515729] [<ffffffff8104a676>] ? kthread+0x7d/0x85 [77092.515729] [<ffffffff810037d4>] ? kernel_thread_helper+0x4/0x10 [77092.515729] [<ffffffff81429040>] ? restore_args+0x0/0x30 [77092.515729] [<ffffffff8104a5f9>] ? kthread+0x0/0x85 [77092.515729] [<ffffffff810037d0>] ? kernel_thread_helper+0x0/0x10 [77092.515729] Code: c9 c3 55 48 89 e5 41 57 41 56 41 55 41 54 49 89 fc bf 01 00 00 00 53 48 83 ec 08 e8 9b 74 e0 ff e8 79 40 00 00 41 89 c6 0f ae f0 <66> 66 90 0f 31 41 89 c5 65 4c 8b 3c 25 48 b5 00 00 0f ae f0 66 [77092.515729] Call Trace: [77092.515729] [<ffffffff81224865>] ? __delay+0xa/0xc [77092.515729] [<ffffffff81228784>] ? do_raw_spin_lock+0xd6/0x135 [77092.515729] [<ffffffff8142821b>] ? _raw_spin_lock+0x5c/0x6f [77092.515729] [<ffffffffa001330a>] ? __ceph_remove_cap+0x8b/0x220 [ceph] [77092.515729] [<ffffffffa001330a>] ? __ceph_remove_cap+0x8b/0x220 [ceph] [77092.515729] [<ffffffffa001409f>] ? ceph_queue_caps_release+0x19c/0x263 [ceph] [77092.515729] [<ffffffffa000634c>] ? ceph_destroy_inode+0x3c/0x1b8 [ceph] [77092.515729] [<ffffffff810c23a2>] ? destroy_inode+0x2a/0x3f [77092.515729] [<ffffffff810c2519>] ? generic_drop_inode+0x52/0x56 [77092.515729] [<ffffffff810c1b1f>] ? iput+0x61/0x65 [77092.515729] [<ffffffff810bf2af>] ? dentry_iput+0xcc/0xe4 [77092.515729] [<ffffffff810bf2eb>] ? d_kill+0x24/0x45 [77092.515729] [<ffffffff810bf538>] ? __shrink_dcache_sb+0x22c/0x2e4 [77092.515729] [<ffffffff810bf6dc>] ? shrink_dcache_memory+0xdd/0x18b [77092.515729] [<ffffffff810bf6ed>] ? shrink_dcache_memory+0xee/0x18b [77092.515729] [<ffffffff81089a6a>] ? shrink_slab+0xdb/0x15e [77092.515729] [<ffffffff8108a61b>] ? kswapd+0x49d/0x682 [77092.515729] [<ffffffff81087c78>] ? isolate_pages_global+0x0/0x209 [77092.515729] [<ffffffff8104a9a8>] ? autoremove_wake_function+0x0/0x38 [77092.515729] [<ffffffff81428d24>] ? _raw_spin_unlock_irqrestore+0x63/0x68 [77092.515729] [<ffffffff8108a17e>] ? kswapd+0x0/0x682 [77092.515729] [<ffffffff8104a676>] ? kthread+0x7d/0x85 [77092.515729] [<ffffffff810037d4>] ? kernel_thread_helper+0x4/0x10 [77092.515729] [<ffffffff81429040>] ? restore_args+0x0/0x30 [77092.515729] [<ffffffff8104a5f9>] ? kthread+0x0/0x85 [77092.515729] [<ffffffff810037d0>] ? kernel_thread_helper+0x0/0x10 [77092.515814] BUG: soft lockup - CPU#1 stuck for 61s! [klogd:1718] [77092.515814] Modules linked in: aes_x86_64 aes_generic ceph [77092.515814] irq event stamp: 110256 [77092.515814] hardirqs last enabled at (110255): [<ffffffff81428d00>] _raw_spin_unlock_irqrestore+0x3f/0x68 [77092.515814] hardirqs last disabled at (110256): [<ffffffff814282c7>] _raw_spin_lock_irq+0x14/0x77 [77092.515814] softirqs last enabled at (110214): [<ffffffff813f701f>] rpc_wake_up_next+0x1b3/0x1be [77092.515814] softirqs last disabled at (110212): [<ffffffff814285de>] _raw_spin_lock_bh+0x13/0x74 [77092.515814] CPU 1 [77092.515814] Modules linked in: aes_x86_64 aes_generic ceph [77092.515814] [77092.515814] Pid: 1718, comm: klogd Tainted: G D 2.6.34-rc6 #27 H8SSL/H8SSL [77092.515814] RIP: 0010:[<ffffffff8122494a>] [<ffffffff8122494a>] delay_tsc+0x2e/0xc4 [77092.515814] RSP: 0000:ffff8800f7acb908 EFLAGS: 00000202 [77092.515814] RAX: 00000000ccce0b90 RBX: ffff8800f7acb938 RCX: 0000000001062560 [77092.515814] RDX: 00000000000099f3 RSI: 0000000000000002 RDI: 0000000000000001 [77092.515814] RBP: ffffffff8100338e R08: ffff71dbf7acb7e7 R09: 0000000000000000 [77092.515814] R10: ffffffff8104eb10 R11: ffffffff8104e54f R12: ffff8800f96a5458 [77092.515814] R13: ffff8800f7f7c440 R14: ffff8800f7aca000 R15: ffff8800f7acbfd8 [77092.515814] FS: 00007faf98a976e0(0000) GS:ffff880002800000(0000) knlGS:0000000000000000 [77092.515814] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [77092.515814] CR2: 00000000004050a8 CR3: 00000000f7edc000 CR4: 00000000000006e0 [77092.515814] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [77092.515814] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [77092.515814] Process klogd (pid: 1718, threadinfo ffff8800f7aca000, task ffff8800f7f7c440) [77092.515814] Stack: [77092.515814] 0000000000000001 ffffffff8169b920 0000000063c06ba4 ffff8800f7f7c440 [77092.515814] <0> 0000000000000000 ffff8800f7f7c7e0 ffff8800f7acb948 ffffffff81224865 [77092.515814] <0> ffff8800f7acb988 ffffffff81228784 0000000082cf3b72 ffffffff8169b938 [77092.515814] Call Trace: [77092.515814] [<ffffffff81224865>] ? __delay+0xa/0xc [77092.515814] [<ffffffff81228784>] ? do_raw_spin_lock+0xd6/0x135 [77092.515814] [<ffffffff8142821b>] ? _raw_spin_lock+0x5c/0x6f [77092.515814] [<ffffffff810c1f08>] ? ifind+0x2c/0xb5 [77092.515814] [<ffffffff81134fd6>] ? nfs_find_actor+0x0/0x70 [77092.515814] [<ffffffff810c1f08>] ? ifind+0x2c/0xb5 [77092.515814] [<ffffffff810c2c40>] ? iget5_locked+0x4b/0x192 [77092.515814] [<ffffffff81133a95>] ? nfs_init_locked+0x0/0x35 [77092.515814] [<ffffffff81134fd6>] ? nfs_find_actor+0x0/0x70 [77092.515814] [<ffffffff811351a9>] ? nfs_fhget+0x50/0x544 [77092.515814] [<ffffffff8112fa07>] ? nfs_lookup+0x122/0x17b [77092.515814] [<ffffffff81057506>] ? print_lock_contention_bug+0x1b/0xe0 [77092.515814] [<ffffffff81428d59>] ? _raw_spin_unlock+0x30/0x4b [77092.515814] [<ffffffff810b810b>] ? do_lookup+0xf9/0x1e7 [77092.515814] [<ffffffff810b9f6e>] ? link_path_walk+0x5c2/0x990 [77092.515814] [<ffffffff810ba916>] ? path_walk+0x51/0xa7 [77092.515814] [<ffffffff810baa10>] ? do_path_lookup+0x29/0x51 [77092.515814] [<ffffffff810bb3e6>] ? user_path_at+0x52/0x93 [77092.515814] [<ffffffff8104dfcf>] ? up_read+0x1e/0x37 [77092.515814] [<ffffffff810b3451>] ? vfs_fstatat+0x35/0x62 [77092.515814] [<ffffffff810b3560>] ? vfs_stat+0x16/0x18 [77092.515814] [<ffffffff810b357c>] ? sys_newstat+0x1a/0x3b [77092.515814] [<ffffffff8109537b>] ? might_fault+0xac/0xb3 [77092.515814] [<ffffffff81427f9d>] ? trace_hardirqs_on_thunk+0x3a/0x3f [77092.515814] [<ffffffff81002a2b>] ? system_call_fastpath+0x16/0x1b [77092.515814] Code: e5 41 57 41 56 41 55 41 54 49 89 fc bf 01 00 00 00 53 48 83 ec 08 e8 9b 74 e0 ff e8 79 40 00 00 41 89 c6 0f ae f0 66 66 90 0f 31 <41> 89 c5 65 4c 8b 3c 25 48 b5 00 00 0f ae f0 66 66 90 0f 31 89 [77092.515814] Call Trace: [77092.515814] [<ffffffff81224865>] ? __delay+0xa/0xc [77092.515814] [<ffffffff81228784>] ? do_raw_spin_lock+0xd6/0x135 [77092.515814] [<ffffffff8142821b>] ? _raw_spin_lock+0x5c/0x6f [77092.515814] [<ffffffff810c1f08>] ? ifind+0x2c/0xb5 [77092.515814] [<ffffffff81134fd6>] ? nfs_find_actor+0x0/0x70 [77092.515814] [<ffffffff810c1f08>] ? ifind+0x2c/0xb5 [77092.515814] [<ffffffff810c2c40>] ? iget5_locked+0x4b/0x192 [77092.515814] [<ffffffff81133a95>] ? nfs_init_locked+0x0/0x35 [77092.515814] [<ffffffff81134fd6>] ? nfs_find_actor+0x0/0x70 [77092.515814] [<ffffffff811351a9>] ? nfs_fhget+0x50/0x544 [77092.515814] [<ffffffff8112fa07>] ? nfs_lookup+0x122/0x17b [77092.515814] [<ffffffff81057506>] ? print_lock_contention_bug+0x1b/0xe0 [77092.515814] [<ffffffff81428d59>] ? _raw_spin_unlock+0x30/0x4b [77092.515814] [<ffffffff810b810b>] ? do_lookup+0xf9/0x1e7 [77092.515814] [<ffffffff810b9f6e>] ? link_path_walk+0x5c2/0x990 [77092.515814] [<ffffffff810ba916>] ? path_walk+0x51/0xa7 [77092.515814] [<ffffffff810baa10>] ? do_path_lookup+0x29/0x51 [77092.515814] [<ffffffff810bb3e6>] ? user_path_at+0x52/0x93 [77092.515814] [<ffffffff8104dfcf>] ? up_read+0x1e/0x37 [77092.515814] [<ffffffff810b3451>] ? vfs_fstatat+0x35/0x62 [77092.515814] [<ffffffff810b3560>] ? vfs_stat+0x16/0x18 [77092.515814] [<ffffffff810b357c>] ? sys_newstat+0x1a/0x3b [77092.515814] [<ffffffff8109537b>] ? might_fault+0xac/0xb3 [77092.515814] [<ffffffff81427f9d>] ? trace_hardirqs_on_thunk+0x3a/0x3f [77092.515814] [<ffffffff81002a2b>] ? system_call_fastpath+0x16/0x1b
#3 Updated by Sage Weil almost 14 years ago
- Priority changed from Normal to High
#4 Updated by Sage Weil almost 14 years ago
- Assignee set to Sage Weil
#5 Updated by Sage Weil almost 14 years ago
finaly caught it!
[ 6800.606705] BUG: unable to handle kernel NULL pointer dereference at 00000000000008a8 [ 6800.608855] IP: [<ffffffff810c31d0>] igrab+0x18/0x40 [ 6800.608855] PGD 6ed4f067 PUD 6ee86067 PMD 0 [ 6800.608855] Oops: 0000 [#1] PREEMPT SMP [ 6800.608855] last sysfs file: /sys/kernel/uevent_seqnum [ 6800.608855] CPU 1 [ 6800.608855] Modules linked in: aes_x86_64 aes_generic ceph fan ac battery psmouse ehci_hcd ohci_hcd ide_pci_generic thermal button processor [ 6800.608855] [ 6800.608855] Pid: 2667, comm: ceph-msgr/1 Not tainted 2.6.34-rc6 #27 H8SSL/H8SSL [ 6800.608855] RIP: 0010:[<ffffffff810c31d0>] [<ffffffff810c31d0>] igrab+0x18/0x40 [ 6800.608855] RSP: 0018:ffff8800f4d1da50 EFLAGS: 00010286 [ 6800.608855] RAX: ffff8800f53d24c0 RBX: 0000000000000450 RCX: ffffffff810c31d0 [ 6800.608855] RDX: 0000000000000001 RSI: ffff8800a66fca90 RDI: 0000000000000001 [ 6800.608855] RBP: ffff8800f4d1da60 R08: 0000000000000000 R09: 0000000000000005 [ 6800.608855] R10: 0000000000000246 R11: ffffffff812290b5 R12: ffff8800a45363c0 [ 6800.608855] R13: ffff8800f7db7438 R14: ffff8800a66fca90 R15: ffff8800a66fca90 [ 6800.608855] FS: 00007f65ba0316e0(0000) GS:ffff880002800000(0000) knlGS:0000000000000000 [ 6800.608855] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 6800.608855] CR2: 00000000000008a8 CR3: 000000006eeb8000 CR4: 00000000000006e0 [ 6800.608855] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 6800.608855] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [ 6800.608855] Process ceph-msgr/1 (pid: 2667, threadinfo ffff8800f4d1c000, task ffff8800f53d24c0) [ 6800.608855] Stack: [ 6800.608855] ffff8800a4536430 ffff8800a4536398 ffff8800f4d1dad0 ffffffffa0091949 [ 6800.608855] <0> 2222222222222222 ffff8800f7db7b98 ffff8800f7db7438 ffffffffa0096bf5 [ 6800.608855] <0> 0000000000000000 0000000000000246 ffff8800f7db7b48 ffff88009dcbaad0 [ 6800.608855] Call Trace: [ 6800.608855] [<ffffffffa0091949>] iterate_session_caps+0x86/0x20e [ceph] [ 6800.608855] [<ffffffffa0096bf5>] ? trim_caps_cb+0x0/0x1be [ceph] [ 6800.608855] [<ffffffffa009583d>] handle_session+0x2c7/0x429 [ceph] [ 6800.608855] [<ffffffffa0095a36>] dispatch+0x97/0x1067 [ceph] [ 6800.608855] [<ffffffff81057f34>] ? mark_held_locks+0x49/0x64 [ 6800.608855] [<ffffffff814265ac>] ? __mutex_unlock_slowpath+0x10d/0x130 [ 6800.608855] [<ffffffff81058062>] ? trace_hardirqs_on_caller+0x113/0x13e [ 6800.608855] [<ffffffff8105809a>] ? trace_hardirqs_on+0xd/0xf [ 6800.608855] [<ffffffffa008da68>] try_read+0xd4a/0x1358 [ceph] [ 6800.608855] [<ffffffff81009a23>] ? native_sched_clock+0x37/0x71 [ 6800.608855] [<ffffffff8104f3ba>] ? sched_clock_local+0x11/0x73 [ 6800.608855] [<ffffffff8105aca5>] ? __lock_acquire+0x7ee/0x851 [ 6800.608855] [<ffffffff81056811>] ? put_lock_stats+0xe/0x27 [ 6800.608855] [<ffffffffa008fa10>] con_work+0x11a/0x6bc [ceph] [ 6800.608855] [<ffffffff810478be>] worker_thread+0x1e8/0x2fa [ 6800.608855] [<ffffffff81047865>] ? worker_thread+0x18f/0x2fa [ 6800.608855] [<ffffffffa008f8f6>] ? con_work+0x0/0x6bc [ceph] [ 6800.608855] [<ffffffff8104a9a8>] ? autoremove_wake_function+0x0/0x38 [ 6800.608855] [<ffffffff810476d6>] ? worker_thread+0x0/0x2fa [ 6800.608855] [<ffffffff8104a676>] kthread+0x7d/0x85 [ 6800.608855] [<ffffffff810037d4>] kernel_thread_helper+0x4/0x10 [ 6800.608855] [<ffffffff81429040>] ? restore_args+0x0/0x30 [ 6800.608855] [<ffffffff8104a5f9>] ? kthread+0x0/0x85 [ 6800.608855] [<ffffffff810037d0>] ? kernel_thread_helper+0x0/0x10 [ 6800.608855] Code: 69 81 e8 7d 5b 36 00 48 89 d8 5b 41 5c 41 5d 41 5e c9 c3 55 48 89 e5 53 48 89 fb 48 c7 c7 20 b9 69 81 48 83 ec 08 e8 ef 4f 36 00 <f6> 83 58 04 00 00 70 74 04 31 db eb 08 48 89 df e8 14 e8 ff ff [ 6800.608855] RIP [<ffffffff810c31d0>] igrab+0x18/0x40 [ 6800.608855] RSP <ffff8800f4d1da50> [ 6800.608855] CR2: 00000000000008a8 [ 6800.957836] ---[ end trace ef6e14cf44ffc20a ]--- [ 6800.962563] note: ceph-msgr/1[2667] exited with preempt_count 2
#6 Updated by Sage Weil almost 14 years ago
- Status changed from In Progress to Resolved
fixed by 'ceph: fix cap removal race' commit:d855b8010914b52d8dd596f6d22c162bf81ccf21
#7 Updated by Sage Weil almost 14 years ago
here are the final two crashes i got on this, presuambly because i had the wrong version of the module loaded:
[ 7843.976466] BUG: unable to handle kernel NULL pointer dereference at 00000000000008a8 [ 7843.980193] IP: [<ffffffff810c31d0>] igrab+0x18/0x40 [ 7843.980193] PGD bd3ea067 PUD 9fc0b067 PMD 0 [ 7843.980193] Oops: 0000 [#1] PREEMPT SMP [ 7843.980193] last sysfs file: /sys/class/net/lo/operstate [ 7843.980193] CPU 1 [ 7843.980193] Modules linked in: aes_x86_64 aes_generic ceph fan ac battery psmouse ehci_hcd ohci_hcd ide_pci_generic thermal button processor [ 7843.980193] [ 7843.980193] Pid: 2664, comm: ceph-msgr/1 Not tainted 2.6.34-rc6 #27 H8SSL/H8SSL [ 7843.980193] RIP: 0010:[<ffffffff810c31d0>] [<ffffffff810c31d0>] igrab+0x18/0x40 [ 7843.980193] RSP: 0018:ffff8800f5baba50 EFLAGS: 00010286 [ 7843.980193] RAX: ffff8800f5ba8200 RBX: 0000000000000450 RCX: ffffffff810c31d0 [ 7843.980193] RDX: 0000000000000001 RSI: ffff8800d43ad850 RDI: 0000000000000001 [ 7843.980193] RBP: ffff8800f5baba60 R08: 0000000000000000 R09: 0000000000000005 [ 7843.980193] R10: 0000000000000246 R11: ffff8800f9803ac0 R12: ffff8800d4393a48 [ 7843.980193] R13: ffff8800f7dda960 R14: ffff8800d43ad850 R15: ffff8800d43ad850 [ 7843.980193] FS: 00007f720fff86e0(0000) GS:ffff880002800000(0000) knlGS:0000000000000000 [ 7843.980193] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 7843.980193] CR2: 00000000000008a8 CR3: 000000009fe5c000 CR4: 00000000000006e0 [ 7844.096105] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 7844.096105] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [ 7844.096105] Process ceph-msgr/1 (pid: 2664, threadinfo ffff8800f5baa000, task ffff8800f5ba8200) [ 7844.096105] Stack: [ 7844.096105] ffff8800d4393ab8 ffff8800d4393a20 ffff8800f5babad0 ffffffffa0091949 [ 7844.096105] <0> 2222222222222222 ffff8800f7ddb0c0 ffff8800f7dda960 ffffffffa0096bf5 [ 7844.096105] <0> 0000000000000000 0000000000000246 ffff8800f7ddb070 ffff88009e438488 [ 7844.096105] Call Trace: [ 7844.096105] [<ffffffffa0091949>] iterate_session_caps+0x86/0x20e [ceph] [ 7844.096105] [<ffffffffa0096bf5>] ? trim_caps_cb+0x0/0x1be [ceph] [ 7844.096105] [<ffffffffa009583d>] handle_session+0x2c7/0x429 [ceph] [ 7844.096105] [<ffffffffa0095a36>] dispatch+0x97/0x1067 [ceph] [ 7844.096105] [<ffffffff81057f34>] ? mark_held_locks+0x49/0x64 [ 7844.096105] [<ffffffff814265ac>] ? __mutex_unlock_slowpath+0x10d/0x130 [ 7844.096105] [<ffffffff81058062>] ? trace_hardirqs_on_caller+0x113/0x13e [ 7844.096105] [<ffffffff8105809a>] ? trace_hardirqs_on+0xd/0xf [ 7844.096105] [<ffffffffa008da68>] try_read+0xd4a/0x1358 [ceph] [ 7844.096105] [<ffffffff81009a23>] ? native_sched_clock+0x37/0x71 [ 7844.096105sy [<ffffffff8104f3ba>] ? sched_clock_local+0x11/0x73 [ 7844.096105] [<ffffffff8105aca5>] ? __lock_acquire+0x7ee/0x851 [ 7844.096105] [<ffffffff81056811>] ? put_lock_stats+0xe/0x27 [ 7844.096105] [<ffffffffa008fa10>] con_work+0x11a/0x6bc [ceph] [ 7844.096105] [<ffffffff810478be>] worker_thread+0x1e8/0x2fa [ 7844.096105] [<ffffffff81047865>] ? worker_thread+0x18f/0x2fa [ 7844.096105] [<ffffffffa008f8f6>] ? con_work+0x0/0x6bc [ceph] [ 7844.096105] [<ffffffff8104a9a8>] ? autoremove_wake_function+0x0/0x38 [ 7844.096105] [<ffffffff810476d6>] ? worker_thread+0x0/0x2fa [ 7844.096105] [<ffffffff8104a676>] kthread+0x7d/0x85 [ 7844.096105] [<ffffffff810037d4>] kernel_thread_helper+0x4/0x10 [ 7844.096105] [<ffffffff81429040>] ? restore_args+0x0/0x30 [ 7844.096105] [<ffffffff8104a5f9>] ? kthread+0x0/0x85 [ 7844.096105] [<ffffffff810037d0>] ? kernel_thread_helper+0x0/0x10 [ 7844.096105] Code: 69 81 e8 7d 5b 36 00 48 89 d8 5b 41 5c 41 5d 41 5e c9 c3 55 48 89 e5 53 48 89 fb 48 c7 c7 20 b9 69 81 48 83 ec 08 e8 ef 4f 3$ [ 7844.096105] RIP [<ffffffff810c31d0>] igrab+0x18/0x40 [ 7844.096105] RSP <ffff8800f5baba50> [ 7844.096105] CR2: 00000000000008a8 [ 7844.322197] ---[ end trace bad341be08865d59 ]--- [28255.195281] BUG: unable to handle kernel NULL pointer dereference at 00000000000008a8 [28255.195938] IP: [<ffffffff810c31d0>] igrab+0x18/0x40 [28255.195938] PGD da4dd067 PUD a1043067 PMD 0 [28255.195938] Oops: 0000 [#1] PREEMPT SMP [28255.195938] last sysfs file: /sys/class/net/lo/operstate [28255.195938] CPU 1 [28255.195938] Modules linked in: aes_x86_64 aes_generic ceph fan ac battery psmouse ide_pci_generic ehci_hcd ohci_hcd thermal button processor [28255.195938] [28255.195938] Pid: 2677, comm: ceph-msgr/1 Not tainted 2.6.34-rc6 #27 H8SSL/H8SSL [28255.195938] RIP: 0010:[<ffffffff810c31d0>] [<ffffffff810c31d0>] igrab+0x18/0x40 [28255.195938] RSP: 0018:ffff8800f633da50 EFLAGS: 00010286 [28255.195938] RAX: ffff8800f6348880 RBX: 0000000000000450 RCX: ffffffff810c31d0 [28255.195938] RDX: 0000000000000001 RSI: ffff88007c5a2510 RDI: 0000000000000001 [28255.195938] RBP: ffff8800f633da60 R08: 0000000000000000 R09: 0000000000000005 [28255.195938] R10: 0000000000000246 R11: ffffffff812290b5 R12: ffff8800deacd290 [28255.195938] R13: ffff8800f7dfede0 R14: ffff88007c5a2510 R15: ffff88007c5a2510 [28255.195938] FS: 00007f31c506f6e0(0000) GS:ffff880002800000(0000) knlGS:0000000000000000 [28255.195938] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [28255.195938] CR2: 00000000000008a8 CR3: 00000000da43d000 CR4: 00000000000006e0 [28255.195938] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [28255.195938] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [28255.195938] Process ceph-msgr/1 (pid: 2677, threadinfo ffff8800f633c000, task ffff8800f6348880) [28255.195938] Stack: [28255.195938] ffff8800deacd1d0 ffff8800deacd268 ffff8800f633dad0 ffffffffa0091949 [28255.195938] <0> 2222222222222222 ffff8800f7dff540 ffff8800f7dfede0 ffffffffa0096bf5 [28255.195938] <0> 0000000000000000 0000000000000246 ffff8800f7dff4f0 ffff8800a10316e0 [28255.195938] Call Trace: [28255.195938] [<ffffffffa0091949>] iterate_session_caps+0x86/0x20e [ceph] [28255.195938] [<ffffffffa0096bf5>] ? trim_caps_cb+0x0/0x1be [ceph] [28255.195938] [<ffffffffa009583d>] handle_session+0x2c7/0x429 [ceph] [28255.195938] [<ffffffffa0095a36>] dispatch+0x97/0x1067 [ceph] [28255.195938] [<ffffffff81057f34>] ? mark_held_locks+0x49/0x64 [28255.195938] [<ffffffff814265ac>] ? __mutex_unlock_slowpath+0x10d/0x130 [28255.195938] [<ffffffff81058062>] ? trace_hardirqs_on_caller+0x113/0x13e [28255.195938] [<ffffffff8105809a>] ? trace_hardirqs_on+0xd/0xf [28255.195938] [<ffffffffa008da68>] try_read+0xd4a/0x1358 [ceph] [28255.195938] [<ffffffff81009a23>] ? native_sched_clock+0x37/0x71 [28255.195938] [<ffffffff8104f3ba>] ? sched_clock_local+0x11/0x73 [28255.195938] [<ffffffff8105aca5>] ? __lock_acquire+0x7ee/0x851 [28255.195938] [<ffffffff81056811>] ? put_lock_stats+0xe/0x27 [28255.195938] [<ffffffffa008fa10>] con_work+0x11a/0x6bc [ceph] [28255.195938] [<ffffffff810478be>] worker_thread+0x1e8/0x2fa [28255.195938] [<ffffffff81047865>] ? worker_thread+0x18f/0x2fa [28255.195938] [<ffffffffa008f8f6>] ? con_work+0x0/0x6bc [ceph] [28255.195938] [<ffffffff8104a9a8>] ? autoremove_wake_function+0x0/0x38 [28255.195938] [<ffffffff810476d6>] ? worker_thread+0x0/0x2fa [28255.195938] [<ffffffff8104a676>] kthread+0x7d/0x85 [28255.195938] [<ffffffff810037d4>] kernel_thread_helper+0x4/0x10 [28255.195938] [<ffffffff81429040>] ? restore_args+0x0/0x30 [28255.195938] [<ffffffff8104a5f9>] ? kthread+0x0/0x85 [28255.195938] [<ffffffff810037d0>] ? kernel_thread_helper+0x0/0x10 [28255.195938] Code: 69 81 e8 7d 5b 36 00 48 89 d8 5b 41 5c 41 5d 41 5e c9 c3 55 48 89 e5 53 48 89 fb 48 c7 c7 20 b9 69 81 48 83 ec 08 e8 ef 4f 3$ [28255.195938] RIP [<ffffffff810c31d0>] igrab+0x18/0x40 [28255.195938] RSP <ffff8800f633da50> [28255.195938] CR2: 00000000000008a8 [28255.541008] ---[ end trace b1b9516d21835bb7 ]---