Ceph » CephFS

From http://permalink.gmane.org/gmane.comp.file-systems.ceph.devel/6237:

Furthermore, a different crash occurs when trying to remove a
non-empty Ceph directory:
******************************************************
root@node33:/mnt# rm debian -r
rm: cannot remove `debian/etc': Directory not empty
Write failed: Broken pipe
******************************************************
The crash trace is shown below:

----------------------------------------------------

[74576.543412] libceph: client0 fsid 9b3222ac-fce2-44eb-8599-d39da02d2393
[74576.651197] libceph: mon0 192.168.2.254:6789 session established
[75143.963663] BUG: unable to handle kernel NULL pointer dereference
at 0000000000000030
[75143.963771] IP: [<ffffffff811061cd>] path_init+0x218/0x2cc
[75143.963827] PGD 37a63067 PUD 37b4b067 PMD 0
[75143.963880] Oops: 0000 [#1] SMP
[75143.963928] CPU 3
[75143.963935] Modules linked in: cbc netconsole loop i5400_edac
snd_pcm edac_core i5k_amb snd_timer snd soundcore evdev snd_page_alloc
tpm_tis tpm processor rng_core thermal_sys pcspkr tpm_bios shpchp
pci_hotplug button sd_mod crc_t10dif usbhid hid ide_cd_mod cdrom
ata_generic uhci_hcd ata_piix libata piix ide_core ehci_hcd usbcore
usb_common tg3 libphy mptsas mptscsih mptbase scsi_transport_sas
scsi_mod [last unloaded: netconsole]
[75143.964390]
[75143.964426] Pid: 3861, comm: rm Not tainted 3.3.4 #1 HP ProLiant DL160 G5
[75143.964485] RIP: 0010:[<ffffffff811061cd>]  [<ffffffff811061cd>]
path_init+0x218/0x2cc
[75143.964570] RSP: 0018:ffff880037b45d58  EFLAGS: 00010202
[75143.964618] RAX: 0000000000000000 RBX: ffff8800b8975000 RCX: ffff880037b45ea8
[75143.964672] RDX: ffff8800b929a900 RSI: ffff880037b45d74 RDI: ffff8800b9d0e830
[75143.964727] RBP: 0000000000000050 R08: ffff880037b45de0 R09: 0000000000000000
[75143.964781] R10: 0000006e7265746c R11: 0000000001406d90 R12: ffff880037b45ea8
[75143.964835] R13: ffff8800b929a900 R14: ffff880037b45de0 R15: 0000000000000003
[75143.964890] FS:  00007fb7a9b47700(0000) GS:ffff8800bfcc0000(0000)
knlGS:0000000000000000
[75143.964974] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[75143.965023] CR2: 0000000000000030 CR3: 00000000379d5000 CR4: 00000000000006e0
[75143.965078] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[75143.965132] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[75143.965187] Process rm (pid: 3861, threadinfo ffff880037b44000,
task ffff8800b91fed00)
[75143.965269] Stack:
[75143.965306]  00000000b9f76000 000000005088218e ffff8800b9f76280
00000000b9f76280
[75143.965397]  ffff880037b45ea8 0000000000000050 ffff8800b8975000
0000000000000010
[75143.965489]  00000000013f2030 ffffffff811072b4 ffffffff81060362
dead000000100100
[75143.965581] Call Trace:
[75143.965622]  [<ffffffff811072b4>] ? path_lookupat+0x2c/0x30b
[75143.965674]  [<ffffffff81060362>] ? try_to_wake_up+0x1a5/0x1a5
[75143.965725]  [<ffffffff811075b1>] ? do_path_lookup+0x1e/0x9a
[75143.965775]  [<ffffffff81107bf1>] ? user_path_parent+0x3a/0x5f
[75143.965826]  [<ffffffff810f3484>] ? virt_to_head_page+0x9/0x2c
[75143.965877]  [<ffffffff81107e96>] ? do_unlinkat+0x1d/0x15e
[75143.965927]  [<ffffffff81109e94>] ? vfs_readdir+0x91/0xa7
[75143.965977]  [<ffffffff8112a9a5>] ? fsnotify_find_inode_mark+0x23/0x2f
[75143.966031]  [<ffffffff810fa97e>] ? filp_close+0x64/0x6c
[75143.966082]  [<ffffffff815d2679>] ? system_call_fastpath+0x16/0x1b
[75143.966133] Code: 04 01 e9 a1 00 00 00 48 8d 74 24 1c e8 ae 6f ff
ff 49 89 c5 b8 f7 ff ff ff 4d 85 ed 0f 84 b0 00 00 00 49 8b 45 18 80
3b 00 74 28 <48> 8b 78 30 b8 ec ff ff ff 0f b7 17 81 e2 00 f0 00 00 81
fa 00
[75143.966507] RIP  [<ffffffff811061cd>] path_init+0x218/0x2cc
[75143.966558]  RSP <ffff880037b45d58>
[75143.966600] CR2: 0000000000000030
[75143.967124] ---[ end trace 18e2f523c5af9a38 ]---
[75143.967322] general protection fault: 0000 [#2] SMP
[75143.967542] CPU 3
[75143.967607] Modules linked in: cbc netconsole loop i5400_edac
snd_pcm edac_core i5k_amb snd_timer snd soundcore evdev snd_page_alloc
tpm_tis tpm processor rng_core thermal_sys pcspkr tpm_bios shpchp
pci_hotplug button sd_mod crc_t10dif usbhid hid ide_cd_mod cdrom
ata_generic uhci_hcd ata_piix libata piix ide_core ehci_hcd usbcore
usb_common tg3 libphy mptsas mptscsih mptbase scsi_transport_sas
scsi_mod [last unloaded: netconsole]
[75143.970715]
[75143.970805] Pid: 3861, comm: rm Tainted: G      D      3.3.4 #1 HP
ProLiant DL160 G5
[75143.971058] RIP: 0010:[<ffffffff810fa947>]  [<ffffffff810fa947>]
filp_close+0x2d/0x6c
[75143.971085] RSP: 0018:ffff880037b45a48  EFLAGS: 00010206
[75143.971085] RAX: 0012080800000000 RBX: ffff8800b910d300 RCX: 0000000000000000
[75143.971085] RDX: 0000000000000000 RSI: ffff8800b9712c00 RDI: ffff8800b910d300
[75143.971085] RBP: ffff8800b9712c00 R08: 0000000000016870 R09: 00007fff71600000
[75143.971085] R10: 0000000000000001 R11: ffff880037b459a8 R12: 0000000000000000
[75143.971085] R13: ffff8800bb51f6c0 R14: 0000000000000004 R15: 0000000000000000
[75143.971085] FS:  0000000000000000(0000) GS:ffff8800bfcc0000(0000)
knlGS:0000000000000000
[75143.971085] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[75143.971085] CR2: 0000000000000030 CR3: 0000000001805000 CR4: 00000000000006e0
[75143.971085] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[75143.971085] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[75143.971085] Process rm (pid: 3861, threadinfo ffff880037b44000,
task ffff8800b91fed00)
[75143.971085] Stack:
[75143.971085]  ffff8800b9712c00 0000000000000007 0000000000000000
ffffffff8103aaad
[75143.971085]  0000000000000009 ffff8800b91fed00 ffff8800b91ff218
0000000000000009
[75143.971085]  ffff8800b8ca0700 ffff8800b92c0880 0000000000000001
ffffffff8103bfe4
[75143.971085] Call Trace:
[75143.971085]  [<ffffffff8103aaad>] ? put_files_struct+0x67/0xbf
[75143.971085]  [<ffffffff8103bfe4>] ? do_exit+0x2aa/0x7e1
[75143.971085]  [<ffffffff810390ee>] ? kmsg_dump+0x53/0xef
[75143.971085]  [<ffffffff815cd01a>] ? oops_end+0x66/0xb7
[75143.971085]  [<ffffffff815cd066>] ? oops_end+0xb2/0xb7
[75143.971085]  [<ffffffff8102eaca>] ? no_context+0x254/0x263
[75143.971085]  [<ffffffff81368f16>] ? ceph_writepages_start+0xbb4/0xbee
[75143.971085]  [<ffffffff815cf1ef>] ? do_page_fault+0x215/0x34c
[75143.971085]  [<ffffffff8136a1e5>] ? __cap_is_valid+0x19/0x9a
[75143.971085]  [<ffffffff8136ba47>] ? ceph_encode_inode_release+0xed/0x2b2
[75143.971085]  [<ffffffff81063c10>] ? update_curr+0xfb/0x130
[75143.971085]  [<ffffffff8100d6fe>] ? __switch_to+0x20b/0x35f
[75143.971085]  [<ffffffff81063c10>] ? update_curr+0xfb/0x130
[75143.971085]  [<ffffffff815cc5f5>] ? page_fault+0x25/0x30
[75143.971085]  [<ffffffff811061cd>] ? path_init+0x218/0x2cc
[75143.971085]  [<ffffffff811061b3>] ? path_init+0x1fe/0x2cc
[75143.971085]  [<ffffffff811072b4>] ? path_lookupat+0x2c/0x30b
[75143.971085]  [<ffffffff81060362>] ? try_to_wake_up+0x1a5/0x1a5
[75143.971085]  [<ffffffff811075b1>] ? do_path_lookup+0x1e/0x9a
[75143.971085]  [<ffffffff81107bf1>] ? user_path_parent+0x3a/0x5f
[75143.971085]  [<ffffffff810f3484>] ? virt_to_head_page+0x9/0x2c
[75143.971085]  [<ffffffff81107e96>] ? do_unlinkat+0x1d/0x15e
[75143.971085]  [<ffffffff81109e94>] ? vfs_readdir+0x91/0xa7
[75143.971085]  [<ffffffff8112a9a5>] ? fsnotify_find_inode_mark+0x23/0x2f
[75143.971085]  [<ffffffff810fa97e>] ? filp_close+0x64/0x6c
[75143.971085]  [<ffffffff815d2679>] ? system_call_fastpath+0x16/0x1b
[75143.971085] Code: 55 48 89 f5 53 48 89 fb 48 8b 47 30 48 85 c0 75
11 48 c7 c7 6e d5 72 81 45 31 e4 e8 f0 fa 4c 00 eb 40 48 8b 47 20 48
85 c0 74 10 <48> 8b 40 60 48 85 c0 74 07 ff d0 41 89 c4 eb 03 45 31 e4
f6 43
[75143.971085] RIP  [<ffffffff810fa947>] filp_close+0x2d/0x6c
[75143.971085]  RSP <ffff880037b45a48>
[75143.988721] ---[ end trace 18e2f523c5af9a39 ]---
[75143.988826] Fixing recursive fault but reboot is needed!
[75146.018276] ------------[ cut here ]------------
[75146.018399] kernel BUG at mm/slub.c:3442!
[75146.018498] invalid opcode: 0000 [#3] SMP
[75146.018718] CPU 1
[75146.018789] Modules linked in: cbc netconsole loop i5400_edac
snd_pcm edac_core i5k_amb snd_timer snd soundcore evdev snd_page_alloc
tpm_tis tpm processor rng_core thermal_sys pcspkr tpm_bios shpchp
pci_hotplug button sd_mod crc_t10dif usbhid hid ide_cd_mod cdrom
ata_generic uhci_hcd ata_piix libata piix ide_core ehci_hcd usbcore
usb_common tg3 libphy mptsas mptscsih mptbase scsi_transport_sas
scsi_mod [last unloaded: netconsole]
[75146.021908]
[75146.021999] Pid: 1137, comm: kworker/1:0 Tainted: G      D
3.3.4 #1 HP ProLiant DL160 G5
[75146.022236] RIP: 0010:[<ffffffff810f55df>]  [<ffffffff810f55df>]
kfree+0x59/0xc2
[75146.022236] RSP: 0018:ffff8800bbb35b90  EFLAGS: 00010246
[75146.022236] RAX: 0100000000000400 RBX: ffff8800bfcdab70 RCX: ffff8800b9f762c8
[75146.022236] RDX: ffff8800bfc4e550 RSI: 0000000000000000 RDI: ffffea0002ff3680
[75146.022236] RBP: ffffffff815a2a50 R08: 0000000000000000 R09: ffffffff814f9620
[75146.022236] R10: 000000000000000d R11: ffff8800b9d0d000 R12: ffffffff815a2a47
[75146.022236] R13: ffff8800ba15d400 R14: ffff8800b9d0d271 R15: ffff8800b9d0d000
[75146.022236] FS:  0000000000000000(0000) GS:ffff8800bfc40000(0000)
knlGS:0000000000000000
[75146.022236] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[75146.022236] CR2: ffffffffff600400 CR3: 0000000037b13000 CR4: 00000000000006e0
[75146.022236] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[75146.022236] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[75146.022236] Process kworker/1:0 (pid: 1137, threadinfo
ffff8800bbb34000, task ffff8800b90b0000)
[75146.022236] Stack:
[75146.022236]  ffff8800b8fde500 ffffffff815a2a50 ffff8800b9f72800
ffffffff815a2a47
[75146.022236]  ffff8800b8fde588 ffffffff81372b8e 0000001b00004040
ffff8800b9f72a68
[75146.022236]  ffff8800b9f72800 ffffffff8137513d ffff8800ba15d400
ffff8800b9f72a68
[75146.022236] Call Trace:
[75146.022236]  [<ffffffff815a2a50>] ? ceph_msg_kfree+0x47/0x47
[75146.022236]  [<ffffffff815a2a47>] ? ceph_msg_kfree+0x3e/0x47
[75146.022236]  [<ffffffff81372b8e>] ? kref_put+0x34/0x3e
[75146.022236]  [<ffffffff8137513d>] ? ceph_mdsc_release_request+0x2f/0x145
[75146.022236]  [<ffffffff8137510e>] ? encode_caps_cb+0x2f9/0x2f9
[75146.022236]  [<ffffffff81372b8e>] ? kref_put+0x34/0x3e
[75146.022236]  [<ffffffff813778d3>] ? dispatch+0xe05/0x132c
[75146.022236]  [<ffffffff814b3d5e>] ? kernel_recvmsg+0x34/0x3f
[75146.022236]  [<ffffffff813dce42>] ? crc32c+0x56/0x7c
[75146.022236]  [<ffffffff815a39c7>] ? ceph_tcp_recvmsg+0x43/0x4f
[75146.022236]  [<ffffffff815a658b>] ? con_work+0x15ac/0x17a8
[75146.022236]  [<ffffffff8104483f>] ? lock_timer_base+0x25/0x49
[75146.022236]  [<ffffffff815a4fdf>] ? ceph_fault+0x2b4/0x2b4
[75146.022236]  [<ffffffff8104ef8a>] ? process_one_work+0x1cd/0x2eb
[75146.022236]  [<ffffffff8104f1d6>] ? worker_thread+0x12e/0x249
[75146.022236]  [<ffffffff8104f0a8>] ? process_one_work+0x2eb/0x2eb
[75146.022236]  [<ffffffff8104f0a8>] ? process_one_work+0x2eb/0x2eb
[75146.022236]  [<ffffffff81052b82>] ? kthread+0x81/0x89
[75146.022236]  [<ffffffff815d3a24>] ? kernel_thread_helper+0x4/0x10
[75146.022236]  [<ffffffff81052b01>] ? kthread_freezable_should_stop+0x53/0x53
[75146.022236]  [<ffffffff815d3a20>] ? gs_change+0x13/0x13
[75146.022236] Code: 00 48 83 c5 10 48 83 7d 00 00 eb e6 48 83 fb 10
76 7d 48 89 df e8 ad de ff ff 48 89 c7 48 8b 00 84 c0 78 14 66 f7 07
00 c0 75 04 <0f> 0b eb fe 5b 5d 41 5c e9 e8 03 fd ff 4c 8b 54 24 18 4c
8b 4f
[75146.022236] RIP  [<ffffffff810f55df>] kfree+0x59/0xc2
[75146.022236]  RSP <ffff8800bbb35b90>
[75146.031675] ---[ end trace 18e2f523c5af9a3a ]---
[75146.031809] BUG: unable to handle kernel paging request at fffffffffffffff8
[75146.032058] IP: [<ffffffff81052783>] kthread_data+0x7/0xc
[75146.032221] PGD 1807067 PUD 1808067 PMD 0
[75146.032494] Oops: 0000 [#4] SMP
[75146.032706] CPU 1
[75146.032771] Modules linked in: cbc netconsole loop i5400_edac
snd_pcm edac_core i5k_amb snd_timer snd soundcore evdev snd_page_alloc
tpm_tis tpm processor rng_core thermal_sys pcspkr tpm_bios shpchp
pci_hotplug button sd_mod crc_t10dif usbhid hid ide_cd_mod cdrom
ata_generic uhci_hcd ata_piix libata piix ide_core ehci_hcd usbcore
usb_common tg3 libphy mptsas mptscsih mptbase scsi_transport_sas
scsi_mod [last unloaded: netconsole]
[75146.035616]
[75146.035616] Pid: 1137, comm: kworker/1:0 Tainted: G      D
3.3.4 #1 HP ProLiant DL160 G5
[75146.035616] RIP: 0010:[<ffffffff81052783>]  [<ffffffff81052783>]
kthread_data+0x7/0xc
[75146.035616] RSP: 0018:ffff8800bbb35900  EFLAGS: 00010002
[75146.035616] RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000001
[75146.035616] RDX: ffffffff819a87b0 RSI: 0000000000000001 RDI: ffff8800b90b0000
[75146.035616] RBP: ffff8800b90b0000 R08: 0000000000000400 R09: ffffffff81013c7c
[75146.035616] R10: ffff8800b90b0000 R11: ffff8800b90b0518 R12: ffff8800b90b02f8
[75146.035616] R13: ffff8800bbb359c8 R14: 0000000000000001 R15: 0000000000000001
[75146.035616] FS:  0000000000000000(0000) GS:ffff8800bfc40000(0000)
knlGS:0000000000000000
[75146.035616] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[75146.035616] CR2: fffffffffffffff8 CR3: 0000000037b13000 CR4: 00000000000006e0
[75146.035616] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[75146.035616] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[75146.035616] Process kworker/1:0 (pid: 1137, threadinfo
ffff8800bbb34000, task ffff8800b90b0000)
[75146.035616] Stack:
[75146.035616]  ffffffff8104e2a2 ffff8800bfc53340 ffffffff815cb1d1
0000000000000001
[75146.035616]  0000000000000296 0000000000013340 ffff8800bbb35fd8
0000000000013340
[75146.035616]  ffff8800bbb35fd8 0000000000013340 ffff8800b90b0000
0000000000013340
[75146.035616] Call Trace:
[75146.035616]  [<ffffffff8104e2a2>] ? wq_worker_sleeping+0x8/0x82
[75146.035616]  [<ffffffff815cb1d1>] ? __schedule+0x166/0x4fc
[75146.035616]  [<ffffffff8103c517>] ? do_exit+0x7dd/0x7e1
[75146.035616]  [<ffffffff815ca46c>] ? printk+0x40/0x4c
[75146.035616]  [<ffffffff815cd01a>] ? oops_end+0x66/0xb7
[75146.035616]  [<ffffffff815cd066>] ? oops_end+0xb2/0xb7
[75146.035616]  [<ffffffff815a2a47>] ? ceph_msg_kfree+0x3e/0x47
[75146.035616]  [<ffffffff8100ef69>] ? do_invalid_op+0x8b/0x95
[75146.035616]  [<ffffffff810f55df>] ? kfree+0x59/0xc2
[75146.035616]  [<ffffffff81515b94>] ? inet_recvmsg+0x64/0x75
[75146.035616]  [<ffffffff815a2a50>] ? ceph_msg_kfree+0x47/0x47
[75146.035616]  [<ffffffff815d389b>] ? invalid_op+0x1b/0x20
[75146.035616]  [<ffffffff815a2a47>] ? ceph_msg_kfree+0x3e/0x47
[75146.035616]  [<ffffffff815a2a50>] ? ceph_msg_kfree+0x47/0x47
[75146.035616]  [<ffffffff814f9620>] ? tcp_recvmsg+0x773/0x95e
[75146.035616]  [<ffffffff810f55df>] ? kfree+0x59/0xc2
[75146.035616]  [<ffffffff815a2a50>] ? ceph_msg_kfree+0x47/0x47
[75146.035616]  [<ffffffff815a2a47>] ? ceph_msg_kfree+0x3e/0x47
[75146.035616]  [<ffffffff81372b8e>] ? kref_put+0x34/0x3e
[75146.035616]  [<ffffffff8137513d>] ? ceph_mdsc_release_request+0x2f/0x145
[75146.035616]  [<ffffffff8137510e>] ? encode_caps_cb+0x2f9/0x2f9
[75146.035616]  [<ffffffff81372b8e>] ? kref_put+0x34/0x3e
[75146.035616]  [<ffffffff813778d3>] ? dispatch+0xe05/0x132c
[75146.035616]  [<ffffffff814b3d5e>] ? kernel_recvmsg+0x34/0x3f
[75146.035616]  [<ffffffff813dce42>] ? crc32c+0x56/0x7c
[75146.035616]  [<ffffffff815a39c7>] ? ceph_tcp_recvmsg+0x43/0x4f
[75146.035616]  [<ffffffff815a658b>] ? con_work+0x15ac/0x17a8
[75146.035616]  [<ffffffff8104483f>] ? lock_timer_base+0x25/0x49
[75146.035616]  [<ffffffff815a4fdf>] ? ceph_fault+0x2b4/0x2b4
[75146.035616]  [<ffffffff8104ef8a>] ? process_one_work+0x1cd/0x2eb
[75146.035616]  [<ffffffff8104f1d6>] ? worker_thread+0x12e/0x249
[75146.035616]  [<ffffffff8104f0a8>] ? process_one_work+0x2eb/0x2eb
[75146.035616]  [<ffffffff8104f0a8>] ? process_one_work+0x2eb/0x2eb
[75146.035616]  [<ffffffff81052b82>] ? kthread+0x81/0x89
[75146.035616]  [<ffffffff815d3a24>] ? kernel_thread_helper+0x4/0x10
[75146.035616]  [<ffffffff81052b01>] ? kthread_freezable_should_stop+0x53/0x53
[75146.035616]  [<ffffffff815d3a20>] ? gs_change+0x13/0x13
[75146.035616] Code: 41 5e 41 5f c3 41 bf ea ff ff ff eb 97 90 90 90
65 48 8b 04 25 c0 c6 00 00 48 8b 80 a0 02 00 00 8b 40 f0 c3 48 8b 87
a0 02 00 00 <48> 8b 40 f8 c3 48 3b 3d 51 5f 95 00 75 08 0f bf 87 6a 06
00 00
[75146.035616] RIP  [<ffffffff81052783>] kthread_data+0x7/0xc
[75146.035616]  RSP <ffff8800bbb35900>
[75146.035616] CR2: fffffffffffffff8
[75146.035616] ---[ end trace 18e2f523c5af9a3b ]---
[75146.035616] Fixing recursive fault but reboot is needed!
[75206.036002] INFO: rcu_sched detected stalls on CPUs/tasks: { 1}
(detected by 3, t=15002 jiffies)
[75206.036265] Pid: 0, comm: swapper/3 Tainted: G      D      3.3.4 #1
[75206.036371] Call Trace:
[75206.036464]  <IRQ>  [<ffffffff8109b7b3>] ? __rcu_pending+0x21a/0x336
[75206.036635]  [<ffffffff81075d59>] ? tick_nohz_handler+0xcb/0xcb
[75206.036740]  [<ffffffff8109b9cc>] ? rcu_check_callbacks+0xa7/0xe7
[75206.036846]  [<ffffffff81075d59>] ? tick_nohz_handler+0xcb/0xcb
[75206.036951]  [<ffffffff81044911>] ? update_process_times+0x31/0x63