Project

General

Profile

Actions

Bug #1696

closed

kclient: crash in ceph_d_prune

Added by Anonymous over 12 years ago. Updated over 12 years ago.

Status:
Resolved
Priority:
Normal
Assignee:
Category:
fs/ceph
Target version:
-
% Done:

0%

Source:
Tags:
Backport:
Regression:
Severity:
Reviewed:
Affected Versions:
ceph-qa-suite:
Crash signature (v1):
Crash signature (v2):

Description

During the 11/08 nighly, several suites:
1606 autotest dbench
1607 workunit direct_io
1608 workunit kclient
1610 workunit misc
1613 workunit suites/fsstress.sh
1614 workunit suites/iozone.sh
1616 workunit suites/tiobench.sh
1617 workunit misc/trivial_sync.sh
1632 locktest

All seem to have failed in similar ways. The logged complaint was
a failed rmdir of /tmp/cephtest due to its not being empty. This was
almost surely due (in all of those cases) to a failure of the umount
of /tmp/cephtest/mnt.0. In each case, /var/log/messages contained a
set of entries very similar to:

Nov  7 00:52:21 sepia21 kernel: [ 2418.233522] Pid: 20532, comm: umount Not tainted 3.1.0-ceph-08929-g15a2015 #1 Supermicro PDSMi/PDSMi+
Nov  7 00:52:21 sepia21 kernel: [ 2418.233678] RIP: 0010:[<ffffffffa02f37b9>]  [<ffffffffa02f37b9>] ceph_d_prune+0x39/0x60 [ceph]
Nov  7 00:52:21 sepia21 kernel: [ 2418.233824] RSP: 0018:ffff8800aa13fd88  EFLAGS: 00010282
Nov  7 00:52:21 sepia21 kernel: [ 2418.233906] RAX: 0000000000000000 RBX: ffff8800c32593e0 RCX: 0000000000000000
Nov  7 00:52:21 sepia21 kernel: [ 2418.234007] RDX: ffff8800c32594b8 RSI: 0000000000000000 RDI: ffff8800c32593e0
Nov  7 00:52:21 sepia21 kernel: [ 2418.234108] RBP: ffff8800aa13fd98 R08: 0000000000000000 R09: 0000000000000000
Nov  7 00:52:21 sepia21 kernel: [ 2418.234209] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8800c3259498
Nov  7 00:52:21 sepia21 kernel: [ 2418.234310] R13: ffffffff81ccb018 R14: ffff880103bc3800 R15: ffff88010204df10
Nov  7 00:52:21 sepia21 kernel: [ 2418.234412] FS:  00007f36a4daa740(0000) GS:ffff88011fc00000(0000) knlGS:0000000000000000
Nov  7 00:52:21 sepia21 kernel: [ 2418.234534] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
Nov  7 00:52:21 sepia21 kernel: [ 2418.234615] CR2: 0000000000000000 CR3: 0000000037251000 CR4: 00000000000006f0
Nov  7 00:52:21 sepia21 kernel: [ 2418.234716] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
Nov  7 00:52:21 sepia21 kernel: [ 2418.240995] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318] Process umount (pid: 20532, threadinfo ffff8800aa13e000, task ffff88010204df10)
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  ffff8801ffffffff ffff8800c32593e0 ffff8800aa13fdb8 ffffffff8118a77a
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  ffff8800c32593e0 ffffffffa03100e0 ffff8800aa13fde8 ffffffff8118b786
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  ffff8800aa13fdd8 00000000844377f8 ffff880103bc3800 ffffffffa03100e0
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff8118a77a>] dentry_lru_prune+0x9a/0xa0
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff8118b786>] shrink_dcache_for_umount_subtree+0x76/0x1e0
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff8118b927>] shrink_dcache_for_umount+0x37/0x60
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff8117789c>] generic_shutdown_super+0x2c/0xe0
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff811779e6>] kill_anon_super+0x16/0x30
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffffa02ee18a>] ceph_kill_sb+0x3a/0x70 [ceph]
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff81178495>] deactivate_locked_super+0x45/0x70
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff8117922a>] deactivate_super+0x4a/0x70
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff8119468d>] mntput_no_expire+0xed/0x140
Nov  7 00:52:21 sepia21 kernel: [ 2418.242318]  [<ffffffff81194bb8>] sys_umount+0x78/0x3b0
Nov  7 00:52:21 sepia21 kernel: [ 2418.340009]  [<ffffffff8160bb42>] system_call_fastpath+0x16/0x1b
Nov  7 00:52:21 sepia21 kernel: [ 2418.340009]  RSP <ffff8800aa13fd88>
Nov  7 00:52:21 sepia21 kernel: [ 2418.360558] ---[ end trace 94b390332507a92a ]---
Nov  7 00:52:21 sepia21 kernel: [ 2418.790542] libceph: mon1 10.3.14.147:6789 session lost, hunting for new mon

When I went back to (multiple of) these systems, and tried to do a user-mode umount of /tmp/cephtest/mnt.0
it failed (not in mtab, not superuser). When I reattempted the umount under sudo, it still reported that
it was not mounted, and returned an exit code of 1, but after that ceph was no longer reported as mounted.
Actions #1

Updated by Sage Weil over 12 years ago

  • Subject changed from test failures unmounting /tmp/cephtest/mnt.0 to kclient: crash in ceph_d_prune
Actions #2

Updated by Sage Weil over 12 years ago

Here is the code:

static void ceph_d_prune(struct dentry *dentry)
{
    struct ceph_dentry_info *di;

    dout("ceph_d_prune %p\n", dentry);

    /* do we have a valid parent? */
    if (!dentry->d_parent || IS_ROOT(dentry))
        return;

    /* if we are not hashed, we don't affect D_COMPLETE */
    if (d_unhashed(dentry))
        return;

    /*
     * we hold d_lock, so d_parent is stable, and d_fsdata is never
     * cleared until d_release
     */
    di = ceph_dentry(dentry->d_parent);
    clear_bit(CEPH_D_COMPLETE, &di->flags);
}

And the objdump (from another machine);

00000000000058c2 <ceph_d_prune>:
 * complete flag on the parent directory.
 *
 * Called under dentry->d_lock.
 */
static void ceph_d_prune(struct dentry *dentry)
{
    58c2:       55                      push   %rbp
    58c3:       48 89 e5                mov    %rsp,%rbp
    58c6:       53                      push   %rbx
    58c7:       48 89 fb                mov    %rdi,%rbx
    58ca:       48 83 ec 08             sub    $0x8,%rsp
        struct ceph_dentry_info *di;

        dout("d_release %p\n", dentry);
    58ce:       80 3d 00 00 00 00 00    cmpb   $0x0,0x0(%rip)        # 58d5 <ceph_d_prune+0x13>
                        58d0: R_X86_64_PC32     __verbose+0x11ef
    58d5:       74 37                   je     590e <ceph_d_prune+0x4c>
    58d7:       be 0e 00 00 00          mov    $0xe,%esi
    58dc:       bf 00 00 00 00          mov    $0x0,%edi
                        58dd: R_X86_64_32       .rodata.str1.1+0x34b6
    58e1:       e8 00 00 00 00          callq  58e6 <ceph_d_prune+0x24>
                        58e2: R_X86_64_PC32     ceph_file_part+0xfffffffffffffffc
    58e6:       51                      push   %rcx
    58e7:       49 89 c0                mov    %rax,%r8
    58ea:       ba 03 00 00 00          mov    $0x3,%edx
    58ef:       41 b9 7a 04 00 00       mov    $0x47a,%r9d
    58f5:       b9 00 00 00 00          mov    $0x0,%ecx
                        58f6: R_X86_64_32       .rodata.str1.1+0x34c4
    58fa:       be 00 00 00 00          mov    $0x0,%esi
                        58fb: R_X86_64_32       .rodata.str1.1+0x34c9
    58ff:       53                      push   %rbx
    5900:       bf 00 00 00 00          mov    $0x0,%edi
                        5901: R_X86_64_32       __verbose+0x11d0
    5905:       31 c0                   xor    %eax,%eax
    5907:       e8 00 00 00 00          callq  590c <ceph_d_prune+0x4a>
                        5908: R_X86_64_PC32     __dynamic_pr_debug+0xfffffffffffffffc
    590c:       58                      pop    %rax
    590d:       5a                      pop    %rdx

        /* do we have a valid parent? */
        if (!dentry->d_parent || IS_ROOT(dentry))
    590e:       48 8b 43 18             mov    0x18(%rbx),%rax
    5912:       48 85 c0                test   %rax,%rax
    5915:       74 16                   je     592d <ceph_d_prune+0x6b>
    5917:       48 39 c3                cmp    %rax,%rbx
    591a:       74 11                   je     592d <ceph_d_prune+0x6b>
                return;

        /* if we are not hashed, we don't affect D_COMPLETE */
        if (d_unhashed(dentry))
    591c:       48 83 7b 10 00          cmpq   $0x0,0x10(%rbx)
    5921:       74 0a                   je     592d <ceph_d_prune+0x6b>
static __always_inline void
clear_bit(int nr, volatile unsigned long *addr)
{
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "andb %1,%0" 
                        : CONST_MASK_ADDR(nr, addr)
    5923:       48 8b 80 90 00 00 00    mov    0x90(%rax),%rax
 */
static __always_inline void
clear_bit(int nr, volatile unsigned long *addr)
{
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "andb %1,%0" 
    592a:       80 20 fd                andb   $0xfd,(%rax)
         * we hold d_lock, so d_parent is stable, and d_fsdata is never
         * cleared until d_release
         */
        di = ceph_dentry(dentry->d_parent);
        clear_bit(CEPH_D_COMPLETE, &di->flags);
}
    592d:       48 8b 5d f8             mov    -0x8(%rbp),%rbx
    5931:       c9                      leaveq 
    5932:       c3                      retq   

Actions #3

Updated by Sage Weil over 12 years ago

  • Description updated (diff)
Actions #4

Updated by Sage Weil over 12 years ago

  • Status changed from New to Resolved
  • Assignee set to Sage Weil

fixed by commit:774ac21da76f5c3018428725074e27a3fd40b128

Actions

Also available in: Atom PDF