Ceph » CephFS

Somehow a path in the long-running cluster got a corrupted number of files/subdirs, and responds to "rm -rf" with "cannot remove, directory not empty". There are no visible files in the directory (and find -type f | xargs rm had been done successfully). scrub_path repair notices but does not resolve the error. Also, its rstats are similarly corrupt.

ls -ld /a/sage-2016-11-12_02:26:45-rados-wip-sage-testing---basic-smithi/541839/remote/smithi059/log

drwxrwxr-x 1 teuthworker teuthworker 18446744073315575061 Jan 11 01:05 /a/sage-2016-11-12_02:26:45-rados-wip-sage-testing---basic-smithi/541839/remote/smithi059/log

excerpt from daemon dump tree:

    {
        "ino": 1100004785907,
        "rdev": 0,
        "ctime": "2017-01-11 01:05:34.995307",
        "btime": "0.000000",
        "mode": 16893,
        "uid": 1001,
        "gid": 1001,
        "nlink": 1,
        "dir_layout": {
            "dir_hash": 2
        },
        "layout": {
            "stripe_unit": 0,
            "stripe_count": 0,
            "object_size": 0,
            "pool_id": -1,
            "pool_ns": "" 
        },
        "old_pools": [],
        "size": 0,
        "truncate_seq": 1,
        "truncate_size": 18446744073709551615,
        "truncate_from": 0,
        "truncate_pending": 0,
        "mtime": "2017-01-11 01:05:34.995307",
        "atime": "2016-11-12 10:08:48.440278",
        "time_warp_seq": 0,
        "change_attr": 9699,
        "client_ranges": [],
        "dirstat": {
            "version": 1,
            "mtime": "2017-01-11 01:05:34.995307",
            "num_files": 18446744073709550716,
            "num_subdirs": 18446744073709551615
        },
        "rstat": {
            "version": 9,
            "rbytes": 18446744073315575061,
            "rfiles": 18446744073709550711,
            "rsubdirs": 0,
            "rsnaprealms": 0,
            "rctime": "2017-01-11 01:05:34.995307" 
        },
        "accounted_rstat": {
            "version": 9,
            "rbytes": 18446744073315575061,
            "rfiles": 18446744073709550711,
            "rsubdirs": 0,
            "rsnaprealms": 0,
            "rctime": "2017-01-11 01:05:34.995307" 
        },
        "version": 56343,
        "file_data_version": 0,
        "xattr_version": 1,
        "backtrace_version": 2,
        "stray_prior_path": "",
        "symlink": "",
        "old_inodes": [],
        "dirfragtree": {
            "splits": []
        },
        "is_auth": true,
        "auth_state": {
            "replicas": {}
        },
        "replica_state": {
            "authority": [
                0,
                -2
            ],
            "replica_nonce": 0
        },
        "auth_pins": 0,
        "nested_auth_pins": 0,
        "is_frozen": false,
        "is_freezing": false,
        "pins": {
            "request": 0,
            "lock": 0,
            "dirfrag": 0,
            "caps": 1,
            "scrubqueue": 0,
            "authpin": 0
        },
        "nref": 1,
        "versionlock": {
            "gather_set": [],
            "num_client_lease": 0,
            "num_rdlocks": 0,
            "num_wrlocks": 0,
            "num_xlocks": 0,
            "xlock_by": {}
        },
        "authlock": {},
        "linklock": {},
        "dirfragtreelock": {},
        "filelock": {
            "gather_set": [],
            "num_client_lease": 0,
            "num_rdlocks": 0,
            "num_wrlocks": 0,
            "num_xlocks": 0,
            "xlock_by": {}
        },
        "xattrlock": {},
        "snaplock": {},
        "nestlock": {
            "gather_set": [],
            "num_client_lease": 0,
            "num_rdlocks": 0,
            "num_wrlocks": 0,
            "num_xlocks": 0,
            "xlock_by": {}
        },
        "flocklock": {},
        "policylock": {},
        "states": [
            "auth" 
        ],
        "client_caps": [
            {
                "client_id": 25937231,
                "pending": "pAsLsXsFsx",
                "issued": "pAsLsXsFsx",
                "wanted": "-",
                "last_sent": "Asx" 
            }
        ],
        "loner": 25937231,
        "want_loner": 25937231,
        "mds_caps_wanted": [],
        "dirfrags": [
            {
                "path": "\/teuthology-archive\/sage-2016-11-12_02:26:45-rados-wip-sage-testing---basic-smithi\/541839\/remote\/smithi059\/log",
                "dirfrag": "1001d64fef3",
                "snapid_first": 2,
                "projected_version": "84312",
                "version": "84312",
                "committing_version": "84312",
                "committed_version": "84312",
                "is_rep": false,
                "dir_auth": "",
                "states": [
                    "auth",
                    "complete" 
                ],
                "is_auth": true,
                "auth_state": {
                    "replicas": {}
                },
                "replica_state": {
                    "authority": [
                        0,
                        -2
                    ],
                    "replica_nonce": 0
                },
                "auth_pins": 0,
                "nested_auth_pins": 0,
                "is_frozen": false,
                "is_freezing": false,
                "pins": {
                    "waiter": 0,
                    "authpin": 0
                },
                "nref": 0,
                "dentries": []
            }
        ]
    }