Ceph » RADOS

I did some test on rbd snap, and found data inconsistent.

cluster status:

[root@CNSZ035541 ~]# ceph -s
    cluster 06cbb412-6367-44ea-9b51-a2befc383c39
     health HEALTH_ERR
            100 pgs inconsistent
            197733 scrub errors
     monmap e1: 1 mons at {CNSZ035541=10.25.89.11:6789/0}
            election epoch 6, quorum 0 CNSZ035541
     osdmap e4014: 4 osds: 4 up, 4 in
            flags sortbitwise,require_jewel_osds
      pgmap v147712: 100 pgs, 1 pools, 1018 GB data, 490 kobjects
            1563 GB used, 2880 GB / 4444 GB avail
                  99 active+clean+inconsistent
                   1 active+clean+scrubbing+deep+inconsistent

[root@CNSZ035541 ~]# rados list-inconsistent-obj 3.0 --format=json-pretty  |more
{
    "epoch": 3854,
    "inconsistents": [
        {
            "object": {
                "name": "rbd_data.b6d62ae8944a.0000000000001817",
                "nspace": "",
                "locator": "",
                "snap": 16,
                "version": 94601
            },
            "errors": [
                "data_digest_mismatch" 
            ],
            "union_shard_errors": [],
            "selected_object_info": "3:019b0426:::rbd_data.b6d62ae8944a.0000000000001817:10(3870'97928 client.46821.0:11972536 [10,f,c,b,a,
9,8,7,6,5,4] dirty|omap_digest s 4108288 uv 94601 od ffffffff alloc_hint [0 0])",
            "shards": [
                {
                    "osd": 0,
                    "errors": [],
                    "size": 4108288,
                    "omap_digest": "0xffffffff",
                    "data_digest": "0x5d9ba027" 
                },
                {
                    "osd": 1,
                    "errors": [],
                    "size": 4108288,
                    "omap_digest": "0xffffffff",
                    "data_digest": "0xb26dd237" 
                },
                {
                    "osd": 2,
                    "errors": [],
                    "size": 4108288,
                    "omap_digest": "0xffffffff",
                    "data_digest": "0x0fc905ed" 
                }
            ]
        },

here is my test script.

#!/bin/sh
j=0
while [ $j -lt 1 ]
do
    for i in `seq 100 100000`
    do

          rbd snap create rbdtest/rbdimg@snap${i};
          date>>snapfuxian_record
          echo "create rbdtest/rbdimg@snap${i}">>snapfuxian_record
          sleep 5;
          if [ $(expr $i % 10) -eq 0 ];then
                start=$(expr $i - 9);
                end=$(expr $i - 2);
                echo "start:${start},end: ${end}" 
                for k in `seq $start $end`
                do
                    echo $k
                    rbd snap rm rbdtest/rbdimg@snap${k};
                    date >>snapfuxian_record
                    echo "rm rbdtest/rbdimg@snap${k}">>snapfuxian_record
                    sleep 3
                done

          fi
    done
    ((j++))
done