Actions
Bug #27205
closedIncorrect stats by "radosgw-admin user stats" after bucket resharding
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
Description of problem:
Steps to Reproduce
1, create a new bucket
2, upload 1024 objects
3, do bucket resharding
4，upload 16 objects
5, verify user stats with uid
ceph version
root@DOL-BJ-1-SSD02:~# ceph -v
ceph version 12.2.7 (3ec878d1e53e1aeb47a9f619c49d9e7c0aa384d5) luminous (stable)
running the following command:
Step 1
root@DOL-BJ-1-SSD02:~# s3cmd mb s3://reshard
Bucket 's3://reshard/' created
Step 2
# for i in `seq 1 1024`;do s3cmd put 16K s3://reshard/16k-$i;done
bucket stats
root@DOL-BJ-1-SSD02:~/data# radosgw-admin bucket stats --bucket reshard
{
"bucket": "reshard",
"zonegroup": "30755d67-e7ef-49f5-9c00-017ad75c6703",
"placement_rule": "default-placement",
"explicit_placement": {
"data_pool": "",
"data_extra_pool": "",
"index_pool": ""
},
"id": "0a27e4b8-a503-452e-9a19-c2e5c0ecfb6a.7920.3",
"marker": "0a27e4b8-a503-452e-9a19-c2e5c0ecfb6a.7920.3",
"index_type": "Normal",
"owner": "liul",
"ver": "0#2049",
"master_ver": "0#0",
"mtime": "2018-08-23 16:38:05.939057",
"max_marker": "0#",
"usage": {
"rgw.main": {
"size": 16777216,
"size_actual": 16777216,
"size_utilized": 16777216,
"size_kb": 16384,
"size_kb_actual": 16384,
"size_kb_utilized": 16384,
"num_objects": 1024
}
},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
}
}
user stats
root@DOL-BJ-1-SSD02:~/data# radosgw-admin user stats --uid liul --sync-stats
{
"stats": {
"total_entries": 1024,
"total_bytes": 16777216,
"total_bytes_rounded": 16777216
},
"last_stats_sync": "2018-08-23 08:46:51.145854Z",
"last_stats_update": "2018-08-23 08:46:51.141904Z"
}
shard status
root@DOL-BJ-1-SSD02:~/data# radosgw-admin bucket limit check --uid liul
[
{
"user_id": "liul",
"buckets": [
{
"bucket": "reshard",
"tenant": "",
"num_objects": 1024,
"num_shards": 0,
"objects_per_shard": 1024,
"fill_status": "OK"
}
]
}
]
Step 3
root@DOL-BJ-1-SSD02:~/data# radosgw-admin bucket reshard --bucket reshard --num-shards 4
*** NOTICE: operation will not remove old bucket index objects ***
*** these will need to be removed manually ***
tenant:
bucket name: reshard
old bucket instance id: 0a27e4b8-a503-452e-9a19-c2e5c0ecfb6a.7920.3
new bucket instance id: 0a27e4b8-a503-452e-9a19-c2e5c0ecfb6a.31404.1
total entries: 1000 1024
root@DOL-BJ-1-SSD02:~/data#
upload 16 objects
root@DOL-BJ-1-SSD02:~/data# for i in `seq 1 16`;do s3cmd put 64K s3://reshard/64k-$i;done
upload: '64K' -> 's3://reshard/64k-1' [1 of 1]
65536 of 65536 100% in 0s 4.48 MB/s done
upload: '64K' -> 's3://reshard/64k-2' [1 of 1]
65536 of 65536 100% in 0s 6.12 MB/s done
upload: '64K' -> 's3://reshard/64k-3' [1 of 1]
65536 of 65536 100% in 0s 5.39 MB/s done
upload: '64K' -> 's3://reshard/64k-4' [1 of 1]
65536 of 65536 100% in 0s 6.05 MB/s done
upload: '64K' -> 's3://reshard/64k-5' [1 of 1]
65536 of 65536 100% in 0s 5.90 MB/s done
upload: '64K' -> 's3://reshard/64k-6' [1 of 1]
65536 of 65536 100% in 0s 5.83 MB/s done
upload: '64K' -> 's3://reshard/64k-7' [1 of 1]
65536 of 65536 100% in 0s 5.37 MB/s done
upload: '64K' -> 's3://reshard/64k-8' [1 of 1]
65536 of 65536 100% in 0s 5.76 MB/s done
upload: '64K' -> 's3://reshard/64k-9' [1 of 1]
65536 of 65536 100% in 0s 5.70 MB/s done
upload: '64K' -> 's3://reshard/64k-10' [1 of 1]
65536 of 65536 100% in 0s 5.68 MB/s done
upload: '64K' -> 's3://reshard/64k-11' [1 of 1]
65536 of 65536 100% in 0s 5.58 MB/s done
upload: '64K' -> 's3://reshard/64k-12' [1 of 1]
65536 of 65536 100% in 0s 5.99 MB/s done
upload: '64K' -> 's3://reshard/64k-13' [1 of 1]
65536 of 65536 100% in 0s 5.70 MB/s done
upload: '64K' -> 's3://reshard/64k-14' [1 of 1]
65536 of 65536 100% in 0s 4.92 MB/s done
upload: '64K' -> 's3://reshard/64k-15' [1 of 1]
65536 of 65536 100% in 0s 5.58 MB/s done
upload: '64K' -> 's3://reshard/64k-16' [1 of 1]
65536 of 65536 100% in 0s 5.77 MB/s done
root@DOL-BJ-1-SSD02:~/data#
now bucket stats
root@DOL-BJ-1-SSD02:~/data# radosgw-admin bucket stats --bucket reshard
{
"bucket": "reshard",
"zonegroup": "30755d67-e7ef-49f5-9c00-017ad75c6703",
"placement_rule": "default-placement",
"explicit_placement": {
"data_pool": "",
"data_extra_pool": "",
"index_pool": ""
},
"id": "0a27e4b8-a503-452e-9a19-c2e5c0ecfb6a.31404.1",
"marker": "0a27e4b8-a503-452e-9a19-c2e5c0ecfb6a.7920.3",
"index_type": "Normal",
"owner": "liul",
"ver": "0#13,1#10,2#16,3#15",
"master_ver": "0#0,1#0,2#0,3#0",
"mtime": "2018-08-23 16:47:59.717105",
"max_marker": "0#,1#,2#,3#",
"usage": {
"rgw.main": {
"size": 17825792,
"size_actual": 17825792,
"size_utilized": 1048576,
"size_kb": 17408,
"size_kb_actual": 17408,
"size_kb_utilized": 1024,
"num_objects": 1040 <<---- Notice this number
}
},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
}
}
now user stats
root@DOL-BJ-1-SSD02:~/data# radosgw-admin user stats --uid liul --sync-stats
{
"stats": {
"total_entries": 1024, <<--- incorrect entry numbers
"total_bytes": 16777216,
"total_bytes_rounded": 16777216
},
"last_stats_sync": "2018-08-23 08:51:15.469328Z",
"last_stats_update": "2018-08-23 08:51:15.465673Z"
}
now shard status
root@DOL-BJ-1-SSD02:~/data# radosgw-admin bucket limit check --uid liul
[
{
"user_id": "liul",
"buckets": [
{
"bucket": "reshard",
"tenant": "",
"num_objects": 1040,
"num_shards": 4,
"objects_per_shard": 260,
"fill_status": "OK"
}
]
}
]
How to safe remove the bucket instance id after bucket resharding ?
This issue also occours when rgw_dynamic_resharding is true
Updated by Lei Liu over 5 years ago
Similar to https://tracker.ceph.com/issues/24505
Updated by Casey Bodley over 5 years ago
- Related to Bug #24505: radosgw-admin user stats are incorrect when dynamic re-sharding is enabled added
Actions