Actions
Bug #49128
closedNon-default storage class results in some garbage
Status:
Resolved
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
100%
Source:
Tags:
multipart storageclass
Backport:
pacific octopus
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
I created a bucket sc-sandbox, whose default storage class was STANDARD with 00000000-default.rgw.buckets.data as the data pool.
And I created Storage Class ANOTHER as well, whose data pool is default.rgw.control.
A file was uploaded in multipart mode and then deleted (rgw_gc_obj_min_wait had been set to 10 so that GC thread was able to remove quick).
However, some 0-size multiparts remained in 00000000-default.rgw.buckets.data even if radosgw-admin gc process was executed manually.
This looks like storage leakage.
Is this a bug or not?
MultipartObjectProcessor::complete() saves the meta into an incorrent pool?
Here are the steps I tried to reproduce the issue:
[root@100 /data/ft_local]# radosgw-admin zonegroup placement add --rgw-zonegroup default --placement-id sc --storage-class ANOTHER
[
{
"key": "default-placement",
"val": {
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
},
{
"key": "sc",
"val": {
"name": "sc",
"tags": [],
"storage_classes": [
"ANOTHER",
"STANDARD"
]
}
}
]
[root@100 /data/ft_local]# radosgw-admin zone placement add --rgw-zone default --placement-id sc --storage-class ANOTHER --data-pool default.rgw.control
{
"id": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"name": "default",
"domain_root": "default.rgw.meta:root",
"control_pool": "default.rgw.control",
"gc_pool": "default.rgw.log:gc",
"lc_pool": "default.rgw.log:lc",
"log_pool": "default.rgw.log",
"intent_log_pool": "default.rgw.log:intent",
"usage_log_pool": "default.rgw.log:usage",
"reshard_pool": "default.rgw.log:reshard",
"user_keys_pool": "default.rgw.meta:users.keys",
"user_email_pool": "default.rgw.meta:users.email",
"user_swift_pool": "default.rgw.meta:users.swift",
"user_uid_pool": "default.rgw.meta:users.uid",
"otp_pool": "default.rgw.otp",
"system_key": {
"access_key": "QBK8NR7HEI8Q0U8PNKB5",
"secret_key": "PrlfAUURn2xCgCggsZ30231rdbolMgTUgcdEFCIp"
},
"placement_pools": [
{
"key": "default-placement",
"val": {
"index_pool": "default.rgw.buckets.index",
"storage_classes": {
"STANDARD": {
"data_pool": "default.rgw.buckets.data"
}
},
"data_tail_pool": "",
"data_extra_pool": "default.rgw.buckets.non-ec",
"index_type": 0
}
},
{
"key": "sc",
"val": {
"index_pool": "00000000-default.rgw.buckets.index",
"storage_classes": {
"ANOTHER": {
"data_pool": "default.rgw.control"
},
"STANDARD": {
"data_pool": "00000000-default.rgw.buckets.data",
"compression_type": "none"
}
},
"data_tail_pool": "",
"data_extra_pool": "00000000-default.rgw.buckets.non-ec",
"index_type": 0
}
}
],
"metadata_heap": "",
"realm_id": ""
}
[root@100 /data/ft_local]# radosgw-admin period update --commit
{
"id": "53eb4a3d-ebd0-4108-97c8-a2201866110f",
"epoch": 4,
"predecessor_uuid": "a898a1f7-b58c-42d6-9382-33ab0a909425",
"sync_status": [],
"period_map": {
"id": "53eb4a3d-ebd0-4108-97c8-a2201866110f",
"zonegroups": [
{
"id": "7b66921d-b898-492c-b500-2841e42564a4",
"name": "default",
"api_name": "default",
"is_master": "true",
"endpoints": [
"http://127.0.0.1"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"zones": [
{
"id": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"name": "default",
"endpoints": [
"http://127.0.0.1"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
},
{
"name": "sc",
"tags": [],
"storage_classes": [
"ANOTHER",
"STANDARD"
]
}
],
"default_placement": "sc",
"realm_id": "faa41a14-7190-471c-9435-8fb3af8bda8d"
}
],
"short_zone_ids": [
{
"key": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"val": 2119278722
}
]
},
"master_zonegroup": "7b66921d-b898-492c-b500-2841e42564a4",
"master_zone": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"period_config": {
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
}
},
"realm_id": "faa41a14-7190-471c-9435-8fb3af8bda8d",
"realm_name": "default",
"realm_epoch": 2
}
2021-02-03 17:50:22.925 7fd487f7c840 2 []removed watcher, disabling cache
[root@100 /data/ft_local]# s3cmd mb s3://sc-sandbox
Bucket 's3://sc-sandbox/' created
[root@100 /data/ft_local]# s3cmd --storage-class=ANOTHER put moira s3://sc-sandbox/moira.a.another
upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 1 of 4, 8MB] [1 of 1]
8388608 of 8388608 100% in 0s 70.11 MB/s done
upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 2 of 4, 8MB] [1 of 1]
8388608 of 8388608 100% in 0s 81.68 MB/s done
upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 3 of 4, 8MB] [1 of 1]
8388608 of 8388608 100% in 0s 71.96 MB/s done
upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 4 of 4, 2MB] [1 of 1]
2264432 of 2264432 100% in 0s 61.20 MB/s done
[root@100 /data/ft_local]# rados -p default.rgw.control ls | grep -v notify
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4
[root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data ls
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1_moira.a.another
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4
[root@100 /data/ft_local]# s3cmd del s3://sc-sandbox/moira.a.another && radosgw-admin gc list --include-all
delete: 's3://sc-sandbox/moira.a.another'
[
{
"tag": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6198.7\u0000",
"time": "2021-02-03 17:51:34.0.631997s",
"objs": [
{
"pool": "default.rgw.control",
"oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1",
"key": "",
"instance": ""
},
{
"pool": "default.rgw.control",
"oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2",
"key": "",
"instance": ""
},
{
"pool": "default.rgw.control",
"oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3",
"key": "",
"instance": ""
},
{
"pool": "default.rgw.control",
"oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4",
"key": "",
"instance": ""
}
]
}
]
2021-02-03 17:51:24.897 7f0e62c65840 2 []removed watcher, disabling cache
[root@100 /data/ft_local]# sleep 10
[root@100 /data/ft_local]# radosgw-admin gc process
2021-02-03 17:51:50.437 7fc3b7950840 2 []all 8 watchers are set, enabling cache
2021-02-03 17:51:50.441 7fc380ff9700 2 []RGWDataChangesLog::ChangesRenewThread: start
2021-02-03 17:51:50.441 7fc3b7950840 0 []WARNING: detected a version of libcurl which contains a bug in curl_multi_wait(). enabling a workaround that may degrade performance slightly.
2021-02-03 17:51:51.785 7fc3b7950840 2 []removed watcher, disabling cache
[root@100 /data/ft_local]# rados -p default.rgw.control ls | grep -v notify
[root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data ls
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4
[root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data stat fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1
00000000-default.rgw.buckets.data/fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1 mtime 2021-02-03 17:50:44.000000, size 0
[root@100 /data/ft_local]#
Actions