Actions
Bug #49128
closedNon-default storage class results in some garbage
Status:
Resolved
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
100%
Source:
Tags:
multipart storageclass
Backport:
pacific octopus
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
I created a bucket sc-sandbox, whose default storage class was STANDARD with 00000000-default.rgw.buckets.data as the data pool.
And I created Storage Class ANOTHER as well, whose data pool is default.rgw.control.
A file was uploaded in multipart mode and then deleted (rgw_gc_obj_min_wait had been set to 10 so that GC thread was able to remove quick).
However, some 0-size multiparts remained in 00000000-default.rgw.buckets.data even if radosgw-admin gc process was executed manually.
This looks like storage leakage.
Is this a bug or not?
MultipartObjectProcessor::complete() saves the meta into an incorrent pool?
Here are the steps I tried to reproduce the issue:
[root@100 /data/ft_local]# radosgw-admin zonegroup placement add --rgw-zonegroup default --placement-id sc --storage-class ANOTHER [ { "key": "default-placement", "val": { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD" ] } }, { "key": "sc", "val": { "name": "sc", "tags": [], "storage_classes": [ "ANOTHER", "STANDARD" ] } } ] [root@100 /data/ft_local]# radosgw-admin zone placement add --rgw-zone default --placement-id sc --storage-class ANOTHER --data-pool default.rgw.control { "id": "fccb890e-16b4-4baf-aa6f-8739b87d7a38", "name": "default", "domain_root": "default.rgw.meta:root", "control_pool": "default.rgw.control", "gc_pool": "default.rgw.log:gc", "lc_pool": "default.rgw.log:lc", "log_pool": "default.rgw.log", "intent_log_pool": "default.rgw.log:intent", "usage_log_pool": "default.rgw.log:usage", "reshard_pool": "default.rgw.log:reshard", "user_keys_pool": "default.rgw.meta:users.keys", "user_email_pool": "default.rgw.meta:users.email", "user_swift_pool": "default.rgw.meta:users.swift", "user_uid_pool": "default.rgw.meta:users.uid", "otp_pool": "default.rgw.otp", "system_key": { "access_key": "QBK8NR7HEI8Q0U8PNKB5", "secret_key": "PrlfAUURn2xCgCggsZ30231rdbolMgTUgcdEFCIp" }, "placement_pools": [ { "key": "default-placement", "val": { "index_pool": "default.rgw.buckets.index", "storage_classes": { "STANDARD": { "data_pool": "default.rgw.buckets.data" } }, "data_tail_pool": "", "data_extra_pool": "default.rgw.buckets.non-ec", "index_type": 0 } }, { "key": "sc", "val": { "index_pool": "00000000-default.rgw.buckets.index", "storage_classes": { "ANOTHER": { "data_pool": "default.rgw.control" }, "STANDARD": { "data_pool": "00000000-default.rgw.buckets.data", "compression_type": "none" } }, "data_tail_pool": "", "data_extra_pool": "00000000-default.rgw.buckets.non-ec", "index_type": 0 } } ], "metadata_heap": "", "realm_id": "" } [root@100 /data/ft_local]# radosgw-admin period update --commit { "id": "53eb4a3d-ebd0-4108-97c8-a2201866110f", "epoch": 4, "predecessor_uuid": "a898a1f7-b58c-42d6-9382-33ab0a909425", "sync_status": [], "period_map": { "id": "53eb4a3d-ebd0-4108-97c8-a2201866110f", "zonegroups": [ { "id": "7b66921d-b898-492c-b500-2841e42564a4", "name": "default", "api_name": "default", "is_master": "true", "endpoints": [ "http://127.0.0.1" ], "hostnames": [], "hostnames_s3website": [], "master_zone": "fccb890e-16b4-4baf-aa6f-8739b87d7a38", "zones": [ { "id": "fccb890e-16b4-4baf-aa6f-8739b87d7a38", "name": "default", "endpoints": [ "http://127.0.0.1" ], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 0, "read_only": "false", "tier_type": "", "sync_from_all": "true", "sync_from": [], "redirect_zone": "" } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD" ] }, { "name": "sc", "tags": [], "storage_classes": [ "ANOTHER", "STANDARD" ] } ], "default_placement": "sc", "realm_id": "faa41a14-7190-471c-9435-8fb3af8bda8d" } ], "short_zone_ids": [ { "key": "fccb890e-16b4-4baf-aa6f-8739b87d7a38", "val": 2119278722 } ] }, "master_zonegroup": "7b66921d-b898-492c-b500-2841e42564a4", "master_zone": "fccb890e-16b4-4baf-aa6f-8739b87d7a38", "period_config": { "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 } }, "realm_id": "faa41a14-7190-471c-9435-8fb3af8bda8d", "realm_name": "default", "realm_epoch": 2 } 2021-02-03 17:50:22.925 7fd487f7c840 2 []removed watcher, disabling cache [root@100 /data/ft_local]# s3cmd mb s3://sc-sandbox Bucket 's3://sc-sandbox/' created [root@100 /data/ft_local]# s3cmd --storage-class=ANOTHER put moira s3://sc-sandbox/moira.a.another upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 1 of 4, 8MB] [1 of 1] 8388608 of 8388608 100% in 0s 70.11 MB/s done upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 2 of 4, 8MB] [1 of 1] 8388608 of 8388608 100% in 0s 81.68 MB/s done upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 3 of 4, 8MB] [1 of 1] 8388608 of 8388608 100% in 0s 71.96 MB/s done upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 4 of 4, 2MB] [1 of 1] 2264432 of 2264432 100% in 0s 61.20 MB/s done [root@100 /data/ft_local]# rados -p default.rgw.control ls | grep -v notify fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4 [root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data ls fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1_moira.a.another fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4 [root@100 /data/ft_local]# s3cmd del s3://sc-sandbox/moira.a.another && radosgw-admin gc list --include-all delete: 's3://sc-sandbox/moira.a.another' [ { "tag": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6198.7\u0000", "time": "2021-02-03 17:51:34.0.631997s", "objs": [ { "pool": "default.rgw.control", "oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1", "key": "", "instance": "" }, { "pool": "default.rgw.control", "oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2", "key": "", "instance": "" }, { "pool": "default.rgw.control", "oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3", "key": "", "instance": "" }, { "pool": "default.rgw.control", "oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4", "key": "", "instance": "" } ] } ] 2021-02-03 17:51:24.897 7f0e62c65840 2 []removed watcher, disabling cache [root@100 /data/ft_local]# sleep 10 [root@100 /data/ft_local]# radosgw-admin gc process 2021-02-03 17:51:50.437 7fc3b7950840 2 []all 8 watchers are set, enabling cache 2021-02-03 17:51:50.441 7fc380ff9700 2 []RGWDataChangesLog::ChangesRenewThread: start 2021-02-03 17:51:50.441 7fc3b7950840 0 []WARNING: detected a version of libcurl which contains a bug in curl_multi_wait(). enabling a workaround that may degrade performance slightly. 2021-02-03 17:51:51.785 7fc3b7950840 2 []removed watcher, disabling cache [root@100 /data/ft_local]# rados -p default.rgw.control ls | grep -v notify [root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data ls fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3 fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4 [root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data stat fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1 00000000-default.rgw.buckets.data/fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1 mtime 2021-02-03 17:50:44.000000, size 0 [root@100 /data/ft_local]#
Actions