Bug #49128
Non-default storage class results in some garbage
Status:
Resolved
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
100%
Source:
Tags:
multipart storageclass
Backport:
pacific octopus
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
I created a bucket sc-sandbox, whose default storage class was STANDARD with 00000000-default.rgw.buckets.data as the data pool.
And I created Storage Class ANOTHER as well, whose data pool is default.rgw.control.
A file was uploaded in multipart mode and then deleted (rgw_gc_obj_min_wait had been set to 10 so that GC thread was able to remove quick).
However, some 0-size multiparts remained in 00000000-default.rgw.buckets.data even if radosgw-admin gc process was executed manually.
This looks like storage leakage.
Is this a bug or not?
MultipartObjectProcessor::complete() saves the meta into an incorrent pool?
Here are the steps I tried to reproduce the issue:
[root@100 /data/ft_local]# radosgw-admin zonegroup placement add --rgw-zonegroup default --placement-id sc --storage-class ANOTHER
[
{
"key": "default-placement",
"val": {
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
}
},
{
"key": "sc",
"val": {
"name": "sc",
"tags": [],
"storage_classes": [
"ANOTHER",
"STANDARD"
]
}
}
]
[root@100 /data/ft_local]# radosgw-admin zone placement add --rgw-zone default --placement-id sc --storage-class ANOTHER --data-pool default.rgw.control
{
"id": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"name": "default",
"domain_root": "default.rgw.meta:root",
"control_pool": "default.rgw.control",
"gc_pool": "default.rgw.log:gc",
"lc_pool": "default.rgw.log:lc",
"log_pool": "default.rgw.log",
"intent_log_pool": "default.rgw.log:intent",
"usage_log_pool": "default.rgw.log:usage",
"reshard_pool": "default.rgw.log:reshard",
"user_keys_pool": "default.rgw.meta:users.keys",
"user_email_pool": "default.rgw.meta:users.email",
"user_swift_pool": "default.rgw.meta:users.swift",
"user_uid_pool": "default.rgw.meta:users.uid",
"otp_pool": "default.rgw.otp",
"system_key": {
"access_key": "QBK8NR7HEI8Q0U8PNKB5",
"secret_key": "PrlfAUURn2xCgCggsZ30231rdbolMgTUgcdEFCIp"
},
"placement_pools": [
{
"key": "default-placement",
"val": {
"index_pool": "default.rgw.buckets.index",
"storage_classes": {
"STANDARD": {
"data_pool": "default.rgw.buckets.data"
}
},
"data_tail_pool": "",
"data_extra_pool": "default.rgw.buckets.non-ec",
"index_type": 0
}
},
{
"key": "sc",
"val": {
"index_pool": "00000000-default.rgw.buckets.index",
"storage_classes": {
"ANOTHER": {
"data_pool": "default.rgw.control"
},
"STANDARD": {
"data_pool": "00000000-default.rgw.buckets.data",
"compression_type": "none"
}
},
"data_tail_pool": "",
"data_extra_pool": "00000000-default.rgw.buckets.non-ec",
"index_type": 0
}
}
],
"metadata_heap": "",
"realm_id": ""
}
[root@100 /data/ft_local]# radosgw-admin period update --commit
{
"id": "53eb4a3d-ebd0-4108-97c8-a2201866110f",
"epoch": 4,
"predecessor_uuid": "a898a1f7-b58c-42d6-9382-33ab0a909425",
"sync_status": [],
"period_map": {
"id": "53eb4a3d-ebd0-4108-97c8-a2201866110f",
"zonegroups": [
{
"id": "7b66921d-b898-492c-b500-2841e42564a4",
"name": "default",
"api_name": "default",
"is_master": "true",
"endpoints": [
"http://127.0.0.1"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"zones": [
{
"id": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"name": "default",
"endpoints": [
"http://127.0.0.1"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false",
"tier_type": "",
"sync_from_all": "true",
"sync_from": [],
"redirect_zone": ""
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": [],
"storage_classes": [
"STANDARD"
]
},
{
"name": "sc",
"tags": [],
"storage_classes": [
"ANOTHER",
"STANDARD"
]
}
],
"default_placement": "sc",
"realm_id": "faa41a14-7190-471c-9435-8fb3af8bda8d"
}
],
"short_zone_ids": [
{
"key": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"val": 2119278722
}
]
},
"master_zonegroup": "7b66921d-b898-492c-b500-2841e42564a4",
"master_zone": "fccb890e-16b4-4baf-aa6f-8739b87d7a38",
"period_config": {
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
}
},
"realm_id": "faa41a14-7190-471c-9435-8fb3af8bda8d",
"realm_name": "default",
"realm_epoch": 2
}
2021-02-03 17:50:22.925 7fd487f7c840 2 []removed watcher, disabling cache
[root@100 /data/ft_local]# s3cmd mb s3://sc-sandbox
Bucket 's3://sc-sandbox/' created
[root@100 /data/ft_local]# s3cmd --storage-class=ANOTHER put moira s3://sc-sandbox/moira.a.another
upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 1 of 4, 8MB] [1 of 1]
8388608 of 8388608 100% in 0s 70.11 MB/s done
upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 2 of 4, 8MB] [1 of 1]
8388608 of 8388608 100% in 0s 81.68 MB/s done
upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 3 of 4, 8MB] [1 of 1]
8388608 of 8388608 100% in 0s 71.96 MB/s done
upload: 'moira' -> 's3://sc-sandbox/moira.a.another' [part 4 of 4, 2MB] [1 of 1]
2264432 of 2264432 100% in 0s 61.20 MB/s done
[root@100 /data/ft_local]# rados -p default.rgw.control ls | grep -v notify
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4
[root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data ls
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1_moira.a.another
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4
[root@100 /data/ft_local]# s3cmd del s3://sc-sandbox/moira.a.another && radosgw-admin gc list --include-all
delete: 's3://sc-sandbox/moira.a.another'
[
{
"tag": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6198.7\u0000",
"time": "2021-02-03 17:51:34.0.631997s",
"objs": [
{
"pool": "default.rgw.control",
"oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1",
"key": "",
"instance": ""
},
{
"pool": "default.rgw.control",
"oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2",
"key": "",
"instance": ""
},
{
"pool": "default.rgw.control",
"oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3",
"key": "",
"instance": ""
},
{
"pool": "default.rgw.control",
"oid": "fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4",
"key": "",
"instance": ""
}
]
}
]
2021-02-03 17:51:24.897 7f0e62c65840 2 []removed watcher, disabling cache
[root@100 /data/ft_local]# sleep 10
[root@100 /data/ft_local]# radosgw-admin gc process
2021-02-03 17:51:50.437 7fc3b7950840 2 []all 8 watchers are set, enabling cache
2021-02-03 17:51:50.441 7fc380ff9700 2 []RGWDataChangesLog::ChangesRenewThread: start
2021-02-03 17:51:50.441 7fc3b7950840 0 []WARNING: detected a version of libcurl which contains a bug in curl_multi_wait(). enabling a workaround that may degrade performance slightly.
2021-02-03 17:51:51.785 7fc3b7950840 2 []removed watcher, disabling cache
[root@100 /data/ft_local]# rados -p default.rgw.control ls | grep -v notify
[root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data ls
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.2
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.3
fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.4
[root@100 /data/ft_local]# rados -p 00000000-default.rgw.buckets.data stat fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1
00000000-default.rgw.buckets.data/fccb890e-16b4-4baf-aa6f-8739b87d7a38.6204.1__multipart_moira.a.another.2~3E_WLd95R2Cqmz8m-Gz1GJX7mFGUZs9.1 mtime 2021-02-03 17:50:44.000000, size 0
[root@100 /data/ft_local]#
Related issues
History
#1 Updated by Casey Bodley almost 3 years ago
- Status changed from New to Triaged
- Tags set to multipart storageclass
- Backport set to pacific octopus nautilus
it looks like the bug happens when uploading a multipart part to a non-default storage class. during upload, we write the object data to the correct pool for its storage class
but when completing the upload and doing the bucket index transaction, there's another write op (with a bunch of setxattrs) to the head object. this write is targeting the standard pool (because non-multipart uploads write their head objects to the default pool). as a result of this setxattrs op, we create a zero-sized object in the wrong pool, which GC never tries to clean up
the bucket index transaction in RGWRados::Object::Write::_do_write_meta() needs to learn the difference between multipart and non-multipart head objects. it's using RGWRados::get_obj_head_ref(), which only looks at the bucket's default placement
#2 Updated by Jeegn Chen over 2 years ago
#3 Updated by Casey Bodley over 2 years ago
- Status changed from Triaged to Fix Under Review
- Pull request ID set to 39934
#4 Updated by Casey Bodley over 2 years ago
- Status changed from Fix Under Review to Pending Backport
- Backport changed from pacific octopus nautilus to pacific octopus
#5 Updated by Backport Bot over 2 years ago
- Copied to Backport #51352: pacific: Non-default storage class results in some garbage added
#6 Updated by Backport Bot over 2 years ago
- Copied to Backport #51353: octopus: Non-default storage class results in some garbage added
#7 Updated by Backport Bot over 1 year ago
- Tags changed from multipart storageclass to multipart storageclass backport_processed
#8 Updated by Konstantin Shalygin about 1 year ago
- Status changed from Pending Backport to Resolved
- % Done changed from 0 to 100
- Tags changed from multipart storageclass backport_processed to multipart storageclass