Actions
Bug #22928
closedRGW will not list contents of Jewel-era buckets: reshard does NOT fix
Status:
Resolved
Priority:
High
Assignee:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
luminous
Regression:
Yes
Severity:
2 - major
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
This is similar to issue #22756 at the outset, however it was reported on the mailing lists by Ingo Reimann <ireimann@dunkel.de>, for new buckets that were created the Jewel release, while #22756 was for much older buckets.
The user noted that the cluster was old (dating back to Argonaut), but the affected RGW user & bucket were created under Jewel. The user has reviewed the output they emailed to me and agreed to it being public.
root@cephrgw01:~# ceph -v
ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous (stable)
root@cephrgw01:~# radosgw-admin metadata get bucket:cephrgw
{
"key": "bucket:cephrgw",
"ver": {
"tag": "_yvk6Y5LsN9JWlo8YvzNkKOM",
"ver": 1
},
"mtime": "2017-06-14 07:11:36.816989Z",
"data": {
"bucket": {
"name": "cephrgw",
"marker": "default.241135083.1",
"bucket_id": "default.241135083.1",
"tenant": "",
"explicit_placement": {
"data_pool": "rgw.buckets",
"data_extra_pool": "",
"index_pool": "rgw.buckets"
}
},
"owner": "DunkelCeph",
"creation_time": "2017-06-14 07:11:33.424648Z",
"linked": "true",
"has_bucket_info": "false"
}
}
root@cephrgw01:~# radosgw-admin metadata get bucket.instance:cephrgw:default.241135083.1
{
"key": "bucket.instance:cephrgw:default.241135083.1",
"ver": {
"tag": "_lhPMGrI03ZPcMw7Mj7z6duy",
"ver": 2
},
"mtime": "2018-01-29 10:33:46.942786Z",
"data": {
"bucket_info": {
"bucket": {
"name": "cephrgw",
"marker": "default.241135083.1",
"bucket_id": "default.241135083.1",
"tenant": "",
"explicit_placement": {
"data_pool": "rgw.buckets",
"data_extra_pool": "",
"index_pool": "rgw.buckets"
}
},
"creation_time": "2017-06-14 07:11:33.424648Z",
"owner": "DunkelCeph",
"flags": 0,
"zonegroup": "default",
"placement_rule": "",
"has_instance_obj": "true",
"quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1024,
"max_size_kb": 0,
"max_objects": -1
},
"num_shards": 8,
"bi_shard_hash_type": 0,
"requester_pays": "false",
"has_website": "false",
"swift_versioning": "false",
"swift_ver_location": "",
"index_type": 0,
"mdsearch_config": [],
"reshard_status": 0,
"new_bucket_instance_id": ""
},
"attrs": [
{
"key": "user.rgw.acl",
"val": "AgLXAQAAAwIcAAAACgAAAER1bmtlbENlcGgKAAAARHVua2VsQ2VwaAMDrwEAAAEBAAAACgAAAER1bmtlbENlcGgPAAAABQAAAAoAAABEdW5rZWxDZXBoBAM8AAAAAgIEAAAAAAAAAAoAAABEdW5rZWxDZXBoAAAAAAAAA
+AACAgQAAAAPAAAACgAAAER1bmtlbENlcGgAAAAACgAAAER1bmtlbENlcGgEAzwAAAACAgQAAAAAAAAACgAAAER1bmtlbENlcGgAAAAAAAAAAAICBAAAAAIAAAAKAAAARHVua2VsQ2VwaAAAAAAKAAAARHVua2VsQ2VwaAQDPAAAAAICBAAAAAAAAAAKA
+AAARHVua2VsQ2VwaAAAAAAAAAAAAgIEAAAAAQAAAAoAAABEdW5rZWxDZXBoAAAAAAoAAABEdW5rZWxDZXBoBAM8AAAAAgIEAAAAAAAAAAoAAABEdW5rZWxDZXBoAAAAAAAAAAACAgQAAAAIAAAACgAAAER1bmtlbENlcGgAAAAACgAAAER1bmtlbENlc
+GgEAzwAAAACAgQAAAAAAAAACgAAAER1bmtlbENlcGgAAAAAAAAAAAICBAAAAAQAAAAKAAAARHVua2VsQ2VwaAAAAAAAAAAA"
},
{
"key": "user.rgw.idtag",
"val": ""
}
]
}
}
root@cephrgw01:~# radosgw-admin bucket stats --bucket=cephrgw
{
"bucket": "cephrgw",
"zonegroup": "default",
"placement_rule": "",
"explicit_placement": {
"data_pool": "rgw.buckets",
"data_extra_pool": "",
"index_pool": "rgw.buckets"
},
"id": "default.241135083.1",
"marker": "default.241135083.1",
"index_type": "Normal",
"owner": "DunkelCeph",
"ver": "0#7,1#4,2#4,3#4,4#4,5#7,6#4,7#4",
"master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0",
"mtime": "2018-01-29 11:33:46.942786",
"max_marker": "0#,1#,2#,3#,4#,5#,6#,7#",
"usage": {},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1024,
"max_size_kb": 0,
"max_objects": -1
}
}
root@cephrgw01:~# rados -p rgw.buckets listomapvals .dir.default.241135083.1.1
testfile
value (212 bytes) :
00000000 08 03 ce 00 00 00 08 00 00 00 74 65 73 74 66 69 |..........testfi|
00000010 6c 65 78 0b 03 00 00 00 00 00 01 04 03 76 00 00 |lex..........v..|
00000020 00 01 37 00 00 00 00 00 00 00 49 e3 40 59 46 39 |..7.......I.@YF9|
00000030 47 03 20 00 00 00 39 38 65 30 30 31 63 31 31 62 |G. ...98e001c11b|
00000040 37 63 31 39 63 64 61 36 35 30 66 34 64 64 64 34 |7c19cda650f4ddd4|
00000050 35 39 34 64 62 63 0a 00 00 00 44 75 6e 6b 65 6c |594dbc....Dunkel|
00000060 43 65 70 68 0a 00 00 00 44 75 6e 6b 65 6c 43 65 |Ceph....DunkelCe|
00000070 70 68 19 00 00 00 74 65 78 74 2f 70 6c 61 69 6e |ph....text/plain|
00000080 3b 20 63 68 61 72 73 65 74 3d 75 74 66 2d 38 37 |; charset=utf-87|
00000090 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 |................|
000000a0 01 06 00 00 00 13 84 78 0b 03 00 02 16 00 00 00 |.......x........|
000000b0 64 65 66 61 75 6c 74 2e 32 34 31 30 39 34 32 32 |default.24109422|
000000c0 36 2e 34 37 37 37 00 00 00 00 00 00 00 00 00 00 |6.4777..........|
000000d0 00 00 00 00 |....|
000000d4
root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw
[]
root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw --debug-rgw=20
(attached)
root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw
[]
You have new mail in /var/mail/root
root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw --debug-rgw=20
(attached)
root@cephrgw01:~# radosgw-admin bucket reshard --bucket=cephrgw --num-shards=8 --yes-i-really-mean-it
*** NOTICE: operation will not remove old bucket index objects ***
*** these will need to be removed manually ***
tenant:
bucket name: cephrgw
old bucket instance id: default.241135083.1
new bucket instance id: default.278815034.1
total entries: 0
root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw
[]
root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw
[]
root@cephrgw01:~# radosgw-admin bucket stats --bucket=cephrgw
{
"bucket": "cephrgw",
"zonegroup": "default",
"placement_rule": "",
"explicit_placement": {
"data_pool": "rgw.buckets",
"data_extra_pool": "",
"index_pool": "rgw.buckets"
},
"id": "default.278815034.1",
"marker": "default.241135083.1",
"index_type": "Normal",
"owner": "DunkelCeph",
"ver": "0#1,1#1,2#1,3#1,4#1,5#1,6#1,7#1",
"master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0",
"mtime": "2018-02-01 19:29:20.845789",
"max_marker": "0#,1#,2#,3#,4#,5#,6#,7#",
"usage": {},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1024,
"max_size_kb": 0,
"max_objects": -1
}
}
root@cephrgw01:~# rados -p rgw.buckets listomapkeys
.dir.default.241135083.1.1
testfile
root@cephrgw01:~# rados -p rgw.buckets stat default.241135083.1_testfile
rgw.buckets/default.241135083.1_testfile mtime 2017-06-14 09:18:33.000000, size 55
root@cephrgw01:~# rados -p rgw.buckets listomapvals
default.241135083.1_testfile
root@cephrgw01:~# rados -p rgw.buckets get default.241135083.1_testfile -
Testdatei zur Prüfung der Funktion der rados-gateways
root@cephrgw01:~# rados -p rgw.buckets listxattr default.241135083.1_testfile
user.rgw.acl
user.rgw.content_type
user.rgw.etag
user.rgw.idtag
user.rgw.manifest
user.rgw.pg_ver
user.rgw.source_zone
user.rgw.x-amz-date
root@cephrgw01:~# rados -p rgw.buckets getxattr default.241135083.1_testfile user.rgw.idtag
default.241094226.4777
root@cephrgw01:~# rados -p rgw.buckets getxattr
default.241135083.1_testfile user.rgw.manifest | xxd
00000000: 0503 6b01 0000 3700 0000 0000 0000 0000 ..k...7.........
00000010: 0000 0005 0388 0000 0007 0000 0063 6570 .............cep
00000020: 6872 6777 0000 0000 0000 0000 0800 0000 hrgw............
00000030: 7465 7374 6669 6c65 0803 5f00 0000 0700 testfile.._.....
00000040: 0000 6365 7068 7267 770b 0000 0072 6777 ..cephrgw....rgw
00000050: 2e62 7563 6b65 7473 1300 0000 6465 6661 .buckets....defa
00000060: 756c 742e 3234 3131 3335 3038 332e 3113 ult.241135083.1.
00000070: 0000 0064 6566 6175 6c74 2e32 3431 3133 ...default.24113
00000080: 3530 3833 2e31 0b00 0000 7267 772e 6275 5083.1....rgw.bu
00000090: 636b 6574 7300 0000 0000 0000 0000 0000 ckets...........
000000a0: 0037 0000 0000 0000 0000 0008 0000 0000 .7..............
000000b0: 0021 0000 002e 5977 5256 375f 4846 3133 .!....YwRV7_HF13
000000c0: 4c79 7371 6146 6e5f 4e5f 4150 664d 3938 LysqaFn_N_APfM98
000000d0: 4e4f 7656 675f 0100 0000 0000 0000 0000 NOvVg_..........
000000e0: 0000 0201 2000 0000 0000 0000 0000 0800 .... ...........
000000f0: 0000 0000 0000 0000 0000 0000 0000 4000 ..............@.
00000100: 0000 0000 0000 0000 0803 5f00 0000 0700 .........._.....
00000110: 0000 6365 7068 7267 770b 0000 0072 6777 ..cephrgw....rgw
00000120: 2e62 7563 6b65 7473 1300 0000 6465 6661 .buckets....defa
00000130: 756c 742e 3234 3131 3335 3038 332e 3113 ult.241135083.1.
00000140: 0000 0064 6566 6175 6c74 2e32 3431 3133 ...default.24113
00000150: 3530 3833 2e31 0b00 0000 7267 772e 6275 5083.1....rgw.bu
00000160: 636b 6574 7300 0000 0000 0000 0000 0000 ckets...........
00000170: 00 .
Files
Updated by Yehuda Sadeh about 6 years ago
The problem is that we fail to go to the explicit placement pool for index operations even if one is set. One workaround would be to copy the objects from the old index pool to the new index pool. Another workaround would be to create a new placement target in the zone that reflects the old placement, and modify the bucket info placement_rule to point at that.
Updated by Yehuda Sadeh about 6 years ago
Updated by Abhishek Lekshmanan about 6 years ago
- Copied to Backport #23106: luminous: RGW will not list contents of Jewel-era buckets: reshard does NOT fix added
Updated by Yuri Weinstein about 6 years ago
Updated by Wido den Hollander over 3 years ago
While upgrading a cluster from Mimic to Nautilus I experienced this as well. This cluster was installed 7 years ago with what I think was Argonaut and has been upgraded ever since.
Some buckets (around 500) are broken when we start the Nautilus gateways so we currently have the gateways running with Mimic.
root@mon01:~# ceph versions
{
"mon": {
"ceph version 14.2.11 (f7fdb2f52131f54b891a2ec99d8205561242cdaf) nautilus (stable)": 3
},
"mgr": {
"ceph version 14.2.11 (f7fdb2f52131f54b891a2ec99d8205561242cdaf) nautilus (stable)": 3
},
"osd": {
"ceph version 14.2.10 (b340acf629a010a74d90da5782a2c5fe0b54ac20) nautilus (stable)": 91
},
"mds": {},
"rgw": {
"ceph version 13.2.10 (564bdc4ae87418a232fc901524470e1a0f76d641) mimic (stable)": 2,
"ceph version 14.2.11 (f7fdb2f52131f54b891a2ec99d8205561242cdaf) nautilus (stable)": 1
},
"overall": {
"ceph version 13.2.10 (564bdc4ae87418a232fc901524470e1a0f76d641) mimic (stable)": 2,
"ceph version 14.2.10 (b340acf629a010a74d90da5782a2c5fe0b54ac20) nautilus (stable)": 91,
"ceph version 14.2.11 (f7fdb2f52131f54b891a2ec99d8205561242cdaf) nautilus (stable)": 7
}
}
root@mon01:~#
I started to investigate and I found one bucket called pbx which is experiencing these problems.
I spawned a Nautilus gateway on one of the Monitors and when I perform a query it fails:
curl -I -H 'Host: pbx.o.auroraobjects.eu' http://localhost:7480/ HTTP/1.1 400 Bad Request Content-Length: 215 Bucket: pbx x-amz-request-id: tx00000000000000000000f-005f802972-1a9fee9a-ams02 Accept-Ranges: bytes Content-Type: application/xml Date: Fri, 09 Oct 2020 09:12:18 GMT Connection: Keep-Alive
This fails with the following message in the log:
2020-10-09 10:36:25.266 7f43c4a17700 0 req 14 0.000s NOTICE: invalid dest placement: 2020-10-09 10:36:25.266 7f43c4a17700 10 req 14 0.000s init_permissions on pbx[ams02.241978.4] failed, ret=-22 2020-10-09 10:36:25.266 7f43c4a17700 20 op->ERRORHANDLER: err_no=-22 new_err_no=-22 2020-10-09 10:36:25.266 7f43c4a17700 2 req 14 0.000s s3:stat_bucket op status=0 2020-10-09 10:36:25.266 7f43c4a17700 2 req 14 0.000s s3:stat_bucket http status=400 2020-10-09 10:36:25.266 7f43c4a17700 1 ====== req done req=0x560e46a70930 op status=0 http_status=400 latency=0s ======
The invalid dest placement comes from rgw_op.cc:
/* init dest placement -- only if bucket exists, otherwise request is either not relevant, or
* it's a create_bucket request, in which case the op will deal with the placement later */
if (s->bucket_exists) {
s->dest_placement.storage_class = s->info.storage_class;
s->dest_placement.inherit_from(s->bucket_info.placement_rule);
if (!store->svc.zone->get_zone_params().valid_placement(s->dest_placement)) {
ldpp_dout(s, 0) << "NOTICE: invalid dest placement: " << s->dest_placement.to_str() << dendl;
return -EINVAL;
}
}
This bucket indeed has no placement rule attached and only has an explicit placement set:
root@mon01:~# radosgw-admin metadata get bucket.instance:pbx:ams02.241978.4
{
"key": "bucket.instance:pbx:ams02.241978.4",
"ver": {
"tag": "_Vucn8pfwzF-qGg9bqip5lM_",
"ver": 1
},
"mtime": "2019-04-16 21:07:48.841780Z",
"data": {
"bucket_info": {
"bucket": {
"name": "pbx",
"marker": "ams02.241978.4",
"bucket_id": "ams02.241978.4",
"tenant": "",
"explicit_placement": {
"data_pool": ".rgw.buckets",
"data_extra_pool": "",
"index_pool": ".rgw.buckets"
}
},
"creation_time": "2014-02-16 12:32:15.000000Z",
"owner": "vdvm",
"flags": 0,
"zonegroup": "eu",
"placement_rule": "",
"has_instance_obj": "true",
"quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"num_shards": 0,
"bi_shard_hash_type": 0,
"requester_pays": "false",
"has_website": "false",
"swift_versioning": "false",
"swift_ver_location": "",
"index_type": 0,
"mdsearch_config": [],
"reshard_status": 0,
"new_bucket_instance_id": ""
},
"attrs": [
{
"key": "user.rgw.acl",
"val": "AgJ3AAAAAgISAAAABAAAAHZkdm0GAAAAQXR0aWxhAwNZAAAAAQEAAAAEAAAAdmR2bQ8AAAABAAAABAAAAHZkdm0DAzIAAAACAgQAAAAAAAAABAAAAHZkdm0AAAAAAAAAAAICBAAAAA8AAAAGAAAAQXR0aWxhAAAAAAAAAAA="
}
]
}
}
Notice that placement_rule is empty.
So I attempted to create a new placement rule called legacy-placement, but that fails:
{
"index_pool": ".rgw.buckets",
"storage_classes": {
"STANDARD": {
"data_pool": ".rgw.buckets"
}
},
"data_extra_pool": ".rgw.buckets.non-ec",
"index_type": 0
}
root@mon01:~# radosgw-admin zone placement add --placement-id=legacy-placement < legacy-placement.json ERROR: index pool not configured, need to specify --index-pool root@mon01:~#
I have also tried to reshard the bucket, but that did not resolve the issue.
On the mailinglist there is a thread from 2019: https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/thread/ULKK5RU2VXLFXNUJMZBMUG7CQ5UCWJCB/#R6CPZ2TEWRFL2JJWP7TT5GX7DPSV5S7Z
There people compiled RGW manually with lines commented. I'm trying to stay away from that as will probably have other users running into this as well.
Updating the bucket.instance metadata is not possible: (note that the ID changed due to the reshard)
radosgw-admin metadata put bucket.instance:pbx:ams02.446941181.1 < pbx_instance.json
In this JSON I manually set placement_rule to default-placement, but such an update is ignored and thus I can't fix it.
Updated by Wido den Hollander over 3 years ago
I looked into the differences between Mimic and Nautilus and I found this commit from Yehuda: https://github.com/ceph/ceph/commit/2a8e8a98d8c56cc374ec671846a20e2b0484bc75
This breaks the buckets ability to work as the placement_rule (null) for this bucket doesn't match any of the placement policies for the zone.
Now the question is:
- Do we modify this if-statement
- Do we enhance 'bucket check --fix' so that it fixes the placement policy
Actions