Actions
Bug #22928
closedRGW will not list contents of Jewel-era buckets: reshard does NOT fix
Status:
Resolved
Priority:
High
Assignee:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
luminous
Regression:
Yes
Severity:
2 - major
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
This is similar to issue #22756 at the outset, however it was reported on the mailing lists by Ingo Reimann <ireimann@dunkel.de>, for new buckets that were created the Jewel release, while #22756 was for much older buckets.
The user noted that the cluster was old (dating back to Argonaut), but the affected RGW user & bucket were created under Jewel. The user has reviewed the output they emailed to me and agreed to it being public.
root@cephrgw01:~# ceph -v ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous (stable) root@cephrgw01:~# radosgw-admin metadata get bucket:cephrgw { "key": "bucket:cephrgw", "ver": { "tag": "_yvk6Y5LsN9JWlo8YvzNkKOM", "ver": 1 }, "mtime": "2017-06-14 07:11:36.816989Z", "data": { "bucket": { "name": "cephrgw", "marker": "default.241135083.1", "bucket_id": "default.241135083.1", "tenant": "", "explicit_placement": { "data_pool": "rgw.buckets", "data_extra_pool": "", "index_pool": "rgw.buckets" } }, "owner": "DunkelCeph", "creation_time": "2017-06-14 07:11:33.424648Z", "linked": "true", "has_bucket_info": "false" } } root@cephrgw01:~# radosgw-admin metadata get bucket.instance:cephrgw:default.241135083.1 { "key": "bucket.instance:cephrgw:default.241135083.1", "ver": { "tag": "_lhPMGrI03ZPcMw7Mj7z6duy", "ver": 2 }, "mtime": "2018-01-29 10:33:46.942786Z", "data": { "bucket_info": { "bucket": { "name": "cephrgw", "marker": "default.241135083.1", "bucket_id": "default.241135083.1", "tenant": "", "explicit_placement": { "data_pool": "rgw.buckets", "data_extra_pool": "", "index_pool": "rgw.buckets" } }, "creation_time": "2017-06-14 07:11:33.424648Z", "owner": "DunkelCeph", "flags": 0, "zonegroup": "default", "placement_rule": "", "has_instance_obj": "true", "quota": { "enabled": false, "check_on_raw": false, "max_size": -1024, "max_size_kb": 0, "max_objects": -1 }, "num_shards": 8, "bi_shard_hash_type": 0, "requester_pays": "false", "has_website": "false", "swift_versioning": "false", "swift_ver_location": "", "index_type": 0, "mdsearch_config": [], "reshard_status": 0, "new_bucket_instance_id": "" }, "attrs": [ { "key": "user.rgw.acl", "val": "AgLXAQAAAwIcAAAACgAAAER1bmtlbENlcGgKAAAARHVua2VsQ2VwaAMDrwEAAAEBAAAACgAAAER1bmtlbENlcGgPAAAABQAAAAoAAABEdW5rZWxDZXBoBAM8AAAAAgIEAAAAAAAAAAoAAABEdW5rZWxDZXBoAAAAAAAAA +AACAgQAAAAPAAAACgAAAER1bmtlbENlcGgAAAAACgAAAER1bmtlbENlcGgEAzwAAAACAgQAAAAAAAAACgAAAER1bmtlbENlcGgAAAAAAAAAAAICBAAAAAIAAAAKAAAARHVua2VsQ2VwaAAAAAAKAAAARHVua2VsQ2VwaAQDPAAAAAICBAAAAAAAAAAKA +AAARHVua2VsQ2VwaAAAAAAAAAAAAgIEAAAAAQAAAAoAAABEdW5rZWxDZXBoAAAAAAoAAABEdW5rZWxDZXBoBAM8AAAAAgIEAAAAAAAAAAoAAABEdW5rZWxDZXBoAAAAAAAAAAACAgQAAAAIAAAACgAAAER1bmtlbENlcGgAAAAACgAAAER1bmtlbENlc +GgEAzwAAAACAgQAAAAAAAAACgAAAER1bmtlbENlcGgAAAAAAAAAAAICBAAAAAQAAAAKAAAARHVua2VsQ2VwaAAAAAAAAAAA" }, { "key": "user.rgw.idtag", "val": "" } ] } } root@cephrgw01:~# radosgw-admin bucket stats --bucket=cephrgw { "bucket": "cephrgw", "zonegroup": "default", "placement_rule": "", "explicit_placement": { "data_pool": "rgw.buckets", "data_extra_pool": "", "index_pool": "rgw.buckets" }, "id": "default.241135083.1", "marker": "default.241135083.1", "index_type": "Normal", "owner": "DunkelCeph", "ver": "0#7,1#4,2#4,3#4,4#4,5#7,6#4,7#4", "master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0", "mtime": "2018-01-29 11:33:46.942786", "max_marker": "0#,1#,2#,3#,4#,5#,6#,7#", "usage": {}, "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1024, "max_size_kb": 0, "max_objects": -1 } } root@cephrgw01:~# rados -p rgw.buckets listomapvals .dir.default.241135083.1.1 testfile value (212 bytes) : 00000000 08 03 ce 00 00 00 08 00 00 00 74 65 73 74 66 69 |..........testfi| 00000010 6c 65 78 0b 03 00 00 00 00 00 01 04 03 76 00 00 |lex..........v..| 00000020 00 01 37 00 00 00 00 00 00 00 49 e3 40 59 46 39 |..7.......I.@YF9| 00000030 47 03 20 00 00 00 39 38 65 30 30 31 63 31 31 62 |G. ...98e001c11b| 00000040 37 63 31 39 63 64 61 36 35 30 66 34 64 64 64 34 |7c19cda650f4ddd4| 00000050 35 39 34 64 62 63 0a 00 00 00 44 75 6e 6b 65 6c |594dbc....Dunkel| 00000060 43 65 70 68 0a 00 00 00 44 75 6e 6b 65 6c 43 65 |Ceph....DunkelCe| 00000070 70 68 19 00 00 00 74 65 78 74 2f 70 6c 61 69 6e |ph....text/plain| 00000080 3b 20 63 68 61 72 73 65 74 3d 75 74 66 2d 38 37 |; charset=utf-87| 00000090 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 |................| 000000a0 01 06 00 00 00 13 84 78 0b 03 00 02 16 00 00 00 |.......x........| 000000b0 64 65 66 61 75 6c 74 2e 32 34 31 30 39 34 32 32 |default.24109422| 000000c0 36 2e 34 37 37 37 00 00 00 00 00 00 00 00 00 00 |6.4777..........| 000000d0 00 00 00 00 |....| 000000d4 root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw [] root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw --debug-rgw=20 (attached) root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw [] You have new mail in /var/mail/root root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw --debug-rgw=20 (attached) root@cephrgw01:~# radosgw-admin bucket reshard --bucket=cephrgw --num-shards=8 --yes-i-really-mean-it *** NOTICE: operation will not remove old bucket index objects *** *** these will need to be removed manually *** tenant: bucket name: cephrgw old bucket instance id: default.241135083.1 new bucket instance id: default.278815034.1 total entries: 0 root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw [] root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw [] root@cephrgw01:~# radosgw-admin bucket stats --bucket=cephrgw { "bucket": "cephrgw", "zonegroup": "default", "placement_rule": "", "explicit_placement": { "data_pool": "rgw.buckets", "data_extra_pool": "", "index_pool": "rgw.buckets" }, "id": "default.278815034.1", "marker": "default.241135083.1", "index_type": "Normal", "owner": "DunkelCeph", "ver": "0#1,1#1,2#1,3#1,4#1,5#1,6#1,7#1", "master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0", "mtime": "2018-02-01 19:29:20.845789", "max_marker": "0#,1#,2#,3#,4#,5#,6#,7#", "usage": {}, "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1024, "max_size_kb": 0, "max_objects": -1 } } root@cephrgw01:~# rados -p rgw.buckets listomapkeys .dir.default.241135083.1.1 testfile root@cephrgw01:~# rados -p rgw.buckets stat default.241135083.1_testfile rgw.buckets/default.241135083.1_testfile mtime 2017-06-14 09:18:33.000000, size 55 root@cephrgw01:~# rados -p rgw.buckets listomapvals default.241135083.1_testfile root@cephrgw01:~# rados -p rgw.buckets get default.241135083.1_testfile - Testdatei zur Prüfung der Funktion der rados-gateways root@cephrgw01:~# rados -p rgw.buckets listxattr default.241135083.1_testfile user.rgw.acl user.rgw.content_type user.rgw.etag user.rgw.idtag user.rgw.manifest user.rgw.pg_ver user.rgw.source_zone user.rgw.x-amz-date root@cephrgw01:~# rados -p rgw.buckets getxattr default.241135083.1_testfile user.rgw.idtag default.241094226.4777 root@cephrgw01:~# rados -p rgw.buckets getxattr default.241135083.1_testfile user.rgw.manifest | xxd 00000000: 0503 6b01 0000 3700 0000 0000 0000 0000 ..k...7......... 00000010: 0000 0005 0388 0000 0007 0000 0063 6570 .............cep 00000020: 6872 6777 0000 0000 0000 0000 0800 0000 hrgw............ 00000030: 7465 7374 6669 6c65 0803 5f00 0000 0700 testfile.._..... 00000040: 0000 6365 7068 7267 770b 0000 0072 6777 ..cephrgw....rgw 00000050: 2e62 7563 6b65 7473 1300 0000 6465 6661 .buckets....defa 00000060: 756c 742e 3234 3131 3335 3038 332e 3113 ult.241135083.1. 00000070: 0000 0064 6566 6175 6c74 2e32 3431 3133 ...default.24113 00000080: 3530 3833 2e31 0b00 0000 7267 772e 6275 5083.1....rgw.bu 00000090: 636b 6574 7300 0000 0000 0000 0000 0000 ckets........... 000000a0: 0037 0000 0000 0000 0000 0008 0000 0000 .7.............. 000000b0: 0021 0000 002e 5977 5256 375f 4846 3133 .!....YwRV7_HF13 000000c0: 4c79 7371 6146 6e5f 4e5f 4150 664d 3938 LysqaFn_N_APfM98 000000d0: 4e4f 7656 675f 0100 0000 0000 0000 0000 NOvVg_.......... 000000e0: 0000 0201 2000 0000 0000 0000 0000 0800 .... ........... 000000f0: 0000 0000 0000 0000 0000 0000 0000 4000 ..............@. 00000100: 0000 0000 0000 0000 0803 5f00 0000 0700 .........._..... 00000110: 0000 6365 7068 7267 770b 0000 0072 6777 ..cephrgw....rgw 00000120: 2e62 7563 6b65 7473 1300 0000 6465 6661 .buckets....defa 00000130: 756c 742e 3234 3131 3335 3038 332e 3113 ult.241135083.1. 00000140: 0000 0064 6566 6175 6c74 2e32 3431 3133 ...default.24113 00000150: 3530 3833 2e31 0b00 0000 7267 772e 6275 5083.1....rgw.bu 00000160: 636b 6574 7300 0000 0000 0000 0000 0000 ckets........... 00000170: 00 .
Files
Actions