Actions
Bug #22928
closedRGW will not list contents of Jewel-era buckets: reshard does NOT fix
Status:
Resolved
Priority:
High
Assignee:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
luminous
Regression:
Yes
Severity:
2 - major
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
This is similar to issue #22756 at the outset, however it was reported on the mailing lists by Ingo Reimann <ireimann@dunkel.de>, for new buckets that were created the Jewel release, while #22756 was for much older buckets.
The user noted that the cluster was old (dating back to Argonaut), but the affected RGW user & bucket were created under Jewel. The user has reviewed the output they emailed to me and agreed to it being public.
root@cephrgw01:~# ceph -v
ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous (stable)
root@cephrgw01:~# radosgw-admin metadata get bucket:cephrgw
{
"key": "bucket:cephrgw",
"ver": {
"tag": "_yvk6Y5LsN9JWlo8YvzNkKOM",
"ver": 1
},
"mtime": "2017-06-14 07:11:36.816989Z",
"data": {
"bucket": {
"name": "cephrgw",
"marker": "default.241135083.1",
"bucket_id": "default.241135083.1",
"tenant": "",
"explicit_placement": {
"data_pool": "rgw.buckets",
"data_extra_pool": "",
"index_pool": "rgw.buckets"
}
},
"owner": "DunkelCeph",
"creation_time": "2017-06-14 07:11:33.424648Z",
"linked": "true",
"has_bucket_info": "false"
}
}
root@cephrgw01:~# radosgw-admin metadata get bucket.instance:cephrgw:default.241135083.1
{
"key": "bucket.instance:cephrgw:default.241135083.1",
"ver": {
"tag": "_lhPMGrI03ZPcMw7Mj7z6duy",
"ver": 2
},
"mtime": "2018-01-29 10:33:46.942786Z",
"data": {
"bucket_info": {
"bucket": {
"name": "cephrgw",
"marker": "default.241135083.1",
"bucket_id": "default.241135083.1",
"tenant": "",
"explicit_placement": {
"data_pool": "rgw.buckets",
"data_extra_pool": "",
"index_pool": "rgw.buckets"
}
},
"creation_time": "2017-06-14 07:11:33.424648Z",
"owner": "DunkelCeph",
"flags": 0,
"zonegroup": "default",
"placement_rule": "",
"has_instance_obj": "true",
"quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1024,
"max_size_kb": 0,
"max_objects": -1
},
"num_shards": 8,
"bi_shard_hash_type": 0,
"requester_pays": "false",
"has_website": "false",
"swift_versioning": "false",
"swift_ver_location": "",
"index_type": 0,
"mdsearch_config": [],
"reshard_status": 0,
"new_bucket_instance_id": ""
},
"attrs": [
{
"key": "user.rgw.acl",
"val": "AgLXAQAAAwIcAAAACgAAAER1bmtlbENlcGgKAAAARHVua2VsQ2VwaAMDrwEAAAEBAAAACgAAAER1bmtlbENlcGgPAAAABQAAAAoAAABEdW5rZWxDZXBoBAM8AAAAAgIEAAAAAAAAAAoAAABEdW5rZWxDZXBoAAAAAAAAA
+AACAgQAAAAPAAAACgAAAER1bmtlbENlcGgAAAAACgAAAER1bmtlbENlcGgEAzwAAAACAgQAAAAAAAAACgAAAER1bmtlbENlcGgAAAAAAAAAAAICBAAAAAIAAAAKAAAARHVua2VsQ2VwaAAAAAAKAAAARHVua2VsQ2VwaAQDPAAAAAICBAAAAAAAAAAKA
+AAARHVua2VsQ2VwaAAAAAAAAAAAAgIEAAAAAQAAAAoAAABEdW5rZWxDZXBoAAAAAAoAAABEdW5rZWxDZXBoBAM8AAAAAgIEAAAAAAAAAAoAAABEdW5rZWxDZXBoAAAAAAAAAAACAgQAAAAIAAAACgAAAER1bmtlbENlcGgAAAAACgAAAER1bmtlbENlc
+GgEAzwAAAACAgQAAAAAAAAACgAAAER1bmtlbENlcGgAAAAAAAAAAAICBAAAAAQAAAAKAAAARHVua2VsQ2VwaAAAAAAAAAAA"
},
{
"key": "user.rgw.idtag",
"val": ""
}
]
}
}
root@cephrgw01:~# radosgw-admin bucket stats --bucket=cephrgw
{
"bucket": "cephrgw",
"zonegroup": "default",
"placement_rule": "",
"explicit_placement": {
"data_pool": "rgw.buckets",
"data_extra_pool": "",
"index_pool": "rgw.buckets"
},
"id": "default.241135083.1",
"marker": "default.241135083.1",
"index_type": "Normal",
"owner": "DunkelCeph",
"ver": "0#7,1#4,2#4,3#4,4#4,5#7,6#4,7#4",
"master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0",
"mtime": "2018-01-29 11:33:46.942786",
"max_marker": "0#,1#,2#,3#,4#,5#,6#,7#",
"usage": {},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1024,
"max_size_kb": 0,
"max_objects": -1
}
}
root@cephrgw01:~# rados -p rgw.buckets listomapvals .dir.default.241135083.1.1
testfile
value (212 bytes) :
00000000 08 03 ce 00 00 00 08 00 00 00 74 65 73 74 66 69 |..........testfi|
00000010 6c 65 78 0b 03 00 00 00 00 00 01 04 03 76 00 00 |lex..........v..|
00000020 00 01 37 00 00 00 00 00 00 00 49 e3 40 59 46 39 |..7.......I.@YF9|
00000030 47 03 20 00 00 00 39 38 65 30 30 31 63 31 31 62 |G. ...98e001c11b|
00000040 37 63 31 39 63 64 61 36 35 30 66 34 64 64 64 34 |7c19cda650f4ddd4|
00000050 35 39 34 64 62 63 0a 00 00 00 44 75 6e 6b 65 6c |594dbc....Dunkel|
00000060 43 65 70 68 0a 00 00 00 44 75 6e 6b 65 6c 43 65 |Ceph....DunkelCe|
00000070 70 68 19 00 00 00 74 65 78 74 2f 70 6c 61 69 6e |ph....text/plain|
00000080 3b 20 63 68 61 72 73 65 74 3d 75 74 66 2d 38 37 |; charset=utf-87|
00000090 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 |................|
000000a0 01 06 00 00 00 13 84 78 0b 03 00 02 16 00 00 00 |.......x........|
000000b0 64 65 66 61 75 6c 74 2e 32 34 31 30 39 34 32 32 |default.24109422|
000000c0 36 2e 34 37 37 37 00 00 00 00 00 00 00 00 00 00 |6.4777..........|
000000d0 00 00 00 00 |....|
000000d4
root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw
[]
root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw --debug-rgw=20
(attached)
root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw
[]
You have new mail in /var/mail/root
root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw --debug-rgw=20
(attached)
root@cephrgw01:~# radosgw-admin bucket reshard --bucket=cephrgw --num-shards=8 --yes-i-really-mean-it
*** NOTICE: operation will not remove old bucket index objects ***
*** these will need to be removed manually ***
tenant:
bucket name: cephrgw
old bucket instance id: default.241135083.1
new bucket instance id: default.278815034.1
total entries: 0
root@cephrgw01:~# radosgw-admin bucket list --bucket=cephrgw
[]
root@cephrgw01:~# radosgw-admin bi list --bucket=cephrgw
[]
root@cephrgw01:~# radosgw-admin bucket stats --bucket=cephrgw
{
"bucket": "cephrgw",
"zonegroup": "default",
"placement_rule": "",
"explicit_placement": {
"data_pool": "rgw.buckets",
"data_extra_pool": "",
"index_pool": "rgw.buckets"
},
"id": "default.278815034.1",
"marker": "default.241135083.1",
"index_type": "Normal",
"owner": "DunkelCeph",
"ver": "0#1,1#1,2#1,3#1,4#1,5#1,6#1,7#1",
"master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0",
"mtime": "2018-02-01 19:29:20.845789",
"max_marker": "0#,1#,2#,3#,4#,5#,6#,7#",
"usage": {},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1024,
"max_size_kb": 0,
"max_objects": -1
}
}
root@cephrgw01:~# rados -p rgw.buckets listomapkeys
.dir.default.241135083.1.1
testfile
root@cephrgw01:~# rados -p rgw.buckets stat default.241135083.1_testfile
rgw.buckets/default.241135083.1_testfile mtime 2017-06-14 09:18:33.000000, size 55
root@cephrgw01:~# rados -p rgw.buckets listomapvals
default.241135083.1_testfile
root@cephrgw01:~# rados -p rgw.buckets get default.241135083.1_testfile -
Testdatei zur Prüfung der Funktion der rados-gateways
root@cephrgw01:~# rados -p rgw.buckets listxattr default.241135083.1_testfile
user.rgw.acl
user.rgw.content_type
user.rgw.etag
user.rgw.idtag
user.rgw.manifest
user.rgw.pg_ver
user.rgw.source_zone
user.rgw.x-amz-date
root@cephrgw01:~# rados -p rgw.buckets getxattr default.241135083.1_testfile user.rgw.idtag
default.241094226.4777
root@cephrgw01:~# rados -p rgw.buckets getxattr
default.241135083.1_testfile user.rgw.manifest | xxd
00000000: 0503 6b01 0000 3700 0000 0000 0000 0000 ..k...7.........
00000010: 0000 0005 0388 0000 0007 0000 0063 6570 .............cep
00000020: 6872 6777 0000 0000 0000 0000 0800 0000 hrgw............
00000030: 7465 7374 6669 6c65 0803 5f00 0000 0700 testfile.._.....
00000040: 0000 6365 7068 7267 770b 0000 0072 6777 ..cephrgw....rgw
00000050: 2e62 7563 6b65 7473 1300 0000 6465 6661 .buckets....defa
00000060: 756c 742e 3234 3131 3335 3038 332e 3113 ult.241135083.1.
00000070: 0000 0064 6566 6175 6c74 2e32 3431 3133 ...default.24113
00000080: 3530 3833 2e31 0b00 0000 7267 772e 6275 5083.1....rgw.bu
00000090: 636b 6574 7300 0000 0000 0000 0000 0000 ckets...........
000000a0: 0037 0000 0000 0000 0000 0008 0000 0000 .7..............
000000b0: 0021 0000 002e 5977 5256 375f 4846 3133 .!....YwRV7_HF13
000000c0: 4c79 7371 6146 6e5f 4e5f 4150 664d 3938 LysqaFn_N_APfM98
000000d0: 4e4f 7656 675f 0100 0000 0000 0000 0000 NOvVg_..........
000000e0: 0000 0201 2000 0000 0000 0000 0000 0800 .... ...........
000000f0: 0000 0000 0000 0000 0000 0000 0000 4000 ..............@.
00000100: 0000 0000 0000 0000 0803 5f00 0000 0700 .........._.....
00000110: 0000 6365 7068 7267 770b 0000 0072 6777 ..cephrgw....rgw
00000120: 2e62 7563 6b65 7473 1300 0000 6465 6661 .buckets....defa
00000130: 756c 742e 3234 3131 3335 3038 332e 3113 ult.241135083.1.
00000140: 0000 0064 6566 6175 6c74 2e32 3431 3133 ...default.24113
00000150: 3530 3833 2e31 0b00 0000 7267 772e 6275 5083.1....rgw.bu
00000160: 636b 6574 7300 0000 0000 0000 0000 0000 ckets...........
00000170: 00 .
Files
Actions