Project

General

Profile

Bug #17372

RGW will not list Argonaut-era bucket via HTTP (but radosgw-admin works)

Added by Robin Johnson 6 months ago. Updated about 2 months ago.

Status:
Resolved
Priority:
High
Target version:
-
Start date:
09/22/2016
Due date:
% Done:

0%

Source:
Community (dev)
Tags:
Backport:
jewel
Regression:
Yes
Severity:
1 - critical
Reviewed:
Affected Versions:
ceph-qa-suite:
Release:
jewel
Needs Doc:
No

Description

DreamObjects has some very old Ceph production buckets: Argonaut or older.
They do list if you use radosgw-admin bucket list, but NOT via HTTP.
This is definitely a regression since Hammer, possibly since sometime in Jewel (not easy to test each release in my environment).

# radosgw-admin bucket stats --bucket=bucket
{
    "bucket": "bucket",
    "pool": ".rgw.data.1",
    "index_pool": ".rgw.data.1",
    "id": "88283.1",
    "marker": "88283.1",
    "owner": "CENSORED",
    "ver": "0#0",
    "master_ver": "0#0",
    "mtime": "0.000000",
    "max_marker": "0#",
    "usage": {
        "rgw.main": {
            "size_kb": 18,
            "size_kb_actual": 20,
            "num_objects": 1
        }
    },
    "bucket_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    }
}
# radosgw-admin metadata get bucket:bucket
{
    "key": "bucket:bucket",
    "ver": {
        "tag": "",
        "ver": 0
    },
    "mtime": "2012-09-12 23:43:46.861813Z",
    "data": {
        "bucket": {
            "name": "",
            "pool": "",
            "data_extra_pool": "",
            "index_pool": "",
            "marker": "",
            "bucket_id": "",
            "tenant": "" 
        },
        "owner": "",
        "creation_time": "0.000000",
        "linked": "false",
        "has_bucket_info": "true",
        "old_bucket_info": {
            "bucket": {
                "name": "bucket",
                "pool": ".rgw.data.1",
                "data_extra_pool": "",
                "index_pool": ".rgw.data.1",
                "marker": "88283.1",
                "bucket_id": "88283.1",
                "tenant": "" 
            },
            "creation_time": "0.000000",
            "owner": "CENSORED",
            "flags": 0,
            "zonegroup": "",
            "placement_rule": "",
            "has_instance_obj": "false",
            "quota": {
                "enabled": false,
                "max_size_kb": -1,
                "max_objects": -1
            },
            "num_shards": 0,
            "bi_shard_hash_type": 0,
            "requester_pays": "false",
            "has_website": "false",
            "swift_versioning": "false",
            "swift_ver_location": "" 
        }
    }
}
# radosgw-admin metadata get bucket.instance:bucket:88283.1
{
    "key": "bucket.instance:bucket:88283.1",
    "ver": {
        "tag": "JqXTogWqCuSUkGsUgj+VMl1w",
        "ver": 7
    },
    "mtime": "2015-07-01 20:12:08.000000Z",
    "data": {
        "bucket_info": {
            "bucket": {
                "name": "bucket",
                "pool": ".rgw.data.1",
                "data_extra_pool": "",
                "index_pool": ".rgw.data.1",
                "marker": "88283.1",
                "bucket_id": "88283.1",
                "tenant": "" 
            },
            "creation_time": "0.000000",
            "owner": "CENSORED",
            "flags": 0,
            "zonegroup": "",
            "placement_rule": "",
            "has_instance_obj": "true",
            "quota": {
                "enabled": false,
                "max_size_kb": 10240000,
                "max_objects": -1
            },
            "num_shards": 0,
            "bi_shard_hash_type": 0,
            "requester_pays": "false",
            "has_website": "false",
            "swift_versioning": "false",
            "swift_ver_location": "" 
        },
        "attrs": [
            {
                "key": "user.rgw.acl",
                "val": "AgK5AAAAAgIUAAAABgAAAG1hdHdvZAYAAABtYXR3b2QDA5kAAAABAQAAAAYAAABtYXR3b2QPAAAAAgAAAAAAAAADAygAAAACAgQAAAACAAAAAAAAAAAAAAAAAAAAAgIEAAAAAQAAAAAAAAABAAAABgAAAG1hdHdvZAMDNAAAAAICBAAAAAAAAAAGAAAAbWF0d29kAAAAAAAAAAACAgQAAAAPAAAABgAAAG1hdHdvZAAAAAABAAAAAQAAAAEAAAA=" 
            },
            {
                "key": "user.rgw.idtag",
                "val": "" 
            },
            {
                "key": "user.rgw.manifest",
                "val": "" 
            }
        ]
    }
}
# radosgw-admin bucket list --bucket=bucket
[
    {
        "name": "object",
        "instance": "",
        "namespace": "",
        "owner": "CENSORED",
        "owner_display_name": "CENSORED",
        "size": 18236,
        "mtime": "2012-05-10 17:49:34.000000Z",
        "etag": "971a661e70295d76f7801516a6767e59",
        "content_type": "image\/png",
        "tag": "",
        "flags": 0
    }
]

And now the RGW debug log:

# GET http://bucket.objects-us-west-1.dream.io/
016-09-22 11:20:50.254766 7fd4362e8700 20 RGWEnv::set(): HTTP_USER_AGENT: curl/7.35.0
2016-09-22 11:20:50.254776 7fd4362e8700 20 RGWEnv::set(): HTTP_ACCEPT: */*
2016-09-22 11:20:50.254777 7fd4362e8700 20 RGWEnv::set(): HTTP_HOST: bucket.objects-us-west-1.dream.io
2016-09-22 11:20:50.254778 7fd4362e8700 20 RGWEnv::set(): REQUEST_METHOD: GET
2016-09-22 11:20:50.254779 7fd4362e8700 20 RGWEnv::set(): REQUEST_URI: /
2016-09-22 11:20:50.254781 7fd4362e8700 20 RGWEnv::set(): QUERY_STRING: 
2016-09-22 11:20:50.254782 7fd4362e8700 20 RGWEnv::set(): REMOTE_USER: 
2016-09-22 11:20:50.254783 7fd4362e8700 20 RGWEnv::set(): SCRIPT_URI: /
2016-09-22 11:20:50.254787 7fd4362e8700 20 RGWEnv::set(): SERVER_PORT: 0
2016-09-22 11:20:50.254788 7fd4362e8700 20 HTTP_ACCEPT=*/*
2016-09-22 11:20:50.254789 7fd4362e8700 20 HTTP_HOST=bucket.objects-us-west-1.dream.io
2016-09-22 11:20:50.254789 7fd4362e8700 20 HTTP_USER_AGENT=curl/7.35.0
2016-09-22 11:20:50.254790 7fd4362e8700 20 QUERY_STRING=
2016-09-22 11:20:50.254790 7fd4362e8700 20 REMOTE_USER=
2016-09-22 11:20:50.254791 7fd4362e8700 20 REQUEST_METHOD=GET
2016-09-22 11:20:50.254791 7fd4362e8700 20 REQUEST_URI=/
2016-09-22 11:20:50.254792 7fd4362e8700 20 SCRIPT_URI=/
2016-09-22 11:20:50.254792 7fd4362e8700 20 SERVER_PORT=0
2016-09-22 11:20:50.254793 7fd4362e8700  1 ====== starting new request req=0x7fd4362e27d0 =====
2016-09-22 11:20:50.254810 7fd4362e8700  2 req 2:0.000017::GET /::initializing for trans_id = tx000000000000000000002-0057e3be92-1084d9eb-default
2016-09-22 11:20:50.254820 7fd4362e8700 10 rgw api priority: s3=4 s3website=-1
2016-09-22 11:20:50.254822 7fd4362e8700 10 host=bucket.objects-us-west-1.dream.io
2016-09-22 11:20:50.254824 7fd4362e8700 20 subdomain=bucket domain=objects-us-west-1.dream.io in_hosted_domain=1 in_hosted_domain_s3website=0
2016-09-22 11:20:50.254826 7fd4362e8700 20 final domain/bucket subdomain=bucket domain=objects-us-west-1.dream.io in_hosted_domain=1 in_hosted_domain_s3website=0 s->info.domain=objects-us-west-1.dream.io s->info.request_uri=/bucket/
2016-09-22 11:20:50.254851 7fd4362e8700 20 get_handler handler=25RGWHandler_REST_Bucket_S3
2016-09-22 11:20:50.254854 7fd4362e8700 10 handler=25RGWHandler_REST_Bucket_S3
2016-09-22 11:20:50.254855 7fd4362e8700  2 req 2:0.000062:s3:GET /::getting op 0
2016-09-22 11:20:50.254862 7fd4362e8700 10 op=25RGWListBucket_ObjStore_S3
2016-09-22 11:20:50.254863 7fd4362e8700  2 req 2:0.000070:s3:GET /:list_bucket:authorizing
2016-09-22 11:20:50.254866 7fd4362e8700  2 req 2:0.000073:s3:GET /:list_bucket:normalizing buckets and tenants
2016-09-22 11:20:50.254868 7fd4362e8700 10 s->object=<NULL> s->bucket=bucket
2016-09-22 11:20:50.254870 7fd4362e8700  2 req 2:0.000077:s3:GET /:list_bucket:init permissions
2016-09-22 11:20:50.254884 7fd4362e8700 20 get_system_obj_state: rctx=0x7fd4362e26e0 obj=.rgw:bucket state=0x7fd5440086d8 s->prefetch_data=0
2016-09-22 11:20:50.254891 7fd4362e8700 10 cache get: name=.rgw+bucket : type miss (requested=22, cached=19)
2016-09-22 11:20:50.254935 7fd4362e8700  1 -- [2607:f298:4:d006::5795]:0/603440092 --> [2607:f298:4:d007::8852]:6860/111972 -- osd_op(client.277141995.0:383 13.3c5bc352 bucket [call version.read,getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e1005544) v7 -- ?+0 0x7fd54400aac0 con 0x7fd576de0710
2016-09-22 11:20:50.259375 7fd227032700  1 -- [2607:f298:4:d006::5795]:0/603440092 <== osd.1010 [2607:f298:4:d007::8853]:6804/481920 2 ==== osd_op_reply(382 obj_delete_at_hint.0000000059 [call] v0'0 uv1283 ondisk = 0) v7 ==== 149+0+15 (2904161573 0 2149983739) 0x7fd4884220c0 con 0x7fd4b00a1ce0
2016-09-22 11:20:50.259537 7fd4bffff700  1 -- [2607:f298:4:d006::5795]:0/603440092 --> [2607:f298:4:d007::8853]:6804/481920 -- osd_op(client.277141995.0:384 18.1d0c86ea obj_delete_at_hint.0000000059 [call lock.unlock] snapc 0=[] ondisk+write+known_if_redirected e1005544) v7 -- ?+0 0x7fd4b00a3590 con 0x7fd4b00a1ce0
2016-09-22 11:20:50.273352 7fd22dea0700  1 -- [2607:f298:4:d006::5795]:0/603440092 <== osd.994 [2607:f298:4:d007::8852]:6860/111972 3 ==== osd_op_reply(383 bucket [call,getxattrs,stat] v0'0 uv5397 ondisk = 0) v7 ==== 210+0+255 (623213301 0 1810348938) 0x7fd43c5e6e40 con 0x7fd576de0710
2016-09-22 11:20:50.273425 7fd4362e8700 10 cache put: name=.rgw+bucket info.flags=22
2016-09-22 11:20:50.273436 7fd4362e8700 10 moving .rgw+bucket to cache LRU end
2016-09-22 11:20:50.273444 7fd4362e8700 10 updating xattr: name=user.rgw.acl bl.length()=191
2016-09-22 11:20:50.273451 7fd4362e8700 20 get_system_obj_state: s->obj_tag was set empty
2016-09-22 11:20:50.273454 7fd4362e8700 20 Read xattr: user.rgw.acl
2016-09-22 11:20:50.273455 7fd4362e8700 20 Read xattr: user.rgw.idtag
2016-09-22 11:20:50.273456 7fd4362e8700 20 Read xattr: user.rgw.manifest
2016-09-22 11:20:50.273460 7fd4362e8700 10 cache get: name=.rgw+bucket : hit (requested=17, cached=23)
2016-09-22 11:20:50.273479 7fd4362e8700 20 rgw_get_bucket_info: old bucket info, bucket=bucket(@.rgw.data.1[88283.1]) owner CENSOR
2016-09-22 11:20:50.273493 7fd4362e8700 15 decode_policy Read AccessControlPolicy<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>CENSOR</ID><DisplayName>CENSOR</DisplayName></Owner><AccessControlList><Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group"><URI>http://acs.amazonaws.com/groups/global/AllUsers</URI></Grantee><Permission>READ</Permission></Grant><Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser"><ID>CENSOR</ID><DisplayName>CENSOR</DisplayName></Grantee><Permission>FULL_CONTROL</Permission></Grant></AccessControlList></AccessControlPolicy>
2016-09-22 11:20:50.273506 7fd4362e8700 10 read_permissions on bucket(@.rgw.data.1[88283.1]) ret=-2
2016-09-22 11:20:50.273509 7fd4362e8700 20 op->ERRORHANDLER: err_no=-2 new_err_no=-2
2016-09-22 11:20:50.273592 7fd4362e8700  2 req 2:0.018799:s3:GET /:list_bucket:op status=0
2016-09-22 11:20:50.273599 7fd4362e8700  2 req 2:0.018805:s3:GET /:list_bucket:http status=404
2016-09-22 11:20:50.273602 7fd4362e8700  1 ====== req done req=0x7fd4362e27d0 op status=0 http_status=404 ======
2016-09-22 11:20:50.273612 7fd4362e8700 20 process_request() returned -2
2016-09-22 11:20:50.273633 7fd4362e8700  1 civetweb: 0x7fd544001ea0: 2607:f298:4:d006::5796 - - [22/Sep/2016:11:20:50 +0000] "GET / HTTP/1.1" 404 0 - curl/7.35.0

# GET http://bucket.objects-us-west-1.dream.io/object
2016-09-22 11:24:36.697643 7fd435ae7700 20 RGWEnv::set(): HTTP_USER_AGENT: curl/7.35.0
2016-09-22 11:24:36.697656 7fd435ae7700 20 RGWEnv::set(): HTTP_ACCEPT: */*
2016-09-22 11:24:36.697659 7fd435ae7700 20 RGWEnv::set(): HTTP_HOST: bucket.objects-us-west-1.dream.io
2016-09-22 11:24:36.697660 7fd435ae7700 20 RGWEnv::set(): REQUEST_METHOD: GET
2016-09-22 11:24:36.697661 7fd435ae7700 20 RGWEnv::set(): REQUEST_URI: /object
2016-09-22 11:24:36.697662 7fd435ae7700 20 RGWEnv::set(): QUERY_STRING: 
2016-09-22 11:24:36.697663 7fd435ae7700 20 RGWEnv::set(): REMOTE_USER: 
2016-09-22 11:24:36.697664 7fd435ae7700 20 RGWEnv::set(): SCRIPT_URI: /object
2016-09-22 11:24:36.697666 7fd435ae7700 20 RGWEnv::set(): SERVER_PORT: 0
2016-09-22 11:24:36.697667 7fd435ae7700 20 HTTP_ACCEPT=*/*
2016-09-22 11:24:36.697668 7fd435ae7700 20 HTTP_HOST=bucket.objects-us-west-1.dream.io
2016-09-22 11:24:36.697668 7fd435ae7700 20 HTTP_USER_AGENT=curl/7.35.0
2016-09-22 11:24:36.697669 7fd435ae7700 20 QUERY_STRING=
2016-09-22 11:24:36.697669 7fd435ae7700 20 REMOTE_USER=
2016-09-22 11:24:36.697670 7fd435ae7700 20 REQUEST_METHOD=GET
2016-09-22 11:24:36.697671 7fd435ae7700 20 REQUEST_URI=/object
2016-09-22 11:24:36.697671 7fd435ae7700 20 SCRIPT_URI=/object
2016-09-22 11:24:36.697672 7fd435ae7700 20 SERVER_PORT=0
2016-09-22 11:24:36.697673 7fd435ae7700  1 ====== starting new request req=0x7fd435ae17d0 =====
2016-09-22 11:24:36.697691 7fd435ae7700  2 req 3:0.000018::GET /object::initializing for trans_id = tx000000000000000000003-0057e3bf74-1084d9eb-default
2016-09-22 11:24:36.697700 7fd435ae7700 10 rgw api priority: s3=4 s3website=-1
2016-09-22 11:24:36.697701 7fd435ae7700 10 host=bucket.objects-us-west-1.dream.io
2016-09-22 11:24:36.697704 7fd435ae7700 20 subdomain=bucket domain=objects-us-west-1.dream.io in_hosted_domain=1 in_hosted_domain_s3website=0
2016-09-22 11:24:36.697706 7fd435ae7700 20 final domain/bucket subdomain=bucket domain=objects-us-west-1.dream.io in_hosted_domain=1 in_hosted_domain_s3website=0 s->info.domain=objects-us-west-1.dream.io s->info.request_uri=/bucket/object
2016-09-22 11:24:36.697729 7fd435ae7700 20 get_handler handler=22RGWHandler_REST_Obj_S3
2016-09-22 11:24:36.697732 7fd435ae7700 10 handler=22RGWHandler_REST_Obj_S3
2016-09-22 11:24:36.697734 7fd435ae7700  2 req 3:0.000060:s3:GET /object::getting op 0
2016-09-22 11:24:36.697741 7fd435ae7700 10 op=21RGWGetObj_ObjStore_S3
2016-09-22 11:24:36.697742 7fd435ae7700  2 req 3:0.000069:s3:GET /object:get_obj:authorizing
2016-09-22 11:24:36.697745 7fd435ae7700  2 req 3:0.000072:s3:GET /object:get_obj:normalizing buckets and tenants
2016-09-22 11:24:36.697747 7fd435ae7700 10 s->object=object s->bucket=bucket
2016-09-22 11:24:36.697749 7fd435ae7700  2 req 3:0.000076:s3:GET /object:get_obj:init permissions
2016-09-22 11:24:36.697765 7fd435ae7700 20 get_system_obj_state: rctx=0x7fd435ae16e0 obj=.rgw:bucket state=0x7fd4e400c6e8 s->prefetch_data=0
2016-09-22 11:24:36.697775 7fd435ae7700 10 cache get: name=.rgw+bucket : hit (requested=22, cached=23)
2016-09-22 11:24:36.697784 7fd435ae7700 20 get_system_obj_state: s->obj_tag was set empty
2016-09-22 11:24:36.697786 7fd435ae7700 20 Read xattr: user.rgw.acl
2016-09-22 11:24:36.697786 7fd435ae7700 20 Read xattr: user.rgw.idtag
2016-09-22 11:24:36.697787 7fd435ae7700 20 Read xattr: user.rgw.manifest
2016-09-22 11:24:36.697788 7fd435ae7700 10 cache get: name=.rgw+bucket : hit (requested=17, cached=23)
2016-09-22 11:24:36.697803 7fd435ae7700 20 rgw_get_bucket_info: old bucket info, bucket=bucket(@.rgw.data.1[88283.1]) owner CENSOR
2016-09-22 11:24:36.697819 7fd435ae7700 15 decode_policy Read AccessControlPolicy<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>CENSOR</ID><DisplayName>CENSOR</DisplayName></Owner><AccessControlList><Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group"><URI>http://acs.amazonaws.com/groups/global/AllUsers</URI></Grantee><Permission>READ</Permission></Grant><Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser"><ID>CENSOR</ID><DisplayName>CENSOR</DisplayName></Grantee><Permission>FULL_CONTROL</Permission></Grant></AccessControlList></AccessControlPolicy>
2016-09-22 11:24:36.697827 7fd435ae7700 10 read_permissions on bucket(@.rgw.data.1[88283.1]) ret=-2
2016-09-22 11:24:36.697829 7fd435ae7700 20 op->ERRORHANDLER: err_no=-2 new_err_no=-2
2016-09-22 11:24:36.697913 7fd435ae7700  2 req 3:0.000240:s3:GET /object:get_obj:op status=0
2016-09-22 11:24:36.697923 7fd435ae7700  2 req 3:0.000250:s3:GET /object:get_obj:http status=404
2016-09-22 11:24:36.697931 7fd435ae7700  1 ====== req done req=0x7fd435ae17d0 op status=0 http_status=404 ======
2016-09-22 11:24:36.697941 7fd435ae7700 20 process_request() returned -2
2016-09-22 11:24:36.697960 7fd435ae7700  1 civetweb: 0x7fd4e4001e20: 2607:f298:4:d006::5796 - - [22/Sep/2016:11:24:36 +0000] "GET /object HTTP/1.1" 404 0 - curl/7.35.0


Related issues

Copied to Backport #17735: jewel: RGW will not list Argonaut-era bucket via HTTP (but radosgw-admin works) Resolved

History

#1 Updated by Robin Johnson 6 months ago

zone get as requested by Orit:

{
    "id": "default",
    "name": "default",
    "domain_root": ".rgw",
    "control_pool": ".rgw.control",
    "gc_pool": ".rgw.gc",
    "log_pool": ".log",
    "intent_log_pool": ".intent-log",
    "usage_log_pool": ".usage",
    "user_keys_pool": ".users",
    "user_email_pool": ".users.email",
    "user_swift_pool": ".users.swift",
    "user_uid_pool": ".users.uid",
    "system_key": {
        "access_key": "CENSOR",
        "secret_key": "CENSOR" 
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": ".rgw.data.1",
                "data_pool": ".rgw.data.1",
                "data_extra_pool": ".rgw.data.1",
                "index_type": 0
            }
        }
    ],
    "metadata_heap": ".rgw.meta",
    "realm_id": "6ed37b4b-66ba-4a6a-a464-80d7490cb310" 
}

And an updated RGW log with the zone definetly correct (in case #17371 was leading to RGWs having a bad config when the zone/zonegroup was eaten).

2016-09-22 11:51:55.750920 7fc522323700 20 RGWEnv::set(): HTTP_USER_AGENT: curl/7.35.0
2016-09-22 11:51:55.750934 7fc522323700 20 RGWEnv::set(): HTTP_ACCEPT: */*
2016-09-22 11:51:55.750936 7fc522323700 20 RGWEnv::set(): HTTP_HOST: bucket.objects-us-west-1.dream.io
2016-09-22 11:51:55.750937 7fc522323700 20 RGWEnv::set(): REQUEST_METHOD: GET
2016-09-22 11:51:55.750944 7fc522323700 20 RGWEnv::set(): REQUEST_URI: /object
2016-09-22 11:51:55.750945 7fc522323700 20 RGWEnv::set(): QUERY_STRING: 
2016-09-22 11:51:55.750946 7fc522323700 20 RGWEnv::set(): REMOTE_USER: 
2016-09-22 11:51:55.750947 7fc522323700 20 RGWEnv::set(): SCRIPT_URI: /object
2016-09-22 11:51:55.750949 7fc522323700 20 RGWEnv::set(): SERVER_PORT: 0
2016-09-22 11:51:55.750950 7fc522323700 20 HTTP_ACCEPT=*/*
2016-09-22 11:51:55.750951 7fc522323700 20 HTTP_HOST=bucket.objects-us-west-1.dream.io
2016-09-22 11:51:55.750951 7fc522323700 20 HTTP_USER_AGENT=curl/7.35.0
2016-09-22 11:51:55.750952 7fc522323700 20 QUERY_STRING=
2016-09-22 11:51:55.750952 7fc522323700 20 REMOTE_USER=
2016-09-22 11:51:55.750952 7fc522323700 20 REQUEST_METHOD=GET
2016-09-22 11:51:55.750953 7fc522323700 20 REQUEST_URI=/object
2016-09-22 11:51:55.750953 7fc522323700 20 SCRIPT_URI=/object
2016-09-22 11:51:55.750954 7fc522323700 20 SERVER_PORT=0
2016-09-22 11:51:55.750957 7fc522323700  1 ====== starting new request req=0x7fc52231d7d0 =====
2016-09-22 11:51:55.750983 7fc522323700  2 req 1:0.000026::GET /object::initializing for trans_id = tx000000000000000000001-0057e3c5db-1084dd40-default
2016-09-22 11:51:55.750994 7fc522323700 10 rgw api priority: s3=4 s3website=-1
2016-09-22 11:51:55.750996 7fc522323700 10 host=bucket.objects-us-west-1.dream.io
2016-09-22 11:51:55.751000 7fc522323700 20 subdomain=bucket domain=objects-us-west-1.dream.io in_hosted_domain=1 in_hosted_domain_s3website=0
2016-09-22 11:51:55.751002 7fc522323700 20 final domain/bucket subdomain=bucket domain=objects-us-west-1.dream.io in_hosted_domain=1 in_hosted_domain_s3website=0 s->info.domain=objects-us-west-1.dream.io s->info.request_uri=/bucket/object
2016-09-22 11:51:55.751045 7fc522323700 20 get_handler handler=22RGWHandler_REST_Obj_S3
2016-09-22 11:51:55.751051 7fc522323700 10 handler=22RGWHandler_REST_Obj_S3
2016-09-22 11:51:55.751053 7fc522323700  2 req 1:0.000096:s3:GET /object::getting op 0
2016-09-22 11:51:55.751060 7fc522323700 10 op=21RGWGetObj_ObjStore_S3
2016-09-22 11:51:55.751062 7fc522323700  2 req 1:0.000105:s3:GET /object:get_obj:authorizing
2016-09-22 11:51:55.751068 7fc522323700  2 req 1:0.000111:s3:GET /object:get_obj:normalizing buckets and tenants
2016-09-22 11:51:55.751071 7fc522323700 10 s->object=object s->bucket=bucket
2016-09-22 11:51:55.751074 7fc522323700  2 req 1:0.000117:s3:GET /object:get_obj:init permissions
2016-09-22 11:51:55.751101 7fc522323700 20 get_system_obj_state: rctx=0x7fc52231d6e0 obj=.rgw:bucket state=0x7fc5bc0d9b48 s->prefetch_data=0
2016-09-22 11:51:55.751115 7fc522323700 10 cache get: name=.rgw+bucket : miss
2016-09-22 11:51:55.751197 7fc522323700  1 -- [2607:f298:4:d006::5795]:0/1259939625 --> [2607:f298:4:d007::8852]:6860/111972 -- osd_op(client.277142848.0:1164 13.3c5bc352 bucket [call version.read,getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e1005544) v7 -- ?+0 0x7fc5bc0dbea0 con 0x7fc5b4050b80
2016-09-22 11:51:55.757522 7fc30e0f4700  1 -- [2607:f298:4:d006::5795]:0/1259939625 <== osd.994 [2607:f298:4:d007::8852]:6860/111972 3 ==== osd_op_reply(1164 bucket [call,getxattrs,stat] v0'0 uv5397 ondisk = 0) v7 ==== 210+0+255 (623213301 0 1810348938) 0x7fc62c36a690 con 0x7fc5b4050b80
2016-09-22 11:51:55.757629 7fc522323700 10 cache put: name=.rgw+bucket info.flags=22
2016-09-22 11:51:55.757645 7fc522323700 10 adding .rgw+bucket to cache LRU end
2016-09-22 11:51:55.757649 7fc522323700 10 updating xattr: name=user.rgw.acl bl.length()=191
2016-09-22 11:51:55.757654 7fc522323700 20 get_system_obj_state: s->obj_tag was set empty
2016-09-22 11:51:55.757656 7fc522323700 20 Read xattr: user.rgw.acl
2016-09-22 11:51:55.757656 7fc522323700 20 Read xattr: user.rgw.idtag
2016-09-22 11:51:55.757657 7fc522323700 20 Read xattr: user.rgw.manifest
2016-09-22 11:51:55.757661 7fc522323700 10 cache get: name=.rgw+bucket : type miss (requested=17, cached=22)
2016-09-22 11:51:55.757664 7fc522323700 20 get_system_obj_state: rctx=0x7fc52231d6e0 obj=.rgw:bucket state=0x7fc5bc0d9b48 s->prefetch_data=0
2016-09-22 11:51:55.757678 7fc522323700 20 rados->read ofs=0 len=524288
2016-09-22 11:51:55.757707 7fc522323700  1 -- [2607:f298:4:d006::5795]:0/1259939625 --> [2607:f298:4:d007::8852]:6860/111972 -- osd_op(client.277142848.0:1165 13.3c5bc352 bucket [call version.read,read 0~524288] snapc 0=[] ack+read+known_if_redirected e1005544) v7 -- ?+0 0x7fc5bc0dafb0 con 0x7fc5b4050b80
2016-09-22 11:51:55.763642 7fc30e0f4700  1 -- [2607:f298:4:d006::5795]:0/1259939625 <== osd.994 [2607:f298:4:d007::8852]:6860/111972 4 ==== osd_op_reply(1165 bucket [call,read 0~73] v0'0 uv5397 ondisk = 0) v7 ==== 168+0+97 (3684936440 0 3958803098) 0x7fc62c36ad00 con 0x7fc5b4050b80
2016-09-22 11:51:55.763756 7fc522323700 20 rados->read r=0 bl.length=73
2016-09-22 11:51:55.763774 7fc522323700 10 cache put: name=.rgw+bucket info.flags=17
2016-09-22 11:51:55.763779 7fc522323700 10 moving .rgw+bucket to cache LRU end
2016-09-22 11:51:55.763806 7fc522323700 20 rgw_get_bucket_info: old bucket info, bucket=bucket(@.rgw.data.1[88283.1]) owner CENSOR
2016-09-22 11:51:55.763842 7fc522323700 15 decode_policy Read AccessControlPolicy<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>CENSOR</ID><DisplayName>CENSOR</DisplayName></Owner><AccessControlList><Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group"><URI>http://acs.amazonaws.com/groups/global/AllUsers</URI></Grantee><Permission>READ</Permission></Grant><Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser"><ID>CENSOR</ID><DisplayName>CENSOR</DisplayName></Grantee><Permission>FULL_CONTROL</Permission></Grant></AccessControlList></AccessControlPolicy>
2016-09-22 11:51:55.763867 7fc522323700 10 read_permissions on bucket(@.rgw.data.1[88283.1]) ret=-2
2016-09-22 11:51:55.763873 7fc522323700 20 op->ERRORHANDLER: err_no=-2 new_err_no=-2
2016-09-22 11:51:55.764004 7fc522323700  2 req 1:0.013047:s3:GET /object:get_obj:op status=0
2016-09-22 11:51:55.764012 7fc522323700  2 req 1:0.013054:s3:GET /object:get_obj:http status=404
2016-09-22 11:51:55.764020 7fc522323700  1 ====== req done req=0x7fc52231d7d0 op status=0 http_status=404 ======
2016-09-22 11:51:55.764041 7fc522323700 20 process_request() returned -2
2016-09-22 11:51:55.764064 7fc522323700  1 civetweb: 0x7fc5bc02cc70: 2607:f298:4:d006::5796 - - [22/Sep/2016:11:51:55 +0000] "GET /object HTTP/1.1" 404 0 - curl/7.35.0

#2 Updated by Robin Johnson 6 months ago

Slightly newer, also broken bucket as reported by user, but gives a 403, not a 404, so possibly not broken in the same way.

# radosgw-admin metadata get bucket:slaptop-backup
{
    "key": "bucket:slaptop-backup",
    "ver": {
        "tag": "_JBL-re6oCBgAc7C4jfTMdwh",
        "ver": 1
    },
    "mtime": "2014-02-27 20:36:50.000000Z",
    "data": {
        "bucket": {
            "name": "slaptop-backup",
            "pool": ".rgw.data.1",
            "data_extra_pool": "",
            "index_pool": ".rgw.data.1",
            "marker": "default.48899569.78404",
            "bucket_id": "default.48899569.78404" 
        },
        "owner": "CENSOR",
        "creation_time": "2014-02-27 20:36:50.000000Z",
        "linked": "true",
        "has_bucket_info": "false" 
    }
}
# radosgw-admin metadata get bucket.instance:slaptop-backup:default.48899569.78404
{
    "key": "bucket.instance:slaptop-backup:default.48899569.78404",
    "ver": {
        "tag": "_mo89bYQARQbPZML0fwWqmQc",
        "ver": 1
    },
    "mtime": "2014-02-27 20:36:50.000000Z",
    "data": {
        "bucket_info": {
            "bucket": {
                "name": "slaptop-backup",
                "pool": ".rgw.data.1",
                "data_extra_pool": "",
                "index_pool": ".rgw.data.1",
                "marker": "default.48899569.78404",
                "bucket_id": "default.48899569.78404" 
            },
            "creation_time": "2014-02-27 20:36:50.000000Z",
            "owner": "CENSOR",
            "flags": 0,
            "zonegroup": "default",
            "placement_rule": "",
            "has_instance_obj": "true",
            "quota": {
                "enabled": false,
                "max_size_kb": -1,
                "max_objects": -1
            },
            "num_shards": 0,
            "bi_shard_hash_type": 0,
            "requester_pays": "false",
            "has_website": "false",
            "swift_versioning": "false",
            "swift_ver_location": "" 
        },
        "attrs": [
            {
                "key": "user.rgw.acl",
                "val": "AgJ\/AAAAAgIUAAAABgAAAGpvcmRhbgYAAABqb3JkYW4DA18AAAABAQAAAAYAAABqb3JkYW4PAAAAAQAAAAYAAABqb3JkYW4DAzQAAAACAgQAAAAAAAAABgAAAGpvcmRhbgAAAAAAAAAAAgIEAAAADwAAAAYAAABqb3JkYW4AAAAAAAAAAA==" 
            },
            {
                "key": "user.rgw.idtag",
                "val": "" 
            },
            {
                "key": "user.rgw.manifest",
                "val": "" 
            }
        ]
    }
}

#3 Updated by Robin Johnson 6 months ago

Patch from Yehuda, from cbodley's idea.
Confirmed working on the cluster.

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 15544c8..f021a08 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -902,7 +902,12 @@ int RGWPeriod::init(CephContext *_cct, RGWRados *_store, bool setup_obj)

 int RGWPeriod::get_zonegroup(RGWZoneGroup& zonegroup, const string& zonegroup_id) {
-  map<string, RGWZoneGroup>::const_iterator iter = period_map.zonegroups.find(zonegroup_id);
+  map<string, RGWZoneGroup>::const_iterator iter;
+  if (!zonegroup_id.empty()) {
+    iter = period_map.zonegroups.find(zonegroup_id);
+  } else {
+    iter = period_map.zonegroups.find("default");
+  }
   if (iter != period_map.zonegroups.end()) {
     zonegroup = iter->second;
     return 0;

#4 Updated by Yehuda Sadeh 6 months ago

  • Backport set to jewel

#5 Updated by Yehuda Sadeh 6 months ago

  • Status changed from New to Need Review

#6 Updated by Yehuda Sadeh 6 months ago

  • Status changed from Need Review to Need Test

#7 Updated by Yehuda Sadeh 6 months ago

  • Status changed from Need Test to Need Review

#8 Updated by Yehuda Sadeh 5 months ago

  • Assignee set to Orit Wasserman

#9 Updated by Yehuda Sadeh 5 months ago

  • Status changed from Need Review to Pending Backport

#10 Updated by Loic Dachary 5 months ago

  • Copied to Backport #17735: jewel: RGW will not list Argonaut-era bucket via HTTP (but radosgw-admin works) added

#11 Updated by Nathan Cutler about 2 months ago

  • Status changed from Pending Backport to Resolved

Also available in: Atom PDF