Bug #19040
rgw: multiple zonegroups: authorization error to post period
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
This is a part of issues discussed on ceph-devel ML: http://marc.info/?t=148671813300008
I tentatively configured multiple zonegroups in a single realm on single ceph cluster for evaluation of behavior of multiple zonegroups:
- zonegroup default (api_name east)
- zone default http://node5:80
- zonegroup jp (master)
- zone jp-east http://node5:8080 (master)
- zone jp-west http://node5:8081
- zonegroup west
- zone osaka http://node5:8082
After the osaka zone was added, authorization errors were periodically logged.
ceph-client.radosgw.osaka.log:
2017-02-22 16:18:33.578536 7fda6ddc3700 20 RGWEnv::set(): HTTP_HOST: node5:8082 2017-02-22 16:18:33.578546 7fda6ddc3700 20 RGWEnv::set(): HTTP_ACCEPT: */* 2017-02-22 16:18:33.578548 7fda6ddc3700 20 RGWEnv::set(): HTTP_TRANSFER_ENCODING: chunked 2017-02-22 16:18:33.578550 7fda6ddc3700 20 RGWEnv::set(): HTTP_AUTHORIZATION: AWS SVLU3EJTNKI5JQ1QNVY6:yjLXbxZOW5d/QdataEHIWCB4W5s= 2017-02-22 16:18:33.578553 7fda6ddc3700 20 RGWEnv::set(): HTTP_DATE: Wed Feb 22 07:18:33 2017 2017-02-22 16:18:33.578554 7fda6ddc3700 20 RGWEnv::set(): HTTP_EXPECT: 100-continue 2017-02-22 16:18:33.578555 7fda6ddc3700 20 RGWEnv::set(): REQUEST_METHOD: POST 2017-02-22 16:18:33.578556 7fda6ddc3700 20 RGWEnv::set(): REQUEST_URI: /admin/realm/period 2017-02-22 16:18:33.578557 7fda6ddc3700 20 RGWEnv::set(): QUERY_STRING: period=0b9ed9e8-1bd6-4474-a1a1-b74bb882d100&epoch=7&rgwx-zonegroup=2500dbe8-450f-47ed-8c2f-ae12c05445dc 2017-02-22 16:18:33.578558 7fda6ddc3700 20 RGWEnv::set(): REMOTE_USER: 2017-02-22 16:18:33.578559 7fda6ddc3700 20 RGWEnv::set(): SCRIPT_URI: /admin/realm/period 2017-02-22 16:18:33.578561 7fda6ddc3700 20 RGWEnv::set(): SERVER_PORT: 8082 2017-02-22 16:18:33.578562 7fda6ddc3700 20 HTTP_ACCEPT=*/* 2017-02-22 16:18:33.578563 7fda6ddc3700 20 HTTP_AUTHORIZATION=AWS SVLU3EJTNKI5JQ1QNVY6:yjLXbxZOW5d/QdataEHIWCB4W5s= 2017-02-22 16:18:33.578563 7fda6ddc3700 20 HTTP_DATE=Wed Feb 22 07:18:33 2017 2017-02-22 16:18:33.578564 7fda6ddc3700 20 HTTP_EXPECT=100-continue 2017-02-22 16:18:33.578564 7fda6ddc3700 20 HTTP_HOST=node5:8082 2017-02-22 16:18:33.578565 7fda6ddc3700 20 HTTP_TRANSFER_ENCODING=chunked 2017-02-22 16:18:33.578565 7fda6ddc3700 20 QUERY_STRING=period=0b9ed9e8-1bd6-4474-a1a1-b74bb882d100&epoch=7&rgwx-zonegroup=2500dbe8-450f-47ed-8c2f-ae12c05445dc 2017-02-22 16:18:33.578566 7fda6ddc3700 20 REMOTE_USER= 2017-02-22 16:18:33.578566 7fda6ddc3700 20 REQUEST_METHOD=POST 2017-02-22 16:18:33.578567 7fda6ddc3700 20 REQUEST_URI=/admin/realm/period 2017-02-22 16:18:33.578567 7fda6ddc3700 20 SCRIPT_URI=/admin/realm/period 2017-02-22 16:18:33.578568 7fda6ddc3700 20 SERVER_PORT=8082 2017-02-22 16:18:33.578571 7fda6ddc3700 1 ====== starting new request req=0x7fda6ddbd710 ===== 2017-02-22 16:18:33.578588 7fda6ddc3700 2 req 1:0.000017::POST /admin/realm/period::initializing for trans_id = tx000000000000000000001-0058ad3b49-1601-osaka 2017-02-22 16:18:33.578595 7fda6ddc3700 10 rgw api priority: s3=5 s3website=4 2017-02-22 16:18:33.578596 7fda6ddc3700 10 host=node5 2017-02-22 16:18:33.578600 7fda6ddc3700 20 subdomain= domain= in_hosted_domain=0 in_hosted_domain_s3website=0 2017-02-22 16:18:33.578602 7fda6ddc3700 20 final domain/bucket subdomain= domain= in_hosted_domain=0 in_hosted_domain_s3website=0 s->info.domain= s->info.request_uri=/admin/realm/period 2017-02-22 16:18:33.578635 7fda6ddc3700 10 handler=17RGWHandler_Period 2017-02-22 16:18:33.578640 7fda6ddc3700 2 req 1:0.000066::POST /admin/realm/period::getting op 4 2017-02-22 16:18:33.578643 7fda6ddc3700 10 op=17RGWOp_Period_Post 2017-02-22 16:18:33.578645 7fda6ddc3700 2 req 1:0.000074::POST /admin/realm/period:post_period:authorizing 2017-02-22 16:18:33.578665 7fda6ddc3700 20 get_system_obj_state: rctx=0x7fda6ddbc250 obj=osaka.rgw.users.keys:SVLU3EJTNKI5JQ1QNVY6 state=0x7fdaa80096e8 s->prefetch_data=0 2017-02-22 16:18:33.578669 7fda6ddc3700 10 cache get: name=osaka.rgw.users.keys+SVLU3EJTNKI5JQ1QNVY6 : miss 2017-02-22 16:18:33.580341 7fda6ddc3700 10 cache put: name=osaka.rgw.users.keys+SVLU3EJTNKI5JQ1QNVY6 info.flags=6 2017-02-22 16:18:33.580353 7fda6ddc3700 10 adding osaka.rgw.users.keys+SVLU3EJTNKI5JQ1QNVY6 to cache LRU end 2017-02-22 16:18:33.580356 7fda6ddc3700 20 get_system_obj_state: s->obj_tag was set empty 2017-02-22 16:18:33.580360 7fda6ddc3700 10 cache get: name=osaka.rgw.users.keys+SVLU3EJTNKI5JQ1QNVY6 : type miss (requested=1, cached=6) 2017-02-22 16:18:33.580362 7fda6ddc3700 20 get_system_obj_state: rctx=0x7fda6ddbc250 obj=osaka.rgw.users.keys:SVLU3EJTNKI5JQ1QNVY6 state=0x7fdaa80096e8 s->prefetch_data=0 2017-02-22 16:18:33.580364 7fda6ddc3700 20 rados->read ofs=0 len=524288 2017-02-22 16:18:33.580876 7fda6ddc3700 20 rados->read r=0 bl.length=13 2017-02-22 16:18:33.580885 7fda6ddc3700 10 cache put: name=osaka.rgw.users.keys+SVLU3EJTNKI5JQ1QNVY6 info.flags=1 2017-02-22 16:18:33.580886 7fda6ddc3700 10 moving osaka.rgw.users.keys+SVLU3EJTNKI5JQ1QNVY6 to cache LRU end 2017-02-22 16:18:33.580900 7fda6ddc3700 20 get_system_obj_state: rctx=0x7fda6ddbbf10 obj=osaka.rgw.users.uid:sync-user state=0x7fdaa800af68 s->prefetch_data=0 2017-02-22 16:18:33.580905 7fda6ddc3700 10 cache get: name=osaka.rgw.users.uid+sync-user : miss 2017-02-22 16:18:33.581731 7fda6ddc3700 10 cache put: name=osaka.rgw.users.uid+sync-user info.flags=22 2017-02-22 16:18:33.581737 7fda6ddc3700 10 adding osaka.rgw.users.uid+sync-user to cache LRU end 2017-02-22 16:18:33.581739 7fda6ddc3700 20 get_system_obj_state: s->obj_tag was set empty 2017-02-22 16:18:33.581742 7fda6ddc3700 10 cache get: name=osaka.rgw.users.uid+sync-user : type miss (requested=17, cached=22) 2017-02-22 16:18:33.581744 7fda6ddc3700 20 get_system_obj_state: rctx=0x7fda6ddbbf10 obj=osaka.rgw.users.uid:sync-user state=0x7fdaa800af68 s->prefetch_data=0 2017-02-22 16:18:33.581759 7fda6ddc3700 20 rados->read ofs=0 len=524288 2017-02-22 16:18:33.582250 7fda6ddc3700 20 rados->read r=0 bl.length=338 2017-02-22 16:18:33.582255 7fda6ddc3700 10 cache put: name=osaka.rgw.users.uid+sync-user info.flags=17 2017-02-22 16:18:33.582261 7fda6ddc3700 10 moving osaka.rgw.users.uid+sync-user to cache LRU end 2017-02-22 16:18:33.582277 7fda6ddc3700 10 chain_cache_entry: cache_locator=osaka.rgw.users.uid+sync-user 2017-02-22 16:18:33.582320 7fda6ddc3700 10 get_canon_resource(): dest=/admin/realm/period 2017-02-22 16:18:33.582336 7fda6ddc3700 10 auth_hdr: POST Wed Feb 22 07:18:33 2017 /admin/realm/period 2017-02-22 16:18:33.582375 7fda6ddc3700 15 calculated digest=1OEyzxkmnrgJbfanbSLlBqyPiRc= 2017-02-22 16:18:33.582381 7fda6ddc3700 15 auth_sign=yjLXbxZOW5d/QdataEHIWCB4W5s= 2017-02-22 16:18:33.582382 7fda6ddc3700 15 compare=72 2017-02-22 16:18:33.582383 7fda6ddc3700 10 failed to authorize request 2017-02-22 16:18:33.582384 7fda6ddc3700 20 handler->ERRORHANDLER: err_no=-2027 new_err_no=-2027 2017-02-22 16:18:33.582468 7fda6ddc3700 2 req 1:0.003897::POST /admin/realm/period:post_period:op status=0 2017-02-22 16:18:33.582475 7fda6ddc3700 2 req 1:0.003905::POST /admin/realm/period:post_period:http status=403 2017-02-22 16:18:33.582479 7fda6ddc3700 1 ====== req done req=0x7fda6ddbd710 op status=0 http_status=403 ====== 2017-02-22 16:18:33.582486 7fda6ddc3700 20 process_request() returned -2027 2017-02-22 16:18:33.582503 7fda6ddc3700 1 civetweb: 0x7fdaa8003e80: 192.168.20.15 - - [22/Feb/2017:16:18:33 +0900] "POST /admin/realm/period HTTP/1.1" 403 0 - -
ceph-client.radosgw.jp-east.log:
2017-02-22 16:18:33.576092 7f8c4c45f700 10 rgw period pusher: pushing period 0b9ed9e8-1bd6-4474-a1a1-b74bb882d100 to 7326745b-f844-4650-804f-1b5f902a270e 2017-02-22 16:18:33.576436 7f8c4c45f700 10 get_canon_resource(): dest=/admin/realm/period 2017-02-22 16:18:33.576443 7f8c4c45f700 10 generated canonical header: POST Wed Feb 22 07:18:33 2017 /admin/realm/period 2017-02-22 16:18:33.582525 7f8c4bc5e700 10 receive_http_header 2017-02-22 16:18:33.582532 7f8c4bc5e700 10 received header:HTTP/1.1 403 Forbidden 2017-02-22 16:18:33.582534 7f8c4bc5e700 10 receive_http_header 2017-02-22 16:18:33.582535 7f8c4bc5e700 10 received header:x-amz-request-id: tx000000000000000000001-0058ad3b49-1601-osaka 2017-02-22 16:18:33.582540 7f8c4bc5e700 10 receive_http_header 2017-02-22 16:18:33.582541 7f8c4bc5e700 10 received header:Content-Length: 119 2017-02-22 16:18:33.582543 7f8c4bc5e700 10 receive_http_header 2017-02-22 16:18:33.582543 7f8c4bc5e700 10 received header:Accept-Ranges: bytes 2017-02-22 16:18:33.582546 7f8c4bc5e700 10 receive_http_header 2017-02-22 16:18:33.582547 7f8c4bc5e700 10 received header:Content-Type: application/json 2017-02-22 16:18:33.582549 7f8c4bc5e700 10 receive_http_header 2017-02-22 16:18:33.582550 7f8c4bc5e700 10 received header:Date: Wed, 22 Feb 2017 07:18:33 GMT 2017-02-22 16:18:33.582552 7f8c4bc5e700 10 receive_http_header 2017-02-22 16:18:33.582553 7f8c4bc5e700 10 received header: 2017-02-22 16:18:33.582595 7f8c4c45f700 5 failed to wait for op, ret=-13: POST http://node5:8082/admin/realm/period?period=0b9ed9e8-1bd6-4474-a1a1-b74bb882d100&epoch=7&rgwx-zonegroup=2500dbe8-450f-47ed-8c2f-ae12c05445dc 2017-02-22 16:18:33.582641 7f8c4c45f700 10 rgw period pusher: waiting 30.000000s for retry..
It seems the system user account is correctly configured.
{
"id": "dfc9230a-5835-4ad3-b317-4217ddcf99a5",
"name": "osaka",
"domain_root": "osaka.rgw.data.root",
"control_pool": "osaka.rgw.control",
"gc_pool": "osaka.rgw.gc",
"log_pool": "osaka.rgw.log",
"intent_log_pool": "osaka.rgw.intent-log",
"usage_log_pool": "osaka.rgw.usage",
"user_keys_pool": "osaka.rgw.users.keys",
"user_email_pool": "osaka.rgw.users.email",
"user_swift_pool": "osaka.rgw.users.swift",
"user_uid_pool": "osaka.rgw.users.uid",
"system_key": {
"access_key": "SVLU3EJTNKI5JQ1QNVY6",
"secret_key": "dzKmZl3H5rbQepXabV3PQOHMVWrxIlzCLucyDCcg"
},
"placement_pools": [
{
"key": "default-placement",
"val": {
"index_pool": "osaka.rgw.buckets.index",
"data_pool": "osaka.rgw.buckets.data",
"data_extra_pool": "osaka.rgw.buckets.non-ec",
"index_type": 0
}
}
],
"metadata_heap": "",
"realm_id": "d2def91a-84c6-4383-84df-c2b30bf91bab"
}
# radosgw-admin user info --uid sync-user --rgw-zonegroup west --rgw-zone osaka
{
"user_id": "sync-user",
"display_name": "Synchronization User",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [],
"keys": [
{
"user": "sync-user",
"access_key": "SVLU3EJTNKI5JQ1QNVY6",
"secret_key": "dzKmZl3H5rQepXabV3PQOHMVWrxIlzCLucyDCcg"
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"system": "true",
"default_placement": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"temp_url_keys": []
}
# radosgw-admin user info --uid sync-user --rgw-zonegroup jp --rgw-zone jp-east
{
"user_id": "sync-user",
"display_name": "Synchronization User",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [],
"keys": [
{
"user": "sync-user",
"access_key": "SVLU3EJTNKI5JQ1QNVY6",
"secret_key": "dzKmZl3H5rbQepXabV3PQOHMVWrxIlzCLucyDCcg"
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"system": "true",
"default_placement": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"temp_url_keys": []
}
History
#1 Updated by Osamu KIMURA about 7 years ago
FYI
# radosgw-admin zonegroup-map get
{
"zonegroups": [
{
"key": "2500dbe8-450f-47ed-8c2f-ae12c05445dc",
"val": {
"id": "2500dbe8-450f-47ed-8c2f-ae12c05445dc",
"name": "jp",
"api_name": "jp",
"is_master": "true",
"endpoints": [
"http:\/\/node5:8080"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "22719b44-532d-41ee-974e-fc89cb93255f",
"zones": [
{
"id": "0e38bb03-c053-4bc4-bb3b-6eec7851c827",
"name": "jp-west",
"endpoints": [
"http:\/\/node5:8081"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 0,
"read_only": "false"
},
{
"id": "22719b44-532d-41ee-974e-fc89cb93255f",
"name": "jp-east",
"endpoints": [
"http:\/\/node5:8080"
],
"log_meta": "true",
"log_data": "true",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "d2def91a-84c6-4383-84df-c2b30bf91bab"
}
},
{
"key": "7326745b-f844-4650-804f-1b5f902a270e",
"val": {
"id": "7326745b-f844-4650-804f-1b5f902a270e",
"name": "west",
"api_name": "west",
"is_master": "false",
"endpoints": [
"http:\/\/node5:8082"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "dfc9230a-5835-4ad3-b317-4217ddcf99a5",
"zones": [
{
"id": "dfc9230a-5835-4ad3-b317-4217ddcf99a5",
"name": "osaka",
"endpoints": [
"http:\/\/node5:8082"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "d2def91a-84c6-4383-84df-c2b30bf91bab"
}
},
{
"key": "b9fb59d2-7c90-4cfd-a5b4-006adbda2af9",
"val": {
"id": "b9fb59d2-7c90-4cfd-a5b4-006adbda2af9",
"name": "default",
"api_name": "east",
"is_master": "false",
"endpoints": [
"http:\/\/node5:80"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "6216eff2-1146-4ba1-9368-43d70dd45975",
"zones": [
{
"id": "6216eff2-1146-4ba1-9368-43d70dd45975",
"name": "default",
"endpoints": [
"http:\/\/node5:80"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "d2def91a-84c6-4383-84df-c2b30bf91bab"
}
}
],
"master_zonegroup": "2500dbe8-450f-47ed-8c2f-ae12c05445dc",
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
}
# radosgw-admin period get
{
"id": "0b9ed9e8-1bd6-4474-a1a1-b74bb882d100",
"epoch": 7,
"predecessor_uuid": "5e7b5c3f-831d-4b83-99cf-03c05634ecf1",
"sync_status": [
"",
"",
...
"",
""
],
"period_map": {
"id": "0b9ed9e8-1bd6-4474-a1a1-b74bb882d100",
"zonegroups": [
{
"id": "2500dbe8-450f-47ed-8c2f-ae12c05445dc",
"name": "jp",
"api_name": "jp",
"is_master": "true",
"endpoints": [
"http:\/\/node5:8080"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "22719b44-532d-41ee-974e-fc89cb93255f",
"zones": [
{
"id": "0e38bb03-c053-4bc4-bb3b-6eec7851c827",
"name": "jp-west",
"endpoints": [
"http:\/\/node5:8081"
],
"log_meta": "false",
"log_data": "true",
"bucket_index_max_shards": 0,
"read_only": "false"
},
{
"id": "22719b44-532d-41ee-974e-fc89cb93255f",
"name": "jp-east",
"endpoints": [
"http:\/\/node5:8080"
],
"log_meta": "true",
"log_data": "true",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "d2def91a-84c6-4383-84df-c2b30bf91bab"
},
{
"id": "7326745b-f844-4650-804f-1b5f902a270e",
"name": "west",
"api_name": "west",
"is_master": "false",
"endpoints": [
"http:\/\/node5:8082"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "dfc9230a-5835-4ad3-b317-4217ddcf99a5",
"zones": [
{
"id": "dfc9230a-5835-4ad3-b317-4217ddcf99a5",
"name": "osaka",
"endpoints": [
"http:\/\/node5:8082"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "d2def91a-84c6-4383-84df-c2b30bf91bab"
},
{
"id": "b9fb59d2-7c90-4cfd-a5b4-006adbda2af9",
"name": "default",
"api_name": "east",
"is_master": "false",
"endpoints": [
"http:\/\/node5:80"
],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "6216eff2-1146-4ba1-9368-43d70dd45975",
"zones": [
{
"id": "6216eff2-1146-4ba1-9368-43d70dd45975",
"name": "default",
"endpoints": [
"http:\/\/node5:80"
],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "d2def91a-84c6-4383-84df-c2b30bf91bab"
}
],
"short_zone_ids": [
{
"key": "0e38bb03-c053-4bc4-bb3b-6eec7851c827",
"val": 3484765632
},
{
"key": "22719b44-532d-41ee-974e-fc89cb93255f",
"val": 3931456753
},
{
"key": "6216eff2-1146-4ba1-9368-43d70dd45975",
"val": 2199015833
},
{
"key": "dfc9230a-5835-4ad3-b317-4217ddcf99a5",
"val": 1386299322
}
]
},
"master_zonegroup": "2500dbe8-450f-47ed-8c2f-ae12c05445dc",
"master_zone": "22719b44-532d-41ee-974e-fc89cb93255f",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "d2def91a-84c6-4383-84df-c2b30bf91bab",
"realm_name": "fj",
"realm_epoch": 2
}
#2 Updated by Yehuda Sadeh about 7 years ago
- Assignee set to Orit Wasserman