Actions
Bug #17051
closedperiod commit loses zonegroup changes: region_map converted repeatedly
% Done:
0%
Source:
Community (dev)
Tags:
jewel
Backport:
jewel
Regression:
No
Severity:
2 - major
Reviewed:
Affected Versions:
ceph-qa-suite:
rgw, upgrade/hammer
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
Reproduced on both Jewel and Jewel-backports 10.2.2-432-gac44947 (commit ac44947). This is trying to upgrade a Hammer cluster, with the next step being upgrading a duplicate of the pool layout from the Congress cluster.
'zonegroup get' shows a zonegroup with a empty realm_id, and many other fields are not being updated with new changes.
Possibly related, of two RGW instances in this cluster:
- one works fine for bucket create, but the other reports 'rest connection is invalid' in the logs and 400 error to the user.
- 'period update --commit' causes BOTH RGW instances to break.
- radosgw-admin realm list
{
"default_info": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
"realms": [
"gold"
]
} - radosgw-admin realm get
{
"id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
"name": "gold",
"current_period": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
"epoch": 2
} - radosgw-admin period list
{
"periods": [
"00ae3a0c-6249-43f7-94d0-e57d740e6e90",
"e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4:staging",
"ea085b17-be40-4371-9cff-694c82ab7b7a"
]
} - radosgw-admin zonegroup list
read_default_id : 0 {
"default_info": "default",
"zonegroups": [
"default"
]
} - radosgw-admin zonegroup get
{
"id": "default",
"name": "default",
"api_name": "us-west-1-staging",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1-staging.dream.io",
"https:\/\/api.benjamin.dhobjects.com"
],
"hostnames": [
"objects-us-west-1-staging.dream.io",
"api.benjamin.dhobjects.com"
],
"hostnames_s3website": [],
"master_zone": "",
"zones": [ {
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [ {
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": ""
} - radosgw-admin zonegroup set --rgw-zonegroup=default --rgw-zone=default --infile zonegroup.json --master --default
{
"id": "default",
"name": "default",
"api_name": "us-west-1-staging",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1-staging.dream.io",
"https:\/\/api.benjamin.dhobjects.com"
],
"hostnames": [
"objects-us-west-1-staging.dream.io",
"api.benjamin.dhobjects.com"
],
"hostnames_s3website": [
"objects-website-us-west-1-staging.dream.io",
"api-www.benjamin.dhobjects.com"
],
"master_zone": "default",
"zones": [ {
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "true",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [ {
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4"
} - radosgw-admin period update
{
"id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4:staging",
"epoch": 9,
"predecessor_uuid": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
"sync_status": [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
""
],
"period_map": {
"id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
"zonegroups": [ {
"id": "default",
"name": "default",
"api_name": "us-west-1-staging",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1-staging.dream.io",
"https:\/\/api.benjamin.dhobjects.com"
],
"hostnames": [
"objects-us-west-1-staging.dream.io",
"api.benjamin.dhobjects.com"
],
"hostnames_s3website": [
"objects-website-us-west-1-staging.dream.io",
"api-www.benjamin.dhobjects.com"
],
"master_zone": "default",
"zones": [ {
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "true",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [ {
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4"
}
],
"short_zone_ids": [ {
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "default",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
"realm_name": "gold",
"realm_epoch": 3
} - radosgw-admin period commit
{
"id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
"epoch": 10,
"predecessor_uuid": "ea085b17-be40-4371-9cff-694c82ab7b7a",
"sync_status": [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
""
],
"period_map": {
"id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
"zonegroups": [ {
"id": "default",
"name": "default",
"api_name": "us-west-1-staging",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1-staging.dream.io",
"https:\/\/api.benjamin.dhobjects.com"
],
"hostnames": [
"objects-us-west-1-staging.dream.io",
"api.benjamin.dhobjects.com"
],
"hostnames_s3website": [
"objects-website-us-west-1-staging.dream.io",
"api-www.benjamin.dhobjects.com"
],
"master_zone": "default",
"zones": [ {
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "true",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [ {
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4"
}
],
"short_zone_ids": [ {
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "default",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
"realm_name": "gold",
"realm_epoch": 2
}
- radosgw-admin zonegroup get
{
"id": "default",
"name": "default",
"api_name": "us-west-1-staging",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1-staging.dream.io",
"https:\/\/api.benjamin.dhobjects.com"
],
"hostnames": [
"objects-us-west-1-staging.dream.io",
"api.benjamin.dhobjects.com"
],
"hostnames_s3website": [],
"master_zone": "",
"zones": [ {
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [ {
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": ""
} - radosgw-admin period update --commit
{
"id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
"epoch": 11,
"predecessor_uuid": "ea085b17-be40-4371-9cff-694c82ab7b7a",
"sync_status": [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
""
],
"period_map": {
"id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
"zonegroups": [],
"short_zone_ids": [ {
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "default",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
"realm_name": "gold",
"realm_epoch": 2
} - radosgw-admin zonegroup get
{
"id": "default",
"name": "default",
"api_name": "us-west-1-staging",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1-staging.dream.io",
"https:\/\/api.benjamin.dhobjects.com"
],
"hostnames": [
"objects-us-west-1-staging.dream.io",
"api.benjamin.dhobjects.com"
],
"hostnames_s3website": [],
"master_zone": "",
"zones": [ {
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [ {
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": ""
}
Actions