Actions
Bug #17364
closedrgw: Unable to commit period zonegroup change
Added by Robin Johnson over 7 years ago. Updated over 7 years ago.
% Done:
0%
Source:
other
Tags:
Backport:
jewel
Regression:
No
Severity:
2 - major
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
Legacy data on the DreamObjects production cluster .rgw.root pool causes it to be impossible to commit a new zonegroup.
Files
| 20160921T1917-period-update-commit-debug.txt (62.3 KB) 20160921T1917-period-update-commit-debug.txt | rgw-admin period update --commit --debug-ms=1 --debug-rgw=10 2>&1 | Robin Johnson, 09/21/2016 07:22 PM |
Updated by Robin Johnson over 7 years ago
# radosgw-admin realm list
{
"default_info": "957adae0-7f18-4e8f-9046-c5c02b873907",
"realms": [
"gold"
]
}
# radosgw-admin period list
{
"periods": [
"957adae0-7f18-4e8f-9046-c5c02b873907:staging",
"d91d342a-9556-4160-825e-abc41f5db858"
]
}
# radosgw-admin zonegroup list
read_default_id : 0
{
"default_info": "default",
"zonegroups": [
"default"
]
}
# radosgw-admin zone list
{
"default_info": "default",
"zones": [
"default"
]
}
# radosgw-admin zonegroup get --rgw-zonegroup=default
{
"id": "default",
"name": "default",
"api_name": "us-west-1",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1.dream.io",
"https:\/\/objects.dreamhost.com"
],
"hostnames": [
"objects-us-west-1.dream.io",
"objects.dreamhost.com"
],
"hostnames_s3website": [
"objects-website-us-west-1.dream.io"
],
"master_zone": "default",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "true",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907"
}
# radosgw-admin zone get --rgw-zonegroup=default
{
"id": "default",
"name": "default",
"domain_root": ".rgw",
"control_pool": ".rgw.control",
"gc_pool": ".rgw.gc",
"log_pool": ".log",
"intent_log_pool": ".intent-log",
"usage_log_pool": ".usage",
"user_keys_pool": ".users",
"user_email_pool": ".users.email",
"user_swift_pool": ".users.swift",
"user_uid_pool": ".users.uid",
"system_key": {
"access_key": "CENSORED",
"secret_key": "CENSORED"
},
"placement_pools": [
{
"key": "default-placement",
"val": {
"index_pool": ".rgw.data.1",
"data_pool": ".rgw.data.1",
"data_extra_pool": ".rgw.data.1",
"index_type": 0
}
}
],
"metadata_heap": ".rgw.meta",
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907"
}
# radosgw-admin period update
{
"id": "957adae0-7f18-4e8f-9046-c5c02b873907:staging",
"epoch": 1,
"predecessor_uuid": "d91d342a-9556-4160-825e-abc41f5db858",
"sync_status": [],
"period_map": {
"id": "d91d342a-9556-4160-825e-abc41f5db858",
"zonegroups": [
{
"id": "default",
"name": "default",
"api_name": "us-west-1",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1.dream.io",
"https:\/\/objects.dreamhost.com"
],
"hostnames": [
"objects-us-west-1.dream.io",
"objects.dreamhost.com"
],
"hostnames_s3website": [
"objects-website-us-west-1.dream.io"
],
"master_zone": "default",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "true",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907"
}
],
"short_zone_ids": [
{
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "default",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907",
"realm_name": "gold",
"realm_epoch": 2
}
# radosgw-admin period update --commit
2016-09-21 19:26:37.433750 7fef47674900 0 Error delete object id default: (2) No such file or directory
2016-09-21 19:26:37.461898 7fef47674900 0 zonegroup default missing zone for master_zone=
couldn't init storage provider
## Now everything failed :-(
# radosgw-admin zonegroup get --rgw-zonegroup=default
{
"id": "default",
"name": "default",
"api_name": "",
"is_master": "true",
"endpoints": [],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907"
}
Updated by Robin Johnson over 7 years ago
Even more minimal case that causes commit to fail.
# radosgw-admin period get
{
"id": "d91d342a-9556-4160-825e-abc41f5db858",
"epoch": 1,
"predecessor_uuid": "",
"sync_status": [],
"period_map": {
"id": "d91d342a-9556-4160-825e-abc41f5db858",
"zonegroups": [
{
"id": "default",
"name": "default",
"api_name": "",
"is_master": "true",
"endpoints": [],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907"
}
],
"short_zone_ids": [
{
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907",
"realm_name": "gold",
"realm_epoch": 1
}
# radosgw-admin zonegroup get
{
"id": "default",
"name": "default",
"api_name": "",
"is_master": "true",
"endpoints": [],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907"
}
# radosgw-admin zone get
{
"id": "default",
"name": "default",
"domain_root": ".rgw",
"control_pool": ".rgw.control",
"gc_pool": ".rgw.gc",
"log_pool": ".log",
"intent_log_pool": ".intent-log",
"usage_log_pool": ".usage",
"user_keys_pool": ".users",
"user_email_pool": ".users.email",
"user_swift_pool": ".users.swift",
"user_uid_pool": ".users.uid",
"system_key": {
"access_key": "",
"secret_key": ""
},
"placement_pools": [],
"metadata_heap": ".rgw.meta",
"realm_id": ""
}
# radosgw-admin zone list
{
"default_info": "default",
"zones": [
"default"
]
}
# radosgw-admin zonegroup list
read_default_id : 0
{
"default_info": "default",
"zonegroups": [
"default"
]
}
# radosgw-admin period update
{
"id": "957adae0-7f18-4e8f-9046-c5c02b873907:staging",
"epoch": 1,
"predecessor_uuid": "d91d342a-9556-4160-825e-abc41f5db858",
"sync_status": [],
"period_map": {
"id": "d91d342a-9556-4160-825e-abc41f5db858",
"zonegroups": [
{
"id": "default",
"name": "default",
"api_name": "",
"is_master": "true",
"endpoints": [],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "default",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "true",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907"
}
],
"short_zone_ids": [
{
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "default",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907",
"realm_name": "gold",
"realm_epoch": 2
}
# radosgw-admin period update --commit
2016-09-21 19:46:04.894228 7f6a5b360900 0 Error delete object id default: (2) No such file or directory
2016-09-21 19:46:04.922625 7f6a5b360900 0 zonegroup default missing zone for master_zone=
couldn't init storage provider
Updated by Robin Johnson over 7 years ago
- Priority changed from Immediate to Normal
- Severity changed from 1 - critical to 2 - major
Downgraded priority & severity. As a workaround, I wiped out the .rgw.root pool, and recreated the content as zonegroup=default,zone=default, which worked. Trying to use other names from fresh also caused a very similar problem to happen, with a "default" zonegroup being created anyway.
Updated by Yehuda Sadeh over 7 years ago
- Subject changed from Unable to commit period zonegroup change to rgw: Unable to commit period zonegroup change
- Assignee set to Orit Wasserman
Updated by Orit Wasserman over 7 years ago
- Status changed from New to Fix Under Review
Updated by Orit Wasserman over 7 years ago
- Status changed from Fix Under Review to Pending Backport
Updated by Loïc Dachary over 7 years ago
- Copied to Backport #18102: jewel: rgw: Unable to commit period zonegroup change added
Updated by Nathan Cutler over 7 years ago
- Status changed from Pending Backport to Resolved
Actions