Actions
Bug #17364
closedrgw: Unable to commit period zonegroup change
% Done:
0%
Source:
other
Tags:
Backport:
jewel
Regression:
No
Severity:
2 - major
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
Legacy data on the DreamObjects production cluster .rgw.root pool causes it to be impossible to commit a new zonegroup.
Files
Updated by Robin Johnson over 7 years ago
# radosgw-admin realm list { "default_info": "957adae0-7f18-4e8f-9046-c5c02b873907", "realms": [ "gold" ] } # radosgw-admin period list { "periods": [ "957adae0-7f18-4e8f-9046-c5c02b873907:staging", "d91d342a-9556-4160-825e-abc41f5db858" ] } # radosgw-admin zonegroup list read_default_id : 0 { "default_info": "default", "zonegroups": [ "default" ] } # radosgw-admin zone list { "default_info": "default", "zones": [ "default" ] } # radosgw-admin zonegroup get --rgw-zonegroup=default { "id": "default", "name": "default", "api_name": "us-west-1", "is_master": "true", "endpoints": [ "https:\/\/objects-us-west-1.dream.io", "https:\/\/objects.dreamhost.com" ], "hostnames": [ "objects-us-west-1.dream.io", "objects.dreamhost.com" ], "hostnames_s3website": [ "objects-website-us-west-1.dream.io" ], "master_zone": "default", "zones": [ { "id": "default", "name": "default", "endpoints": [], "log_meta": "true", "log_data": "false", "bucket_index_max_shards": 31, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907" } # radosgw-admin zone get --rgw-zonegroup=default { "id": "default", "name": "default", "domain_root": ".rgw", "control_pool": ".rgw.control", "gc_pool": ".rgw.gc", "log_pool": ".log", "intent_log_pool": ".intent-log", "usage_log_pool": ".usage", "user_keys_pool": ".users", "user_email_pool": ".users.email", "user_swift_pool": ".users.swift", "user_uid_pool": ".users.uid", "system_key": { "access_key": "CENSORED", "secret_key": "CENSORED" }, "placement_pools": [ { "key": "default-placement", "val": { "index_pool": ".rgw.data.1", "data_pool": ".rgw.data.1", "data_extra_pool": ".rgw.data.1", "index_type": 0 } } ], "metadata_heap": ".rgw.meta", "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907" } # radosgw-admin period update { "id": "957adae0-7f18-4e8f-9046-c5c02b873907:staging", "epoch": 1, "predecessor_uuid": "d91d342a-9556-4160-825e-abc41f5db858", "sync_status": [], "period_map": { "id": "d91d342a-9556-4160-825e-abc41f5db858", "zonegroups": [ { "id": "default", "name": "default", "api_name": "us-west-1", "is_master": "true", "endpoints": [ "https:\/\/objects-us-west-1.dream.io", "https:\/\/objects.dreamhost.com" ], "hostnames": [ "objects-us-west-1.dream.io", "objects.dreamhost.com" ], "hostnames_s3website": [ "objects-website-us-west-1.dream.io" ], "master_zone": "default", "zones": [ { "id": "default", "name": "default", "endpoints": [], "log_meta": "true", "log_data": "false", "bucket_index_max_shards": 31, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907" } ], "short_zone_ids": [ { "key": "default", "val": 2610307010 } ] }, "master_zonegroup": "default", "master_zone": "default", "period_config": { "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } }, "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907", "realm_name": "gold", "realm_epoch": 2 } # radosgw-admin period update --commit 2016-09-21 19:26:37.433750 7fef47674900 0 Error delete object id default: (2) No such file or directory 2016-09-21 19:26:37.461898 7fef47674900 0 zonegroup default missing zone for master_zone= couldn't init storage provider ## Now everything failed :-( # radosgw-admin zonegroup get --rgw-zonegroup=default { "id": "default", "name": "default", "api_name": "", "is_master": "true", "endpoints": [], "hostnames": [], "hostnames_s3website": [], "master_zone": "", "zones": [ { "id": "default", "name": "default", "endpoints": [], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 0, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907" }
Updated by Robin Johnson over 7 years ago
Even more minimal case that causes commit to fail.
# radosgw-admin period get { "id": "d91d342a-9556-4160-825e-abc41f5db858", "epoch": 1, "predecessor_uuid": "", "sync_status": [], "period_map": { "id": "d91d342a-9556-4160-825e-abc41f5db858", "zonegroups": [ { "id": "default", "name": "default", "api_name": "", "is_master": "true", "endpoints": [], "hostnames": [], "hostnames_s3website": [], "master_zone": "", "zones": [ { "id": "default", "name": "default", "endpoints": [], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 0, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907" } ], "short_zone_ids": [ { "key": "default", "val": 2610307010 } ] }, "master_zonegroup": "default", "master_zone": "", "period_config": { "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } }, "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907", "realm_name": "gold", "realm_epoch": 1 } # radosgw-admin zonegroup get { "id": "default", "name": "default", "api_name": "", "is_master": "true", "endpoints": [], "hostnames": [], "hostnames_s3website": [], "master_zone": "", "zones": [ { "id": "default", "name": "default", "endpoints": [], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 0, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907" } # radosgw-admin zone get { "id": "default", "name": "default", "domain_root": ".rgw", "control_pool": ".rgw.control", "gc_pool": ".rgw.gc", "log_pool": ".log", "intent_log_pool": ".intent-log", "usage_log_pool": ".usage", "user_keys_pool": ".users", "user_email_pool": ".users.email", "user_swift_pool": ".users.swift", "user_uid_pool": ".users.uid", "system_key": { "access_key": "", "secret_key": "" }, "placement_pools": [], "metadata_heap": ".rgw.meta", "realm_id": "" } # radosgw-admin zone list { "default_info": "default", "zones": [ "default" ] } # radosgw-admin zonegroup list read_default_id : 0 { "default_info": "default", "zonegroups": [ "default" ] } # radosgw-admin period update { "id": "957adae0-7f18-4e8f-9046-c5c02b873907:staging", "epoch": 1, "predecessor_uuid": "d91d342a-9556-4160-825e-abc41f5db858", "sync_status": [], "period_map": { "id": "d91d342a-9556-4160-825e-abc41f5db858", "zonegroups": [ { "id": "default", "name": "default", "api_name": "", "is_master": "true", "endpoints": [], "hostnames": [], "hostnames_s3website": [], "master_zone": "default", "zones": [ { "id": "default", "name": "default", "endpoints": [], "log_meta": "true", "log_data": "false", "bucket_index_max_shards": 0, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907" } ], "short_zone_ids": [ { "key": "default", "val": 2610307010 } ] }, "master_zonegroup": "default", "master_zone": "default", "period_config": { "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } }, "realm_id": "957adae0-7f18-4e8f-9046-c5c02b873907", "realm_name": "gold", "realm_epoch": 2 } # radosgw-admin period update --commit 2016-09-21 19:46:04.894228 7f6a5b360900 0 Error delete object id default: (2) No such file or directory 2016-09-21 19:46:04.922625 7f6a5b360900 0 zonegroup default missing zone for master_zone= couldn't init storage provider
Updated by Robin Johnson over 7 years ago
- Priority changed from Immediate to Normal
- Severity changed from 1 - critical to 2 - major
Downgraded priority & severity. As a workaround, I wiped out the .rgw.root pool, and recreated the content as zonegroup=default,zone=default, which worked. Trying to use other names from fresh also caused a very similar problem to happen, with a "default" zonegroup being created anyway.
Updated by Yehuda Sadeh over 7 years ago
- Subject changed from Unable to commit period zonegroup change to rgw: Unable to commit period zonegroup change
- Assignee set to Orit Wasserman
Updated by Orit Wasserman over 7 years ago
- Status changed from New to Fix Under Review
Updated by Orit Wasserman over 7 years ago
- Status changed from Fix Under Review to Pending Backport
Updated by Loïc Dachary over 7 years ago
- Copied to Backport #18102: jewel: rgw: Unable to commit period zonegroup change added
Updated by Nathan Cutler over 7 years ago
- Status changed from Pending Backport to Resolved
Actions