Actions
Bug #17371
closedRGW loses realm/period/zonegroup/zone data: period overwritten if somewhere in the cluster is still running Hammer
% Done:
0%
Source:
Community (dev)
Tags:
Backport:
jewel
Regression:
No
Severity:
2 - major
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
Something in RGW is causing the active realm/zonegroup/zone data to be silently lost or overwritten. I thought I was losing my mind, but then I checked scrollback, and found that it DID show the period data set correctly.
Here is the output of period update --commit, from 2016-09-22/00:35:22.020358 UTC.
root@peon5752:/home/rjohnson/dho-config/congress/tmp# radosgw-admin period update --commit
2016-09-22 00:35:16.649594 7f3d250a9900 0 RGWZoneParams::create(): error creating default zone params: (17) File exists
2016-09-22 00:35:18.660110 7f3d250a9900 0 error read_lastest_epoch .rgw.root:periods.68a89bcc-6769-498e-9da1-91fe9bfb72e5.latest_epoch
2016-09-22 00:35:22.020358 7f3d250a9900 1 Set the period's master zonegroup default as the default
{
"id": "68a89bcc-6769-498e-9da1-91fe9bfb72e5",
"epoch": 1,
"predecessor_uuid": "fb6c314c-9e34-4273-8779-d0ab16043532",
"sync_status": [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
""
],
"period_map": {
"id": "68a89bcc-6769-498e-9da1-91fe9bfb72e5",
"zonegroups": [
{
"id": "default",
"name": "default",
"api_name": "us-west-1",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1.dream.io",
"https:\/\/objects.dreamhost.com"
],
"hostnames": [
"objects-us-west-1.dream.io",
"objects.dreamhost.com"
],
"hostnames_s3website": [
"objects-website-us-west-1.dream.io"
],
"master_zone": "default",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "true",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "6ed37b4b-66ba-4a6a-a464-80d7490cb310"
}
],
"short_zone_ids": [
{
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "default",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "6ed37b4b-66ba-4a6a-a464-80d7490cb310",
"realm_name": "gold",
"realm_epoch": 2
}
Here is the output from period list & period get as of 2016-09-22/00:56:13.064602 UTC, still good.
rjohnson@peon5752:~$ sudo radosgw-admin period list
{
"periods": [
"68a89bcc-6769-498e-9da1-91fe9bfb72e5",
"6ed37b4b-66ba-4a6a-a464-80d7490cb310:staging",
"9e123a4f-5546-4645-b11d-6b21244a5b67",
"fb6c314c-9e34-4273-8779-d0ab16043532"
]
}
rjohnson@peon5752:~$ sudo radosgw-admin period get
{
"id": "68a89bcc-6769-498e-9da1-91fe9bfb72e5",
"epoch": 1,
"predecessor_uuid": "fb6c314c-9e34-4273-8779-d0ab16043532",
"sync_status": [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
""
],
"period_map": {
"id": "68a89bcc-6769-498e-9da1-91fe9bfb72e5",
"zonegroups": [
{
"id": "default",
"name": "default",
"api_name": "us-west-1",
"is_master": "true",
"endpoints": [
"https:\/\/objects-us-west-1.dream.io",
"https:\/\/objects.dreamhost.com"
],
"hostnames": [
"objects-us-west-1.dream.io",
"objects.dreamhost.com"
],
"hostnames_s3website": [
"objects-website-us-west-1.dream.io"
],
"master_zone": "default",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "true",
"log_data": "false",
"bucket_index_max_shards": 31,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "6ed37b4b-66ba-4a6a-a464-80d7490cb310"
}
],
"short_zone_ids": [
{
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "default",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "6ed37b4b-66ba-4a6a-a464-80d7490cb310",
"realm_name": "gold",
"realm_epoch": 2
}
Follow followed by the output of the same as of 2016-09-22/10:34:08 same day, now it's GONE.
root@peon5752:/home/rjohnson# date -u
Thu Sep 22 10:34:08 UTC 2016
root@peon5752:/home/rjohnson# radosgw-admin period list
{
"periods": [
"68a89bcc-6769-498e-9da1-91fe9bfb72e5",
"6ed37b4b-66ba-4a6a-a464-80d7490cb310:staging",
"9e123a4f-5546-4645-b11d-6b21244a5b67",
"fb6c314c-9e34-4273-8779-d0ab16043532"
]
}
root@peon5752:/home/rjohnson# radosgw-admin period get
{
"id": "68a89bcc-6769-498e-9da1-91fe9bfb72e5",
"epoch": 1,
"predecessor_uuid": "fb6c314c-9e34-4273-8779-d0ab16043532",
"sync_status": [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
""
],
"period_map": {
"id": "68a89bcc-6769-498e-9da1-91fe9bfb72e5",
"zonegroups": [
{
"id": "default",
"name": "default",
"api_name": "",
"is_master": "true",
"endpoints": [],
"hostnames": [],
"hostnames_s3website": [],
"master_zone": "",
"zones": [
{
"id": "default",
"name": "default",
"endpoints": [],
"log_meta": "false",
"log_data": "false",
"bucket_index_max_shards": 0,
"read_only": "false"
}
],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement",
"realm_id": "6ed37b4b-66ba-4a6a-a464-80d7490cb310"
}
],
"short_zone_ids": [
{
"key": "default",
"val": 2610307010
}
]
},
"master_zonegroup": "default",
"master_zone": "default",
"period_config": {
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
}
},
"realm_id": "6ed37b4b-66ba-4a6a-a464-80d7490cb310",
"realm_name": "gold",
"realm_epoch": 2
}
Actions