Project

General

Profile

Actions
Copy link

Bug #17051

closed

period commit loses zonegroup changes: region_map converted repeatedly

Added by Robin Johnson over 7 years ago. Updated over 7 years ago.

Status:
Resolved
Priority:
High
Assignee:
Casey Bodley
Target version:
-
% Done:

0%

Source:
Community (dev)
Tags:
jewel
Backport:
jewel
Regression:
No
Severity:
2 - major
Reviewed:
Affected Versions:
Ceph - v10.2.2
ceph-qa-suite:
rgw, upgrade/hammer
Pull request ID:
Crash signature (v1):
Crash signature (v2):

Description

Reproduced on both Jewel and Jewel-backports 10.2.2-432-gac44947 (commit ac44947). This is trying to upgrade a Hammer cluster, with the next step being upgrading a duplicate of the pool layout from the Congress cluster.

'zonegroup get' shows a zonegroup with a empty realm_id, and many other fields are not being updated with new changes.
Possibly related, of two RGW instances in this cluster:
- one works fine for bucket create, but the other reports 'rest connection is invalid' in the logs and 400 error to the user.
- 'period update --commit' causes BOTH RGW instances to break.

  1. radosgw-admin realm list {
    "default_info": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
    "realms": [
    "gold"
    ]
    }
  2. radosgw-admin realm get {
    "id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
    "name": "gold",
    "current_period": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
    "epoch": 2
    }
  3. radosgw-admin period list {
    "periods": [
    "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
    "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4:staging",
    "ea085b17-be40-4371-9cff-694c82ab7b7a"
    ]
    }
  4. radosgw-admin zonegroup list
    read_default_id : 0 {
    "default_info": "default",
    "zonegroups": [
    "default"
    ]
    }
  5. radosgw-admin zonegroup get {
    "id": "default",
    "name": "default",
    "api_name": "us-west-1-staging",
    "is_master": "true",
    "endpoints": [
    "https:\/\/objects-us-west-1-staging.dream.io",
    "https:\/\/api.benjamin.dhobjects.com"
    ],
    "hostnames": [
    "objects-us-west-1-staging.dream.io",
    "api.benjamin.dhobjects.com"
    ],
    "hostnames_s3website": [],
    "master_zone": "",
    "zones": [ {
    "id": "default",
    "name": "default",
    "endpoints": [],
    "log_meta": "false",
    "log_data": "false",
    "bucket_index_max_shards": 31,
    "read_only": "false"
    }
    ],
    "placement_targets": [ {
    "name": "default-placement",
    "tags": []
    }
    ],
    "default_placement": "default-placement",
    "realm_id": ""
    }
  6. radosgw-admin zonegroup set --rgw-zonegroup=default --rgw-zone=default --infile zonegroup.json --master --default {
    "id": "default",
    "name": "default",
    "api_name": "us-west-1-staging",
    "is_master": "true",
    "endpoints": [
    "https:\/\/objects-us-west-1-staging.dream.io",
    "https:\/\/api.benjamin.dhobjects.com"
    ],
    "hostnames": [
    "objects-us-west-1-staging.dream.io",
    "api.benjamin.dhobjects.com"
    ],
    "hostnames_s3website": [
    "objects-website-us-west-1-staging.dream.io",
    "api-www.benjamin.dhobjects.com"
    ],
    "master_zone": "default",
    "zones": [ {
    "id": "default",
    "name": "default",
    "endpoints": [],
    "log_meta": "true",
    "log_data": "false",
    "bucket_index_max_shards": 31,
    "read_only": "false"
    }
    ],
    "placement_targets": [ {
    "name": "default-placement",
    "tags": []
    }
    ],
    "default_placement": "default-placement",
    "realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4"
    }
  7. radosgw-admin period update {
    "id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4:staging",
    "epoch": 9,
    "predecessor_uuid": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
    "sync_status": [
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    ""
    ],
    "period_map": {
    "id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
    "zonegroups": [ {
    "id": "default",
    "name": "default",
    "api_name": "us-west-1-staging",
    "is_master": "true",
    "endpoints": [
    "https:\/\/objects-us-west-1-staging.dream.io",
    "https:\/\/api.benjamin.dhobjects.com"
    ],
    "hostnames": [
    "objects-us-west-1-staging.dream.io",
    "api.benjamin.dhobjects.com"
    ],
    "hostnames_s3website": [
    "objects-website-us-west-1-staging.dream.io",
    "api-www.benjamin.dhobjects.com"
    ],
    "master_zone": "default",
    "zones": [ {
    "id": "default",
    "name": "default",
    "endpoints": [],
    "log_meta": "true",
    "log_data": "false",
    "bucket_index_max_shards": 31,
    "read_only": "false"
    }
    ],
    "placement_targets": [ {
    "name": "default-placement",
    "tags": []
    }
    ],
    "default_placement": "default-placement",
    "realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4"
    }
    ],
    "short_zone_ids": [ {
    "key": "default",
    "val": 2610307010
    }
    ]
    },
    "master_zonegroup": "default",
    "master_zone": "default",
    "period_config": {
    "bucket_quota": {
    "enabled": false,
    "max_size_kb": -1,
    "max_objects": -1
    },
    "user_quota": {
    "enabled": false,
    "max_size_kb": -1,
    "max_objects": -1
    }
    },
    "realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
    "realm_name": "gold",
    "realm_epoch": 3
    }
  8. radosgw-admin period commit {
    "id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
    "epoch": 10,
    "predecessor_uuid": "ea085b17-be40-4371-9cff-694c82ab7b7a",
    "sync_status": [
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    ""
    ],
    "period_map": {
    "id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
    "zonegroups": [ {
    "id": "default",
    "name": "default",
    "api_name": "us-west-1-staging",
    "is_master": "true",
    "endpoints": [
    "https:\/\/objects-us-west-1-staging.dream.io",
    "https:\/\/api.benjamin.dhobjects.com"
    ],
    "hostnames": [
    "objects-us-west-1-staging.dream.io",
    "api.benjamin.dhobjects.com"
    ],
    "hostnames_s3website": [
    "objects-website-us-west-1-staging.dream.io",
    "api-www.benjamin.dhobjects.com"
    ],
    "master_zone": "default",
    "zones": [ {
    "id": "default",
    "name": "default",
    "endpoints": [],
    "log_meta": "true",
    "log_data": "false",
    "bucket_index_max_shards": 31,
    "read_only": "false"
    }
    ],
    "placement_targets": [ {
    "name": "default-placement",
    "tags": []
    }
    ],
    "default_placement": "default-placement",
    "realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4"
    }
    ],
    "short_zone_ids": [ {
    "key": "default",
    "val": 2610307010
    }
    ]
    },
    "master_zonegroup": "default",
    "master_zone": "default",
    "period_config": {
    "bucket_quota": {
    "enabled": false,
    "max_size_kb": -1,
    "max_objects": -1
    },
    "user_quota": {
    "enabled": false,
    "max_size_kb": -1,
    "max_objects": -1
    }
    },
    "realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
    "realm_name": "gold",
    "realm_epoch": 2
    }
  1. radosgw-admin zonegroup get {
    "id": "default",
    "name": "default",
    "api_name": "us-west-1-staging",
    "is_master": "true",
    "endpoints": [
    "https:\/\/objects-us-west-1-staging.dream.io",
    "https:\/\/api.benjamin.dhobjects.com"
    ],
    "hostnames": [
    "objects-us-west-1-staging.dream.io",
    "api.benjamin.dhobjects.com"
    ],
    "hostnames_s3website": [],
    "master_zone": "",
    "zones": [ {
    "id": "default",
    "name": "default",
    "endpoints": [],
    "log_meta": "false",
    "log_data": "false",
    "bucket_index_max_shards": 31,
    "read_only": "false"
    }
    ],
    "placement_targets": [ {
    "name": "default-placement",
    "tags": []
    }
    ],
    "default_placement": "default-placement",
    "realm_id": ""
    }
  2. radosgw-admin period update --commit {
    "id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
    "epoch": 11,
    "predecessor_uuid": "ea085b17-be40-4371-9cff-694c82ab7b7a",
    "sync_status": [
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    "",
    ""
    ],
    "period_map": {
    "id": "00ae3a0c-6249-43f7-94d0-e57d740e6e90",
    "zonegroups": [],
    "short_zone_ids": [ {
    "key": "default",
    "val": 2610307010
    }
    ]
    },
    "master_zonegroup": "default",
    "master_zone": "default",
    "period_config": {
    "bucket_quota": {
    "enabled": false,
    "max_size_kb": -1,
    "max_objects": -1
    },
    "user_quota": {
    "enabled": false,
    "max_size_kb": -1,
    "max_objects": -1
    }
    },
    "realm_id": "e6e42c97-cb51-4e6e-bfbd-24b3bbec4ab4",
    "realm_name": "gold",
    "realm_epoch": 2
    }
  3. radosgw-admin zonegroup get {
    "id": "default",
    "name": "default",
    "api_name": "us-west-1-staging",
    "is_master": "true",
    "endpoints": [
    "https:\/\/objects-us-west-1-staging.dream.io",
    "https:\/\/api.benjamin.dhobjects.com"
    ],
    "hostnames": [
    "objects-us-west-1-staging.dream.io",
    "api.benjamin.dhobjects.com"
    ],
    "hostnames_s3website": [],
    "master_zone": "",
    "zones": [ {
    "id": "default",
    "name": "default",
    "endpoints": [],
    "log_meta": "false",
    "log_data": "false",
    "bucket_index_max_shards": 31,
    "read_only": "false"
    }
    ],
    "placement_targets": [ {
    "name": "default-placement",
    "tags": []
    }
    ],
    "default_placement": "default-placement",
    "realm_id": ""
    }

Related issues 1 (0 open — 1 closed)

Copied to rgw - Backport #17140: jewel: period commit loses zonegroup changes: region_map converted repeatedlyResolvedLoïc DacharyActions
Actions
Copy link

Also available in: Atom PDF