Ceph » rgw

We've got two buckets where the Hammer fixup check-head-obj-locator claims it needs fixing, but it can't fix it, and bucket rm fails to remove the bucket.

1. radosgw-admin bucket check --check-head-obj-locator --bucket="jenkins-jucxk1jxn18013k9jftva-202" --fix
   ERROR: fix_head_object_locator() returned ret=-22016-04-28 18:43:18.428011 7f8f011d1780 -1 ERROR: ioctx.operate(oid=default.192572025.16587____) returned ret=-2

{
"bucket": "jenkins-jucxk1jxn18013k9jftva-202",
"check_objects": [ {
"key": {
"type": "head",
"name": "__",
"instance": ""
},
"oid": "default.192572025.16587____",
"locator": "default.192572025.16587___",
"needs_fixing": true,
"status": "needs_fixing"
}, {
"key": {
"type": "tail",
"name": "__",
"instance": ""
},
"needs_fixing": false,
"status": "ok"
}

]
}

1. radosgw-admin --bucket jenkins-jucxk1jxn18013k9jftva-202 bucket stats {
   "bucket": "jenkins-jucxk1jxn18013k9jftva-202",
   "pool": ".rgw.data.1",
   "index_pool": ".rgw.data.1",
   "id": "default.192572025.16587",
   "marker": "default.192572025.16587",
   "owner": "jenkins-congress-main",
   "ver": "0#28",
   "master_ver": "0#0",
   "mtime": "2016-02-14 10:20:46.000000",
   "max_marker": "0#",
   "usage": {
   "rgw.main": {
   "size_kb": 0,
   "size_kb_actual": 0,
   "num_objects": 1
   }
   },
   "bucket_quota": {
   "enabled": false,
   "max_size_kb": -1,
   "max_objects": -1
   }
   }

1. radosgw-admin --bucket jenkins-e6fttd5hpwcaof3izpevp-203 bucket check --check-head-obj-locator --fix
   ERROR: fix_head_object_locator() returned ret=-22016-04-28 18:46:36.330858 7fcf699f1780 -1 ERROR: ioctx.operate(oid=default.192572025.7454____) returned ret=-2

{
"bucket": "jenkins-e6fttd5hpwcaof3izpevp-203",
"check_objects": [ {
"key": {
"type": "head",
"name": "__",
"instance": ""
},
"oid": "default.192572025.7454____",
"locator": "default.192572025.7454___",
"needs_fixing": true,
"status": "needs_fixing"
}, {
"key": {
"type": "tail",
"name": "__",
"instance": ""
},
"needs_fixing": false,
"status": "ok"
}

]
}

1. radosgw-admin --bucket jenkins-e6fttd5hpwcaof3izpevp-203 bucket stats {
   "bucket": "jenkins-e6fttd5hpwcaof3izpevp-203",
   "pool": ".rgw.data.1",
   "index_pool": ".rgw.data.1",
   "id": "default.192572025.7454",
   "marker": "default.192572025.7454",
   "owner": "jenkins-congress-main",
   "ver": "0#27",
   "master_ver": "0#0",
   "mtime": "2016-02-11 23:49:54.000000",
   "max_marker": "0#",
   "usage": {
   "rgw.main": {
   "size_kb": 0,
   "size_kb_actual": 0,
   "num_objects": 1
   }
   },
   "bucket_quota": {
   "enabled": false,
   "max_size_kb": -1,
   "max_objects": -1
   }
   }