Project

General

Profile

Bug #47868

rbd-target-api / one of two service crash

Added by Mike Widmo 9 days ago. Updated 7 days ago.

Status:
New
Priority:
Normal
Assignee:
-
Category:
librbd
Target version:
% Done:

0%

Source:
Community (user)
Tags:
iscsi rbd-target-api
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature:

Description

Service rbd-target-api restarts / stops frequently.

Log from service:

Oct 15 14:35:16 ceph03 rbd-target-api[16038]: Traceback (most recent call last):
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: File "/usr/bin/rbd-target-api", line 2951, in <module>
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: main()
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: File "/usr/bin/rbd-target-api", line 2866, in main
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: ceph_gw.define()
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: File "/usr/lib/python3.6/site-packages/ceph_iscsi_config/gateway.py", line 242, in define
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: self.define_targets()
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: File "/usr/lib/python3.6/site-packages/ceph_iscsi_config/gateway.py", line 215, in define_targets
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: target = self.define_target(iqn, target.get('ip_list', {}))
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: File "/usr/lib/python3.6/site-packages/ceph_iscsi_config/gateway.py", line 172, in define_target
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: target.manage('target')
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: File "/usr/lib/python3.6/site-packages/ceph_iscsi_config/target.py", line 586, in manage
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: self.load_config()
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: File "/usr/lib/python3.6/site-packages/ceph_iscsi_config/target.py", line 388, in load_config
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: ip_address = list(tpg.network_portals)[0].ip_address
Oct 15 14:35:16 ceph03 rbd-target-api[16038]: IndexError: list index out of range
Oct 15 14:35:16 ceph03 systemd[1]: rbd-target-api.service: Main process exited, code=exited, status=1/FAILURE
Oct 15 14:35:16 ceph03 systemd[1]: rbd-target-api.service: Failed with result 'exit-code'.

Source of error: /usr/lib/python3.6/site-packages/ceph_iscsi_config/target.py

           # there could/should be multiple tpg's for the target
for tpg in self.target.tpgs:
self.tpg_list.append(tpg)
ip_address = list(tpg.network_portals)[0].ip_address
gateway_name = self._get_gateway_name(ip_address)
if gateway_name:
self.tpg_tag_by_gateway_name[gateway_name] = tpg.tag

gateway.conf from rbd:

@ {
"created": "2020/05/21 10:44:29",
"discovery_auth": {
"mutual_password": "",
"mutual_password_encryption_enabled": false,
"mutual_username": "",
"password": "",
"password_encryption_enabled": false,
"username": ""
},
"disks": {
"rbd/veeam_copy": {
"allocating_host": "ceph01",
"backstore": "user:rbd",
"backstore_object_name": "rbd.veeam_copy",
"controls": {},
"created": "2020/05/21 10:54:50",
"image": "veeam_copy",
"owner": "ceph01",
"pool": "rbd",
"pool_id": 2,
"updated": "2020/05/21 10:54:51",
"wwn": "c1fddb87-e9fb-4993-b7d8-9a9f0427f1dd"
}
},
"epoch": 11,
"gateways": {
"ceph01": {
"active_luns": 1,
"created": "2020/05/21 10:48:52",
"updated": "2020/05/21 10:54:51"
},
"ceph03": {
"active_luns": 0,
"created": "2020/05/21 10:48:57",
"updated": "2020/05/21 10:48:57"
}
},
"targets": {
"iqn.2020-04.com.mit.iscsi-gw:iscsi-01": {
"acl_enabled": true,
"auth": {
"mutual_password": "",
"mutual_password_encryption_enabled": false,
"mutual_username": "",
"password": "",
"password_encryption_enabled": false,
"username": ""
},
"clients": {
"iqn.1991-05.com.microsoft:XXXXXXXXXXXXXXXXXXX": {
"auth": {
"mutual_password": "",
"mutual_password_encryption_enabled": false,
"mutual_username": "",
"password": "XXXXXXXXXXXXX-YYYYYYYYYYYYY",
"password_encryption_enabled": false,
"username": "XXXXXXXXXXXXX-YYYYYYYYYYYYY"
},
"group_name": "",
"luns": {
"rbd/veeam_copy": {
"lun_id": 0
}
}
}
},
"controls": {},
"created": "2020/05/21 10:47:06",
"disks": {
"rbd/veeam_copy": {
"lun_id": 0
}
},
"groups": {},
"ip_list": [
"172.30.0.41",
"172.30.0.43"
],
"portals": {
"ceph01": {
"gateway_ip_list": [
"172.30.0.41",
"172.30.0.43"
],
"inactive_portal_ips": [
"172.30.0.43"
],
"portal_ip_addresses": [
"172.30.0.41"
],
"tpgs": 2
},
"ceph03": {
"gateway_ip_list": [
"172.30.0.41",
"172.30.0.43"
],
"inactive_portal_ips": [
"172.30.0.41"
],
"portal_ip_addresses": [
"172.30.0.43"
],
"tpgs": 2
}
},
"updated": "2020/05/21 10:54:51"
}
},
"updated": "2020/05/21 11:09:38",
"version": 11
@

History

#1 Updated by Mike Widmo 7 days ago

Update:

I removed faulty host/service from gwcli, service is up, and running, but during reconnect gateway:

create gateway_name=ceph03 ip_addresses=172.30.0.43

i got error:

Adding gateway, sync'ing 1 disk(s) and 1 client(s)
Failed : Gateway creation failed on ceph03. Unhandled exception: list index out of range

Also available in: Atom PDF