Actions
Bug #64452
openTeuthology runs into "TypeError: expected string or bytes-like object" during log scraping
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Crash signature (v1):
Crash signature (v2):
Description
/a/lflores-2024-02-12_21:56:27-rados-wip-yuri10-testing-2024-02-08-0854-pacific-distro-default-smithi/7557107
2024-02-13T00:44:58.988 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9a8420ca-ca04-11ee-95b9-87774f69a715 --force
2024-02-13T00:44:59.776 DEBUG:teuthology.orchestra.run.smithi160:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9a8420ca-ca04-11ee-95b9-87774f69a715 --force
2024-02-13T00:45:00.441 INFO:tasks.cephadm:Removing cephadm ...
2024-02-13T00:45:00.441 DEBUG:teuthology.orchestra.run.smithi059:> rm -rf /home/ubuntu/cephtest/cephadm
2024-02-13T00:45:00.464 DEBUG:teuthology.orchestra.run.smithi160:> rm -rf /home/ubuntu/cephtest/cephadm
2024-02-13T00:45:00.483 INFO:tasks.cephadm:Teardown complete
2024-02-13T00:45:00.484 ERROR:teuthology.run_tasks:Manager failed: cephadm
Traceback (most recent call last):
File "/home/teuthworker/src/git.ceph.com_teuthology_d9fdb2209e15b39d9f061fd85399f352ce0f0894/teuthology/run_tasks.py", line 154, in run_tasks
suppress = manager.__exit__(*exc_info)
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File "/home/teuthworker/src/github.com_ljflores_ceph_84e714279aa684636ac645daf0a1a85c2094cfb3/qa/tasks/cephadm.py", line 1508, in task
log.info('Teardown begin')
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File "/home/teuthworker/src/git.ceph.com_teuthology_d9fdb2209e15b39d9f061fd85399f352ce0f0894/teuthology/contextutil.py", line 54, in nested
raise exc[1]
File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "/home/teuthworker/src/github.com_ljflores_ceph_84e714279aa684636ac645daf0a1a85c2094cfb3/qa/tasks/cephadm.py", line 1376, in initialize_config
yield
File "/home/teuthworker/src/git.ceph.com_teuthology_d9fdb2209e15b39d9f061fd85399f352ce0f0894/teuthology/contextutil.py", line 46, in nested
if exit(*exc):
File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File "/home/teuthworker/src/github.com_ljflores_ceph_84e714279aa684636ac645daf0a1a85c2094cfb3/qa/tasks/cephadm.py", line 238, in ceph_log
if first_in_ceph_log('\[ERR\]|\[WRN\]|\[SEC\]',
File "/home/teuthworker/src/github.com_ljflores_ceph_84e714279aa684636ac645daf0a1a85c2094cfb3/qa/tasks/cephadm.py", line 229, in first_in_ceph_log
r = ctx.ceph[cluster_name].bootstrap_remote.run(
File "/home/teuthworker/src/git.ceph.com_teuthology_d9fdb2209e15b39d9f061fd85399f352ce0f0894/teuthology/orchestra/remote.py", line 523, in run
r = self._runner(client=self.ssh, name=self.shortname, **kwargs)
File "/home/teuthworker/src/git.ceph.com_teuthology_d9fdb2209e15b39d9f061fd85399f352ce0f0894/teuthology/orchestra/run.py", line 447, in run
r = RemoteProcess(client, args, check_status=check_status, hostname=name,
File "/home/teuthworker/src/git.ceph.com_teuthology_d9fdb2209e15b39d9f061fd85399f352ce0f0894/teuthology/orchestra/run.py", line 65, in __init__
self.command = quote(args)
File "/home/teuthworker/src/git.ceph.com_teuthology_d9fdb2209e15b39d9f061fd85399f352ce0f0894/teuthology/orchestra/run.py", line 257, in quote
return ' '.join(_quote(args))
File "/home/teuthworker/src/git.ceph.com_teuthology_d9fdb2209e15b39d9f061fd85399f352ce0f0894/teuthology/orchestra/run.py", line 255, in _quote
yield pipes.quote(a)
File "/usr/lib/python3.8/shlex.py", line 325, in quote
if _find_unsafe(s) is None:
TypeError: expected string or bytes-like object
Updated by Zack Cerza 2 months ago
args contains a value of None, which is not something that run() can interpret.
Updated by Kamoltat (Junior) Sirivadhna 2 months ago
def first_in_ceph_log(pattern, excludes):
.
.
args = [
'sudo',
'egrep', pattern,
'/var/log/ceph/{fsid}/ceph.log'.format(
fsid=fsid),
]
if excludes:
for exclude in excludes:
args.extend([run.Raw('|'), 'egrep', '-v', exclude])
args.extend([
run.Raw('|'), 'head', '-n', '1',
])
r = ctx.ceph[cluster_name].bootstrap_remote.run(
stdout=StringIO(),
args=args,
)
stdout = r.stdout.getvalue()
if stdout != '':
return stdout
return None
.
.
.
teuthology/orchestra/run.py
def quote(args):
"""
Internal quote wrapper.
"""
def _quote(args):
"""
Handle quoted string, testing for raw charaters.
"""
for a in args:
if isinstance(a, Raw):
yield a.value
else:
yield pipes.quote(a)
if isinstance(args, list):
return ' '.join(_quote(args))
else:
return args
it died here because quote does not expect a NONE type
Updated by Kamoltat (Junior) Sirivadhna 2 months ago
cause of this is due to https://github.com/ceph/ceph/pull/55523#discussion_r1499641955
where there's a typo in yaml file ...
maybe teuthology should handle weird inputs better?
Updated by Laura Flores 2 months ago
- Status changed from New to Fix Under Review
- Assignee set to Kamoltat (Junior) Sirivadhna
Fix is in https://github.com/ceph/ceph/pull/55717.
Actions