Infrastructure

/a/yuriw-2023-03-15_21:14:59-upgrade:pacific-x-quincy-release-distro-default-smithi/7209145

2023-03-16T00:26:43.808 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Mar 15 23:17:13 smithi033 bash[46202]: level=error ts=2023-03-15T23:17:13.509Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://172.21.15.145:8443/api/prometheus_receiver: x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" context_err="context deadline exceeded" 
2023-03-16T00:26:45.216 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Mar 15 23:17:13 smithi033 bash[46202]: level=error ts=2023-03-15T23:17:13.509Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://172.21.15.33:8443/api/prometheus_receiver: x509: cannot validate certificate for 172.21.15.33 because it doesn't contain any IP SANs" context_err="context deadline exceeded" 
2023-03-16T00:26:45.216 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Mar 15 23:17:13 smithi033 bash[46202]: level=error ts=2023-03-15T23:17:13.509Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://172.21.15.145:8443/api/prometheus_receiver: x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; Post https://172.21.15.33:8443/api/prometheus_receiver: x509: cannot validate certificate for 172.21.15.33 because it doesn't contain any IP SANs" 
2023-03-16T00:26:45.217 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Mar 15 23:17:13 smithi033 bash[46202]: level=error ts=2023-03-15T23:17:13.513Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://172.21.15.145:8443/api/prometheus_receiver: x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" context_err="context deadline exceeded" 
2023-03-16T00:26:45.217 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Mar 15 23:17:13 smithi033 bash[46202]: level=error ts=2023-03-15T23:17:13.513Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://172.21.15.33:8443/api/prometheus_receiver: x509: cannot validate certificate for 172.21.15.33 because it doesn't contain any IP SANs" context_err="context deadline exceeded" 
2023-03-16T00:26:45.217 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Mar 15 23:17:13 smithi033 bash[46202]: level=error ts=2023-03-15T23:17:13.513Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://172.21.15.33:8443/api/prometheus_receiver: x509: cannot validate certificate for 172.21.15.33 because it doesn't contain any IP SANs; Post https://172.21.15.145:8443/api/prometheus_receiver: x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 
2023-03-16T00:26:45.219 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/run_tasks.py", line 103, in run_tasks
    manager = run_one_task(taskname, ctx=ctx, config=config)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/run_tasks.py", line 82, in run_one_task
    return task(**kwargs)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/task/parallel.py", line 56, in task
    p.spawn(_run_spawned, ctx, confg, taskname)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/parallel.py", line 84, in __exit__
    for result in self:
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/parallel.py", line 98, in __next__
    resurrect_traceback(result)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/parallel.py", line 30, in resurrect_traceback
    raise exc.exc_info[1]
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/parallel.py", line 23, in capture_traceback
    return func(*args, **kwargs)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/task/parallel.py", line 64, in _run_spawned
    mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/run_tasks.py", line 82, in run_one_task
    return task(**kwargs)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/task/sequential.py", line 47, in task
    mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/run_tasks.py", line 82, in run_one_task
    return task(**kwargs)
  File "/home/teuthworker/src/github.com_ceph_ceph-c_714f8ff94ab1a8a5b10ea54247535614e53b7234/qa/tasks/cephadm.py", line 1149, in shell
    _shell(ctx, cluster_name, remote,
  File "/home/teuthworker/src/github.com_ceph_ceph-c_714f8ff94ab1a8a5b10ea54247535614e53b7234/qa/tasks/cephadm.py", line 37, in _shell
    return remote.run(
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/orchestra/remote.py", line 525, in run
    r = self._runner(client=self.ssh, name=self.shortname, **kwargs)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/orchestra/run.py", line 450, in run
    r.execute()
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/teuthology/orchestra/run.py", line 100, in execute
    self.client.exec_command(self.command)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/virtualenv/lib/python3.8/site-packages/paramiko/client.py", line 510, in exec_command
    chan = self._transport.open_session(timeout=timeout)
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 919, in open_session
    return self.open_channel(
  File "/home/teuthworker/src/git.ceph.com_teuthology_5c9057f4b1ecae7b7410e418e40b739ff0616d25/virtualenv/lib/python3.8/site-packages/paramiko/transport.py", line 1054, in open_channel
    raise SSHException("Timeout opening channel.")
paramiko.ssh_exception.SSHException: Timeout opening channel.