Ceph » Orchestrator

Right now, if you try to add a host to the cluster but the pub key was never set as an authorized key on the host you get an error like this

[ceph: root@vm-00 /]# ceph orch host add vm-01 192.168.122.248
Error EINVAL: Traceback (most recent call last):
  File "/usr/share/ceph/mgr/mgr_module.py", line 1761, in _handle_command
    return self.handle_command(inbuf, cmd)
  File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 171, in handle_command
    return dispatch[cmd['prefix']].call(self, cmd, inbuf)
  File "/usr/share/ceph/mgr/mgr_module.py", line 462, in call
    return self.func(mgr, **kwargs)
  File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 107, in <lambda>
    wrapper_copy = lambda *l_args, **l_kwargs: wrapper(*l_args, **l_kwargs)  # noqa: E731
  File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 96, in wrapper
    return func(*args, **kwargs)
  File "/usr/share/ceph/mgr/orchestrator/module.py", line 453, in _add_host
    return self._apply_misc([s], False, Format.plain)
  File "/usr/share/ceph/mgr/orchestrator/module.py", line 1248, in _apply_misc
    raise_if_exception(completion)
  File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 225, in raise_if_exception
    e = pickle.loads(c.serialized_exception)
TypeError: __init__() missing 2 required positional arguments: 'hostname' and 'addr'

which makes it seem like the user made a mistake entering the command. In the logs however, we see

2023-03-15T16:13:59.418677+0000 mgr.vm-00.ipgxuq [DBG] Opening connection to root@192.168.122.248 with ssh options '-F /tmp/cephadm-conf-s6fzkw9n -i /tmp/cephadm-identity-4j5oqipn'
2023-03-15T16:13:59.469637+0000 mgr.vm-00.ipgxuq [DBG] Failed to connect to vm-01 (192.168.122.248). Permission denied
Log: Opening SSH connection to 192.168.122.248, port 22
[conn=2] Connected to SSH server at 192.168.122.248, port 22
[conn=2]   Local address: 192.168.122.95, port 47714
[conn=2]   Peer address: 192.168.122.248, port 22
[conn=2] Beginning auth for user root
[conn=2] Auth failed for user root
[conn=2] Connection failure: Permission denied
[conn=2] Aborting connection

2023-03-15T16:13:59.470326+0000 mgr.vm-00.ipgxuq [ERR] Failed to connect to vm-01 (192.168.122.248). Permission denied
Log: Opening SSH connection to 192.168.122.248, port 22
[conn=2] Connected to SSH server at 192.168.122.248, port 22
[conn=2]   Local address: 192.168.122.95, port 47714
[conn=2]   Peer address: 192.168.122.248, port 22
[conn=2] Beginning auth for user root
[conn=2] Auth failed for user root
[conn=2] Connection failure: Permission denied
[conn=2] Aborting connection
Traceback (most recent call last):
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 111, in redirect_log
    yield
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 90, in _remote_connection
    preferred_auth=['publickey'], options=ssh_options)
  File "/lib/python3.6/site-packages/asyncssh/connection.py", line 6804, in connect
    'Opening SSH connection to')
  File "/lib/python3.6/site-packages/asyncssh/connection.py", line 303, in _connect
    await conn.wait_established()
  File "/lib/python3.6/site-packages/asyncssh/connection.py", line 2243, in wait_established
    await self._waiter
asyncssh.misc.PermissionDenied: Permission denied

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 125, in wrapper
    return OrchResult(f(*args, **kwargs))
  File "/usr/share/ceph/mgr/cephadm/module.py", line 2973, in apply
    results.append(self._apply(spec))
  File "/usr/share/ceph/mgr/cephadm/module.py", line 2723, in _apply
    return self._add_host(cast(HostSpec, spec))
  File "/usr/share/ceph/mgr/cephadm/module.py", line 1518, in _add_host
    ip_addr = self._check_valid_addr(spec.hostname, spec.addr)
  File "/usr/share/ceph/mgr/cephadm/module.py", line 1499, in _check_valid_addr
    error_ok=True, no_fsid=True))
  File "/usr/share/ceph/mgr/cephadm/module.py", line 690, in wait_async
    return self.event_loop.get_result(coro)
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 56, in get_result
    return asyncio.run_coroutine_threadsafe(coro, self._loop).result()
  File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result
    return self.__get_result()
  File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 1433, in _run_cephadm
    await self.mgr.ssh._remote_connection(host, addr)
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 96, in _remote_connection
    raise
  File "/lib64/python3.6/contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 123, in redirect_log
    raise HostConnectionError(msg, host, addr)
cephadm.ssh.HostConnectionError: Failed to connect to vm-01 (192.168.122.248). Permission denied
Log: Opening SSH connection to 192.168.122.248, port 22
[conn=2] Connected to SSH server at 192.168.122.248, port 22
[conn=2]   Local address: 192.168.122.95, port 47714
[conn=2]   Peer address: 192.168.122.248, port 22
[conn=2] Beginning auth for user root
[conn=2] Auth failed for user root
[conn=2] Connection failure: Permission denied
[conn=2] Aborting connection

which indicates a permission error on the connection (i.e. the actual issue). We need to make sure the fact that it was a permission issue reaches the user and also just not say that it was a problem with missing required arguments