Actions
Bug #52717
closedcephadm fails to probe devices on osd nodes
% Done:
0%
Source:
Tags:
Backport:
pacific,octopus
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
after a cluster has been migrated from ceph-ansible to cephadm, `ceph health detail` shows the following message:
some ceph commands outputs:
[WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices
host osd0 `cephadm ceph-volume` failed: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
# `ceph -s`
```
cluster:
id: f3a8a06d-6519-458c-bf94-57d92886f3d5
health: HEALTH_WARN
mon is allowing insecure global_id reclaim
failed to probe daemons or devices
services:
mon: 1 daemons, quorum mon0 (age 118m)
mgr: mon0(active, since 117m)
osd: 2 osds: 2 up (since 116m), 2 in (since 14h)
data:
pools: 2 pools, 384 pgs
objects: 320 objects, 1.1 GiB
usage: 26 GiB used, 49 GiB / 75 GiB avail
pgs: 384 active+clean
io:
client: 852 B/s rd, 0 op/s rd, 0 op/s wr
```
# `ceph health detail`
```
HEALTH_WARN mon is allowing insecure global_id reclaim; failed to probe daemons or devices
[WRN] AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED: mon is allowing insecure global_id reclaim
mon.mon0 has auth_allow_insecure_global_id_reclaim set to true
[WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices
host osd0 `cephadm ceph-volume` failed: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
```
# `ceph orch ls`
```
service_type: crash
service_name: crash
placement:
label: ceph
status:
created: '2021-09-23T07:04:54.296080Z'
last_refresh: '2021-09-23T08:52:08.265911Z'
running: 2
size: 2
events:
- 2021-09-23T07:04:54.305073Z service:crash [INFO] "service was created"
---
service_type: iscsi
service_id: rbd
service_name: iscsi.rbd
placement:
count: 1
label: iscsigws
spec:
api_password: admin
api_user: admin
pool: rbd
trusted_ip_list: 192.168.122.1
status:
created: '2021-09-23T07:04:30.586251Z'
last_refresh: '2021-09-23T08:51:04.602553Z'
running: 1
size: 1
events:
- 2021-09-23T07:04:30.597913Z service:iscsi.rbd [INFO] "service was created"
---
service_type: mgr
service_name: mgr
placement:
count: 1
label: mgrs
status:
created: '2021-09-23T07:05:04.145113Z'
last_refresh: '2021-09-23T08:52:08.438048Z'
running: 1
size: 1
events:
- 2021-09-23T07:05:04.152839Z service:mgr [INFO] "service was created"
---
service_type: mon
service_name: mon
placement:
count: 1
label: mons
status:
created: '2021-09-23T07:04:59.190229Z'
last_refresh: '2021-09-23T08:52:08.437444Z'
running: 1
size: 1
events:
- 2021-09-23T07:04:59.198914Z service:mon [INFO] "service was created"
---
service_type: osd
service_id: unmanaged
service_name: osd.unmanaged
placement: {}
unmanaged: true
spec:
filter_logic: AND
objectstore: bluestore
status:
container_image_id: edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e
container_image_name: quay.ceph.io/ceph-ci/daemon:latest-pacific
last_refresh: '2021-09-23T08:52:08.264705Z'
running: 2
size: 2
```
# `ceph orch ps`
```
daemon_type: crash
daemon_id: mon0
service_name: crash
hostname: mon0
container_id: d4108a994a11
container_image_id: edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e
container_image_name: quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
container_image_digests:
- quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
memory_usage: 2143289
version: 16.2.6
status: 1
status_desc: running
is_active: false
ports: []
last_refresh: '2021-09-23T08:52:08.438403Z'
created: '2021-09-23T07:05:01.352134Z'
started: '2021-09-23T07:05:00.713812Z'
events:
- 2021-09-23T07:05:03.065793Z daemon:crash.mon0 [INFO] "Deployed crash.mon0 on host
'mon0'"
---
daemon_type: crash
daemon_id: osd0
service_name: crash
hostname: osd0
container_id: 6d5646a0f3b9
container_image_id: edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e
container_image_name: quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
container_image_digests:
- quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
memory_usage: 7602176
version: 16.2.6
status: 1
status_desc: running
is_active: false
ports: []
last_refresh: '2021-09-23T08:52:08.265911Z'
created: '2021-09-23T07:05:07.020314Z'
started: '2021-09-23T07:05:06.364774Z'
events:
- 2021-09-23T07:05:08.500229Z daemon:crash.osd0 [INFO] "Deployed crash.osd0 on host
'osd0'"
---
daemon_type: iscsi
daemon_id: rbd.iscsi-gw0.agwyjo
service_name: iscsi.rbd
hostname: iscsi-gw0
container_id: 7cd11d5a1e25
container_image_id: edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e
container_image_name: quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
container_image_digests:
- quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
memory_usage: 154035814
version: '3.5'
status: 1
status_desc: running
is_active: false
ports: []
last_refresh: '2021-09-23T08:51:04.602553Z'
created: '2021-09-23T07:04:37.285256Z'
started: '2021-09-23T07:04:36.944144Z'
events:
- 2021-09-23T07:04:37.403392Z daemon:iscsi.rbd.iscsi-gw0.agwyjo [INFO] "Deployed iscsi.rbd.iscsi-gw0.agwyjo
on host 'iscsi-gw0'"
---
daemon_type: mgr
daemon_id: mon0
hostname: mon0
container_id: 89a068a4f35c
container_image_id: edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e
container_image_name: quay.ceph.io/ceph-ci/daemon:latest-pacific
container_image_digests:
- quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
memory_usage: 71366082
version: 16.2.6
status: 1
status_desc: running
is_active: true
last_refresh: '2021-09-23T08:52:08.438048Z'
created: '2021-09-23T07:03:03.853190Z'
started: '2021-09-23T07:02:35.299669Z'
events:
- 2021-09-23T07:03:06.101328Z daemon:mgr.mon0 [INFO] "Reconfigured mgr.mon0 on host
'mon0'"
---
daemon_type: mon
daemon_id: mon0
hostname: mon0
container_id: f9380ed241d8
container_image_id: edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e
container_image_name: quay.ceph.io/ceph-ci/daemon:latest-pacific
container_image_digests:
- quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
memory_usage: 545783808
memory_request: 2147483648
version: 16.2.6
status: 1
status_desc: running
is_active: false
last_refresh: '2021-09-23T08:52:08.437444Z'
created: '2021-09-23T07:02:59.614156Z'
started: '2021-09-23T07:02:18.309813Z'
events:
- 2021-09-23T07:03:01.954729Z daemon:mon.mon0 [INFO] "Reconfigured mon.mon0 on host
'mon0'"
---
daemon_type: osd
daemon_id: '0'
hostname: osd0
container_id: 6bcc8d18af38
container_image_id: edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e
container_image_name: quay.ceph.io/ceph-ci/daemon:latest-pacific
container_image_digests:
- quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
memory_usage: 74239180
memory_request: 4294967296
version: 16.2.6
status: 1
status_desc: running
osdspec_affinity: ''
is_active: false
last_refresh: '2021-09-23T08:52:08.264705Z'
created: '2021-09-23T07:05:15.690383Z'
started: '2021-09-23T07:03:37.375734Z'
events:
- 2021-09-23T07:05:17.741088Z daemon:osd.0 [INFO] "Reconfigured osd.0 on host 'osd0'"
---
daemon_type: osd
daemon_id: '1'
hostname: osd0
container_id: 1d8e487d91b7
container_image_id: edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e
container_image_name: quay.ceph.io/ceph-ci/daemon:latest-pacific
container_image_digests:
- quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9
memory_usage: 79597404
memory_request: 4294967296
version: 16.2.6
status: 1
status_desc: running
osdspec_affinity: ''
is_active: false
last_refresh: '2021-09-23T08:52:08.265569Z'
created: '2021-09-23T07:05:19.376413Z'
started: '2021-09-23T07:03:48.033507Z'
events:
- 2021-09-23T07:05:21.580366Z daemon:osd.1 [INFO] "Reconfigured osd.1 on host 'osd0'"
```
# `ceph orch host ls`
```
addr: 192.168.121.43
hostname: iscsi-gw0
labels:
- iscsigws
status: ''
---
addr: 192.168.121.226
hostname: mon0
labels:
- mgrs
- mons
- ceph
status: ''
---
addr: 192.168.121.162
hostname: osd0
labels:
- osds
- ceph
status: ''
```
# `ceph orch device ls`
```
addr: iscsi-gw0
devices: []
labels: []
name: iscsi-gw0
---
addr: mon0
devices: []
labels: []
name: mon0
---
addr: osd0
devices: []
labels: []
name: osd0
```
# `ceph orch upgrade status`
```
{
"target_image": null,
"in_progress": false,
"services_complete": [],
"progress": null,
"message": ""
}
```
# `ceph log last cephadm`
```
2021-09-23T08:39:28.669384+0000 mgr.mon0 (mgr.24100) 3609 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:40:31.726862+0000 mgr.mon0 (mgr.24100) 3647 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:41:37.150585+0000 mgr.mon0 (mgr.24100) 3688 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:42:40.038274+0000 mgr.mon0 (mgr.24100) 3726 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:43:42.842593+0000 mgr.mon0 (mgr.24100) 3764 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:44:45.783846+0000 mgr.mon0 (mgr.24100) 3804 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:45:48.678666+0000 mgr.mon0 (mgr.24100) 3842 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:46:51.576870+0000 mgr.mon0 (mgr.24100) 3881 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:47:55.352946+0000 mgr.mon0 (mgr.24100) 3920 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:48:58.301041+0000 mgr.mon0 (mgr.24100) 3959 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:50:01.411823+0000 mgr.mon0 (mgr.24100) 3998 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:51:04.398474+0000 mgr.mon0 (mgr.24100) 4036 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:52:10.118699+0000 mgr.mon0 (mgr.24100) 4077 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:53:12.940788+0000 mgr.mon0 (mgr.24100) 4116 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:54:16.084152+0000 mgr.mon0 (mgr.24100) 4156 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:55:19.184141+0000 mgr.mon0 (mgr.24100) 4195 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:56:22.116773+0000 mgr.mon0 (mgr.24100) 4234 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:57:24.967725+0000 mgr.mon0 (mgr.24100) 4272 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:58:28.756625+0000 mgr.mon0 (mgr.24100) 4312 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T08:59:31.616613+0000 mgr.mon0 (mgr.24100) 4351 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
2021-09-23T09:00:34.352640+0000 mgr.mon0 (mgr.24100) 4394 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
Traceback (most recent call last):
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1366, in _remote_connection
yield (conn, connr)
File "/usr/share/ceph/mgr/cephadm/serve.py", line 1263, in _run_cephadm
code, '\n'.join(err)))
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Inferring config /var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config
ERROR: [Errno 2] No such file or directory: '/var/lib/ceph/f3a8a06d-6519-458c-bf94-57d92886f3d5/mon.osd0/config'
```
# `cephadm ls`
```
[
{
"style": "legacy",
"name": "mon.osd0",
"fsid": "f3a8a06d-6519-458c-bf94-57d92886f3d5",
"systemd_unit": "ceph-mon@osd0",
"enabled": false,
"state": "stopped",
"host_version": null
},
{
"style": "legacy",
"name": "mds.osd0",
"fsid": "f3a8a06d-6519-458c-bf94-57d92886f3d5",
"systemd_unit": "ceph-mds@osd0",
"enabled": false,
"state": "stopped",
"host_version": null
},
{
"style": "legacy",
"name": "mgr.osd0",
"fsid": "f3a8a06d-6519-458c-bf94-57d92886f3d5",
"systemd_unit": "ceph-mgr@osd0",
"enabled": false,
"state": "stopped",
"host_version": null
},
{
"style": "cephadm:v1",
"name": "osd.0",
"fsid": "f3a8a06d-6519-458c-bf94-57d92886f3d5",
"systemd_unit": "ceph-f3a8a06d-6519-458c-bf94-57d92886f3d5@osd.0",
"enabled": true,
"state": "running",
"memory_request": null,
"memory_limit": null,
"ports": null,
"container_id": "6bcc8d18af385c5c612eb885fcf4f05fe04365c6e31343487caa55f994913712",
"container_image_name": "quay.ceph.io/ceph-ci/daemon:latest-pacific",
"container_image_id": "edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e",
"container_image_digests": [
"quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9"
],
"memory_usage": 73756835,
"version": "16.2.6",
"started": "2021-09-23T07:03:37.375734Z",
"created": "2021-09-23T07:05:15.690383Z",
"deployed": "2021-09-23T07:03:33.654564Z",
"configured": "2021-09-23T07:05:15.690383Z"
},
{
"style": "cephadm:v1",
"name": "osd.1",
"fsid": "f3a8a06d-6519-458c-bf94-57d92886f3d5",
"systemd_unit": "ceph-f3a8a06d-6519-458c-bf94-57d92886f3d5@osd.1",
"enabled": true,
"state": "running",
"memory_request": null,
"memory_limit": null,
"ports": null,
"container_id": "1d8e487d91b7e15d5e068fd7609aba0d76f1a305cc3cdcae0d8340f4c1548bde",
"container_image_name": "quay.ceph.io/ceph-ci/daemon:latest-pacific",
"container_image_id": "edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e",
"container_image_digests": [
"quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9"
],
"memory_usage": 77762396,
"version": "16.2.6",
"started": "2021-09-23T07:03:48.033507Z",
"created": "2021-09-23T07:05:19.376413Z",
"deployed": "2021-09-23T07:03:44.726653Z",
"configured": "2021-09-23T07:05:19.376413Z"
},
{
"style": "cephadm:v1",
"name": "crash.osd0",
"fsid": "f3a8a06d-6519-458c-bf94-57d92886f3d5",
"systemd_unit": "ceph-f3a8a06d-6519-458c-bf94-57d92886f3d5@crash.osd0",
"enabled": true,
"state": "running",
"service_name": "crash",
"ports": [],
"ip": null,
"deployed_by": [
"quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9"
],
"rank": null,
"rank_generation": null,
"memory_request": null,
"memory_limit": null,
"container_id": "6d5646a0f3b9e40c6c5af77cd7d3e1e826362acf8cdf76dc983f91f40c515fac",
"container_image_name": "quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9",
"container_image_id": "edb69fcbdaf4fcfdde8cfc592fde55a6bf0342cc486ec0e3ed15cc616f98ce7e",
"container_image_digests": [
"quay.ceph.io/ceph-ci/daemon@sha256:188304b973beb92b9a256dbf2747f50a33b42be5e6725c76ce637e7ca00c30b9"
],
"memory_usage": 7559184,
"version": "16.2.6",
"started": "2021-09-23T07:05:06.364774Z",
"created": "2021-09-23T07:05:07.020314Z",
"deployed": "2021-09-23T07:05:05.005298Z",
"configured": "2021-09-23T07:05:07.020314Z"
}
]
```
Actions