|
# systemctl -l status ceph\*
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.1.service - Ceph osd.1 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:24:20 +04; 4 days ago
|
|
Process: 36100 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.1.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 36102 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.1/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 36104 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.1/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 36447 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.1.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.2G
|
|
CPU: 30min 9.772s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.1.service
|
|
├─libpod-payload-7f9edd875d3224f9460dfc2feedb075c522d3dae83331e92b7ca86a125854332
|
|
│ ├─13906 /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─13996 /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─13895 /usr/bin/conmon --api-version 1 -c 7f9edd875d3224f9460dfc2feedb075c522d3dae83331e92b7ca86a125854332 -u 7f9edd875d3224f9460dfc2feedb075c522d3dae83331e92b7ca86a125854332 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/7f9edd875d3224f9460dfc2feedb075c522d3dae83331e92b7ca86a125854332/userdata -p /run/containers/storage/overlay-containers/7f9edd875d3224f9460dfc2feedb075c522d3dae83331e92b7ca86a125854332/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-1 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/7f9edd875d3224f9460dfc2feedb075c522d3dae83331e92b7ca86a125854332/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.1.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 7f9edd875d3224f9460dfc2feedb075c522d3dae83331e92b7ca86a125854332
|
|
|
|
Jan 18 07:08:38 cep1.domain.com sudo[1781044]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:39 cep1.domain.com sudo[1781047]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdn
|
|
Jan 18 07:08:39 cep1.domain.com sudo[1781047]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:39 cep1.domain.com sudo[1781047]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:39 cep1.domain.com sudo[1781047]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:56:16 cep1.domain.com ceph-osd[13996]: log_channel(cluster) log [DBG] : 7.220 scrub ok
|
|
Jan 18 08:01:48 cep1.domain.com ceph-osd[13996]: log_channel(cluster) log [DBG] : 7.24b scrub ok
|
|
Jan 18 08:22:08 cep1.domain.com ceph-osd[13996]: log_channel(cluster) log [DBG] : 7.179 deep-scrub ok
|
|
Jan 18 09:14:25 cep1.domain.com ceph-osd[13996]: log_channel(cluster) log [DBG] : 5.1d scrub ok
|
|
Jan 18 09:42:15 cep1.domain.com ceph-osd[13996]: log_channel(cluster) log [DBG] : 7.3b3 scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@node-exporter.cep1.service - Ceph node-exporter.cep1 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:20:29 +04; 4 days ago
|
|
Process: 34111 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@node-exporter.cep1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@node-exporter.cep1.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34116 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/node-exporter.cep1/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34121 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/node-exporter.cep1/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 34126 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@node-exporter.cep1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@node-exporter.cep1.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 72 (limit: 411291)
|
|
Memory: 60.8M
|
|
CPU: 8h 40min 8.647s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@node-exporter.cep1.service
|
|
├─libpod-payload-0f3900bb99ef72da00e31f32a5e5d415f6e6d12f59d286a340778be643aa6b7d
|
|
│ ├─ 8781 /run/podman-init -- /bin/node_exporter --no-collector.timex --web.listen-address=192.0.2.11:9100 --path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs
|
|
│ └─10474 /bin/node_exporter --no-collector.timex --web.listen-address=192.0.2.11:9100 --path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs
|
|
└─runtime
|
|
└─8774 /usr/bin/conmon --api-version 1 -c 0f3900bb99ef72da00e31f32a5e5d415f6e6d12f59d286a340778be643aa6b7d -u 0f3900bb99ef72da00e31f32a5e5d415f6e6d12f59d286a340778be643aa6b7d -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/0f3900bb99ef72da00e31f32a5e5d415f6e6d12f59d286a340778be643aa6b7d/userdata -p /run/containers/storage/overlay-containers/0f3900bb99ef72da00e31f32a5e5d415f6e6d12f59d286a340778be643aa6b7d/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-node-exporter-cep1 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/0f3900bb99ef72da00e31f32a5e5d415f6e6d12f59d286a340778be643aa6b7d/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@node-exporter.cep1.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 0f3900bb99ef72da00e31f32a5e5d415f6e6d12f59d286a340778be643aa6b7d
|
|
|
|
Notice: journal has been rotated since unit was started, output may be incomplete.
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.5.service - Ceph osd.5 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:23:29 +04; 4 days ago
|
|
Process: 34835 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.5.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.5.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34838 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.5/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34841 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.5/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 35502 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.5.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.5.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.7G
|
|
CPU: 28min 39.554s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.5.service
|
|
├─libpod-payload-5d7452f25567ec855e68e0f69637fa864a56db0618d91acf24abab0478a3e708
|
|
│ ├─13502 /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─13579 /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─13495 /usr/bin/conmon --api-version 1 -c 5d7452f25567ec855e68e0f69637fa864a56db0618d91acf24abab0478a3e708 -u 5d7452f25567ec855e68e0f69637fa864a56db0618d91acf24abab0478a3e708 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/5d7452f25567ec855e68e0f69637fa864a56db0618d91acf24abab0478a3e708/userdata -p /run/containers/storage/overlay-containers/5d7452f25567ec855e68e0f69637fa864a56db0618d91acf24abab0478a3e708/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-5 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/5d7452f25567ec855e68e0f69637fa864a56db0618d91acf24abab0478a3e708/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.5.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 5d7452f25567ec855e68e0f69637fa864a56db0618d91acf24abab0478a3e708
|
|
|
|
Jan 18 07:08:47 cep1.domain.com sudo[1781092]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:47 cep1.domain.com sudo[1781095]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdn
|
|
Jan 18 07:08:47 cep1.domain.com sudo[1781095]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:47 cep1.domain.com sudo[1781095]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:47 cep1.domain.com sudo[1781095]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 08:21:03 cep1.domain.com ceph-osd[13579]: log_channel(cluster) log [DBG] : 7.136 scrub ok
|
|
Jan 18 08:43:50 cep1.domain.com ceph-osd[13579]: log_channel(cluster) log [DBG] : 7.100 scrub ok
|
|
Jan 18 09:46:42 cep1.domain.com ceph-osd[13579]: log_channel(cluster) log [DBG] : 7.14f deep-scrub ok
|
|
Jan 18 09:55:52 cep1.domain.com ceph-osd[13579]: log_channel(cluster) log [DBG] : 7.127 scrub ok
|
|
Jan 18 09:58:30 cep1.domain.com ceph-osd[13579]: log_channel(cluster) log [DBG] : 7.66 scrub ok
|
|
|
|
● ceph.target - All Ceph clusters and services
|
|
Loaded: loaded (/etc/systemd/system/ceph.target; enabled; vendor preset: disabled)
|
|
Active: active since Fri 2023-01-13 16:58:27 +04; 4 days ago
|
|
Until: Fri 2023-01-13 16:58:27 +04; 4 days ago
|
|
|
|
Notice: journal has been rotated since unit was started, output may be incomplete.
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.10.service - Ceph osd.10 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:23:29 +04; 4 days ago
|
|
Process: 34834 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.10.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.10.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34837 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.10/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34840 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.10/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 35471 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.10.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.10.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.2G
|
|
CPU: 31min 2.791s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.10.service
|
|
├─libpod-payload-a5224ee22d33d68ff9e00555c742adf3a0a2faba9455c9abdb91e32e0f55b8ca
|
|
│ ├─15240 /run/podman-init -- /usr/bin/ceph-osd -n osd.10 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─15941 /usr/bin/ceph-osd -n osd.10 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─15238 /usr/bin/conmon --api-version 1 -c a5224ee22d33d68ff9e00555c742adf3a0a2faba9455c9abdb91e32e0f55b8ca -u a5224ee22d33d68ff9e00555c742adf3a0a2faba9455c9abdb91e32e0f55b8ca -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/a5224ee22d33d68ff9e00555c742adf3a0a2faba9455c9abdb91e32e0f55b8ca/userdata -p /run/containers/storage/overlay-containers/a5224ee22d33d68ff9e00555c742adf3a0a2faba9455c9abdb91e32e0f55b8ca/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-10 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/a5224ee22d33d68ff9e00555c742adf3a0a2faba9455c9abdb91e32e0f55b8ca/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.10.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg a5224ee22d33d68ff9e00555c742adf3a0a2faba9455c9abdb91e32e0f55b8ca
|
|
|
|
Jan 18 07:43:16 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : 7.1b8 scrub ok
|
|
Jan 18 07:49:02 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : 7.393 scrub ok
|
|
Jan 18 07:49:08 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : 7.199 scrub ok
|
|
Jan 18 08:02:34 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : purged_snaps scrub starts
|
|
Jan 18 08:02:34 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : purged_snaps scrub ok
|
|
Jan 18 08:32:50 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : 7.2d5 scrub ok
|
|
Jan 18 08:33:08 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : 4.7 scrub ok
|
|
Jan 18 08:38:02 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : 7.243 scrub ok
|
|
Jan 18 08:45:50 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : 6.10 scrub ok
|
|
Jan 18 10:06:34 cep1.domain.com ceph-osd[15941]: log_channel(cluster) log [DBG] : 7.2e2 deep-scrub ok
|
|
|
|
● ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mgr.cep1.domain.com.asmugh.service - Ceph mgr.cep1.domain.com.asmugh for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: active (running) since Fri 2023-01-13 17:06:22 +04; 4 days ago
|
|
Process: 24380 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mgr.cep1.domain.com.asmugh.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mgr.cep1.domain.com.asmugh.service-cid (code=exited, status=0/SUCCESS)
|
|
Process: 24381 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/mgr.cep1.domain.com.asmugh/unit.run (code=exited, status=0/SUCCESS)
|
|
Main PID: 24583 (conmon)
|
|
Tasks: 47 (limit: 411291)
|
|
Memory: 487.0M
|
|
CPU: 43min 57.173s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mgr.cep1.domain.com.asmugh.service
|
|
├─libpod-payload-8cc07c233522b2a75eb7eee76da7066a0b10e278f888162b453ea7ed67be9e75
|
|
│ ├─24585 /run/podman-init -- /usr/bin/ceph-mgr -n mgr.cep1.domain.com.asmugh -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─24587 /usr/bin/ceph-mgr -n mgr.cep1.domain.com.asmugh -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─24583 /usr/bin/conmon --api-version 1 -c 8cc07c233522b2a75eb7eee76da7066a0b10e278f888162b453ea7ed67be9e75 -u 8cc07c233522b2a75eb7eee76da7066a0b10e278f888162b453ea7ed67be9e75 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/8cc07c233522b2a75eb7eee76da7066a0b10e278f888162b453ea7ed67be9e75/userdata -p /run/containers/storage/overlay-containers/8cc07c233522b2a75eb7eee76da7066a0b10e278f888162b453ea7ed67be9e75/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-mgr-cep1-domain-com-asmugh --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/8cc07c233522b2a75eb7eee76da7066a0b10e278f888162b453ea7ed67be9e75/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mgr.cep1.domain.com.asmugh.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 8cc07c233522b2a75eb7eee76da7066a0b10e278f888162b453ea7ed67be9e75
|
|
|
|
Jan 18 10:15:05 cep1.domain.com ceph-mgr[24587]: [dashboard INFO root] Redirecting to active 'https://192.0.2.12:8443/'
|
|
Jan 18 10:15:05 cep1.domain.com ceph-mgr[24587]: [dashboard INFO request] [::ffff:192.0.2.12:60052] [GET] [302] [0.001s] [101.0B] [c14444e6-dd50-423a-8be2-5a1c2b44c39d] /
|
|
Jan 18 10:15:07 cep1.domain.com ceph-mgr[24587]: [dashboard INFO root] Redirecting to active 'https://192.0.2.12:8443/'
|
|
Jan 18 10:15:07 cep1.domain.com ceph-mgr[24587]: [dashboard INFO request] [::ffff:192.0.2.11:38378] [GET] [302] [0.001s] [101.0B] [d7099ccc-5fbf-46a6-ad02-e2dd3bb92a45] /
|
|
Jan 18 10:15:09 cep1.domain.com ceph-mgr[24587]: [dashboard INFO root] Redirecting to active 'https://192.0.2.12:8443/'
|
|
Jan 18 10:15:09 cep1.domain.com ceph-mgr[24587]: [dashboard INFO request] [::ffff:192.0.2.12:36254] [GET] [302] [0.001s] [101.0B] [a83d4c59-a38f-4975-a34c-068f42e18e1f] /
|
|
Jan 18 10:15:09 cep1.domain.com ceph-mgr[24587]: [dashboard INFO root] Redirecting to active 'https://192.0.2.12:8443/'
|
|
Jan 18 10:15:09 cep1.domain.com ceph-mgr[24587]: [dashboard INFO request] [::ffff:192.0.2.13:33738] [GET] [302] [0.001s] [101.0B] [724e8b5a-7441-4d45-8325-25b3bc3074da] /
|
|
Jan 18 10:15:10 cep1.domain.com ceph-mgr[24587]: [prometheus INFO cherrypy.access.139923854642928] ::ffff:192.0.2.12 - - [18/Jan/2023:03:15:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4"
|
|
Jan 18 10:15:10 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-mgr-cep1-domain-com-asmugh[24583]: ::ffff:192.0.2.12 - - [18/Jan/2023:03:15:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4"
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.8.service - Ceph osd.8 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:23:29 +04; 4 days ago
|
|
Process: 34836 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.8.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.8.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34839 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.8/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34842 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.8/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 35500 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.8.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.8.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.3G
|
|
CPU: 31min 10.610s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.8.service
|
|
├─libpod-payload-923d0277c2da5a2a092fc883563c56942c9a4ccb7857b667bf00d22e2a18a5a6
|
|
│ ├─15161 /run/podman-init -- /usr/bin/ceph-osd -n osd.8 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─15179 /usr/bin/ceph-osd -n osd.8 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─15159 /usr/bin/conmon --api-version 1 -c 923d0277c2da5a2a092fc883563c56942c9a4ccb7857b667bf00d22e2a18a5a6 -u 923d0277c2da5a2a092fc883563c56942c9a4ccb7857b667bf00d22e2a18a5a6 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/923d0277c2da5a2a092fc883563c56942c9a4ccb7857b667bf00d22e2a18a5a6/userdata -p /run/containers/storage/overlay-containers/923d0277c2da5a2a092fc883563c56942c9a4ccb7857b667bf00d22e2a18a5a6/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-8 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/923d0277c2da5a2a092fc883563c56942c9a4ccb7857b667bf00d22e2a18a5a6/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.8.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 923d0277c2da5a2a092fc883563c56942c9a4ccb7857b667bf00d22e2a18a5a6
|
|
|
|
Jan 18 07:08:54 cep1.domain.com sudo[1781245]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdm
|
|
Jan 18 07:08:54 cep1.domain.com sudo[1781245]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:54 cep1.domain.com sudo[1781245]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:54 cep1.domain.com sudo[1781245]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:21:23 cep1.domain.com ceph-osd[15179]: log_channel(cluster) log [DBG] : 7.fe deep-scrub ok
|
|
Jan 18 07:36:30 cep1.domain.com ceph-osd[15179]: log_channel(cluster) log [DBG] : 6.14 scrub ok
|
|
Jan 18 08:09:08 cep1.domain.com ceph-osd[15179]: log_channel(cluster) log [DBG] : 7.3ff scrub ok
|
|
Jan 18 08:46:47 cep1.domain.com ceph-osd[15179]: log_channel(cluster) log [DBG] : 7.12a scrub ok
|
|
Jan 18 09:27:32 cep1.domain.com ceph-osd[15179]: log_channel(cluster) log [DBG] : 7.219 scrub ok
|
|
Jan 18 09:44:10 cep1.domain.com ceph-osd[15179]: log_channel(cluster) log [DBG] : 7.12f scrub ok
|
|
|
|
● ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@alertmanager.cep1.service - Ceph alertmanager.cep1 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: active (running) since Fri 2023-01-13 17:06:22 +04; 4 days ago
|
|
Process: 24278 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@alertmanager.cep1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@alertmanager.cep1.service-cid (code=exited, status=0/SUCCESS)
|
|
Process: 24279 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/alertmanager.cep1/unit.run (code=exited, status=0/SUCCESS)
|
|
Main PID: 24465 (conmon)
|
|
Tasks: 41 (limit: 411291)
|
|
Memory: 41.3M
|
|
CPU: 22min 854ms
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@alertmanager.cep1.service
|
|
├─libpod-payload-96e9f81b3fb8bb800576c4c7b7a4d01e58d22006584397c43235c075d88685c6
|
|
│ ├─24468 /run/podman-init -- /bin/alertmanager --cluster.listen-address=:9094 --web.listen-address=192.0.2.11:9093 --config.file=/etc/alertmanager/alertmanager.yml
|
|
│ └─24470 /bin/alertmanager --cluster.listen-address=:9094 --web.listen-address=192.0.2.11:9093 --config.file=/etc/alertmanager/alertmanager.yml
|
|
└─runtime
|
|
└─24465 /usr/bin/conmon --api-version 1 -c 96e9f81b3fb8bb800576c4c7b7a4d01e58d22006584397c43235c075d88685c6 -u 96e9f81b3fb8bb800576c4c7b7a4d01e58d22006584397c43235c075d88685c6 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/96e9f81b3fb8bb800576c4c7b7a4d01e58d22006584397c43235c075d88685c6/userdata -p /run/containers/storage/overlay-containers/96e9f81b3fb8bb800576c4c7b7a4d01e58d22006584397c43235c075d88685c6/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-alertmanager-cep1 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/96e9f81b3fb8bb800576c4c7b7a4d01e58d22006584397c43235c075d88685c6/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@alertmanager.cep1.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 96e9f81b3fb8bb800576c4c7b7a4d01e58d22006584397c43235c075d88685c6
|
|
|
|
Notice: journal has been rotated since unit was started, output may be incomplete.
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@prometheus.cep1.service - Ceph prometheus.cep1 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:20:29 +04; 4 days ago
|
|
Process: 34112 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@prometheus.cep1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@prometheus.cep1.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34117 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/prometheus.cep1/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34122 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/prometheus.cep1/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 34127 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@prometheus.cep1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@prometheus.cep1.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 41 (limit: 411291)
|
|
Memory: 1.3G
|
|
CPU: 1h 52min 54.603s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@prometheus.cep1.service
|
|
├─libpod-payload-b356b8eb057b6833d88afdd9dc5f089cfceb90221f64372fd6326d4a1c8f0a89
|
|
│ ├─6134 /run/podman-init -- /bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.listen-address=192.0.2.11:9095 --web.external-url=http://cep1.domain.com:9095
|
|
│ └─8832 /bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.listen-address=192.0.2.11:9095 --web.external-url=http://cep1.domain.com:9095
|
|
└─runtime
|
|
└─6132 /usr/bin/conmon --api-version 1 -c b356b8eb057b6833d88afdd9dc5f089cfceb90221f64372fd6326d4a1c8f0a89 -u b356b8eb057b6833d88afdd9dc5f089cfceb90221f64372fd6326d4a1c8f0a89 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/b356b8eb057b6833d88afdd9dc5f089cfceb90221f64372fd6326d4a1c8f0a89/userdata -p /run/containers/storage/overlay-containers/b356b8eb057b6833d88afdd9dc5f089cfceb90221f64372fd6326d4a1c8f0a89/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/b356b8eb057b6833d88afdd9dc5f089cfceb90221f64372fd6326d4a1c8f0a89/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@prometheus.cep1.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg b356b8eb057b6833d88afdd9dc5f089cfceb90221f64372fd6326d4a1c8f0a89
|
|
|
|
Jan 18 06:00:10 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-17T23:00:10.643Z caller=compact.go:460 level=info component=tsdb msg="compact blocks" count=3 mint=1673913600138 maxt=1673978400000 ulid=01GQ0ZC0E2PFS17CZD106JA9QF sources="[01GPZP5JXYVG93YZYK47ZERZDM 01GQ0ART5EN380T1ASAT8456MX 01GQ0ZBYVB29DX8GKNF6RHVY8F]" duration=7.505021286s
|
|
Jan 18 06:00:10 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-17T23:00:10.653Z caller=db.go:1287 level=info component=tsdb msg="Deleting obsolete block" block=01GQ0ART5EN380T1ASAT8456MX
|
|
Jan 18 06:00:10 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-17T23:00:10.659Z caller=db.go:1287 level=info component=tsdb msg="Deleting obsolete block" block=01GPZP5JXYVG93YZYK47ZERZDM
|
|
Jan 18 06:00:10 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-17T23:00:10.666Z caller=db.go:1287 level=info component=tsdb msg="Deleting obsolete block" block=01GQ0ZBYVB29DX8GKNF6RHVY8F
|
|
Jan 18 08:00:01 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-18T01:00:01.515Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1673992800138 maxt=1674000000000 ulid=01GQ167N45TDPHM2KETMTJB1X8 duration=997.8641ms
|
|
Jan 18 08:00:01 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-18T01:00:01.531Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=14.032177ms
|
|
Jan 18 08:00:01 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-18T01:00:01.533Z caller=checkpoint.go:98 level=info component=tsdb msg="Creating checkpoint" from_segment=1671 to_segment=1672 mint=1674000000000
|
|
Jan 18 08:00:02 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-18T01:00:02.600Z caller=head.go:996 level=info component=tsdb msg="WAL checkpoint complete" first=1671 last=1672 duration=1.068042488s
|
|
Jan 18 10:00:01 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-18T03:00:01.094Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1674000000138 maxt=1674007200000 ulid=01GQ1D3CC56DFKE42DA9H9BXJS duration=577.798043ms
|
|
Jan 18 10:00:01 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-prometheus-cep1[6132]: ts=2023-01-18T03:00:01.110Z caller=head.go:827 level=info component=tsdb msg="Head GC completed" duration=13.43343ms
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.6.service - Ceph osd.6 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:24:26 +04; 4 days ago
|
|
Process: 36632 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.6.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.6.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 36633 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.6/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 36634 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.6/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 36821 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.6.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.6.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.1G
|
|
CPU: 28min 28.097s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.6.service
|
|
├─libpod-payload-1fa0b8e4d93727ffd05319b5a4e7620e71848818a517e9b8cbb1f1aa31afdfdd
|
|
│ ├─15175 /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─15202 /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─15173 /usr/bin/conmon --api-version 1 -c 1fa0b8e4d93727ffd05319b5a4e7620e71848818a517e9b8cbb1f1aa31afdfdd -u 1fa0b8e4d93727ffd05319b5a4e7620e71848818a517e9b8cbb1f1aa31afdfdd -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/1fa0b8e4d93727ffd05319b5a4e7620e71848818a517e9b8cbb1f1aa31afdfdd/userdata -p /run/containers/storage/overlay-containers/1fa0b8e4d93727ffd05319b5a4e7620e71848818a517e9b8cbb1f1aa31afdfdd/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-6 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/1fa0b8e4d93727ffd05319b5a4e7620e71848818a517e9b8cbb1f1aa31afdfdd/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.6.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 1fa0b8e4d93727ffd05319b5a4e7620e71848818a517e9b8cbb1f1aa31afdfdd
|
|
|
|
Jan 18 07:08:49 cep1.domain.com sudo[1781104]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:49 cep1.domain.com sudo[1781104]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:49 cep1.domain.com sudo[1781104]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:49 cep1.domain.com sudo[1781107]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdm
|
|
Jan 18 07:08:49 cep1.domain.com sudo[1781107]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:49 cep1.domain.com sudo[1781107]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:49 cep1.domain.com sudo[1781107]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:22:22 cep1.domain.com ceph-osd[15202]: log_channel(cluster) log [DBG] : 7.b0 scrub ok
|
|
Jan 18 09:06:37 cep1.domain.com ceph-osd[15202]: log_channel(cluster) log [DBG] : 7.3f3 scrub ok
|
|
Jan 18 09:53:09 cep1.domain.com ceph-osd[15202]: log_channel(cluster) log [DBG] : 7.37a scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.11.service - Ceph osd.11 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:24:20 +04; 4 days ago
|
|
Process: 36101 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.11.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.11.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 36103 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.11/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 36105 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.11/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 36446 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.11.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.11.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.2G
|
|
CPU: 27min 16.845s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.11.service
|
|
├─libpod-payload-98303d0e1cca2c5400b21513595c0f3f902f15e8c4f0ddd5481a43ab7dd1ce04
|
|
│ ├─13623 /run/podman-init -- /usr/bin/ceph-osd -n osd.11 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─14206 /usr/bin/ceph-osd -n osd.11 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─13621 /usr/bin/conmon --api-version 1 -c 98303d0e1cca2c5400b21513595c0f3f902f15e8c4f0ddd5481a43ab7dd1ce04 -u 98303d0e1cca2c5400b21513595c0f3f902f15e8c4f0ddd5481a43ab7dd1ce04 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/98303d0e1cca2c5400b21513595c0f3f902f15e8c4f0ddd5481a43ab7dd1ce04/userdata -p /run/containers/storage/overlay-containers/98303d0e1cca2c5400b21513595c0f3f902f15e8c4f0ddd5481a43ab7dd1ce04/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-11 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/98303d0e1cca2c5400b21513595c0f3f902f15e8c4f0ddd5481a43ab7dd1ce04/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.11.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 98303d0e1cca2c5400b21513595c0f3f902f15e8c4f0ddd5481a43ab7dd1ce04
|
|
|
|
Jan 18 07:09:00 cep1.domain.com sudo[1781278]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:09:00 cep1.domain.com sudo[1781278]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:09:01 cep1.domain.com sudo[1781281]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdm
|
|
Jan 18 07:09:01 cep1.domain.com sudo[1781281]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:09:01 cep1.domain.com sudo[1781281]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:09:01 cep1.domain.com sudo[1781281]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:30:32 cep1.domain.com ceph-osd[14206]: log_channel(cluster) log [DBG] : 7.3db scrub ok
|
|
Jan 18 08:16:01 cep1.domain.com ceph-osd[14206]: log_channel(cluster) log [DBG] : 7.138 scrub ok
|
|
Jan 18 08:17:04 cep1.domain.com ceph-osd[14206]: log_channel(cluster) log [DBG] : 6.11 scrub ok
|
|
Jan 18 09:19:16 cep1.domain.com ceph-osd[14206]: log_channel(cluster) log [DBG] : 7.10 scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@grafana.cep1.service - Ceph grafana.cep1 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:20:29 +04; 4 days ago
|
|
Process: 34109 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@grafana.cep1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@grafana.cep1.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34114 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/grafana.cep1/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34119 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/grafana.cep1/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 34124 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@grafana.cep1.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@grafana.cep1.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 41 (limit: 411291)
|
|
Memory: 299.1M
|
|
CPU: 1h 7min 39.734s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@grafana.cep1.service
|
|
├─libpod-payload-aadcd354a6128aa8c7cd1b7abfa64231df9fd22ad40d9f59c98801a2fb9e9ac8
|
|
│ ├─6095 /run/podman-init -- /bin/sh -c "grafana-server --config=/etc/grafana/grafana.ini" /bin/bash
|
|
│ └─6113 grafana-server --config=/etc/grafana/grafana.ini
|
|
└─runtime
|
|
└─6093 /usr/bin/conmon --api-version 1 -c aadcd354a6128aa8c7cd1b7abfa64231df9fd22ad40d9f59c98801a2fb9e9ac8 -u aadcd354a6128aa8c7cd1b7abfa64231df9fd22ad40d9f59c98801a2fb9e9ac8 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/aadcd354a6128aa8c7cd1b7abfa64231df9fd22ad40d9f59c98801a2fb9e9ac8/userdata -p /run/containers/storage/overlay-containers/aadcd354a6128aa8c7cd1b7abfa64231df9fd22ad40d9f59c98801a2fb9e9ac8/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-grafana-cep1 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/aadcd354a6128aa8c7cd1b7abfa64231df9fd22ad40d9f59c98801a2fb9e9ac8/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@grafana.cep1.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg aadcd354a6128aa8c7cd1b7abfa64231df9fd22ad40d9f59c98801a2fb9e9ac8
|
|
|
|
Jan 18 10:13:29 cep1.domain.com ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-grafana-cep1[6093]: t=2023-01-18T03:13:29+0000 lvl=eror msg="Request error" logger=context userId=0 orgId=1 uname= error="Context.HTML:write tcp 192.0.2.11:3000->192.0.2.11:40444: write: connection reset by peer" stack="/drone/src/pkg/macaron/context.go:103 (0xdbd339)\n/drone/src/pkg/api/index.go:647 (0x2133790)\n/usr/local/go/src/reflect/value.go:556 (0x4e65a4)\n/usr/local/go/src/reflect/value.go:339 (0x4e5b24)\n/drone/src/pkg/macaron/inject.go:159 (0xdbe166)\n/drone/src/pkg/macaron/inject.go:119 (0xdbddb4)\n/drone/src/pkg/macaron/context.go:67 (0xdbd04b)\n/drone/src/pkg/macaron/context.go:62 (0x16456d3)\n/drone/src/pkg/middleware/request_metrics.go:55 (0x16456c2)\n/usr/local/go/src/reflect/value.go:556 (0x4e65a4)\n/usr/local/go/src/reflect/value.go:339 (0x4e5b24)\n/drone/src/pkg/macaron/inject.go:159 (0xdbe166)\n/drone/src/pkg/macaron/inject.go:119 (0xdbddb4)\n/drone/src/pkg/macaron/context.go:67 (0xdbd04b)\n/drone/src/pkg/macaron/context.go:62 (0xdc517a)\n/drone/src/pkg/macaron/macaron.go:155 (0xdc516c)\n/usr/local/go/src/net/http/server.go:2047 (0x6d990e)\n/drone/src/pkg/middleware/csp.go:22 (0x16419b2)\n/usr/local/go/src/reflect/value.go:556 (0x4e65a4)\n/usr/local/go/src/reflect/value.go:339 (0x4e5b24)\n/drone/src/pkg/macaron/inject.go:159 (0xdbe166)\n/drone/src/pkg/macaron/inject.go:119 (0xdbddb4)\n/drone/src/pkg/macaron/context.go:67 (0xdbd04b)\n/drone/src/pkg/macaron/context.go:62 (0xdc517a)\n/drone/src/pkg/macaron/macaron.go:155 (0xdc516c)\n/usr/local/go/src/net/http/server.go:2047 (0x6d990e)\n/drone/src/pkg/macaron/render.go:44 (0xdbfe26)\n/usr/local/go/src/reflect/value.go:556 (0x4e65a4)\n/usr/local/go/src/reflect/value.go:339 (0x4e5b24)\n/drone/src/pkg/macaron/inject.go:159 (0xdbe166)\n/drone/src/pkg/macaron/inject.go:119 (0xdbddb4)\n/drone/src/pkg/macaron/context.go:67 (0xdbd04b)\n/drone/src/pkg/macaron/context.go:62 (0xdc517a)\n/drone/src/pkg/macaron/macaron.go:155 (0xdc516c)\n/usr/local/go/src/net/http/server.go:2047 (0x6d990e)\n/drone/src/pkg/middleware/csrf.go:19 (0x1641d2f)\n/usr/local/go/src/reflect/value.go:556 (0x4e65a4)\n/usr/local/go/src/reflect/value.go:339 (0x4e5b24)\n/drone/src/pkg/macaron/inject.go:159 (0xdbe166)\n/drone/src/pkg/macaron/inject.go:119 (0xdbddb4)\n/drone/src/pkg/macaron/context.go:67 (0xdbd04b)\n/drone/src/pkg/macaron/context.go:62 (0x1644a2a)\n/drone/src/pkg/middleware/recovery.go:167 (0x1644a21)\n/drone/src/pkg/macaron/context.go:32 (0xdbcf3b)\n/drone/src/pkg/macaron/inject.go:139 (0xdbded4)\n/drone/src/pkg/macaron/inject.go:117 (0xdbddfc)\n/drone/src/pkg/macaron/context.go:67 (0xdbd04b)\n/drone/src/pkg/macaron/context.go:62 (0x16428e4)\n/drone/src/pkg/middleware/logger.go:33 (0x16428cd)\n/usr/local/go/src/reflect/value.go:556 (0x4e65a4)\n/usr/local/go/src/reflect/value.go:339 (0x4e5b24)\n/drone/src/pkg/macaron/inject.go:159 (0xdbe166)\n/drone/src/pkg/macaron/inject.go:119 (0xdbddb4)\n/drone/src/pkg/macaron/context.go:67 (0xdbd04b)\n/drone/src/pkg/macaron/context.go:62 (0x16472b9)\n/drone/src/pkg/middleware/request_tracing.go:57 (0x16472a8)\n/usr/local/go/src/reflect/value.go:556 (0x4e65a4)\n/usr/local/go/src/reflect/value.go:339 (0x4e5b24)\n/drone/src/pkg/macaron/inject.go:159 (0xdbe166)\n/drone/src/pkg/macaron/inject.go:119 (0xdbddb4)\n/drone/src/pkg/macaron/context.go:67 (0xdbd04b)\n/drone/src/pkg/macaron/router.go:156 (0xdc179d)\n/drone/src/pkg/macaron/router.go:213 (0xdc2455)\n/drone/src/pkg/macaron/macaron.go:189 (0xdbf770)\n/usr/local/go/src/net/http/server.go:2879 (0x6dce7a)\n/usr/local/go/src/net/http/server.go:1930 (0x6d89e7)\n/usr/local/go/src/runtime/asm_amd64.s:1581 (0x473060)\n"
|
|
[too many like this so snip, I *HATE* these messages in the log repeating every *5 seconds* because of monitoring, so excessive logging!!]
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.0.service - Ceph osd.0 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:23:17 +04; 4 days ago
|
|
Process: 34482 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.0.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.0.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34483 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.0/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34484 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.0/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 34651 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.0.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.0.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.3G
|
|
CPU: 31min 39.005s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.0.service
|
|
├─libpod-payload-4967c9a06c1dff0308a2d9665d042b63edbeaca1379739e0acdf5cfa449759a3
|
|
│ ├─15939 /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─15973 /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─15937 /usr/bin/conmon --api-version 1 -c 4967c9a06c1dff0308a2d9665d042b63edbeaca1379739e0acdf5cfa449759a3 -u 4967c9a06c1dff0308a2d9665d042b63edbeaca1379739e0acdf5cfa449759a3 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/4967c9a06c1dff0308a2d9665d042b63edbeaca1379739e0acdf5cfa449759a3/userdata -p /run/containers/storage/overlay-containers/4967c9a06c1dff0308a2d9665d042b63edbeaca1379739e0acdf5cfa449759a3/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-0 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/4967c9a06c1dff0308a2d9665d042b63edbeaca1379739e0acdf5cfa449759a3/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.0.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 4967c9a06c1dff0308a2d9665d042b63edbeaca1379739e0acdf5cfa449759a3
|
|
|
|
Jan 18 07:08:36 cep1.domain.com sudo[1781032]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:36 cep1.domain.com sudo[1781035]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdn
|
|
Jan 18 07:08:36 cep1.domain.com sudo[1781035]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:36 cep1.domain.com sudo[1781035]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:36 cep1.domain.com sudo[1781035]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:26:11 cep1.domain.com ceph-osd[15973]: log_channel(cluster) log [DBG] : 7.40 scrub ok
|
|
Jan 18 08:12:38 cep1.domain.com ceph-osd[15973]: log_channel(cluster) log [DBG] : 7.19e scrub ok
|
|
Jan 18 09:21:12 cep1.domain.com ceph-osd[15973]: log_channel(cluster) log [DBG] : 4.1e scrub ok
|
|
Jan 18 09:25:27 cep1.domain.com ceph-osd[15973]: log_channel(cluster) log [DBG] : 7.4 deep-scrub ok
|
|
Jan 18 09:49:25 cep1.domain.com ceph-osd[15973]: log_channel(cluster) log [DBG] : 7.39c scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mon.cep1.domain.com.service - Ceph mon.cep1.domain.com for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:20:29 +04; 4 days ago
|
|
Process: 34110 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mon.cep1.domain.com.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mon.cep1.domain.com.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34115 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/mon.cep1.domain.com/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34120 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/mon.cep1.domain.com/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 34125 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mon.cep1.domain.com.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mon.cep1.domain.com.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 27 (limit: 411291)
|
|
Memory: 459.1M
|
|
CPU: 52min 48.783s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mon.cep1.domain.com.service
|
|
├─libpod-payload-7c9fe2039f561d51bb1d0d797414a3a151ac74a383f102ff92d782775553f7bc
|
|
│ ├─7674 /run/podman-init -- /usr/bin/ceph-mon -n mon.cep1.domain.com -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false
|
|
│ └─7835 /usr/bin/ceph-mon -n mon.cep1.domain.com -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false
|
|
└─runtime
|
|
└─7669 /usr/bin/conmon --api-version 1 -c 7c9fe2039f561d51bb1d0d797414a3a151ac74a383f102ff92d782775553f7bc -u 7c9fe2039f561d51bb1d0d797414a3a151ac74a383f102ff92d782775553f7bc -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/7c9fe2039f561d51bb1d0d797414a3a151ac74a383f102ff92d782775553f7bc/userdata -p /run/containers/storage/overlay-containers/7c9fe2039f561d51bb1d0d797414a3a151ac74a383f102ff92d782775553f7bc/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-mon-cep1-domain-com --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/7c9fe2039f561d51bb1d0d797414a3a151ac74a383f102ff92d782775553f7bc/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@mon.cep1.domain.com.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 7c9fe2039f561d51bb1d0d797414a3a151ac74a383f102ff92d782775553f7bc
|
|
|
|
Jan 18 10:16:30 cep1.domain.com ceph-mon[7835]: pgmap v225873: 1217 pgs: 1217 active+clean; 684 GiB data, 4.6 TiB used, 260 TiB / 265 TiB avail
|
|
Jan 18 10:16:30 cep1.domain.com ceph-mon[7835]: from='mgr.95324 ' entity='mgr.cep2.atrumb'
|
|
Jan 18 10:16:30 cep1.domain.com ceph-mon[7835]: from='mgr.95324 192.0.2.12:0/2758914751' entity='mgr.cep2.atrumb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch
|
|
Jan 18 10:16:32 cep1.domain.com ceph-mon[7835]: pgmap v225874: 1217 pgs: 1217 active+clean; 684 GiB data, 4.6 TiB used, 260 TiB / 265 TiB avail
|
|
Jan 18 10:16:34 cep1.domain.com ceph-mon[7835]: pgmap v225875: 1217 pgs: 1217 active+clean; 684 GiB data, 4.6 TiB used, 260 TiB / 265 TiB avail
|
|
Jan 18 10:16:36 cep1.domain.com ceph-mon[7835]: pgmap v225876: 1217 pgs: 1217 active+clean; 684 GiB data, 4.6 TiB used, 260 TiB / 265 TiB avail
|
|
Jan 18 10:16:39 cep1.domain.com ceph-mon[7835]: pgmap v225877: 1217 pgs: 1217 active+clean; 684 GiB data, 4.6 TiB used, 260 TiB / 265 TiB avail
|
|
Jan 18 10:16:40 cep1.domain.com ceph-mon[7835]: pgmap v225878: 1217 pgs: 1217 active+clean; 684 GiB data, 4.6 TiB used, 260 TiB / 265 TiB avail
|
|
Jan 18 10:16:41 cep1.domain.com ceph-mon[7835]: pgmap v225879: 1217 pgs: 1217 active+clean; 684 GiB data, 4.6 TiB used, 260 TiB / 265 TiB avail
|
|
Jan 18 10:16:44 cep1.domain.com ceph-mon[7835]: pgmap v225880: 1217 pgs: 1217 active+clean; 684 GiB data, 4.6 TiB used, 260 TiB / 265 TiB avail
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.2.service - Ceph osd.2 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:23:30 +04; 4 days ago
|
|
Process: 35214 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.2.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.2.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 35215 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.2/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 35216 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.2/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 35542 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.2.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.2.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.2G
|
|
CPU: 29min 39.511s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.2.service
|
|
├─libpod-payload-201fd4a9d42d811469e23fe8b913d1dda0a08000e038c8beaa69212d7082d509
|
|
│ ├─14390 /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─14511 /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─14387 /usr/bin/conmon --api-version 1 -c 201fd4a9d42d811469e23fe8b913d1dda0a08000e038c8beaa69212d7082d509 -u 201fd4a9d42d811469e23fe8b913d1dda0a08000e038c8beaa69212d7082d509 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/201fd4a9d42d811469e23fe8b913d1dda0a08000e038c8beaa69212d7082d509/userdata -p /run/containers/storage/overlay-containers/201fd4a9d42d811469e23fe8b913d1dda0a08000e038c8beaa69212d7082d509/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-2 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/201fd4a9d42d811469e23fe8b913d1dda0a08000e038c8beaa69212d7082d509/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.2.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 201fd4a9d42d811469e23fe8b913d1dda0a08000e038c8beaa69212d7082d509
|
|
|
|
Jan 18 07:08:40 cep1.domain.com sudo[1781053]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:40 cep1.domain.com sudo[1781056]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/sdn
|
|
Jan 18 07:08:40 cep1.domain.com sudo[1781056]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:40 cep1.domain.com sudo[1781056]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:40 cep1.domain.com sudo[1781056]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:41 cep1.domain.com sudo[1781059]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdn
|
|
Jan 18 07:08:41 cep1.domain.com sudo[1781059]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:41 cep1.domain.com sudo[1781059]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:41 cep1.domain.com sudo[1781059]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:22:05 cep1.domain.com ceph-osd[14511]: log_channel(cluster) log [DBG] : 7.4d scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.3.service - Ceph osd.3 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:24:24 +04; 4 days ago
|
|
Process: 36448 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.3.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.3.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 36449 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.3/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 36450 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.3/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 36631 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.3.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.3.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.2G
|
|
CPU: 30min 42.941s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.3.service
|
|
├─libpod-payload-d0704e4945dee2c05ced669090606c9b8bd62a3d50dd352640e760ff2a2fc8a9
|
|
│ ├─13831 /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─14543 /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─13814 /usr/bin/conmon --api-version 1 -c d0704e4945dee2c05ced669090606c9b8bd62a3d50dd352640e760ff2a2fc8a9 -u d0704e4945dee2c05ced669090606c9b8bd62a3d50dd352640e760ff2a2fc8a9 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/d0704e4945dee2c05ced669090606c9b8bd62a3d50dd352640e760ff2a2fc8a9/userdata -p /run/containers/storage/overlay-containers/d0704e4945dee2c05ced669090606c9b8bd62a3d50dd352640e760ff2a2fc8a9/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-3 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/d0704e4945dee2c05ced669090606c9b8bd62a3d50dd352640e760ff2a2fc8a9/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.3.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg d0704e4945dee2c05ced669090606c9b8bd62a3d50dd352640e760ff2a2fc8a9
|
|
|
|
Jan 18 07:08:42 cep1.domain.com sudo[1781068]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:42 cep1.domain.com sudo[1781068]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:42 cep1.domain.com sudo[1781068]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:43 cep1.domain.com sudo[1781071]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdn
|
|
Jan 18 07:08:43 cep1.domain.com sudo[1781071]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:43 cep1.domain.com sudo[1781071]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:43 cep1.domain.com sudo[1781071]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:57:29 cep1.domain.com ceph-osd[14543]: log_channel(cluster) log [DBG] : 7.1fe scrub ok
|
|
Jan 18 09:40:17 cep1.domain.com ceph-osd[14543]: log_channel(cluster) log [DBG] : 7.69 scrub ok
|
|
Jan 18 09:57:23 cep1.domain.com ceph-osd[14543]: log_channel(cluster) log [DBG] : 7.363 deep-scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.7.service - Ceph osd.7 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:23:23 +04; 4 days ago
|
|
Process: 34654 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.7.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.7.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34655 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.7/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34656 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.7/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 34833 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.7.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.7.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.3G
|
|
CPU: 30min 23.161s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.7.service
|
|
├─libpod-payload-2813158cefb124b293f489578ccf27a46ff8e103ff9e518a1aa6d740ebb06acb
|
|
│ ├─14419 /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─14434 /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─14417 /usr/bin/conmon --api-version 1 -c 2813158cefb124b293f489578ccf27a46ff8e103ff9e518a1aa6d740ebb06acb -u 2813158cefb124b293f489578ccf27a46ff8e103ff9e518a1aa6d740ebb06acb -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/2813158cefb124b293f489578ccf27a46ff8e103ff9e518a1aa6d740ebb06acb/userdata -p /run/containers/storage/overlay-containers/2813158cefb124b293f489578ccf27a46ff8e103ff9e518a1aa6d740ebb06acb/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-7 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/2813158cefb124b293f489578ccf27a46ff8e103ff9e518a1aa6d740ebb06acb/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.7.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 2813158cefb124b293f489578ccf27a46ff8e103ff9e518a1aa6d740ebb06acb
|
|
|
|
Jan 18 07:08:52 cep1.domain.com sudo[1781119]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdm
|
|
Jan 18 07:08:52 cep1.domain.com sudo[1781119]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:52 cep1.domain.com sudo[1781119]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:52 cep1.domain.com sudo[1781119]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:36:45 cep1.domain.com ceph-osd[14434]: log_channel(cluster) log [DBG] : 7.3c9 scrub ok
|
|
Jan 18 08:16:52 cep1.domain.com ceph-osd[14434]: log_channel(cluster) log [DBG] : 7.3f8 scrub ok
|
|
Jan 18 08:20:34 cep1.domain.com ceph-osd[14434]: log_channel(cluster) log [DBG] : 7.217 scrub ok
|
|
Jan 18 08:51:28 cep1.domain.com ceph-osd[14434]: log_channel(cluster) log [DBG] : 7.f scrub ok
|
|
Jan 18 09:25:07 cep1.domain.com ceph-osd[14434]: log_channel(cluster) log [DBG] : 7.e1 scrub ok
|
|
Jan 18 10:12:32 cep1.domain.com ceph-osd[14434]: log_channel(cluster) log [DBG] : 7.68 scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.9.service - Ceph osd.9 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:24:03 +04; 4 days ago
|
|
Process: 35717 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.9.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.9.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 35718 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.9/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 35719 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.9/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 35884 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.9.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.9.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.3G
|
|
CPU: 27min 8.856s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.9.service
|
|
├─libpod-payload-1a1eabd0eee25c89fa8f67dedb20c4f1b2df3e6f013ff66d01aacf2e200014cc
|
|
│ ├─13697 /run/podman-init -- /usr/bin/ceph-osd -n osd.9 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─13787 /usr/bin/ceph-osd -n osd.9 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─13687 /usr/bin/conmon --api-version 1 -c 1a1eabd0eee25c89fa8f67dedb20c4f1b2df3e6f013ff66d01aacf2e200014cc -u 1a1eabd0eee25c89fa8f67dedb20c4f1b2df3e6f013ff66d01aacf2e200014cc -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/1a1eabd0eee25c89fa8f67dedb20c4f1b2df3e6f013ff66d01aacf2e200014cc/userdata -p /run/containers/storage/overlay-containers/1a1eabd0eee25c89fa8f67dedb20c4f1b2df3e6f013ff66d01aacf2e200014cc/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-9 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/1a1eabd0eee25c89fa8f67dedb20c4f1b2df3e6f013ff66d01aacf2e200014cc/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.9.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 1a1eabd0eee25c89fa8f67dedb20c4f1b2df3e6f013ff66d01aacf2e200014cc
|
|
|
|
Jan 18 07:08:56 cep1.domain.com sudo[1781254]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/sdm
|
|
Jan 18 07:08:56 cep1.domain.com sudo[1781254]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:56 cep1.domain.com sudo[1781254]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:56 cep1.domain.com sudo[1781254]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:56 cep1.domain.com sudo[1781257]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdm
|
|
Jan 18 07:08:56 cep1.domain.com sudo[1781257]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:56 cep1.domain.com sudo[1781257]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:56 cep1.domain.com sudo[1781257]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:45:47 cep1.domain.com ceph-osd[13787]: log_channel(cluster) log [DBG] : purged_snaps scrub starts
|
|
Jan 18 07:45:47 cep1.domain.com ceph-osd[13787]: log_channel(cluster) log [DBG] : purged_snaps scrub ok
|
|
|
|
● ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30.target - Ceph cluster 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30.target; enabled; vendor preset: disabled)
|
|
Active: active since Fri 2023-01-13 16:58:27 +04; 4 days ago
|
|
Until: Fri 2023-01-13 16:58:27 +04; 4 days ago
|
|
|
|
Notice: journal has been rotated since unit was started, output may be incomplete.
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.4.service - Ceph osd.4 for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:24:00 +04; 4 days ago
|
|
Process: 35543 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.4.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.4.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 35544 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.4/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 35545 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/osd.4/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 35716 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.4.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.4.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 66 (limit: 411291)
|
|
Memory: 1.2G
|
|
CPU: 30min 39.913s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.4.service
|
|
├─libpod-payload-8a98581f2c117d026c3381d316546ad868d2bba2099028f438245d992e01002e
|
|
│ ├─16002 /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─16023 /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─15989 /usr/bin/conmon --api-version 1 -c 8a98581f2c117d026c3381d316546ad868d2bba2099028f438245d992e01002e -u 8a98581f2c117d026c3381d316546ad868d2bba2099028f438245d992e01002e -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/8a98581f2c117d026c3381d316546ad868d2bba2099028f438245d992e01002e/userdata -p /run/containers/storage/overlay-containers/8a98581f2c117d026c3381d316546ad868d2bba2099028f438245d992e01002e/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-osd-4 --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/8a98581f2c117d026c3381d316546ad868d2bba2099028f438245d992e01002e/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@osd.4.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 8a98581f2c117d026c3381d316546ad868d2bba2099028f438245d992e01002e
|
|
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781080]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781080]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781083]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdn
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781083]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781083]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781083]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:13:50 cep1.domain.com ceph-osd[16023]: log_channel(cluster) log [DBG] : 7.2ea scrub ok
|
|
Jan 18 08:43:09 cep1.domain.com ceph-osd[16023]: log_channel(cluster) log [DBG] : 7.30e scrub ok
|
|
Jan 18 09:13:52 cep1.domain.com ceph-osd[16023]: log_channel(cluster) log [DBG] : 7.d2 deep-scrub ok
|
|
Jan 18 09:22:26 cep1.domain.com ceph-osd[16023]: log_channel(cluster) log [DBG] : 4.0 scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service - Ceph rgw.pool.cep1.abmahl for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:20:29 +04; 4 days ago
|
|
Process: 34113 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34118 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/rgw.pool.cep1.abmahl/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34123 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/rgw.pool.cep1.abmahl/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 34128 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 605 (limit: 411291)
|
|
Memory: 167.0M
|
|
CPU: 16min 8.665s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service
|
|
├─libpod-payload-44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049
|
|
│ ├─7821 /run/podman-init -- /usr/bin/radosgw -n client.rgw.pool.cep1.abmahl -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─9305 /usr/bin/radosgw -n client.rgw.pool.cep1.abmahl -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─7819 /usr/bin/conmon --api-version 1 -c 44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049 -u 44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049/userdata -p /run/containers/storage/overlay-containers/44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-rgw-pool-cep1-abmahl --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049/userdata/oci-log --conmon-ommand-arg --rm --exit-command-arg 8a98581f2c117d026c3381d316546ad868d2bba2099028f438245d992e01002e
|
|
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781080]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781080]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781083]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme ata smart-log-add --json /dev/sdn
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781083]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781083]: pam_unix(sudo:session): session opened for user root by (uid=0)
|
|
Jan 18 07:08:45 cep1.domain.com sudo[1781083]: pam_unix(sudo:session): session closed for user root
|
|
Jan 18 07:13:50 cep1.domain.com ceph-osd[16023]: log_channel(cluster) log [DBG] : 7.2ea scrub ok
|
|
Jan 18 08:43:09 cep1.domain.com ceph-osd[16023]: log_channel(cluster) log [DBG] : 7.30e scrub ok
|
|
Jan 18 09:13:52 cep1.domain.com ceph-osd[16023]: log_channel(cluster) log [DBG] : 7.d2 deep-scrub ok
|
|
Jan 18 09:22:26 cep1.domain.com ceph-osd[16023]: log_channel(cluster) log [DBG] : 4.0 scrub ok
|
|
|
|
× ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service - Ceph rgw.pool.cep1.abmahl for 5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30
|
|
Loaded: loaded (/etc/systemd/system/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@.service; enabled; vendor preset: disabled)
|
|
Active: failed (Result: exit-code) since Fri 2023-01-13 17:20:29 +04; 4 days ago
|
|
Process: 34113 ExecStartPre=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-cid (code=exited, status=219/CGROUP)
|
|
Process: 34118 ExecStart=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/rgw.pool.cep1.abmahl/unit.run (code=exited, status=219/CGROUP)
|
|
Process: 34123 ExecStopPost=/bin/bash /var/lib/ceph/5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30/rgw.pool.cep1.abmahl/unit.poststop (code=exited, status=0/SUCCESS)
|
|
Process: 34128 ExecStopPost=/bin/rm -f /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-pid /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-cid (code=exited, status=0/SUCCESS)
|
|
Tasks: 605 (limit: 411291)
|
|
Memory: 167.0M
|
|
CPU: 16min 8.665s
|
|
CGroup: /system.slice/system-ceph\x2d90397970\x2d29c0\x2d11ed\x2d9131\x2d6805ca3210e9.slice/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service
|
|
├─libpod-payload-44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049
|
|
│ ├─7821 /run/podman-init -- /usr/bin/radosgw -n client.rgw.pool.cep1.abmahl -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
│ └─9305 /usr/bin/radosgw -n client.rgw.pool.cep1.abmahl -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false
|
|
└─runtime
|
|
└─7819 /usr/bin/conmon --api-version 1 -c 44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049 -u 44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049 -r /usr/bin/crun -b /var/lib/containers/storage/overlay-containers/44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049/userdata -p /run/containers/storage/overlay-containers/44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049/userdata/pidfile -n ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30-rgw-pool-cep1-abmahl --exit-dir /run/libpod/exits --full-attach -l journald --log-level warning --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/containers/storage/overlay-containers/44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049/userdata/oci-log --conmon-pidfile /run/ceph-5c0752e5-73f2-411f-8c8f-e9c2d0fd7d30@rgw.pool.cep1.abmahl.service-pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /var/lib/containers/storage/volumes --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg overlay --exit-command-arg --storage-opt --exit-command-arg overlay.mountopt=nodev,metacopy=on --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 44faf1ecd556573d5758bb142ade3005a9fe1e1e699e86a972e9262d088c5049
|
|
|
|
Jan 18 08:08:15 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 08:53:18 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 08:53:18 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 09:08:19 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 09:08:19 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 09:23:20 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 09:38:21 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 09:38:21 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 09:53:23 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|
|
Jan 18 10:08:24 cep1.domain.com radosgw[9305]: ERROR: client_io->complete_request() returned Connection reset by peer
|