Ceph » cephsqlite

I have a bunch of errors, about 39 crashes of this mgr.

Context: 4 hosts, 3 monitors. This happened after a ton of IO and writes.

[ceph: root@CEPHADM-1 /]# ceph crash info 2023-12-29T00:01:42.093170Z_a77e687d-8da4-4d21-b000-253419e3c1fa
{
    "backtrace": [
        "  File \"/usr/share/ceph/mgr/devicehealth/module.py\", line 373, in serve\n    self.scrape_all()",
        "  File \"/usr/share/ceph/mgr/devicehealth/module.py\", line 425, in scrape_all\n    self.put_device_metrics(device, data)",
        "  File \"/usr/share/ceph/mgr/devicehealth/module.py\", line 500, in put_device_metrics\n    self._create_device(devid)",
        "  File \"/usr/share/ceph/mgr/devicehealth/module.py\", line 487, in _create_device\n    cursor = self.db.execute(SQL, (devid,))",
        "sqlite3.InternalError: unknown operation" 
    ],
    "ceph_version": "18.2.0",
    "crash_id": "2023-12-29T00:01:42.093170Z_a77e687d-8da4-4d21-b000-253419e3c1fa",
    "entity_name": "mgr.CEPHADM-1.zzrhqn",
    "mgr_module": "devicehealth",
    "mgr_module_caller": "PyModuleRunner::serve",
    "mgr_python_exception": "InternalError",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mgr",
    "stack_sig": "92fb822a43775ec1de7d41c75c8d0ec0bbb72ba5429a46000f34101c1bc6524e",
    "timestamp": "2023-12-29T00:01:42.093170Z",
    "utsname_hostname": "CEPHADM-1",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-521.el8.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Mon Oct 30 18:21:17 UTC 2023" 
}

[ceph: root@CEPHADM-1 /]# ceph crash info 2023-12-29T00:20:05.025432Z_bfddb25f-2351-4b63-8236-ebe4adb459e6
{
    "backtrace": [
        "  File \"/usr/share/ceph/mgr/devicehealth/module.py\", line 764, in get_recent_device_metrics\n    return self._get_device_metrics(devid, min_sample=min_sample)",
        "  File \"/usr/share/ceph/mgr/devicehealth/module.py\", line 557, in _get_device_metrics\n    cursor = self.db.execute(SQL_MIN, (devid, imin_sample))",
        "sqlite3.InternalError: unknown operation" 
    ],
    "ceph_version": "18.2.0",
    "crash_id": "2023-12-29T00:20:05.025432Z_bfddb25f-2351-4b63-8236-ebe4adb459e6",
    "entity_name": "mgr.CEPHADM-1.zzrhqn",
    "mgr_module": "devicehealth",
    "mgr_module_caller": "ActivePyModule::dispatch_remote get_recent_device_metrics",
    "mgr_python_exception": "InternalError",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mgr",
    "stack_sig": "bc6cf684446bc2b6e8263b0d6c7095626c5459cac2cbbfda38c790c945ff419b",
    "timestamp": "2023-12-29T00:20:05.025432Z",
    "utsname_hostname": "CEPHADM-1",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-521.el8.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Mon Oct 30 18:21:17 UTC 2023" 
}

[ceph: root@CEPHADM-1 /]# ceph crash info 2023-12-29T00:20:05.028504Z_f274a5e6-93d5-4829-9e80-b21b46796857
{
    "backtrace": [
        "  File \"/usr/share/ceph/mgr/devicehealth/module.py\", line 764, in get_recent_device_metrics\n    return self._get_device_metrics(devid, min_sample=min_sample)",
        "  File \"/usr/share/ceph/mgr/devicehealth/module.py\", line 557, in _get_device_metrics\n    cursor = self.db.execute(SQL_MIN, (devid, imin_sample))",
        "sqlite3.InternalError: unknown operation" 
    ],
    "ceph_version": "18.2.0",
    "crash_id": "2023-12-29T00:20:05.028504Z_f274a5e6-93d5-4829-9e80-b21b46796857",
    "entity_name": "mgr.CEPHADM-1.zzrhqn",
    "mgr_module": "devicehealth",
    "mgr_module_caller": "ActivePyModule::dispatch_remote get_recent_device_metrics",
    "mgr_python_exception": "InternalError",
    "os_id": "centos",
    "os_name": "CentOS Stream",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mgr",
    "stack_sig": "bc6cf684446bc2b6e8263b0d6c7095626c5459cac2cbbfda38c790c945ff419b",
    "timestamp": "2023-12-29T00:20:05.028504Z",
    "utsname_hostname": "CEPHADM-1",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-521.el8.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Mon Oct 30 18:21:17 UTC 2023" 
}