Project

General

Profile

Bug #38592

mon,osd: src/common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded.empty()) on shutdown

Added by Sage Weil 6 months ago. Updated 6 months ago.

Status:
Duplicate
Priority:
Urgent
Assignee:
-
Category:
-
Target version:
-
Start date:
03/05/2019
Due date:
% Done:

0%

Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:

Description

root@reesi002:~# ceph crash info 2019-03-05_16:15:11.094344Z_e5a27293-4f23-452f-bde8-f07efd303a71
{
    "crash_id": "2019-03-05_16:15:11.094344Z_e5a27293-4f23-452f-bde8-f07efd303a71",
    "timestamp": "2019-03-05 16:15:11.094344Z",
    "process_name": "ceph-osd",
    "entity_name": "osd.132",
    "ceph_version": "14.1.0-101-gdddb858",
    "utsname_hostname": "reesi006.front.sepia.ceph.com",
    "utsname_sysname": "Linux",
    "utsname_release": "4.4.0-112-generic",
    "utsname_version": "#135-Ubuntu SMP Fri Jan 19 11:48:36 UTC 2018",
    "utsname_machine": "x86_64",
    "os_name": "Ubuntu",
    "os_id": "ubuntu",
    "os_version_id": "16.04",
    "os_version": "16.04.4 LTS (Xenial Xerus)",
    "assert_condition": "(sharded_in_flight_list.back())->ops_in_flight_sharded.empty()",
    "assert_func": "OpTracker::~OpTracker()",
    "assert_file": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc",
    "assert_line": 163,
    "assert_thread_name": "ceph-osd",
    "assert_msg": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: In function 'OpTracker::~OpTracker()' thread 7f070d150f00 time 2019-03-05 16:15:11.073923\n/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded.empty())\n",
    "backtrace": [
        "(()+0x11390) [0x7f070a9df390]",
        "(gsignal()+0x38) [0x7f0709f0a428]",
        "(abort()+0x16a) [0x7f0709f0c02a]",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x84f89d]",
        "(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x84fa27]",
        "/usr/bin/ceph-osd() [0xc30da7]",
        "(OSD::~OSD()+0x539) [0x9b8e09]",
        "(OSD::~OSD()+0x9) [0x9b9189]",
        "(main()+0x409b) [0x856e0b]",
        "(__libc_start_main()+0xf0) [0x7f0709ef5830]",
        "(_start()+0x29) [0x93b7f9]" 
    ]
}
root@reesi002:~# ceph crash info 2019-03-05_16:15:20.013302Z_ea62ef16-08a0-4d92-b518-b206d9a940a9
{
    "crash_id": "2019-03-05_16:15:20.013302Z_ea62ef16-08a0-4d92-b518-b206d9a940a9",
    "timestamp": "2019-03-05 16:15:20.013302Z",
    "process_name": "ceph-mon",
    "entity_name": "mon.reesi005",
    "ceph_version": "14.1.0-101-gdddb858",
    "utsname_hostname": "reesi005",
    "utsname_sysname": "Linux",
    "utsname_release": "4.4.0-116-generic",
    "utsname_version": "#140-Ubuntu SMP Mon Feb 12 21:23:04 UTC 2018",
    "utsname_machine": "x86_64",
    "os_name": "Ubuntu",
    "os_id": "ubuntu",
    "os_version_id": "16.04",
    "os_version": "16.04.4 LTS (Xenial Xerus)",
    "assert_condition": "(sharded_in_flight_list.back())->ops_in_flight_sharded.empty()",
    "assert_func": "OpTracker::~OpTracker()",
    "assert_file": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc",
    "assert_line": 163,
    "assert_thread_name": "ceph-mon",
    "assert_msg": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: In function 'OpTracker::~OpTracker()' thread 7f4179419040 time 2019-03-05 16:15:19.990460\n/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded.empty())\n",
    "backtrace": [
        "(()+0x11390) [0x7f416f35d390]",
        "(gsignal()+0x38) [0x7f416eaaa428]",
        "(abort()+0x16a) [0x7f416eaac02a]",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f4170557053]",
        "(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7f41705571dd]",
        "(()+0x3796e7) [0x7f41705ef6e7]",
        "(Monitor::~Monitor()+0x2bb) [0x6a2fdb]",
        "(Monitor::~Monitor()+0x9) [0x6a36f9]",
        "(main()+0x2834) [0x57eda4]",
        "(__libc_start_main()+0xf0) [0x7f416ea95830]",
        "(_start()+0x29) [0x6619d9]" 
    ]
}
root@reesi002:~# ceph crash info 2019-03-05_16:15:27.936967Z_26dc7a32-e008-416b-ab19-2becaf1bfc38
{
    "crash_id": "2019-03-05_16:15:27.936967Z_26dc7a32-e008-416b-ab19-2becaf1bfc38",
    "timestamp": "2019-03-05 16:15:27.936967Z",
    "process_name": "ceph-mon",
    "entity_name": "mon.reesi003",
    "ceph_version": "14.1.0-101-gdddb858",
    "utsname_hostname": "reesi003",
    "utsname_sysname": "Linux",
    "utsname_release": "4.4.0-116-generic",
    "utsname_version": "#140-Ubuntu SMP Mon Feb 12 21:23:04 UTC 2018",
    "utsname_machine": "x86_64",
    "os_name": "Ubuntu",
    "os_id": "ubuntu",
    "os_version_id": "16.04",
    "os_version": "16.04.4 LTS (Xenial Xerus)",
    "assert_condition": "(sharded_in_flight_list.back())->ops_in_flight_sharded.empty()",
    "assert_func": "OpTracker::~OpTracker()",
    "assert_file": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc",
    "assert_line": 163,
    "assert_thread_name": "ceph-mon",
    "assert_msg": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: In function 'OpTracker::~OpTracker()' thread 7f9e28861040 time 2019-03-05 16:15:27.909141\n/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded.empty())\n",
    "backtrace": [
        "(()+0x11390) [0x7f9e1e7a5390]",
        "(gsignal()+0x38) [0x7f9e1def2428]",
        "(abort()+0x16a) [0x7f9e1def402a]",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f9e1f99f053]",
        "(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7f9e1f99f1dd]",
        "(()+0x3796e7) [0x7f9e1fa376e7]",
        "(Monitor::~Monitor()+0x2bb) [0x6a2fdb]",
        "(Monitor::~Monitor()+0x9) [0x6a36f9]",
        "(main()+0x2834) [0x57eda4]",
        "(__libc_start_main()+0xf0) [0x7f9e1dedd830]",
        "(_start()+0x29) [0x6619d9]" 
    ]
}

Related issues

Related to RADOS - Bug #36546: common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded. empty()) Duplicate 10/22/2018
Duplicates RADOS - Bug #38377: OpTracker destruct assert when OSD destruct Resolved 02/19/2019

History

#1 Updated by Neha Ojha 6 months ago

#2 Updated by Sage Weil 6 months ago

  • Related to Bug #36546: common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded. empty()) added

#3 Updated by Sage Weil 6 months ago

  • Duplicates Bug #38377: OpTracker destruct assert when OSD destruct added

#4 Updated by Sage Weil 6 months ago

  • Status changed from Verified to Duplicate

Also available in: Atom PDF