Bug #38592
mon,osd: src/common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded.empty()) on shutdown
Status:
Duplicate
Priority:
Urgent
Assignee:
-
Category:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
root@reesi002:~# ceph crash info 2019-03-05_16:15:11.094344Z_e5a27293-4f23-452f-bde8-f07efd303a71 { "crash_id": "2019-03-05_16:15:11.094344Z_e5a27293-4f23-452f-bde8-f07efd303a71", "timestamp": "2019-03-05 16:15:11.094344Z", "process_name": "ceph-osd", "entity_name": "osd.132", "ceph_version": "14.1.0-101-gdddb858", "utsname_hostname": "reesi006.front.sepia.ceph.com", "utsname_sysname": "Linux", "utsname_release": "4.4.0-112-generic", "utsname_version": "#135-Ubuntu SMP Fri Jan 19 11:48:36 UTC 2018", "utsname_machine": "x86_64", "os_name": "Ubuntu", "os_id": "ubuntu", "os_version_id": "16.04", "os_version": "16.04.4 LTS (Xenial Xerus)", "assert_condition": "(sharded_in_flight_list.back())->ops_in_flight_sharded.empty()", "assert_func": "OpTracker::~OpTracker()", "assert_file": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc", "assert_line": 163, "assert_thread_name": "ceph-osd", "assert_msg": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: In function 'OpTracker::~OpTracker()' thread 7f070d150f00 time 2019-03-05 16:15:11.073923\n/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded.empty())\n", "backtrace": [ "(()+0x11390) [0x7f070a9df390]", "(gsignal()+0x38) [0x7f0709f0a428]", "(abort()+0x16a) [0x7f0709f0c02a]", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x84f89d]", "(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x84fa27]", "/usr/bin/ceph-osd() [0xc30da7]", "(OSD::~OSD()+0x539) [0x9b8e09]", "(OSD::~OSD()+0x9) [0x9b9189]", "(main()+0x409b) [0x856e0b]", "(__libc_start_main()+0xf0) [0x7f0709ef5830]", "(_start()+0x29) [0x93b7f9]" ] } root@reesi002:~# ceph crash info 2019-03-05_16:15:20.013302Z_ea62ef16-08a0-4d92-b518-b206d9a940a9 { "crash_id": "2019-03-05_16:15:20.013302Z_ea62ef16-08a0-4d92-b518-b206d9a940a9", "timestamp": "2019-03-05 16:15:20.013302Z", "process_name": "ceph-mon", "entity_name": "mon.reesi005", "ceph_version": "14.1.0-101-gdddb858", "utsname_hostname": "reesi005", "utsname_sysname": "Linux", "utsname_release": "4.4.0-116-generic", "utsname_version": "#140-Ubuntu SMP Mon Feb 12 21:23:04 UTC 2018", "utsname_machine": "x86_64", "os_name": "Ubuntu", "os_id": "ubuntu", "os_version_id": "16.04", "os_version": "16.04.4 LTS (Xenial Xerus)", "assert_condition": "(sharded_in_flight_list.back())->ops_in_flight_sharded.empty()", "assert_func": "OpTracker::~OpTracker()", "assert_file": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc", "assert_line": 163, "assert_thread_name": "ceph-mon", "assert_msg": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: In function 'OpTracker::~OpTracker()' thread 7f4179419040 time 2019-03-05 16:15:19.990460\n/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded.empty())\n", "backtrace": [ "(()+0x11390) [0x7f416f35d390]", "(gsignal()+0x38) [0x7f416eaaa428]", "(abort()+0x16a) [0x7f416eaac02a]", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f4170557053]", "(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7f41705571dd]", "(()+0x3796e7) [0x7f41705ef6e7]", "(Monitor::~Monitor()+0x2bb) [0x6a2fdb]", "(Monitor::~Monitor()+0x9) [0x6a36f9]", "(main()+0x2834) [0x57eda4]", "(__libc_start_main()+0xf0) [0x7f416ea95830]", "(_start()+0x29) [0x6619d9]" ] } root@reesi002:~# ceph crash info 2019-03-05_16:15:27.936967Z_26dc7a32-e008-416b-ab19-2becaf1bfc38 { "crash_id": "2019-03-05_16:15:27.936967Z_26dc7a32-e008-416b-ab19-2becaf1bfc38", "timestamp": "2019-03-05 16:15:27.936967Z", "process_name": "ceph-mon", "entity_name": "mon.reesi003", "ceph_version": "14.1.0-101-gdddb858", "utsname_hostname": "reesi003", "utsname_sysname": "Linux", "utsname_release": "4.4.0-116-generic", "utsname_version": "#140-Ubuntu SMP Mon Feb 12 21:23:04 UTC 2018", "utsname_machine": "x86_64", "os_name": "Ubuntu", "os_id": "ubuntu", "os_version_id": "16.04", "os_version": "16.04.4 LTS (Xenial Xerus)", "assert_condition": "(sharded_in_flight_list.back())->ops_in_flight_sharded.empty()", "assert_func": "OpTracker::~OpTracker()", "assert_file": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc", "assert_line": 163, "assert_thread_name": "ceph-mon", "assert_msg": "/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: In function 'OpTracker::~OpTracker()' thread 7f9e28861040 time 2019-03-05 16:15:27.909141\n/build/ceph-14.1.0-101-gdddb858/src/common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded.empty())\n", "backtrace": [ "(()+0x11390) [0x7f9e1e7a5390]", "(gsignal()+0x38) [0x7f9e1def2428]", "(abort()+0x16a) [0x7f9e1def402a]", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f9e1f99f053]", "(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7f9e1f99f1dd]", "(()+0x3796e7) [0x7f9e1fa376e7]", "(Monitor::~Monitor()+0x2bb) [0x6a2fdb]", "(Monitor::~Monitor()+0x9) [0x6a36f9]", "(main()+0x2834) [0x57eda4]", "(__libc_start_main()+0xf0) [0x7f9e1dedd830]", "(_start()+0x29) [0x6619d9]" ] }
Related issues
History
#1 Updated by Neha Ojha about 5 years ago
Is this related to http://tracker.ceph.com/issues/36546?
#2 Updated by Sage Weil about 5 years ago
- Related to Bug #36546: common/TrackedOp.cc: 163: FAILED ceph_assert((sharded_in_flight_list.back())->ops_in_flight_sharded. empty()) added
#3 Updated by Sage Weil about 5 years ago
- Duplicates Bug #38377: OpTracker destruct assert when OSD destruct added
#4 Updated by Sage Weil about 5 years ago
- Status changed from 12 to Duplicate