Actions
Bug #52234
opencrash in Throttle::get()
Status:
New
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
0%
Source:
Telemetry
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
56629a5f97eb83418851e3c9f1cd748af3f19902ddbcd4c8f26fe2fd719c0fa6
6dadff6c079acfcfa84282b695ce3885e86d94636b6852629d675eb008db1ca1
801d5b960db942824dfbe2814e34362d46b88645f9694edbf202cf754e5b7adc
8ef370db2590d515039724ec4efec4a5c8f152c7d90bde10fc8e653dd4b1c8f0
e4bd909221bbc951e8c54eee1fb9cf0a4e7b16c8a819a928f01e57a11dfe275f
ee9f576579ca7da4020a64078393ab6b76981893f12d6bdcac15bb141b726365
Description
Sanitized backtrace:
/lib64/libpthread.so.0( pthread_cond_wait() std::condition_variable::wait(std::unique_lock<std::mutex>&) Throttle::_wait(long, std::unique_lock<std::mutex>&) Throttle::get(long, long) BlueStore::BlueStoreThrottle::try_start_transaction(KeyValueDB&, BlueStore::TransContext&, std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >) BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*) ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*) non-virtual thunk to PrimaryLogPG::queue_transaction(ceph::os::Transaction&&, boost::intrusive_ptr<OpRequest>) ECBackend::dispatch_recovery_messages(RecoveryMessages&, int) ECBackend::_handle_message(boost::intrusive_ptr<OpRequest>) PGBackend::handle_message(boost::intrusive_ptr<OpRequest>) PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&) OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&) ceph::osd::scheduler::PGRecoveryMsg::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&) OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) ShardedThreadPool::shardedthreadpool_worker(unsigned int) ShardedThreadPool::WorkThreadSharded::entry() /lib64/libpthread.so.0( clone()
Crash dump sample:
{ "backtrace": [ "/lib64/libpthread.so.0(+0x12b20) [0x7f76f0cc1b20]", "pthread_cond_wait()", "(std::condition_variable::wait(std::unique_lock<std::mutex>&)+0x10) [0x7f76f03068f0]", "(Throttle::_wait(long, std::unique_lock<std::mutex>&)+0x10d) [0x55f74bfc712d]", "(Throttle::get(long, long)+0xb9) [0x55f74bfc7b49]", "(BlueStore::BlueStoreThrottle::try_start_transaction(KeyValueDB&, BlueStore::TransContext&, std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >)+0x29) [0x55f74be0d409]", "(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x854) [0x55f74be8e144]", "(ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x85) [0x55f74b9b0ee5]", "(non-virtual thunk to PrimaryLogPG::queue_transaction(ceph::os::Transaction&&, boost::intrusive_ptr<OpRequest>)+0x53) [0x55f74baea793]", "(ECBackend::dispatch_recovery_messages(RecoveryMessages&, int)+0xb85) [0x55f74bd02d65]", "(ECBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x22c) [0x55f74bd13d6c]", "(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x52) [0x55f74bb1ddd2]", "(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x5de) [0x55f74bac0e5e]", "(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x309) [0x55f74b94a299]", "(ceph::osd::scheduler::PGRecoveryMsg::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x68) [0x55f74bba72e8]", "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xa58) [0x55f74b96a2c8]", "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x55f74bfd1074]", "(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55f74bfd3d14]", "/lib64/libpthread.so.0(+0x814a) [0x7f76f0cb714a]", "clone()" ], "ceph_version": "16.2.5", "crash_id": "2021-08-02T04:47:51.939022Z_4532de0f-310a-462a-b2f0-fc74fcef7703", "entity_name": "osd.12bf74eb311d97792ca579e48eb22537fea92e63", "os_id": "centos", "os_name": "CentOS Linux", "os_version": "8", "os_version_id": "8", "process_name": "ceph-osd", "stack_sig": "6dadff6c079acfcfa84282b695ce3885e86d94636b6852629d675eb008db1ca1", "timestamp": "2021-08-02T04:47:51.939022Z", "utsname_machine": "x86_64", "utsname_release": "4.18.0-305.10.2.el8_4.x86_64", "utsname_sysname": "Linux", "utsname_version": "#1 SMP Mon Jul 12 04:43:18 EDT 2021" }
Updated by Telemetry Bot over 2 years ago
Updated by Neha Ojha over 2 years ago
- Has duplicate Bug #52205: crash: /lib64/libpthread.so.0( added
Updated by Neha Ojha over 2 years ago
- Has duplicate Bug #52203: crash: /lib64/libpthread.so.0( added
Updated by Neha Ojha over 2 years ago
- Subject changed from crash: /lib64/libpthread.so.0( to crash in Throttle::get()
Updated by Igor Fedotov over 2 years ago
Relevant thread at ceph-users: https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/thread/GNBVS2AWHCOVMOTXPHZNIG2KKLC4VNYA/
Updated by Igor Fedotov over 2 years ago
So far kernels 4.18 and 5.11 are mentioned among suffering ones
Updated by Igor Fedotov over 2 years ago
May be this modification is relevant as it appeared in Pacific and touches throttling mechanics... https://github.com/ceph/ceph/pull/30027
Updated by Telemetry Bot about 2 years ago
Updated by Telemetry Bot about 2 years ago
Updated by Telemetry Bot about 2 years ago
Actions