Ceph : Issues
https://tracker.ceph.com/
https://tracker.ceph.com/favicon.ico
2023-05-15T03:00:40Z
Ceph
Redmine
mgr - Bug #61146 (New): crash: File "mgr/diskprediction_local/module.py", in predict_life_expecta...
https://tracker.ceph.com/issues/61146
2023-05-15T03:00:40Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2603a41926f5f1a1e7662b08a0b4886eff8402e88352bddba268827b72f6370d">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2603a41926f5f1a1e7662b08a0b4886eff8402e88352bddba268827b72f6370d</a></p>
<p>Sanitized backtrace:<br /><pre> File "mgr/diskprediction_local/module.py", in predict_life_expectancy: result = self._predict_life_expectancy(devid)
File "mgr/diskprediction_local/module.py", in _predict_life_expectancy: predicted_result = obj_predictor.predict(predict_datas)
File "mgr/diskprediction_local/predictor.py", in predict: pred = clf.predict(ordered_data)
File "/lib/python3/dist-packages/sklearn/svm/_base.py", in predict: if self.break_ties and self.decision_function_shape == 'ovo':
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
" File \"/usr/share/ceph/mgr/diskprediction_local/module.py\", line 209, in predict_life_expectancy\n result = self._predict_life_expectancy(devid)",
" File \"/usr/share/ceph/mgr/diskprediction_local/module.py\", line 205, in _predict_life_expectancy\n predicted_result = obj_predictor.predict(predict_datas)",
" File \"/usr/share/ceph/mgr/diskprediction_local/predictor.py\", line 475, in predict\n pred = clf.predict(ordered_data)",
" File \"/lib/python3/dist-packages/sklearn/svm/_base.py\", line 606, in predict\n if self.break_ties and self.decision_function_shape == 'ovo':",
"<redacted>"
],
"ceph_version": "17.2.5",
"crash_id": "2023-05-11T09:59:59.493520Z_3075a2e3-eb45-49ad-b54d-213029638cf6",
"entity_name": "mgr.244b507d063016063c303fbaf7143db92fde47b2",
"mgr_module": "diskprediction_local",
"mgr_module_caller": "ActivePyModule::dispatch_remote predict_life_expectancy",
"mgr_python_exception": "AttributeError",
"os_id": "11",
"os_name": "Debian GNU/Linux 11 (bullseye)",
"os_version": "11 (bullseye)",
"os_version_id": "11",
"process_name": "ceph-mgr",
"stack_sig": "a5b7ed95629c8e9969a9b9a7daaa828bb7c6c2caee23133bdf387bf81ab5b085",
"timestamp": "2023-05-11T09:59:59.493520Z",
"utsname_machine": "x86_64",
"utsname_release": "5.15.60-2-pve",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP PVE 5.15.60-2 (Tue, 04 Oct 2022 16:52:28 +0200)"
}</pre></p>
mgr - Bug #61145 (New): crash: File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in ...
https://tracker.ceph.com/issues/61145
2023-05-15T03:00:38Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2b4dd9701698005f416fae427ae8079249b3c658c8bc182a82e78ecad105aaa1">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2b4dd9701698005f416fae427ae8079249b3c658c8bc182a82e78ecad105aaa1</a></p>
<p>Sanitized backtrace:<br /><pre> File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in _new_conn: (self._dns_host, self.port), self.timeout, **extra_kw)
File "/usr/lib/python3.6/site-packages/urllib3/util/connection.py", in create_connection: raise err
File "/usr/lib/python3.6/site-packages/urllib3/util/connection.py", in create_connection: sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection
During handling of the above exception, another exception
Traceback (most recent call
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: chunked=chunked)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in _make_request: self._validate_conn(conn)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in _validate_conn: conn.connect()
File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in connect: conn = self._new_conn()
File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in _new_conn: self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x7f5f3f9aa208>: Failed to establish a new connection: [Errno 111] Connection
During handling of the above exception, another exception
Traceback (most recent call
File "mgr/rook/module.py", in available: self.k8s.list_namespaced_pod(self._rook_env.namespace)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py", in list_namespaced_pod: (data) = self.list_namespaced_pod_with_http_info(namespace, **kwargs) # noqa: E501
File "/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py", in list_namespaced_pod_with_http_info: collection_formats=collection_formats)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in call_api: _preload_content, _request_timeout)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in __call_api: _request_timeout=_request_timeout)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in request: headers=headers)
File "/usr/lib/python3.6/site-packages/kubernetes/client/rest.py", in GET: query_params=query_params)
File "/usr/lib/python3.6/site-packages/kubernetes/client/rest.py", in request: headers=headers)
File "/usr/lib/python3.6/site-packages/urllib3/request.py", in request: **urlopen_kw)
File "/usr/lib/python3.6/site-packages/urllib3/request.py", in request_encode_url: return self.urlopen(method, url, **extra_kw)
File "/usr/lib/python3.6/site-packages/urllib3/poolmanager.py", in urlopen: response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: _stacktrace=sys.exc_info()[2])
File "/usr/lib/python3.6/site-packages/urllib3/util/retry.py", in increment: raise MaxRetryError(_pool, url, error or ResponseError(cause))
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-04-13 11:44:20.992490",
"backtrace": [
" File \"/usr/lib/python3.6/site-packages/urllib3/connection.py\", line 162, in _new_conn\n (self._dns_host, self.port), self.timeout, **extra_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/util/connection.py\", line 80, in create_connection\n raise err",
" File \"/usr/lib/python3.6/site-packages/urllib3/util/connection.py\", line 70, in create_connection\n sock.connect(sa)",
"ConnectionRefusedError: [Errno 111] Connection refused",
"\nDuring handling of the above exception, another exception occurred:\n",
"Traceback (most recent call last):",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\n chunked=chunked)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 343, in _make_request\n self._validate_conn(conn)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 839, in _validate_conn\n conn.connect()",
" File \"/usr/lib/python3.6/site-packages/urllib3/connection.py\", line 315, in connect\n conn = self._new_conn()",
" File \"/usr/lib/python3.6/site-packages/urllib3/connection.py\", line 171, in _new_conn\n self, \"Failed to establish a new connection: %s\" % e)",
"urllib3.exceptions.NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x7f5f3f9aa208>: Failed to establish a new connection: [Errno 111] Connection refused",
"\nDuring handling of the above exception, another exception occurred:\n",
"Traceback (most recent call last):",
" File \"/usr/share/ceph/mgr/rook/module.py\", line 108, in available\n self.k8s.list_namespaced_pod(self._rook_env.namespace)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py\", line 12803, in list_namespaced_pod\n (data) = self.list_namespaced_pod_with_http_info(namespace, **kwargs) # noqa: E501",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py\", line 12905, in list_namespaced_pod_with_http_info\n collection_formats=collection_formats)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 345, in call_api\n _preload_content, _request_timeout)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 176, in __call_api\n _request_timeout=_request_timeout)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 366, in request\n headers=headers)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/rest.py\", line 241, in GET\n query_params=query_params)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/rest.py\", line 214, in request\n headers=headers)",
" File \"/usr/lib/python3.6/site-packages/urllib3/request.py\", line 68, in request\n **urlopen_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/request.py\", line 89, in request_encode_url\n return self.urlopen(method, url, **extra_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/poolmanager.py\", line 324, in urlopen\n response = conn.urlopen(method, u.request_uri, **kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\n _stacktrace=sys.exc_info()[2])",
" File \"/usr/lib/python3.6/site-packages/urllib3/util/retry.py\", line 399, in increment\n raise MaxRetryError(_pool, url, error or ResponseError(cause))",
"<redacted>"
],
"ceph_version": "17.2.5",
"crash_id": "2023-04-13T11:26:44.614190Z_6f7e8a57-3184-4dec-b928-fce88f0092c5",
"entity_name": "mgr.85c499e3732d8025117e6274ba277695590fb5f7",
"mgr_module": "rook",
"mgr_module_caller": "ActivePyModule::dispatch_remote available",
"mgr_python_exception": "MaxRetryError",
"os_id": "centos",
"os_name": "CentOS Stream",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-mgr",
"stack_sig": "429ad003244431b8d84310ed74666ab8172821e8588998b67e576c9f6e9f6bb7",
"timestamp": "2023-04-13T11:26:44.614190Z",
"utsname_machine": "x86_64",
"utsname_release": "5.15.106-talos",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Wed Apr 5 15:23:01 UTC 2023"
}</pre></p>
bluestore - Bug #61144 (New): crash: pread64()
https://tracker.ceph.com/issues/61144
2023-05-15T03:00:37Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=4e23a6b2eeff85f26c96608922764ba35ef05c58e25700b1b24c6bf9aff53133">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=4e23a6b2eeff85f26c96608922764ba35ef05c58e25700b1b24c6bf9aff53133</a></p>
<p>Sanitized backtrace:<br /><pre> pread64()
KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)
BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::list*, char*)
BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)
rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)
rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const
rocksdb::BlockPrefetcher::PrefetchIfNeeded(rocksdb::BlockBasedTable::Rep const*, rocksdb::BlockHandle const&, unsigned long, bool)
rocksdb::BlockBasedTableIterator::InitDataBlock()
rocksdb::BlockBasedTableIterator::FindBlockForward()
rocksdb::BlockBasedTableIterator::FindKeyForward()
rocksdb::BlockBasedTableIterator::Next()
rocksdb::BlockBasedTableIterator::NextAndGetResult(rocksdb::IterateResult*)
rocksdb::IteratorWrapperBase<rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)
rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()
rocksdb::MergingIterator::Next()
rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)
rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()
rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*)
rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*)
rocksdb::DBIter::Next()
rocksdb::ArenaWrappedDBIter::Next()
ShardMergeIteratorImpl::next()
BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
PGBackend::objects_list_partial(hobject_t const&, int, int, std::vector<hobject_t, std::allocator<hobject_t> >*, hobject_t*)
PrimaryLogPG::scan_range(int, int, BackfillInterval*, ThreadPool::TPHandle&)
PrimaryLogPG::update_range(BackfillInterval*, ThreadPool::TPHandle&)
PrimaryLogPG::recover_backfill(unsigned long, ThreadPool::TPHandle&, bool*)
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fbd7c4fe420]",
"pread64()",
"(KernelDevice::read(unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, IOContext*, bool)+0x109) [0x560dd65dc359]",
"(BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, char*)+0x560) [0x560dd60e0a60]",
"(BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)+0x1f) [0x560dd610826f]",
"(rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x60) [0x560dd6667d54]",
"(rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const+0x75) [0x560dd694d11f]",
"(rocksdb::BlockPrefetcher::PrefetchIfNeeded(rocksdb::BlockBasedTable::Rep const*, rocksdb::BlockHandle const&, unsigned long, bool)+0x168) [0x560dd6b092b8]",
"(rocksdb::BlockBasedTableIterator::InitDataBlock()+0x171) [0x560dd6b07079]",
"(rocksdb::BlockBasedTableIterator::FindBlockForward()+0x281) [0x560dd6b07777]",
"(rocksdb::BlockBasedTableIterator::FindKeyForward()+0x92) [0x560dd6b074f2]",
"(rocksdb::BlockBasedTableIterator::Next()+0x8c) [0x560dd6b06d76]",
"(rocksdb::BlockBasedTableIterator::NextAndGetResult(rocksdb::IterateResult*)+0x21) [0x560dd6b06da9]",
"(rocksdb::IteratorWrapperBase<rocksdb::Slice>::NextAndGetResult(rocksdb::IterateResult*)+0x79) [0x560dd682bcc7]",
"/usr/bin/ceph-osd(+0x1b2e2d4) [0x560dd68012d4]",
"(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x75) [0x560dd674a059]",
"(rocksdb::MergingIterator::Next()+0xc2) [0x560dd698d288]",
"(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x2c) [0x560dd698d3e0]",
"(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x75) [0x560dd674a059]",
"(rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*)+0x12eb) [0x560dd6740f25]",
"(rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*)+0x83) [0x560dd673fbef]",
"(rocksdb::DBIter::Next()+0x2f9) [0x560dd673fa4d]",
"(rocksdb::ArenaWrappedDBIter::Next()+0x20) [0x560dd6a4b372]",
"(ShardMergeIteratorImpl::next()+0x4d) [0x560dd66149cd]",
"/usr/bin/ceph-osd(+0x129d48a) [0x560dd5f7048a]",
"(BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0x1895) [0x560dd5fb4535]",
"(BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0xbf) [0x560dd5fb59ff]",
"(PGBackend::objects_list_partial(hobject_t const&, int, int, std::vector<hobject_t, std::allocator<hobject_t> >*, hobject_t*)+0xdc2) [0x560dd5bf2cc2]",
"(PrimaryLogPG::scan_range(int, int, BackfillInterval*, ThreadPool::TPHandle&)+0x100) [0x560dd5b7db50]",
"(PrimaryLogPG::update_range(BackfillInterval*, ThreadPool::TPHandle&)+0x98) [0x560dd5b80a48]",
"(PrimaryLogPG::recover_backfill(unsigned long, ThreadPool::TPHandle&, bool*)+0x3a65) [0x560dd5b852d5]"
],
"ceph_version": "17.2.5",
"crash_id": "2023-05-09T09:24:32.693019Z_1552a3d6-8386-4f7b-b930-17d0ce2ce110",
"entity_name": "osd.2be64aa30c54163e9db5f1a1b0ffa460c48516ac",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.5 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "ceph-osd",
"stack_sig": "dc29991b9763df0eedaa64f8a6cc5abbb8c53ff70512cb73cd9c7fa29533b416",
"timestamp": "2023-05-09T09:24:32.693019Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-144-generic",
"utsname_sysname": "Linux",
"utsname_version": "#161-Ubuntu SMP Fri Feb 3 14:49:04 UTC 2023"
}</pre></p>
RADOS - Bug #61143 (New): crash: MOSDRepOp::encode_payload(unsigned long)
https://tracker.ceph.com/issues/61143
2023-05-15T03:00:35Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=5b6a45a1fcd86db7edc24b7c93d179448b8f22eebd5834360b1f628d31a6cbb4">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=5b6a45a1fcd86db7edc24b7c93d179448b8f22eebd5834360b1f628d31a6cbb4</a></p>
<p>Sanitized backtrace:<br /><pre> MOSDRepOp::encode_payload(unsigned long)
Message::encode(unsigned long, int, bool)
ProtocolV2::prepare_send_message(unsigned long, Message*)
ProtocolV2::send_message(Message*)
AsyncConnection::send_message(Message*)
OSDService::send_message_osd_cluster(int, Message*, unsigned int)
ReplicatedBackend::issue_op(hobject_t const&, eversion_t const&, unsigned long, osd_reqid_t, eversion_t, eversion_t, hobject_t, hobject_t, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, std::optional<pg_hit_set_history_t>&, ReplicatedBackend::InProgressOp*, ceph::os::Transaction&)
ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> >&&, std::optional<pg_hit_set_history_t>&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>)
PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)
PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)
PrimaryLogPG::agent_maybe_evict(std::shared_ptr<ObjectContext>&, bool)
PrimaryLogPG::agent_work(int, int)
OSDService::agent_entry()
OSDService::AgentThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-05-09 16:08:56.965903",
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fac6b80e420]",
"gsignal()",
"abort()",
"/lib/x86_64-linux-gnu/libc.so.6(+0x22729) [0x7fac6b279729]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x33fd6) [0x7fac6b28afd6]",
"(MOSDRepOp::encode_payload(unsigned long)+0x2d0) [0x564825e07350]",
"(Message::encode(unsigned long, int, bool)+0x2e) [0x5648262cf7be]",
"(ProtocolV2::prepare_send_message(unsigned long, Message*)+0x44) [0x5648265848b4]",
"(ProtocolV2::send_message(Message*)+0x25e) [0x564826584fae]",
"(AsyncConnection::send_message(Message*)+0x18e) [0x5648265560ee]",
"(OSDService::send_message_osd_cluster(int, Message*, unsigned int)+0x225) [0x5648259ae0a5]",
"(ReplicatedBackend::issue_op(hobject_t const&, eversion_t const&, unsigned long, osd_reqid_t, eversion_t, eversion_t, hobject_t, hobject_t, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, std::optional<pg_hit_set_history_t>&, ReplicatedBackend::InProgressOp*, ceph::os::Transaction&)+0x7b5) [0x564825dee925]",
"(ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> >&&, std::optional<pg_hit_set_history_t>&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>)+0x5fd) [0x564825df111d]",
"(PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0xecf) [0x564825af6a3f]",
"(PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)+0x5a) [0x564825af8dfa]",
"(PrimaryLogPG::agent_maybe_evict(std::shared_ptr<ObjectContext>&, bool)+0x950) [0x564825b27820]",
"(PrimaryLogPG::agent_work(int, int)+0x15b4) [0x564825b32c34]",
"(OSDService::agent_entry()+0x198) [0x564825989118]",
"(OSDService::AgentThread::entry()+0x11) [0x564825a5edc1]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7fac6b802609]",
"clone()"
],
"ceph_version": "17.2.6",
"crash_id": "2023-05-09T01:26:38.066176Z_fe17817a-46e5-4b21-ae8c-8fd5fe98f90a",
"entity_name": "osd.7047d449e9ccc5475a4d5b067cc9d60b3c791047",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.6 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "ceph-osd",
"stack_sig": "fc22286141c6a6763629cc097909e1ca9fc54b14a8b8f5ebf1a387223e3a4270",
"timestamp": "2023-05-09T01:26:38.066176Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-109-generic",
"utsname_sysname": "Linux",
"utsname_version": "#123-Ubuntu SMP Fri Apr 8 09:10:54 UTC 2022"
}</pre></p>
mgr - Bug #61142 (New): crash: File "mgr/volumes/module.py", in __init__: self.vc = VolumeClient(...
https://tracker.ceph.com/issues/61142
2023-05-15T03:00:34Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=662e13538f28e275e212fec2af64c0fc0d4d86c605567c81b19a431c25838a48">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=662e13538f28e275e212fec2af64c0fc0d4d86c605567c81b19a431c25838a48</a></p>
<p>Sanitized backtrace:<br /><pre> File "mgr/volumes/module.py", in __init__: self.vc = VolumeClient(self)
File "mgr/volumes/fs/volume.py", in __init__: self.volspec = VolSpec(mgr.rados.conf_get('client_snapdir'))
File "mgr/mgr_module.py", in rados: self._rados.connect()
File "rados.pyx", in rados.Rados.connect
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-05-08 15:23:47.840836",
"backtrace": [
" File \"/usr/share/ceph/mgr/volumes/module.py\", line 499, in __init__\n self.vc = VolumeClient(self)",
" File \"/usr/share/ceph/mgr/volumes/fs/volume.py\", line 56, in __init__\n self.volspec = VolSpec(mgr.rados.conf_get('client_snapdir'))",
" File \"/usr/share/ceph/mgr/mgr_module.py\", line 2127, in rados\n self._rados.connect()",
" File \"rados.pyx\", line 680, in rados.Rados.connect",
"<redacted>"
],
"ceph_version": "17.2.5",
"crash_id": "2023-05-08T10:06:29.463752Z_d5430edb-eb19-4f77-9651-47e14fe0e383",
"entity_name": "mgr.5c926e6e1977b4d8d5108875f1086da94dd6def7",
"mgr_module": "volumes",
"mgr_module_caller": "ActivePyModule::load",
"mgr_python_exception": "PermissionError",
"os_id": "centos",
"os_name": "CentOS Stream",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-mgr",
"stack_sig": "c791711774b0ae6a8b2cfe72b9f41569d3263beda64f4c92da3f42344de6f3c2",
"timestamp": "2023-05-08T10:06:29.463752Z",
"utsname_machine": "x86_64",
"utsname_release": "6.2.14-200.fc37.x86_64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP PREEMPT_DYNAMIC Mon May 1 00:57:50 UTC 2023"
}</pre></p>
mgr - Bug #61141 (New): crash: File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py",...
https://tracker.ceph.com/issues/61141
2023-05-15T03:00:32Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=8108892b9b7fb2850a0a8c66ca233862ecfac1f6f270b9f8771e34a7e1abac6c">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=8108892b9b7fb2850a0a8c66ca233862ecfac1f6f270b9f8771e34a7e1abac6c</a></p>
<p>Sanitized backtrace:<br /><pre> File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: chunked=chunked)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in _make_request: self._validate_conn(conn)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in _validate_conn: conn.connect()
File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in connect: ssl_context=context)
File "/usr/lib/python3.6/site-packages/urllib3/util/ssl_.py", in ssl_wrap_socket: return context.wrap_socket(sock)
File "/usr/lib64/python3.6/ssl.py", in wrap_socket: _context=self, _session=session)
File "/usr/lib64/python3.6/ssl.py", in __init__: self.do_handshake()
File "/usr/lib64/python3.6/ssl.py", in do_handshake: self._sslobj.do_handshake()
File "/usr/lib64/python3.6/ssl.py", in do_handshake: self._sslobj.do_handshake()
OSError: [Errno 0]
During handling of the above exception, another exception
Traceback (most recent call
File "mgr/rook/module.py", in serve: self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py", in list_namespaced_pod: (data) = self.list_namespaced_pod_with_http_info(namespace, **kwargs) # noqa: E501
File "/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py", in list_namespaced_pod_with_http_info: collection_formats=collection_formats)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in call_api: _preload_content, _request_timeout)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in __call_api: _request_timeout=_request_timeout)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in request: headers=headers)
File "/usr/lib/python3.6/site-packages/kubernetes/client/rest.py", in GET: query_params=query_params)
File "/usr/lib/python3.6/site-packages/kubernetes/client/rest.py", in request: headers=headers)
File "/usr/lib/python3.6/site-packages/urllib3/request.py", in request: **urlopen_kw)
File "/usr/lib/python3.6/site-packages/urllib3/request.py", in request_encode_url: return self.urlopen(method, url, **extra_kw)
File "/usr/lib/python3.6/site-packages/urllib3/poolmanager.py", in urlopen: response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: _stacktrace=sys.exc_info()[2])
File "/usr/lib/python3.6/site-packages/urllib3/util/retry.py", in increment: raise MaxRetryError(_pool, url, error or ResponseError(cause))
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-05-08 08:22:22.815734",
"backtrace": [
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\n chunked=chunked)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 343, in _make_request\n self._validate_conn(conn)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 839, in _validate_conn\n conn.connect()",
" File \"/usr/lib/python3.6/site-packages/urllib3/connection.py\", line 358, in connect\n ssl_context=context)",
" File \"/usr/lib/python3.6/site-packages/urllib3/util/ssl_.py\", line 367, in ssl_wrap_socket\n return context.wrap_socket(sock)",
" File \"/usr/lib64/python3.6/ssl.py\", line 365, in wrap_socket\n _context=self, _session=session)",
" File \"/usr/lib64/python3.6/ssl.py\", line 776, in __init__\n self.do_handshake()",
" File \"/usr/lib64/python3.6/ssl.py\", line 1036, in do_handshake\n self._sslobj.do_handshake()",
" File \"/usr/lib64/python3.6/ssl.py\", line 648, in do_handshake\n self._sslobj.do_handshake()",
"OSError: [Errno 0] Error",
"\nDuring handling of the above exception, another exception occurred:\n",
"Traceback (most recent call last):",
" File \"/usr/share/ceph/mgr/rook/module.py\", line 195, in serve\n self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py\", line 12803, in list_namespaced_pod\n (data) = self.list_namespaced_pod_with_http_info(namespace, **kwargs) # noqa: E501",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py\", line 12905, in list_namespaced_pod_with_http_info\n collection_formats=collection_formats)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 345, in call_api\n _preload_content, _request_timeout)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 176, in __call_api\n _request_timeout=_request_timeout)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 366, in request\n headers=headers)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/rest.py\", line 241, in GET\n query_params=query_params)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/rest.py\", line 214, in request\n headers=headers)",
" File \"/usr/lib/python3.6/site-packages/urllib3/request.py\", line 68, in request\n **urlopen_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/request.py\", line 89, in request_encode_url\n return self.urlopen(method, url, **extra_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/poolmanager.py\", line 324, in urlopen\n response = conn.urlopen(method, u.request_uri, **kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\n _stacktrace=sys.exc_info()[2])",
" File \"/usr/lib/python3.6/site-packages/urllib3/util/retry.py\", line 399, in increment\n raise MaxRetryError(_pool, url, error or ResponseError(cause))",
"<redacted>"
],
"ceph_version": "17.2.5",
"crash_id": "2023-05-08T07:07:59.315920Z_d3b0b897-48ae-42b6-a10b-03c0f592dcba",
"entity_name": "mgr.921774b62f5dbd9a51e8d47d23567d10fbec0bcd",
"mgr_module": "rook",
"mgr_module_caller": "PyModuleRunner::serve",
"mgr_python_exception": "MaxRetryError",
"os_id": "centos",
"os_name": "CentOS Stream",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-mgr",
"stack_sig": "baacb966a37238ff502947f9227e01866165c66fb7b79ac8e3d6d05a4785e1d3",
"timestamp": "2023-05-08T07:07:59.315920Z",
"utsname_machine": "x86_64",
"utsname_release": "5.10.0-22-cloud-amd64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Debian 5.10.178-3 (2023-04-22)"
}</pre></p>
RADOS - Bug #61139 (New): crash: void ConnectionTracker::notify_rank_removed(int, int): assert(ra...
https://tracker.ceph.com/issues/61139
2023-05-15T03:00:29Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=ba4154b83e9e098001b354035b171995abb6e5df31c0a32ddd5fbdcb3ef35f54">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=ba4154b83e9e098001b354035b171995abb6e5df31c0a32ddd5fbdcb3ef35f54</a></p>
<p>Assert condition: rank == new_rank<br />Assert function: void ConnectionTracker::notify_rank_removed(int, int)</p>
<p>Sanitized backtrace:<br /><pre> ConnectionTracker::notify_rank_removed(int, int)
Elector::notify_rank_removed(int, int)
Monitor::notify_new_monmap(bool, bool)
Elector::dispatch(boost::intrusive_ptr<MonOpRequest>)
Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)
Monitor::_ms_dispatch(Message*)
Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)
DispatchQueue::entry()
DispatchQueue::DispatchThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "rank == new_rank",
"assert_file": "mon/ConnectionTracker.cc",
"assert_func": "void ConnectionTracker::notify_rank_removed(int, int)",
"assert_line": 252,
"assert_msg": "mon/ConnectionTracker.cc: In function 'void ConnectionTracker::notify_rank_removed(int, int)' thread 7ff7e6ea5700 time 2023-05-03T13:28:23.195969+0000\nmon/ConnectionTracker.cc: 252: FAILED ceph_assert(rank == new_rank)",
"assert_thread_name": "ms_dispatch",
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7ff7eeb5b420]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1b7) [0x7ff7ef02f8e0]",
"/usr/lib/ceph/libceph-common.so.2(+0x27da8a) [0x7ff7ef02fa8a]",
"(ConnectionTracker::notify_rank_removed(int, int)+0x13ba) [0x55ea7d05b9ca]",
"(Elector::notify_rank_removed(int, int)+0x57) [0x55ea7d047b47]",
"(Monitor::notify_new_monmap(bool, bool)+0x491) [0x55ea7cf73261]",
"(Elector::dispatch(boost::intrusive_ptr<MonOpRequest>)+0x104d) [0x55ea7d049cfd]",
"(Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0xf87) [0x55ea7cf8e9c7]",
"(Monitor::_ms_dispatch(Message*)+0xa68) [0x55ea7cf8fbd8]",
"(Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x5d) [0x55ea7cfc9c4d]",
"(Messenger::ms_deliver_dispatch(boost::intrusive_ptr<Message> const&)+0x460) [0x7ff7ef2cdda0]",
"(DispatchQueue::entry()+0x58f) [0x7ff7ef2cb63f]",
"(DispatchQueue::DispatchThread::entry()+0x11) [0x7ff7ef39cb61]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7ff7eeb4f609]",
"clone()"
],
"ceph_version": "17.2.6",
"crash_id": "2023-05-03T13:28:23.213147Z_a6e58cbe-557e-4b3b-bde3-fc5b054fa9b4",
"entity_name": "mon.deebf35037c20c3f658a23e462354e1e10fdbd10",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.6 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "ceph-mon",
"stack_sig": "8f4ea9d6415ee418312e4ba9c478982208b2fac1fda0e40b6f7468cffef5fe70",
"timestamp": "2023-05-03T13:28:23.213147Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-148-generic",
"utsname_sysname": "Linux",
"utsname_version": "#165-Ubuntu SMP Tue Apr 18 08:53:12 UTC 2023"
}</pre></p>
RADOS - Bug #61138 (New): crash: MOSDRepOp::encode_payload(unsigned long)
https://tracker.ceph.com/issues/61138
2023-05-15T03:00:28Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=152c9f3a4d3b4c7cf434838df292f4dda5d44f72ad7064b9c6f1424f0501bcdf">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=152c9f3a4d3b4c7cf434838df292f4dda5d44f72ad7064b9c6f1424f0501bcdf</a></p>
<p>Sanitized backtrace:<br /><pre> MOSDRepOp::encode_payload(unsigned long)
Message::encode(unsigned long, int, bool)
ProtocolV2::send_message(Message*)
AsyncConnection::send_message(Message*)
OSDService::send_message_osd_cluster(int, Message*, unsigned int)
ReplicatedBackend::issue_op(hobject_t const&, eversion_t const&, unsigned long, osd_reqid_t, eversion_t, eversion_t, hobject_t, hobject_t, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, std::optional<pg_hit_set_history_t>&, ReplicatedBackend::InProgressOp*, ceph::os::Transaction&)
ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> >&&, std::optional<pg_hit_set_history_t>&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>)
PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)
PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)
PrimaryLogPG::agent_maybe_evict(std::shared_ptr<ObjectContext>&, bool)
PrimaryLogPG::agent_work(int, int)
OSDService::agent_entry()
OSDService::AgentThread::entry()
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x13140) [0x7f195a81f140]",
"gsignal()",
"abort()",
"/lib/x86_64-linux-gnu/libc.so.6(+0x2240f) [0x7f195a2c240f]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x31662) [0x7f195a2d1662]",
"(MOSDRepOp::encode_payload(unsigned long)+0x2d0) [0x55eb1ddaa4e0]",
"(Message::encode(unsigned long, int, bool)+0x2a) [0x55eb1e20cc7a]",
"(ProtocolV2::send_message(Message*)+0x25e) [0x55eb1e489aae]",
"(AsyncConnection::send_message(Message*)+0x186) [0x55eb1e463c76]",
"(OSDService::send_message_osd_cluster(int, Message*, unsigned int)+0x1ae) [0x55eb1d9ad44e]",
"(ReplicatedBackend::issue_op(hobject_t const&, eversion_t const&, unsigned long, osd_reqid_t, eversion_t, eversion_t, hobject_t, hobject_t, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, std::optional<pg_hit_set_history_t>&, ReplicatedBackend::InProgressOp*, ceph::os::Transaction&)+0x649) [0x55eb1dd90709]",
"(ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr<PGTransaction, std::default_delete<PGTransaction> >&&, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> >&&, std::optional<pg_hit_set_history_t>&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr<OpRequest>)+0x5d4) [0x55eb1dd93c74]",
"(PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0x50d) [0x55eb1dae661d]",
"(PrimaryLogPG::simple_opc_submit(std::unique_ptr<PrimaryLogPG::OpContext, std::default_delete<PrimaryLogPG::OpContext> >)+0x56) [0x55eb1dae82b6]",
"(PrimaryLogPG::agent_maybe_evict(std::shared_ptr<ObjectContext>&, bool)+0x841) [0x55eb1db14da1]",
"(PrimaryLogPG::agent_work(int, int)+0x10b0) [0x55eb1db1a840]",
"(OSDService::agent_entry()+0x19b) [0x55eb1d9e707b]",
"(OSDService::AgentThread::entry()+0xd) [0x55eb1da4ddbd]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x7ea7) [0x7f195a813ea7]",
"clone()"
],
"ceph_version": "17.2.5",
"crash_id": "2023-05-04T15:01:12.568204Z_78aaad7d-5b85-4385-9b4e-3fb4e25d68af",
"entity_name": "osd.31c7d8c587cef38206d8a10fb04724c9fa455a18",
"os_id": "11",
"os_name": "Debian GNU/Linux 11 (bullseye)",
"os_version": "11 (bullseye)",
"os_version_id": "11",
"process_name": "ceph-osd",
"stack_sig": "acf991f216ba25d59c5cc83ce2cf79f65e2e54e4555a2a42d3fcbb58b6d82b55",
"timestamp": "2023-05-04T15:01:12.568204Z",
"utsname_machine": "x86_64",
"utsname_release": "5.13.19-6-pve",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP PVE 5.13.19-15 (Tue, 29 Mar 2022 15:59:50 +0200)"
}</pre></p>
bluestore - Bug #61137 (New): crash: rocksdb::Slice::Slice(char const*, unsigned long)
https://tracker.ceph.com/issues/61137
2023-05-15T03:00:27Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=7fc54fa6109bd26c25208bb301dbdb276a55e5c711a54bc88d61efbccd45215a">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=7fc54fa6109bd26c25208bb301dbdb276a55e5c711a54bc88d61efbccd45215a</a></p>
<p>Sanitized backtrace:<br /><pre> rocksdb::Slice::Slice(char const*, unsigned long)
rocksdb::ExtractUserKey(rocksdb::Slice const&)
rocksdb::InternalKeyComparator::Compare(rocksdb::Slice const&, rocksdb::Slice const&) const
rocksdb::MinIteratorComparator::operator()(rocksdb::IteratorWrapperBase<rocksdb::Slice>*, rocksdb::IteratorWrapperBase<rocksdb::Slice>*) const
rocksdb::BinaryHeap<rocksdb::IteratorWrapperBase<rocksdb::Slice>*, rocksdb::MinIteratorComparator>::downheap(unsigned long)
rocksdb::BinaryHeap<rocksdb::IteratorWrapperBase<rocksdb::Slice>*, rocksdb::MinIteratorComparator>::replace_top(rocksdb::IteratorWrapperBase<rocksdb::Slice>* const&)
rocksdb::MergingIterator::Next()
rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)
rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()
rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*)
rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*)
rocksdb::DBIter::Next()
rocksdb::ArenaWrappedDBIter::Next()
RocksDBStore::RocksDBWholeSpaceIteratorImpl::next()
BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)
PGBackend::objects_list_partial(hobject_t const&, int, int, std::vector<hobject_t, std::allocator<hobject_t> >*, hobject_t*)
PrimaryLogPG::scan_range(int, int, BackfillInterval*, ThreadPool::TPHandle&)
PrimaryLogPG::do_scan(boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)
PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)
OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)
ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
ShardedThreadPool::shardedthreadpool_worker(unsigned int)
ShardedThreadPool::WorkThreadSharded::entry()
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fa668a38420]",
"(rocksdb::Slice::Slice(char const*, unsigned long)+0x20) [0x5579ae9e2970]",
"(rocksdb::ExtractUserKey(rocksdb::Slice const&)+0x82) [0x5579aea103b7]",
"(rocksdb::InternalKeyComparator::Compare(rocksdb::Slice const&, rocksdb::Slice const&) const+0x3c) [0x5579aeb01694]",
"(rocksdb::MinIteratorComparator::operator()(rocksdb::IteratorWrapperBase<rocksdb::Slice>*, rocksdb::IteratorWrapperBase<rocksdb::Slice>*) const+0x7b) [0x5579aed3e1d1]",
"(rocksdb::BinaryHeap<rocksdb::IteratorWrapperBase<rocksdb::Slice>*, rocksdb::MinIteratorComparator>::downheap(unsigned long)+0x174) [0x5579aed41122]",
"(rocksdb::BinaryHeap<rocksdb::IteratorWrapperBase<rocksdb::Slice>*, rocksdb::MinIteratorComparator>::replace_top(rocksdb::IteratorWrapperBase<rocksdb::Slice>* const&)+0x77) [0x5579aed403c3]",
"(rocksdb::MergingIterator::Next()+0x14e) [0x5579aed3f314]",
"(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x2c) [0x5579aed3f3e0]",
"(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x75) [0x5579aeafc059]",
"(rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*)+0x12eb) [0x5579aeaf2f25]",
"(rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*)+0x83) [0x5579aeaf1bef]",
"(rocksdb::DBIter::Next()+0x2f9) [0x5579aeaf1a4d]",
"(rocksdb::ArenaWrappedDBIter::Next()+0x20) [0x5579aedfd372]",
"(RocksDBStore::RocksDBWholeSpaceIteratorImpl::next()+0x32) [0x5579ae9a8ff2]",
"/usr/bin/ceph-osd(+0x129d086) [0x5579ae322086]",
"(BlueStore::_collection_list(BlueStore::Collection*, ghobject_t const&, ghobject_t const&, int, bool, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0x1895) [0x5579ae366535]",
"(BlueStore::collection_list(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, ghobject_t const&, int, std::vector<ghobject_t, std::allocator<ghobject_t> >*, ghobject_t*)+0xbf) [0x5579ae3679ff]",
"(PGBackend::objects_list_partial(hobject_t const&, int, int, std::vector<hobject_t, std::allocator<hobject_t> >*, hobject_t*)+0xdc2) [0x5579adfa4cc2]",
"(PrimaryLogPG::scan_range(int, int, BackfillInterval*, ThreadPool::TPHandle&)+0x100) [0x5579adf2fb50]",
"(PrimaryLogPG::do_scan(boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x8e5) [0x5579adf31205]",
"(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x684) [0x5579adf31f14]",
"(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x1c0) [0x5579add7cca0]",
"(ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x6a) [0x5579ae099a1a]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x935) [0x5579adda9105]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x403) [0x5579ae5398b3]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x5579ae53c8f4]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7fa668a2c609]",
"clone()"
],
"ceph_version": "17.2.5",
"crash_id": "2023-05-03T07:59:42.749515Z_2254a193-b508-475c-88c6-69d4e1a3e96a",
"entity_name": "osd.12ff32981d78a9142d29bb45d450dd1b51493352",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.5 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "ceph-osd",
"stack_sig": "9e5343640d5cad3824db9feafafcabd9c1baa851a30d64d1857351d7110291f8",
"timestamp": "2023-05-03T07:59:42.749515Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-144-generic",
"utsname_sysname": "Linux",
"utsname_version": "#161-Ubuntu SMP Fri Feb 3 14:49:04 UTC 2023"
}</pre></p>
rgw - Bug #61136 (New): crash: tcmalloc::allocate_full_cpp_throw_oom(unsigned long)
https://tracker.ceph.com/issues/61136
2023-05-15T03:00:25Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=387321a62cfa06d39eded6c7b263b3be1cdd0e36c389a87ca68aab2ea9b158fe">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=387321a62cfa06d39eded6c7b263b3be1cdd0e36c389a87ca68aab2ea9b158fe</a></p>
<p>Sanitized backtrace:<br /><pre> tcmalloc::allocate_full_cpp_throw_oom(unsigned long)
RGWSI_Notify::unwatch(RGWSI_RADOS::Obj&, unsigned long)
RGWSI_Notify::finalize_watch()
RGWServices_Def::shutdown()
RGWRados::finalize()
StoreManager::close_storage(rgw::sal::Store*)
radosgw_Main(int, char const**)
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-05-02 19:43:44.952701",
"backtrace": [
"/lib64/libpthread.so.0(+0x12cf0) [0x7f66b4763cf0]",
"gsignal()",
"abort()",
"/lib64/libstdc++.so.6(+0x9009b) [0x7f66b375309b]",
"/lib64/libstdc++.so.6(+0x9653c) [0x7f66b375953c]",
"/lib64/libstdc++.so.6(+0x96597) [0x7f66b3759597]",
"/lib64/libstdc++.so.6(+0x967f8) [0x7f66b37597f8]",
"/lib64/libtcmalloc.so.4(+0x19fa4) [0x7f66b5a00fa4]",
"(tcmalloc::allocate_full_cpp_throw_oom(unsigned long)+0x146) [0x7f66b5a22c96]",
"/lib64/librados.so.2(+0x6fb52) [0x7f66b7f8ab52]",
"/lib64/librados.so.2(+0xbcd29) [0x7f66b7fd7d29]",
"/lib64/librados.so.2(+0xaef3a) [0x7f66b7fc9f3a]",
"/lib64/librados.so.2(+0xaf9e9) [0x7f66b7fca9e9]",
"(RGWSI_Notify::unwatch(RGWSI_RADOS::Obj&, unsigned long)+0x30) [0x7f66b752a520]",
"(RGWSI_Notify::finalize_watch()+0x40) [0x7f66b752b4f0]",
"/lib64/libradosgw.so.2(+0xe1c555) [0x7f66b752b555]",
"(RGWServices_Def::shutdown()+0x4a) [0x7f66b6f35e1a]",
"(RGWRados::finalize()+0x145) [0x7f66b728c5d5]",
"(StoreManager::close_storage(rgw::sal::Store*)+0x16) [0x7f66b738b2f6]",
"(radosgw_Main(int, char const**)+0x4bca) [0x7f66b6eb441a]",
"__libc_start_main()",
"_start()"
],
"ceph_version": "17.2.6",
"crash_id": "2023-05-02T19:37:23.501486Z_bcdce274-72fb-47ea-b017-357ff2133b6f",
"entity_name": "client.6be05bba6a525195eb1e7c7fdab1b4d06f80b22b",
"os_id": "centos",
"os_name": "CentOS Stream",
"os_version": "8",
"os_version_id": "8",
"process_name": "radosgw",
"stack_sig": "b1ee9d73db53426d18182b6517d82d3cdb91bcf126e2375528d31ae968aa3064",
"timestamp": "2023-05-02T19:37:23.501486Z",
"utsname_machine": "x86_64",
"utsname_release": "5.10.0-21-amd64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Debian 5.10.162-1 (2023-01-21)"
}</pre></p>
mgr - Bug #61135 (New): crash: File "mgr/snap_schedule/module.py", in __init__: self.client = Sna...
https://tracker.ceph.com/issues/61135
2023-05-15T03:00:23Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2dd736b7cd7afdcd429049f05783fe9684331538c72645ecf0884b30ca3e5014">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2dd736b7cd7afdcd429049f05783fe9684331538c72645ecf0884b30ca3e5014</a></p>
<p>Sanitized backtrace:<br /><pre> File "mgr/snap_schedule/module.py", in __init__: self.client = SnapSchedClient(self)
File "mgr/snap_schedule/fs/schedule_client.py", in __init__: with self.get_schedule_db(fs_name) as conn_mgr:
File "mgr/snap_schedule/fs/schedule_client.py", in get_schedule_db: assert poolid, f'fs "{fs}" not found'
</pre><br />Crash dump sample:<br /><pre>{
"archived": "2023-04-28 12:03:53.408902",
"backtrace": [
" File \"/usr/share/ceph/mgr/snap_schedule/module.py\", line 38, in __init__\n self.client = SnapSchedClient(self)",
" File \"/usr/share/ceph/mgr/snap_schedule/fs/schedule_client.py\", line 169, in __init__\n with self.get_schedule_db(fs_name) as conn_mgr:",
" File \"/usr/share/ceph/mgr/snap_schedule/fs/schedule_client.py\", line 188, in get_schedule_db\n assert poolid, f'fs \"{fs}\" not found'",
"<redacted>"
],
"ceph_version": "17.2.5",
"crash_id": "2023-04-28T07:54:06.645788Z_ad4bdfdf-76a7-4691-b581-ab8f1d449666",
"entity_name": "mgr.4805a613b2c50f1df82ca403c4207ff34fcb41fc",
"mgr_module": "snap_schedule",
"mgr_module_caller": "ActivePyModule::load",
"mgr_python_exception": "AssertionError",
"os_id": "11",
"os_name": "Debian GNU/Linux 11 (bullseye)",
"os_version": "11 (bullseye)",
"os_version_id": "11",
"process_name": "ceph-mgr",
"stack_sig": "49bbf6fc120c8305de543da3703aca04c1a31cb576cc1c427c8a96dd0a610494",
"timestamp": "2023-04-28T07:54:06.645788Z",
"utsname_machine": "x86_64",
"utsname_release": "5.15.104-1-pve",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP PVE 5.15.104-1 (2023-03-29T15:51Z)"
}</pre></p>
bluestore - Bug #61134 (New): crash: int64_t BlueFS::_read_random(BlueFS::FileReader*, uint64_t, ...
https://tracker.ceph.com/issues/61134
2023-05-15T03:00:22Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=aca178d9daa3aa999ba91c87545da93f40c5b0a34d1a358c4a960be4a3b13b52">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=aca178d9daa3aa999ba91c87545da93f40c5b0a34d1a358c4a960be4a3b13b52</a></p>
<p>Assert condition: r == 0<br />Assert function: int64_t BlueFS::_read_random(BlueFS::FileReader*, uint64_t, uint64_t, char*)</p>
<p>Sanitized backtrace:<br /><pre> BlueFS::_read_random(BlueFS::FileReader*, unsigned long, unsigned long, char*)
BlueRocksRandomAccessFile::Read(unsigned long, unsigned long, rocksdb::Slice*, char*) const
rocksdb::LegacyRandomAccessFileWrapper::Read(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::Slice*, char*, rocksdb::IODebugContext*) const
rocksdb::RandomAccessFileReader::Read(rocksdb::IOOptions const&, unsigned long, unsigned long, rocksdb::Slice*, char*, std::unique_ptr<char [], std::default_delete<char []> >*, bool) const
rocksdb::FilePrefetchBuffer::Prefetch(rocksdb::IOOptions const&, rocksdb::RandomAccessFileReader*, unsigned long, unsigned long, bool)
rocksdb::FilePrefetchBuffer::TryReadFromCache(rocksdb::IOOptions const&, unsigned long, unsigned long, rocksdb::Slice*, bool)
rocksdb::BlockFetcher::ReadBlockContents()
rocksdb::Status rocksdb::BlockBasedTable::RetrieveBlock<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, bool, bool) const
rocksdb::BlockBasedTableIterator::InitDataBlock()
rocksdb::BlockBasedTableIterator::FindBlockForward()
rocksdb::BlockBasedTableIterator::Next()
rocksdb::BlockBasedTableIterator::NextAndGetResult(rocksdb::IterateResult*)
rocksdb::MergingIterator::Next()
rocksdb::CompactionIterator::NextFromInput()
rocksdb::CompactionIterator::Next()
rocksdb::CompactionJob::ProcessKeyValueCompaction(rocksdb::CompactionJob::SubcompactionState*)
rocksdb::CompactionJob::Run()
rocksdb::DBImpl::BackgroundCompaction(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::DBImpl::PrepickedCompaction*, rocksdb::Env::Priority)
rocksdb::DBImpl::BackgroundCallCompaction(rocksdb::DBImpl::PrepickedCompaction*, rocksdb::Env::Priority)
rocksdb::DBImpl::BGWorkCompaction(void*)
rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)
rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)
</pre><br />Crash dump sample:<br /><pre>{
"assert_condition": "r == 0",
"assert_file": "os/bluestore/BlueFS.cc",
"assert_func": "int64_t BlueFS::_read_random(BlueFS::FileReader*, uint64_t, uint64_t, char*)",
"assert_line": 2103,
"assert_msg": "os/bluestore/BlueFS.cc: In function 'int64_t BlueFS::_read_random(BlueFS::FileReader*, uint64_t, uint64_t, char*)' thread 7f6678194700 time 2023-05-04T03:22:04.486901+0000\nos/bluestore/BlueFS.cc: 2103: FAILED ceph_assert(r == 0)",
"assert_thread_name": "rocksdb:low0",
"backtrace": [
"/lib64/libpthread.so.0(+0x12cf0) [0x7f6687db8cf0]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x55ec876aa729]",
"ceph-osd(+0x598895) [0x55ec876aa895]",
"(BlueFS::_read_random(BlueFS::FileReader*, unsigned long, unsigned long, char*)+0xe48) [0x55ec87e8ae68]",
"(BlueRocksRandomAccessFile::Read(unsigned long, unsigned long, rocksdb::Slice*, char*) const+0x24) [0x55ec87e9ae84]",
"(rocksdb::LegacyRandomAccessFileWrapper::Read(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::Slice*, char*, rocksdb::IODebugContext*) const+0x29) [0x55ec883cc449]",
"(rocksdb::RandomAccessFileReader::Read(rocksdb::IOOptions const&, unsigned long, unsigned long, rocksdb::Slice*, char*, std::unique_ptr<char [], std::default_delete<char []> >*, bool) const+0xce4) [0x55ec886277a4]",
"(rocksdb::FilePrefetchBuffer::Prefetch(rocksdb::IOOptions const&, rocksdb::RandomAccessFileReader*, unsigned long, unsigned long, bool)+0x117) [0x55ec886266f7]",
"(rocksdb::FilePrefetchBuffer::TryReadFromCache(rocksdb::IOOptions const&, unsigned long, unsigned long, rocksdb::Slice*, bool)+0x97) [0x55ec88626a27]",
"(rocksdb::BlockFetcher::ReadBlockContents()+0xc4c) [0x55ec885713bc]",
"ceph-osd(+0x14300f5) [0x55ec885420f5]",
"(rocksdb::Status rocksdb::BlockBasedTable::RetrieveBlock<rocksdb::Block>(rocksdb::FilePrefetchBuffer*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::UncompressionDict const&, rocksdb::CachableEntry<rocksdb::Block>*, rocksdb::BlockType, rocksdb::GetContext*, rocksdb::BlockCacheLookupContext*, bool, bool) const+0x1af) [0x55ec8855a06f]",
"(rocksdb::BlockBasedTableIterator::InitDataBlock()+0x25a) [0x55ec88643f0a]",
"(rocksdb::BlockBasedTableIterator::FindBlockForward()+0x21d) [0x55ec88644fad]",
"(rocksdb::BlockBasedTableIterator::Next()+0x3a) [0x55ec8864517a]",
"(rocksdb::BlockBasedTableIterator::NextAndGetResult(rocksdb::IterateResult*)+0x1a) [0x55ec886451ba]",
"ceph-osd(+0x138ec21) [0x55ec884a0c21]",
"(rocksdb::MergingIterator::Next()+0x32) [0x55ec8857a862]",
"(rocksdb::CompactionIterator::NextFromInput()+0x869) [0x55ec885f28e9]",
"(rocksdb::CompactionIterator::Next()+0x19a) [0x55ec885f340a]",
"(rocksdb::CompactionJob::ProcessKeyValueCompaction(rocksdb::CompactionJob::SubcompactionState*)+0x80e) [0x55ec88602e3e]",
"(rocksdb::CompactionJob::Run()+0x33c) [0x55ec8860613c]",
"(rocksdb::DBImpl::BackgroundCompaction(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::DBImpl::PrepickedCompaction*, rocksdb::Env::Priority)+0x1031) [0x55ec883f92a1]",
"(rocksdb::DBImpl::BackgroundCallCompaction(rocksdb::DBImpl::PrepickedCompaction*, rocksdb::Env::Priority)+0xd7) [0x55ec883fdb37]",
"(rocksdb::DBImpl::BGWorkCompaction(void*)+0x3b) [0x55ec883fe21b]",
"(rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x259) [0x55ec885a8179]",
"(rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x5e) [0x55ec885a837e]",
"/lib64/libstdc++.so.6(+0xc2b13) [0x7f6687400b13]",
"/lib64/libpthread.so.0(+0x81ca) [0x7f6687dae1ca]"
],
"ceph_version": "17.2.6",
"crash_id": "2023-05-04T03:22:04.533470Z_9b73bb9e-09d8-4ff0-a0b6-f1d43628ce7d",
"entity_name": "osd.9b82654d240a369091aa604d5c6b7839460c6c92",
"os_id": "centos",
"os_name": "CentOS Stream",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-osd",
"stack_sig": "fd9f81d4edaba8c8160e3c209f3b4ae4a83edf844fc2d7e0b7d6fdce6f351d1f",
"timestamp": "2023-05-04T03:22:04.533470Z",
"utsname_machine": "x86_64",
"utsname_release": "3.10.0-1160.88.1.el7.x86_64",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Tue Mar 7 15:41:52 UTC 2023"
}</pre></p>
bluestore - Bug #61133 (New): crash: pread64()
https://tracker.ceph.com/issues/61133
2023-05-15T03:00:21Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=bb1e0f509ab037eedbbe07dcb03b6e1da1d202c81ce4bf9febd05e2767d5948d">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=bb1e0f509ab037eedbbe07dcb03b6e1da1d202c81ce4bf9febd05e2767d5948d</a></p>
<p>Sanitized backtrace:<br /><pre> pread64()
KernelDevice::read(unsigned long, unsigned long, ceph::buffer::list*, IOContext*, bool)
BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::list*, char*)
BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)
rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)
rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const
rocksdb::BlockPrefetcher::PrefetchIfNeeded(rocksdb::BlockBasedTable::Rep const*, rocksdb::BlockHandle const&, unsigned long, bool)
rocksdb::BlockBasedTableIterator::InitDataBlock()
rocksdb::BlockBasedTableIterator::FindBlockForward()
rocksdb::BlockBasedTableIterator::FindKeyForward()
rocksdb::BlockBasedTableIterator::Next()
rocksdb::BlockBasedTableIterator::NextAndGetResult(rocksdb::IterateResult*)
rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()
rocksdb::MergingIterator::Next()
rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)
rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()
rocksdb::DBIter::Next()
rocksdb::ArenaWrappedDBIter::Next()
RocksDBStore::RocksDBTransactionImpl::rm_range_keys(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
BlueStore::_do_omap_clear(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Onode>&)
BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)
BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)
BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)
BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)
ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)
OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr<OSDMap const>, ThreadPool::TPHandle*)
OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)
ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
ShardedThreadPool::shardedthreadpool_worker(unsigned int)
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f491a26e420]",
"pread64()",
"(KernelDevice::read(unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, IOContext*, bool)+0x109) [0x55c273d41359]",
"(BlueFS::_read(BlueFS::FileReader*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, char*)+0x560) [0x55c273845a60]",
"(BlueRocksRandomAccessFile::Prefetch(unsigned long, unsigned long)+0x1f) [0x55c27386d26f]",
"(rocksdb::LegacyRandomAccessFileWrapper::Prefetch(unsigned long, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x60) [0x55c273dccd54]",
"(rocksdb::RandomAccessFileReader::Prefetch(unsigned long, unsigned long) const+0x75) [0x55c2740b211f]",
"(rocksdb::BlockPrefetcher::PrefetchIfNeeded(rocksdb::BlockBasedTable::Rep const*, rocksdb::BlockHandle const&, unsigned long, bool)+0x168) [0x55c27426e2b8]",
"(rocksdb::BlockBasedTableIterator::InitDataBlock()+0x171) [0x55c27426c079]",
"(rocksdb::BlockBasedTableIterator::FindBlockForward()+0x281) [0x55c27426c777]",
"(rocksdb::BlockBasedTableIterator::FindKeyForward()+0x92) [0x55c27426c4f2]",
"(rocksdb::BlockBasedTableIterator::Next()+0x8c) [0x55c27426bd76]",
"(rocksdb::BlockBasedTableIterator::NextAndGetResult(rocksdb::IterateResult*)+0x21) [0x55c27426bda9]",
"(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x75) [0x55c273eaf059]",
"(rocksdb::MergingIterator::Next()+0xc2) [0x55c2740f2288]",
"(rocksdb::MergingIterator::NextAndGetResult(rocksdb::IterateResult*)+0x2c) [0x55c2740f23e0]",
"(rocksdb::IteratorWrapperBase<rocksdb::Slice>::Next()+0x75) [0x55c273eaf059]",
"(rocksdb::DBIter::Next()+0x1ef) [0x55c273ea4943]",
"(rocksdb::ArenaWrappedDBIter::Next()+0x20) [0x55c2741b0372]",
"(RocksDBStore::RocksDBTransactionImpl::rm_range_keys(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1c3) [0x55c273d754d3]",
"(BlueStore::_do_omap_clear(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Onode>&)+0xdb) [0x55c27370eacb]",
"(BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0x10b5) [0x55c273740975]",
"(BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x89) [0x55c2737414e9]",
"(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x19ab) [0x55c27376e59b]",
"(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2fd) [0x55c27377482d]",
"(ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x87) [0x55c2731a7997]",
"(OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr<OSDMap const>, ThreadPool::TPHandle*)+0x1f3) [0x55c27312b393]",
"(OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x3a5) [0x55c2731721f5]",
"(ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x56) [0x55c27344cbe6]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x935) [0x55c27315c105]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x403) [0x55c2738ec8b3]"
],
"ceph_version": "17.2.5",
"crash_id": "2023-05-02T13:37:59.937766Z_8de7eb28-d695-4010-818e-ed16efe75d89",
"entity_name": "osd.ab3ce4841c1f4a745439e8b8fe5c6967d215979f",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.5 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "ceph-osd",
"stack_sig": "651adbdb495b9ef0d01e8487d37f4e1c7e7397e9fd5ddc3580e5ad470f2fbeac",
"timestamp": "2023-05-02T13:37:59.937766Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-146-generic",
"utsname_sysname": "Linux",
"utsname_version": "#163-Ubuntu SMP Fri Mar 17 18:26:02 UTC 2023"
}</pre></p>
bluestore - Bug #61132 (New): crash: rocksdb::IteratorWrapperBase<rocksdb::Slice>::PrepareValue()
https://tracker.ceph.com/issues/61132
2023-05-15T03:00:19Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=10dd9a6108bc31504a9a6ed8ed65ca7c09033d261bbb1abbb6a016c76c3b6eaf">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=10dd9a6108bc31504a9a6ed8ed65ca7c09033d261bbb1abbb6a016c76c3b6eaf</a></p>
<p>Sanitized backtrace:<br /><pre> rocksdb::IteratorWrapperBase<rocksdb::Slice>::PrepareValue()
rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*)
rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*)
rocksdb::DBIter::Next()
rocksdb::ArenaWrappedDBIter::Next()
RocksDBStore::RocksDBTransactionImpl::rm_range_keys(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
BlueStore::_do_omap_clear(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Onode>&)
BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)
BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)
BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)
BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)
ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)
OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr<OSDMap const>, ThreadPool::TPHandle*)
OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)
ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
ShardedThreadPool::shardedthreadpool_worker(unsigned int)
ShardedThreadPool::WorkThreadSharded::entry()
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fd7c02cd420]",
"(rocksdb::IteratorWrapperBase<rocksdb::Slice>::PrepareValue()+0x17) [0x55aa23fa117f]",
"(rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*)+0x5b0) [0x55aa23f971ea]",
"(rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*)+0x83) [0x55aa23f96bef]",
"(rocksdb::DBIter::Next()+0x2f9) [0x55aa23f96a4d]",
"(rocksdb::ArenaWrappedDBIter::Next()+0x20) [0x55aa242a2372]",
"(RocksDBStore::RocksDBTransactionImpl::rm_range_keys(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1c3) [0x55aa23e674d3]",
"(BlueStore::_do_omap_clear(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Onode>&)+0xdb) [0x55aa23800acb]",
"(BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0x10b5) [0x55aa23832975]",
"(BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x89) [0x55aa238334e9]",
"(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x19ab) [0x55aa2386059b]",
"(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2fd) [0x55aa2386682d]",
"(ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x87) [0x55aa23299997]",
"(OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr<OSDMap const>, ThreadPool::TPHandle*)+0x1f3) [0x55aa2321d393]",
"(OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x3a5) [0x55aa232641f5]",
"(ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x56) [0x55aa2353ebe6]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x935) [0x55aa2324e105]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x403) [0x55aa239de8b3]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x55aa239e18f4]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7fd7c02c1609]",
"clone()"
],
"ceph_version": "17.2.5",
"crash_id": "2023-05-02T11:49:43.560661Z_635596c2-f2a1-4234-8853-70b5a926b67c",
"entity_name": "osd.ab3ce4841c1f4a745439e8b8fe5c6967d215979f",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.5 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "ceph-osd",
"stack_sig": "57a2d91f49e64e6569e641bd812e2610628002e5cb7193218ce43bd5c4c06eff",
"timestamp": "2023-05-02T11:49:43.560661Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-146-generic",
"utsname_sysname": "Linux",
"utsname_version": "#163-Ubuntu SMP Fri Mar 17 18:26:02 UTC 2023"
}</pre></p>
mgr - Bug #61131 (New): crash: File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in ...
https://tracker.ceph.com/issues/61131
2023-05-15T03:00:18Z
Telemetry Bot
<p><a class="external" href="http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=0ac99a32d4de148143c090afe8417494c48cec9581c14364fd05a89c7d95ebb4">http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=0ac99a32d4de148143c090afe8417494c48cec9581c14364fd05a89c7d95ebb4</a></p>
<p>Sanitized backtrace:<br /><pre> File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in _new_conn: (self._dns_host, self.port), self.timeout, **extra_kw)
File "/usr/lib/python3.6/site-packages/urllib3/util/connection.py", in create_connection: raise err
File "/usr/lib/python3.6/site-packages/urllib3/util/connection.py", in create_connection: sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection
During handling of the above exception, another exception
Traceback (most recent call
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: chunked=chunked)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in _make_request: self._validate_conn(conn)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in _validate_conn: conn.connect()
File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in connect: conn = self._new_conn()
File "/usr/lib/python3.6/site-packages/urllib3/connection.py", in _new_conn: self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x7f6d05a8b080>: Failed to establish a new connection: [Errno 111] Connection
During handling of the above exception, another exception
Traceback (most recent call
File "mgr/rook/module.py", in serve: self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py", in list_namespaced_pod: (data) = self.list_namespaced_pod_with_http_info(namespace, **kwargs) # noqa: E501
File "/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py", in list_namespaced_pod_with_http_info: collection_formats=collection_formats)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in call_api: _preload_content, _request_timeout)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in __call_api: _request_timeout=_request_timeout)
File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", in request: headers=headers)
File "/usr/lib/python3.6/site-packages/kubernetes/client/rest.py", in GET: query_params=query_params)
File "/usr/lib/python3.6/site-packages/kubernetes/client/rest.py", in request: headers=headers)
File "/usr/lib/python3.6/site-packages/urllib3/request.py", in request: **urlopen_kw)
File "/usr/lib/python3.6/site-packages/urllib3/request.py", in request_encode_url: return self.urlopen(method, url, **extra_kw)
File "/usr/lib/python3.6/site-packages/urllib3/poolmanager.py", in urlopen: response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: **response_kw)
File "/usr/lib/python3.6/site-packages/urllib3/connectionpool.py", in urlopen: _stacktrace=sys.exc_info()[2])
File "/usr/lib/python3.6/site-packages/urllib3/util/retry.py", in increment: raise MaxRetryError(_pool, url, error or ResponseError(cause))
</pre><br />Crash dump sample:<br /><pre>{
"backtrace": [
" File \"/usr/lib/python3.6/site-packages/urllib3/connection.py\", line 162, in _new_conn\n (self._dns_host, self.port), self.timeout, **extra_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/util/connection.py\", line 80, in create_connection\n raise err",
" File \"/usr/lib/python3.6/site-packages/urllib3/util/connection.py\", line 70, in create_connection\n sock.connect(sa)",
"ConnectionRefusedError: [Errno 111] Connection refused",
"\nDuring handling of the above exception, another exception occurred:\n",
"Traceback (most recent call last):",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\n chunked=chunked)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 343, in _make_request\n self._validate_conn(conn)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 839, in _validate_conn\n conn.connect()",
" File \"/usr/lib/python3.6/site-packages/urllib3/connection.py\", line 315, in connect\n conn = self._new_conn()",
" File \"/usr/lib/python3.6/site-packages/urllib3/connection.py\", line 171, in _new_conn\n self, \"Failed to establish a new connection: %s\" % e)",
"urllib3.exceptions.NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x7f6d05a8b080>: Failed to establish a new connection: [Errno 111] Connection refused",
"\nDuring handling of the above exception, another exception occurred:\n",
"Traceback (most recent call last):",
" File \"/usr/share/ceph/mgr/rook/module.py\", line 195, in serve\n self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py\", line 12803, in list_namespaced_pod\n (data) = self.list_namespaced_pod_with_http_info(namespace, **kwargs) # noqa: E501",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py\", line 12905, in list_namespaced_pod_with_http_info\n collection_formats=collection_formats)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 345, in call_api\n _preload_content, _request_timeout)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 176, in __call_api\n _request_timeout=_request_timeout)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py\", line 366, in request\n headers=headers)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/rest.py\", line 241, in GET\n query_params=query_params)",
" File \"/usr/lib/python3.6/site-packages/kubernetes/client/rest.py\", line 214, in request\n headers=headers)",
" File \"/usr/lib/python3.6/site-packages/urllib3/request.py\", line 68, in request\n **urlopen_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/request.py\", line 89, in request_encode_url\n return self.urlopen(method, url, **extra_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/poolmanager.py\", line 324, in urlopen\n response = conn.urlopen(method, u.request_uri, **kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 667, in urlopen\n **response_kw)",
" File \"/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\n _stacktrace=sys.exc_info()[2])",
" File \"/usr/lib/python3.6/site-packages/urllib3/util/retry.py\", line 399, in increment\n raise MaxRetryError(_pool, url, error or ResponseError(cause))",
"<redacted>"
],
"ceph_version": "17.2.6",
"crash_id": "2023-05-02T10:32:32.466217Z_7b1c3acc-bd25-4c9c-8576-f1943bf4b34e",
"entity_name": "mgr.7e2b35ba789a5cafbc84e1babdbae75ebc270300",
"mgr_module": "rook",
"mgr_module_caller": "PyModuleRunner::serve",
"mgr_python_exception": "MaxRetryError",
"os_id": "centos",
"os_name": "CentOS Stream",
"os_version": "8",
"os_version_id": "8",
"process_name": "ceph-mgr",
"stack_sig": "00b116c1a146dbc8f779a8229708f0464162cfea7ecf607d91577cb8f813dd16",
"timestamp": "2023-05-02T10:32:32.466217Z",
"utsname_machine": "x86_64",
"utsname_release": "5.4.0-148-generic",
"utsname_sysname": "Linux",
"utsname_version": "#165-Ubuntu SMP Tue Apr 18 08:53:12 UTC 2023"
}</pre></p>