Project

General

Profile

Support #38995 ยป api-ceph.log

Log from dask.dataframe.to_parquet - dpetrov dpetrov, 03/28/2019 07:07 AM

 
---------------------------------------------------------------------------
timeout Traceback (most recent call last)
~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
599 body=body, headers=headers,
--> 600 chunked=chunked)
601

~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
353 else:
--> 354 conn.request(method, url, **httplib_request_kw)
355

~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
1238 """Send a complete request to the server."""
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
1240

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_request(self, method, url, body, headers, *args, **kwargs)
124 rval = super(AWSConnection, self)._send_request(
--> 125 method, url, body, headers, *args, **kwargs)
126 self._expect_header_set = False

~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
1284 body = _encode(body, 'body')
-> 1285 self.endheaders(body, encode_chunked=encode_chunked)
1286

~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in endheaders(self, message_body, encode_chunked)
1233 raise CannotSendHeader()
-> 1234 self._send_output(message_body, encode_chunked=encode_chunked)
1235

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_output(self, message_body, *args, **kwargs)
158 if urllib3.util.wait_for_read(self.sock, 1):
--> 159 self._handle_expect_response(message_body)
160 return

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _handle_expect_response(self, message_body)
201 "now sending request body.")
--> 202 self._send_message_body(message_body)
203 elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_message_body(self, message_body)
228 if message_body is not None:
--> 229 self.send(message_body)
230

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in send(self, str)
235 return
--> 236 return super(AWSConnection, self).send(str)
237

~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in send(self, data)
982 datablock = datablock.encode("iso-8859-1")
--> 983 self.sock.sendall(datablock)
984 return

timeout: timed out

During handling of the above exception, another exception occurred:

ProtocolError Traceback (most recent call last)
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/httpsession.py in send(self, request)
257 preload_content=False,
--> 258 decode_content=False,
259 )

~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
637 retries = retries.increment(method, url, error=e, _pool=self,
--> 638 _stacktrace=sys.exc_info()[2])
639 retries.sleep()

~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
342 # Disabled, indicate to re-raise the error.
--> 343 raise six.reraise(type(error), error, _stacktrace)
344

~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/packages/six.py in reraise(tp, value, tb)
684 if value.__traceback__ is not tb:
--> 685 raise value.with_traceback(tb)
686 raise value

~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
599 body=body, headers=headers,
--> 600 chunked=chunked)
601

~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
353 else:
--> 354 conn.request(method, url, **httplib_request_kw)
355

~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
1238 """Send a complete request to the server."""
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
1240

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_request(self, method, url, body, headers, *args, **kwargs)
124 rval = super(AWSConnection, self)._send_request(
--> 125 method, url, body, headers, *args, **kwargs)
126 self._expect_header_set = False

~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
1284 body = _encode(body, 'body')
-> 1285 self.endheaders(body, encode_chunked=encode_chunked)
1286

~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in endheaders(self, message_body, encode_chunked)
1233 raise CannotSendHeader()
-> 1234 self._send_output(message_body, encode_chunked=encode_chunked)
1235

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_output(self, message_body, *args, **kwargs)
158 if urllib3.util.wait_for_read(self.sock, 1):
--> 159 self._handle_expect_response(message_body)
160 return

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _handle_expect_response(self, message_body)
201 "now sending request body.")
--> 202 self._send_message_body(message_body)
203 elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_message_body(self, message_body)
228 if message_body is not None:
--> 229 self.send(message_body)
230

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in send(self, str)
235 return
--> 236 return super(AWSConnection, self).send(str)
237

~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in send(self, data)
982 datablock = datablock.encode("iso-8859-1")
--> 983 self.sock.sendall(datablock)
984 return

ProtocolError: ('Connection aborted.', timeout('timed out',))

During handling of the above exception, another exception occurred:

ConnectionClosedError Traceback (most recent call last)
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in flush(self, force, retries)
1411 PartNumber=part, UploadId=self.mpu['UploadId'],
-> 1412 Body=self.buffer.read(), Key=self.key)
1413 break

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *kwarglist, **kwargs)
1173 return self.s3._call_s3(method, self.s3_additional_kwargs, *kwarglist,
-> 1174 **kwargs)
1175

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
198 **kwargs)
--> 199 return method(**additional_kwargs)
200

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
647 http, parsed_response = self._make_request(
--> 648 operation_model, request_dict, request_context)
649

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_request(self, operation_model, request_dict, request_context)
666 try:
--> 667 return self._endpoint.make_request(operation_model, request_dict)
668 except Exception as e:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
101 operation_model, request_dict)
--> 102 return self._send_request(request_dict, operation_model)
103

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
136 while self._needs_retry(attempts, operation_model, request_dict,
--> 137 success_response, exception):
138 attempts += 1

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
230 operation=operation_model, attempts=attempts,
--> 231 caught_exception=caught_exception, request_dict=request_dict)
232 handler_response = first_non_none_response(responses)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
355 aliased_event_name = self._alias_event_name(event_name)
--> 356 return self._emitter.emit(aliased_event_name, **kwargs)
357

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
227 """
--> 228 return self._emit(event_name, kwargs)
229

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
210 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 211 response = handler(**kwargs)
212 responses.append((handler, response))

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempts, response, caught_exception, **kwargs)
182 """
--> 183 if self._checker(attempts, response, caught_exception):
184 result = self._action(attempts=attempts)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
250 should_retry = self._should_retry(attempt_number, response,
--> 251 caught_exception)
252 if should_retry:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
276 # propogate if one has occurred.
--> 277 return self._checker(attempt_number, response, caught_exception)
278

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
316 checker_response = checker(attempt_number, response,
--> 317 caught_exception)
318 if checker_response:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
222 return self._check_caught_exception(
--> 223 attempt_number, caught_exception)
224 else:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
358 # then this exception just propogates out past the retry code.
--> 359 raise caught_exception

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _do_get_response(self, request, operation_model)
199 if http_response is None:
--> 200 http_response = self._send(request)
201 except HTTPClientError as e:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send(self, request)
243 def _send(self, request):
--> 244 return self.http_session.send(request)
245

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/httpsession.py in send(self, request)
288 request=request,
--> 289 endpoint_url=request.url
290 )

ConnectionClosedError: Connection was closed before we received a valid response from endpoint URL: "http://ceph:5434/databuck/file.pq/part.6.parquet?partNumber=1&uploadId=2~9PePeZsqE8oYN9E66w2GtwUTLd15_-7".

During handling of the above exception, another exception occurred:

OSError Traceback (most recent call last)
~/develop/python/osiris/.venv/lib/python3.6/site-packages/fastparquet/writer.py in make_part_file(f, data, schema, compression, fmd)
630 f.write(MARKER)
--> 631 rg = make_row_group(f, data, schema, compression=compression)
632 if fmd is None:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/fastparquet/writer.py in make_row_group(f, data, schema, compression)
618 chunk = write_column(f, data[column.name], column,
--> 619 compression=comp)
620 rg.columns.append(chunk)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/fastparquet/writer.py in write_column(f, data, selement, compression)
552
--> 553 f.write(bdata)
554

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in write(self, data)
1362 if self.buffer.tell() >= self.blocksize:
-> 1363 self.flush()
1364 return out

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in flush(self, force, retries)
1423 except Exception as e:
-> 1424 raise IOError('Write failed', self, e)
1425 self.parts.append({'PartNumber': part, 'ETag': out['ETag']})

OSError: [Errno Write failed] <S3File databuck/file.pq/part.6.parquet>: ConnectionClosedError('Connection was closed before we received a valid response from endpoint URL: "http://ceph:5434/databuck/file.pq/part.6.parquet?partNumber=1&uploadId=2~9PePeZsqE8oYN9E66w2GtwUTLd15_-7".',)

During handling of the above exception, another exception occurred:

ReadTimeoutError Traceback (most recent call last)
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/dataframe/io/parquet.py in _write_partition_fastparquet(df, fs, path, filename, fmd, compression, partition_on)
547 rgs = make_part_file(fil, df, fmd.schema, compression=compression,
--> 548 fmd=fmd)
549 return rgs

~/develop/python/osiris/.venv/lib/python3.6/site-packages/fastparquet/writer.py in make_part_file(f, data, schema, compression, fmd)
644 f.write(struct.pack(b"<i", foot_size))
--> 645 f.write(MARKER)
646 return rg

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in __exit__(self, *args)
1486 def __exit__(self, *args):
-> 1487 self.close()
1488

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in close(self)
1451 Bucket=self.bucket, Key=self.key,
-> 1452 Body=self.buffer.getvalue(), ACL=self.acl)
1453 except (ClientError, ParamValidationError) as e:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *kwarglist, **kwargs)
1173 return self.s3._call_s3(method, self.s3_additional_kwargs, *kwarglist,
-> 1174 **kwargs)
1175

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
198 **kwargs)
--> 199 return method(**additional_kwargs)
200

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
647 http, parsed_response = self._make_request(
--> 648 operation_model, request_dict, request_context)
649

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_request(self, operation_model, request_dict, request_context)
666 try:
--> 667 return self._endpoint.make_request(operation_model, request_dict)
668 except Exception as e:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
101 operation_model, request_dict)
--> 102 return self._send_request(request_dict, operation_model)
103

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
136 while self._needs_retry(attempts, operation_model, request_dict,
--> 137 success_response, exception):
138 attempts += 1

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
230 operation=operation_model, attempts=attempts,
--> 231 caught_exception=caught_exception, request_dict=request_dict)
232 handler_response = first_non_none_response(responses)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
355 aliased_event_name = self._alias_event_name(event_name)
--> 356 return self._emitter.emit(aliased_event_name, **kwargs)
357

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
227 """
--> 228 return self._emit(event_name, kwargs)
229

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
210 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 211 response = handler(**kwargs)
212 responses.append((handler, response))

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempts, response, caught_exception, **kwargs)
182 """
--> 183 if self._checker(attempts, response, caught_exception):
184 result = self._action(attempts=attempts)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
250 should_retry = self._should_retry(attempt_number, response,
--> 251 caught_exception)
252 if should_retry:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
276 # propogate if one has occurred.
--> 277 return self._checker(attempt_number, response, caught_exception)
278

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
316 checker_response = checker(attempt_number, response,
--> 317 caught_exception)
318 if checker_response:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
222 return self._check_caught_exception(
--> 223 attempt_number, caught_exception)
224 else:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
358 # then this exception just propogates out past the retry code.
--> 359 raise caught_exception

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _do_get_response(self, request, operation_model)
199 if http_response is None:
--> 200 http_response = self._send(request)
201 except HTTPClientError as e:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send(self, request)
243 def _send(self, request):
--> 244 return self.http_session.send(request)
245

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/httpsession.py in send(self, request)
283 except URLLib3ReadTimeoutError as e:
--> 284 raise ReadTimeoutError(endpoint_url=request.url, error=e)
285 except ProtocolError as e:

ReadTimeoutError: Read timeout on endpoint URL: "http://ceph:5434/databuck/file.pq/part.6.parquet"

During handling of the above exception, another exception occurred:

ReadTimeoutError Traceback (most recent call last)
<timed eval> in <module>

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/dataframe/core.py in to_parquet(self, path, *args, **kwargs)
2828 """ See dd.to_parquet docstring for more information """
2829 from .io import to_parquet
-> 2830 return to_parquet(self, path, *args, **kwargs)
2831
2832 @derived_from(pd.DataFrame)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/dataframe/io/parquet.py in to_parquet(df, path, engine, compression, write_index, append, ignore_divisions, partition_on, storage_options, compute, **kwargs)
1232
1233 if compute:
-> 1234 out.compute()
1235 return None
1236 return out

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/base.py in compute(self, **kwargs)
154 dask.base.compute
155 """
--> 156 (result,) = compute(self, traverse=False, **kwargs)
157 return result
158

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/base.py in compute(*args, **kwargs)
396 keys = [x.__dask_keys__() for x in collections]
397 postcomputes = [x.__dask_postcompute__() for x in collections]
--> 398 results = schedule(dsk, keys, **kwargs)
399 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
400

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, pool, **kwargs)
74 results = get_async(pool.apply_async, len(pool._pool), dsk, result,
75 cache=cache, get_id=_thread_get_id,
---> 76 pack_exception=pack_exception, **kwargs)
77
78 # Cleanup pools associated to dead threads

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
460 _execute_task(task, data) # Re-execute locally
461 else:
--> 462 raise_exception(exc, tb)
463 res, worker_id = loads(res_info)
464 state['cache'][key] = res

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/compatibility.py in reraise(exc, tb)
110 if exc.__traceback__ is not tb:
111 raise exc.with_traceback(tb)
--> 112 raise exc
113
114 import pickle as cPickle

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
228 try:
229 task, data = loads(task_info)
--> 230 result = _execute_task(task, data)
231 id = get_id()
232 result = dumps((result, id))

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
117 func, args = arg[0], arg[1:]
118 args2 = [_execute_task(a, cache) for a in args]
--> 119 return func(*args2)
120 elif not ishashable(arg):
121 return arg

~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/dataframe/io/parquet.py in _write_partition_fastparquet(df, fs, path, filename, fmd, compression, partition_on)
546 with fs.open(fs.sep.join([path, filename]), 'wb') as fil:
547 rgs = make_part_file(fil, df, fmd.schema, compression=compression,
--> 548 fmd=fmd)
549 return rgs
550

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in __exit__(self, *args)
1485
1486 def __exit__(self, *args):
-> 1487 self.close()
1488
1489 # Implementations of BufferedIOBase stub methods

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in close(self)
1450 self.s3.s3.put_object,
1451 Bucket=self.bucket, Key=self.key,
-> 1452 Body=self.buffer.getvalue(), ACL=self.acl)
1453 except (ClientError, ParamValidationError) as e:
1454 raise_from(IOError('Write failed: %s' % self.path), e)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *kwarglist, **kwargs)
1172 def _call_s3(self, method, *kwarglist, **kwargs):
1173 return self.s3._call_s3(method, self.s3_additional_kwargs, *kwarglist,
-> 1174 **kwargs)
1175
1176 def info(self, **kwargs):

~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
197 additional_kwargs = self._get_s3_method_kwargs(method, *akwarglist,
198 **kwargs)
--> 199 return method(**additional_kwargs)
200
201 def _get_s3_method_kwargs(self, method, *akwarglist, **kwargs):

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
355 "%s() only accepts keyword arguments." % py_operation_name)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358
359 _api_call.__name__ = str(py_operation_name)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
646 else:
647 http, parsed_response = self._make_request(
--> 648 operation_model, request_dict, request_context)
649
650 self.meta.events.emit(

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_request(self, operation_model, request_dict, request_context)
665 def _make_request(self, operation_model, request_dict, request_context):
666 try:
--> 667 return self._endpoint.make_request(operation_model, request_dict)
668 except Exception as e:
669 self.meta.events.emit(

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
100 logger.debug("Making request for %s with params: %s",
101 operation_model, request_dict)
--> 102 return self._send_request(request_dict, operation_model)
103
104 def create_request(self, params, operation_model=None):

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
135 request, operation_model, context)
136 while self._needs_retry(attempts, operation_model, request_dict,
--> 137 success_response, exception):
138 attempts += 1
139 # If there is a stream associated with the request, we need

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
229 event_name, response=response, endpoint=self,
230 operation=operation_model, attempts=attempts,
--> 231 caught_exception=caught_exception, request_dict=request_dict)
232 handler_response = first_non_none_response(responses)
233 if handler_response is None:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
354 def emit(self, event_name, **kwargs):
355 aliased_event_name = self._alias_event_name(event_name)
--> 356 return self._emitter.emit(aliased_event_name, **kwargs)
357
358 def emit_until_response(self, event_name, **kwargs):

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
226 handlers.
227 """
--> 228 return self._emit(event_name, kwargs)
229
230 def emit_until_response(self, event_name, **kwargs):

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
209 for handler in handlers_to_call:
210 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 211 response = handler(**kwargs)
212 responses.append((handler, response))
213 if stop_on_response and response is not None:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempts, response, caught_exception, **kwargs)
181
182 """
--> 183 if self._checker(attempts, response, caught_exception):
184 result = self._action(attempts=attempts)
185 logger.debug("Retry needed, action of: %s", result)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
249 def __call__(self, attempt_number, response, caught_exception):
250 should_retry = self._should_retry(attempt_number, response,
--> 251 caught_exception)
252 if should_retry:
253 if attempt_number >= self._max_attempts:

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
275 # If we've exceeded the max attempts we just let the exception
276 # propogate if one has occurred.
--> 277 return self._checker(attempt_number, response, caught_exception)
278
279

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
315 for checker in self._checkers:
316 checker_response = checker(attempt_number, response,
--> 317 caught_exception)
318 if checker_response:
319 return checker_response

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
221 elif caught_exception is not None:
222 return self._check_caught_exception(
--> 223 attempt_number, caught_exception)
224 else:
225 raise ValueError("Both response and caught_exception are None.")

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
357 # the MaxAttemptsDecorator is not interested in retrying the exception
358 # then this exception just propogates out past the retry code.
--> 359 raise caught_exception

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _do_get_response(self, request, operation_model)
198 http_response = first_non_none_response(responses)
199 if http_response is None:
--> 200 http_response = self._send(request)
201 except HTTPClientError as e:
202 return (None, e)

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send(self, request)
242
243 def _send(self, request):
--> 244 return self.http_session.send(request)
245
246

~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/httpsession.py in send(self, request)
282 raise ConnectTimeoutError(endpoint_url=request.url, error=e)
283 except URLLib3ReadTimeoutError as e:
--> 284 raise ReadTimeoutError(endpoint_url=request.url, error=e)
285 except ProtocolError as e:
286 raise ConnectionClosedError(

ReadTimeoutError: Read timeout on endpoint URL: "http://ceph:5434/databuck/file.pq/part.6.parquet"
    (1-1/1)