|
---------------------------------------------------------------------------
|
|
timeout Traceback (most recent call last)
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
|
|
599 body=body, headers=headers,
|
|
--> 600 chunked=chunked)
|
|
601
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
|
|
353 else:
|
|
--> 354 conn.request(method, url, **httplib_request_kw)
|
|
355
|
|
|
|
~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
|
|
1238 """Send a complete request to the server."""
|
|
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
|
|
1240
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_request(self, method, url, body, headers, *args, **kwargs)
|
|
124 rval = super(AWSConnection, self)._send_request(
|
|
--> 125 method, url, body, headers, *args, **kwargs)
|
|
126 self._expect_header_set = False
|
|
|
|
~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
|
|
1284 body = _encode(body, 'body')
|
|
-> 1285 self.endheaders(body, encode_chunked=encode_chunked)
|
|
1286
|
|
|
|
~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in endheaders(self, message_body, encode_chunked)
|
|
1233 raise CannotSendHeader()
|
|
-> 1234 self._send_output(message_body, encode_chunked=encode_chunked)
|
|
1235
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_output(self, message_body, *args, **kwargs)
|
|
158 if urllib3.util.wait_for_read(self.sock, 1):
|
|
--> 159 self._handle_expect_response(message_body)
|
|
160 return
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _handle_expect_response(self, message_body)
|
|
201 "now sending request body.")
|
|
--> 202 self._send_message_body(message_body)
|
|
203 elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_message_body(self, message_body)
|
|
228 if message_body is not None:
|
|
--> 229 self.send(message_body)
|
|
230
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in send(self, str)
|
|
235 return
|
|
--> 236 return super(AWSConnection, self).send(str)
|
|
237
|
|
|
|
~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in send(self, data)
|
|
982 datablock = datablock.encode("iso-8859-1")
|
|
--> 983 self.sock.sendall(datablock)
|
|
984 return
|
|
|
|
timeout: timed out
|
|
|
|
During handling of the above exception, another exception occurred:
|
|
|
|
ProtocolError Traceback (most recent call last)
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/httpsession.py in send(self, request)
|
|
257 preload_content=False,
|
|
--> 258 decode_content=False,
|
|
259 )
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
|
|
637 retries = retries.increment(method, url, error=e, _pool=self,
|
|
--> 638 _stacktrace=sys.exc_info()[2])
|
|
639 retries.sleep()
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
|
|
342 # Disabled, indicate to re-raise the error.
|
|
--> 343 raise six.reraise(type(error), error, _stacktrace)
|
|
344
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/packages/six.py in reraise(tp, value, tb)
|
|
684 if value.__traceback__ is not tb:
|
|
--> 685 raise value.with_traceback(tb)
|
|
686 raise value
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
|
|
599 body=body, headers=headers,
|
|
--> 600 chunked=chunked)
|
|
601
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
|
|
353 else:
|
|
--> 354 conn.request(method, url, **httplib_request_kw)
|
|
355
|
|
|
|
~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
|
|
1238 """Send a complete request to the server."""
|
|
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
|
|
1240
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_request(self, method, url, body, headers, *args, **kwargs)
|
|
124 rval = super(AWSConnection, self)._send_request(
|
|
--> 125 method, url, body, headers, *args, **kwargs)
|
|
126 self._expect_header_set = False
|
|
|
|
~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
|
|
1284 body = _encode(body, 'body')
|
|
-> 1285 self.endheaders(body, encode_chunked=encode_chunked)
|
|
1286
|
|
|
|
~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in endheaders(self, message_body, encode_chunked)
|
|
1233 raise CannotSendHeader()
|
|
-> 1234 self._send_output(message_body, encode_chunked=encode_chunked)
|
|
1235
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_output(self, message_body, *args, **kwargs)
|
|
158 if urllib3.util.wait_for_read(self.sock, 1):
|
|
--> 159 self._handle_expect_response(message_body)
|
|
160 return
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _handle_expect_response(self, message_body)
|
|
201 "now sending request body.")
|
|
--> 202 self._send_message_body(message_body)
|
|
203 elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in _send_message_body(self, message_body)
|
|
228 if message_body is not None:
|
|
--> 229 self.send(message_body)
|
|
230
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/awsrequest.py in send(self, str)
|
|
235 return
|
|
--> 236 return super(AWSConnection, self).send(str)
|
|
237
|
|
|
|
~/.pyenv/versions/3.6.6/lib/python3.6/http/client.py in send(self, data)
|
|
982 datablock = datablock.encode("iso-8859-1")
|
|
--> 983 self.sock.sendall(datablock)
|
|
984 return
|
|
|
|
ProtocolError: ('Connection aborted.', timeout('timed out',))
|
|
|
|
During handling of the above exception, another exception occurred:
|
|
|
|
ConnectionClosedError Traceback (most recent call last)
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in flush(self, force, retries)
|
|
1411 PartNumber=part, UploadId=self.mpu['UploadId'],
|
|
-> 1412 Body=self.buffer.read(), Key=self.key)
|
|
1413 break
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *kwarglist, **kwargs)
|
|
1173 return self.s3._call_s3(method, self.s3_additional_kwargs, *kwarglist,
|
|
-> 1174 **kwargs)
|
|
1175
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
|
|
198 **kwargs)
|
|
--> 199 return method(**additional_kwargs)
|
|
200
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
|
|
356 # The "self" in this scope is referring to the BaseClient.
|
|
--> 357 return self._make_api_call(operation_name, kwargs)
|
|
358
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
|
|
647 http, parsed_response = self._make_request(
|
|
--> 648 operation_model, request_dict, request_context)
|
|
649
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_request(self, operation_model, request_dict, request_context)
|
|
666 try:
|
|
--> 667 return self._endpoint.make_request(operation_model, request_dict)
|
|
668 except Exception as e:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
|
|
101 operation_model, request_dict)
|
|
--> 102 return self._send_request(request_dict, operation_model)
|
|
103
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
|
|
136 while self._needs_retry(attempts, operation_model, request_dict,
|
|
--> 137 success_response, exception):
|
|
138 attempts += 1
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
|
|
230 operation=operation_model, attempts=attempts,
|
|
--> 231 caught_exception=caught_exception, request_dict=request_dict)
|
|
232 handler_response = first_non_none_response(responses)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
|
|
355 aliased_event_name = self._alias_event_name(event_name)
|
|
--> 356 return self._emitter.emit(aliased_event_name, **kwargs)
|
|
357
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
|
|
227 """
|
|
--> 228 return self._emit(event_name, kwargs)
|
|
229
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
|
|
210 logger.debug('Event %s: calling handler %s', event_name, handler)
|
|
--> 211 response = handler(**kwargs)
|
|
212 responses.append((handler, response))
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempts, response, caught_exception, **kwargs)
|
|
182 """
|
|
--> 183 if self._checker(attempts, response, caught_exception):
|
|
184 result = self._action(attempts=attempts)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
250 should_retry = self._should_retry(attempt_number, response,
|
|
--> 251 caught_exception)
|
|
252 if should_retry:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
|
|
276 # propogate if one has occurred.
|
|
--> 277 return self._checker(attempt_number, response, caught_exception)
|
|
278
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
316 checker_response = checker(attempt_number, response,
|
|
--> 317 caught_exception)
|
|
318 if checker_response:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
222 return self._check_caught_exception(
|
|
--> 223 attempt_number, caught_exception)
|
|
224 else:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
|
|
358 # then this exception just propogates out past the retry code.
|
|
--> 359 raise caught_exception
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _do_get_response(self, request, operation_model)
|
|
199 if http_response is None:
|
|
--> 200 http_response = self._send(request)
|
|
201 except HTTPClientError as e:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send(self, request)
|
|
243 def _send(self, request):
|
|
--> 244 return self.http_session.send(request)
|
|
245
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/httpsession.py in send(self, request)
|
|
288 request=request,
|
|
--> 289 endpoint_url=request.url
|
|
290 )
|
|
|
|
ConnectionClosedError: Connection was closed before we received a valid response from endpoint URL: "http://ceph:5434/databuck/file.pq/part.6.parquet?partNumber=1&uploadId=2~9PePeZsqE8oYN9E66w2GtwUTLd15_-7".
|
|
|
|
During handling of the above exception, another exception occurred:
|
|
|
|
OSError Traceback (most recent call last)
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/fastparquet/writer.py in make_part_file(f, data, schema, compression, fmd)
|
|
630 f.write(MARKER)
|
|
--> 631 rg = make_row_group(f, data, schema, compression=compression)
|
|
632 if fmd is None:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/fastparquet/writer.py in make_row_group(f, data, schema, compression)
|
|
618 chunk = write_column(f, data[column.name], column,
|
|
--> 619 compression=comp)
|
|
620 rg.columns.append(chunk)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/fastparquet/writer.py in write_column(f, data, selement, compression)
|
|
552
|
|
--> 553 f.write(bdata)
|
|
554
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in write(self, data)
|
|
1362 if self.buffer.tell() >= self.blocksize:
|
|
-> 1363 self.flush()
|
|
1364 return out
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in flush(self, force, retries)
|
|
1423 except Exception as e:
|
|
-> 1424 raise IOError('Write failed', self, e)
|
|
1425 self.parts.append({'PartNumber': part, 'ETag': out['ETag']})
|
|
|
|
OSError: [Errno Write failed] <S3File databuck/file.pq/part.6.parquet>: ConnectionClosedError('Connection was closed before we received a valid response from endpoint URL: "http://ceph:5434/databuck/file.pq/part.6.parquet?partNumber=1&uploadId=2~9PePeZsqE8oYN9E66w2GtwUTLd15_-7".',)
|
|
|
|
During handling of the above exception, another exception occurred:
|
|
|
|
ReadTimeoutError Traceback (most recent call last)
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/dataframe/io/parquet.py in _write_partition_fastparquet(df, fs, path, filename, fmd, compression, partition_on)
|
|
547 rgs = make_part_file(fil, df, fmd.schema, compression=compression,
|
|
--> 548 fmd=fmd)
|
|
549 return rgs
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/fastparquet/writer.py in make_part_file(f, data, schema, compression, fmd)
|
|
644 f.write(struct.pack(b"<i", foot_size))
|
|
--> 645 f.write(MARKER)
|
|
646 return rg
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in __exit__(self, *args)
|
|
1486 def __exit__(self, *args):
|
|
-> 1487 self.close()
|
|
1488
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in close(self)
|
|
1451 Bucket=self.bucket, Key=self.key,
|
|
-> 1452 Body=self.buffer.getvalue(), ACL=self.acl)
|
|
1453 except (ClientError, ParamValidationError) as e:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *kwarglist, **kwargs)
|
|
1173 return self.s3._call_s3(method, self.s3_additional_kwargs, *kwarglist,
|
|
-> 1174 **kwargs)
|
|
1175
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
|
|
198 **kwargs)
|
|
--> 199 return method(**additional_kwargs)
|
|
200
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
|
|
356 # The "self" in this scope is referring to the BaseClient.
|
|
--> 357 return self._make_api_call(operation_name, kwargs)
|
|
358
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
|
|
647 http, parsed_response = self._make_request(
|
|
--> 648 operation_model, request_dict, request_context)
|
|
649
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_request(self, operation_model, request_dict, request_context)
|
|
666 try:
|
|
--> 667 return self._endpoint.make_request(operation_model, request_dict)
|
|
668 except Exception as e:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
|
|
101 operation_model, request_dict)
|
|
--> 102 return self._send_request(request_dict, operation_model)
|
|
103
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
|
|
136 while self._needs_retry(attempts, operation_model, request_dict,
|
|
--> 137 success_response, exception):
|
|
138 attempts += 1
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
|
|
230 operation=operation_model, attempts=attempts,
|
|
--> 231 caught_exception=caught_exception, request_dict=request_dict)
|
|
232 handler_response = first_non_none_response(responses)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
|
|
355 aliased_event_name = self._alias_event_name(event_name)
|
|
--> 356 return self._emitter.emit(aliased_event_name, **kwargs)
|
|
357
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
|
|
227 """
|
|
--> 228 return self._emit(event_name, kwargs)
|
|
229
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
|
|
210 logger.debug('Event %s: calling handler %s', event_name, handler)
|
|
--> 211 response = handler(**kwargs)
|
|
212 responses.append((handler, response))
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempts, response, caught_exception, **kwargs)
|
|
182 """
|
|
--> 183 if self._checker(attempts, response, caught_exception):
|
|
184 result = self._action(attempts=attempts)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
250 should_retry = self._should_retry(attempt_number, response,
|
|
--> 251 caught_exception)
|
|
252 if should_retry:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
|
|
276 # propogate if one has occurred.
|
|
--> 277 return self._checker(attempt_number, response, caught_exception)
|
|
278
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
316 checker_response = checker(attempt_number, response,
|
|
--> 317 caught_exception)
|
|
318 if checker_response:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
222 return self._check_caught_exception(
|
|
--> 223 attempt_number, caught_exception)
|
|
224 else:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
|
|
358 # then this exception just propogates out past the retry code.
|
|
--> 359 raise caught_exception
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _do_get_response(self, request, operation_model)
|
|
199 if http_response is None:
|
|
--> 200 http_response = self._send(request)
|
|
201 except HTTPClientError as e:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send(self, request)
|
|
243 def _send(self, request):
|
|
--> 244 return self.http_session.send(request)
|
|
245
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/httpsession.py in send(self, request)
|
|
283 except URLLib3ReadTimeoutError as e:
|
|
--> 284 raise ReadTimeoutError(endpoint_url=request.url, error=e)
|
|
285 except ProtocolError as e:
|
|
|
|
ReadTimeoutError: Read timeout on endpoint URL: "http://ceph:5434/databuck/file.pq/part.6.parquet"
|
|
|
|
During handling of the above exception, another exception occurred:
|
|
|
|
ReadTimeoutError Traceback (most recent call last)
|
|
<timed eval> in <module>
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/dataframe/core.py in to_parquet(self, path, *args, **kwargs)
|
|
2828 """ See dd.to_parquet docstring for more information """
|
|
2829 from .io import to_parquet
|
|
-> 2830 return to_parquet(self, path, *args, **kwargs)
|
|
2831
|
|
2832 @derived_from(pd.DataFrame)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/dataframe/io/parquet.py in to_parquet(df, path, engine, compression, write_index, append, ignore_divisions, partition_on, storage_options, compute, **kwargs)
|
|
1232
|
|
1233 if compute:
|
|
-> 1234 out.compute()
|
|
1235 return None
|
|
1236 return out
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/base.py in compute(self, **kwargs)
|
|
154 dask.base.compute
|
|
155 """
|
|
--> 156 (result,) = compute(self, traverse=False, **kwargs)
|
|
157 return result
|
|
158
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/base.py in compute(*args, **kwargs)
|
|
396 keys = [x.__dask_keys__() for x in collections]
|
|
397 postcomputes = [x.__dask_postcompute__() for x in collections]
|
|
--> 398 results = schedule(dsk, keys, **kwargs)
|
|
399 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
|
|
400
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, pool, **kwargs)
|
|
74 results = get_async(pool.apply_async, len(pool._pool), dsk, result,
|
|
75 cache=cache, get_id=_thread_get_id,
|
|
---> 76 pack_exception=pack_exception, **kwargs)
|
|
77
|
|
78 # Cleanup pools associated to dead threads
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
|
|
460 _execute_task(task, data) # Re-execute locally
|
|
461 else:
|
|
--> 462 raise_exception(exc, tb)
|
|
463 res, worker_id = loads(res_info)
|
|
464 state['cache'][key] = res
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/compatibility.py in reraise(exc, tb)
|
|
110 if exc.__traceback__ is not tb:
|
|
111 raise exc.with_traceback(tb)
|
|
--> 112 raise exc
|
|
113
|
|
114 import pickle as cPickle
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
|
|
228 try:
|
|
229 task, data = loads(task_info)
|
|
--> 230 result = _execute_task(task, data)
|
|
231 id = get_id()
|
|
232 result = dumps((result, id))
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
|
|
117 func, args = arg[0], arg[1:]
|
|
118 args2 = [_execute_task(a, cache) for a in args]
|
|
--> 119 return func(*args2)
|
|
120 elif not ishashable(arg):
|
|
121 return arg
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/dask/dataframe/io/parquet.py in _write_partition_fastparquet(df, fs, path, filename, fmd, compression, partition_on)
|
|
546 with fs.open(fs.sep.join([path, filename]), 'wb') as fil:
|
|
547 rgs = make_part_file(fil, df, fmd.schema, compression=compression,
|
|
--> 548 fmd=fmd)
|
|
549 return rgs
|
|
550
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in __exit__(self, *args)
|
|
1485
|
|
1486 def __exit__(self, *args):
|
|
-> 1487 self.close()
|
|
1488
|
|
1489 # Implementations of BufferedIOBase stub methods
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in close(self)
|
|
1450 self.s3.s3.put_object,
|
|
1451 Bucket=self.bucket, Key=self.key,
|
|
-> 1452 Body=self.buffer.getvalue(), ACL=self.acl)
|
|
1453 except (ClientError, ParamValidationError) as e:
|
|
1454 raise_from(IOError('Write failed: %s' % self.path), e)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *kwarglist, **kwargs)
|
|
1172 def _call_s3(self, method, *kwarglist, **kwargs):
|
|
1173 return self.s3._call_s3(method, self.s3_additional_kwargs, *kwarglist,
|
|
-> 1174 **kwargs)
|
|
1175
|
|
1176 def info(self, **kwargs):
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
|
|
197 additional_kwargs = self._get_s3_method_kwargs(method, *akwarglist,
|
|
198 **kwargs)
|
|
--> 199 return method(**additional_kwargs)
|
|
200
|
|
201 def _get_s3_method_kwargs(self, method, *akwarglist, **kwargs):
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
|
|
355 "%s() only accepts keyword arguments." % py_operation_name)
|
|
356 # The "self" in this scope is referring to the BaseClient.
|
|
--> 357 return self._make_api_call(operation_name, kwargs)
|
|
358
|
|
359 _api_call.__name__ = str(py_operation_name)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
|
|
646 else:
|
|
647 http, parsed_response = self._make_request(
|
|
--> 648 operation_model, request_dict, request_context)
|
|
649
|
|
650 self.meta.events.emit(
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/client.py in _make_request(self, operation_model, request_dict, request_context)
|
|
665 def _make_request(self, operation_model, request_dict, request_context):
|
|
666 try:
|
|
--> 667 return self._endpoint.make_request(operation_model, request_dict)
|
|
668 except Exception as e:
|
|
669 self.meta.events.emit(
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
|
|
100 logger.debug("Making request for %s with params: %s",
|
|
101 operation_model, request_dict)
|
|
--> 102 return self._send_request(request_dict, operation_model)
|
|
103
|
|
104 def create_request(self, params, operation_model=None):
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
|
|
135 request, operation_model, context)
|
|
136 while self._needs_retry(attempts, operation_model, request_dict,
|
|
--> 137 success_response, exception):
|
|
138 attempts += 1
|
|
139 # If there is a stream associated with the request, we need
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
|
|
229 event_name, response=response, endpoint=self,
|
|
230 operation=operation_model, attempts=attempts,
|
|
--> 231 caught_exception=caught_exception, request_dict=request_dict)
|
|
232 handler_response = first_non_none_response(responses)
|
|
233 if handler_response is None:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
|
|
354 def emit(self, event_name, **kwargs):
|
|
355 aliased_event_name = self._alias_event_name(event_name)
|
|
--> 356 return self._emitter.emit(aliased_event_name, **kwargs)
|
|
357
|
|
358 def emit_until_response(self, event_name, **kwargs):
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
|
|
226 handlers.
|
|
227 """
|
|
--> 228 return self._emit(event_name, kwargs)
|
|
229
|
|
230 def emit_until_response(self, event_name, **kwargs):
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
|
|
209 for handler in handlers_to_call:
|
|
210 logger.debug('Event %s: calling handler %s', event_name, handler)
|
|
--> 211 response = handler(**kwargs)
|
|
212 responses.append((handler, response))
|
|
213 if stop_on_response and response is not None:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempts, response, caught_exception, **kwargs)
|
|
181
|
|
182 """
|
|
--> 183 if self._checker(attempts, response, caught_exception):
|
|
184 result = self._action(attempts=attempts)
|
|
185 logger.debug("Retry needed, action of: %s", result)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
249 def __call__(self, attempt_number, response, caught_exception):
|
|
250 should_retry = self._should_retry(attempt_number, response,
|
|
--> 251 caught_exception)
|
|
252 if should_retry:
|
|
253 if attempt_number >= self._max_attempts:
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
|
|
275 # If we've exceeded the max attempts we just let the exception
|
|
276 # propogate if one has occurred.
|
|
--> 277 return self._checker(attempt_number, response, caught_exception)
|
|
278
|
|
279
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
315 for checker in self._checkers:
|
|
316 checker_response = checker(attempt_number, response,
|
|
--> 317 caught_exception)
|
|
318 if checker_response:
|
|
319 return checker_response
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
|
|
221 elif caught_exception is not None:
|
|
222 return self._check_caught_exception(
|
|
--> 223 attempt_number, caught_exception)
|
|
224 else:
|
|
225 raise ValueError("Both response and caught_exception are None.")
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
|
|
357 # the MaxAttemptsDecorator is not interested in retrying the exception
|
|
358 # then this exception just propogates out past the retry code.
|
|
--> 359 raise caught_exception
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _do_get_response(self, request, operation_model)
|
|
198 http_response = first_non_none_response(responses)
|
|
199 if http_response is None:
|
|
--> 200 http_response = self._send(request)
|
|
201 except HTTPClientError as e:
|
|
202 return (None, e)
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/endpoint.py in _send(self, request)
|
|
242
|
|
243 def _send(self, request):
|
|
--> 244 return self.http_session.send(request)
|
|
245
|
|
246
|
|
|
|
~/develop/python/osiris/.venv/lib/python3.6/site-packages/botocore/httpsession.py in send(self, request)
|
|
282 raise ConnectTimeoutError(endpoint_url=request.url, error=e)
|
|
283 except URLLib3ReadTimeoutError as e:
|
|
--> 284 raise ReadTimeoutError(endpoint_url=request.url, error=e)
|
|
285 except ProtocolError as e:
|
|
286 raise ConnectionClosedError(
|
|
|
|
ReadTimeoutError: Read timeout on endpoint URL: "http://ceph:5434/databuck/file.pq/part.6.parquet"
|