http://qa-proxy.ceph.com/teuthology/amaredia-2019-05-15_20:26:39-rgw:verify-master-distro-basic-smithi/3958169/teuthology.log

2019-05-15T23:13:05.493 INFO:teuthology.orchestra.run.smithi191.stdout:> Task :test
2019-05-15T23:13:05.494 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:05.494 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testAbortMultipartUploadLLAPIAWS4 STARTED
2019-05-15T23:13:29.993 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:29.993 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testAbortMultipartUploadLLAPIAWS4 PASSED
2019-05-15T23:13:31.393 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:31.393 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testAbortMultipartUploadNotFoundLLAPIAWS4 STARTED
2019-05-15T23:13:31.993 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:31.993 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testAbortMultipartUploadNotFoundLLAPIAWS4 PASSED
2019-05-15T23:13:33.093 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:33.093 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testDownloadHLAPIAWS4 STARTED
2019-05-15T23:13:51.492 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:51.493 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testDownloadHLAPIAWS4 PASSED
2019-05-15T23:13:52.792 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:52.792 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testDownloadNoBucketHLAPIAWS4 STARTED
2019-05-15T23:13:52.992 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:52.993 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testDownloadNoBucketHLAPIAWS4 PASSED
2019-05-15T23:13:53.192 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:53.193 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testDownloadNoKeyHLAPIAWS4 STARTED
2019-05-15T23:13:53.692 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:53.693 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testDownloadNoKeyHLAPIAWS4 PASSED
2019-05-15T23:13:54.992 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:13:54.992 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartCopyHLAPIAWS4 STARTED
2019-05-15T23:14:18.392 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:14:18.392 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartCopyHLAPIAWS4 PASSED
2019-05-15T23:14:20.893 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:14:20.893 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartCopyMultipleSizesLLAPIAWS4 STARTED
2019-05-15T23:15:24.291 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:24.291 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartCopyMultipleSizesLLAPIAWS4 PASSED
2019-05-15T23:15:27.091 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:27.091 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartCopyNoSRCBucketHLAPIAWS4 STARTED
2019-05-15T23:15:27.491 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:27.491 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartCopyNoSRCBucketHLAPIAWS4 PASSED
2019-05-15T23:15:28.691 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:28.691 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartCopyNoSRCKeyHLAPIAWS4 STARTED
2019-05-15T23:15:29.591 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:29.591 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartCopyNoSRCKeyHLAPIAWS4 PASSED
2019-05-15T23:15:31.591 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:31.591 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartDownloadHLAPIAWS4 STARTED
2019-05-15T23:15:49.691 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:49.691 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartDownloadHLAPIAWS4 PASSED
2019-05-15T23:15:50.990 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:50.991 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartDownloadNoBucketHLAPIAWS4 STARTED
2019-05-15T23:15:51.191 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:51.191 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartDownloadNoBucketHLAPIAWS4 PASSED
2019-05-15T23:15:51.590 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:15:51.591 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartDownloadWithPauseHLAPIAWS4 STARTED
2019-05-15T23:16:10.690 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:16:10.690 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartDownloadWithPauseHLAPIAWS4 PASSED
2019-05-15T23:16:12.290 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:16:12.290 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadHLAPIAWS4 STARTED
2019-05-15T23:16:57.589 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:16:57.590 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadHLAPIAWS4 PASSED
2019-05-15T23:17:01.089 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:17:01.089 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadIncorrectMissingPartLLAPIAWS4 STARTED
2019-05-15T23:17:22.189 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:17:22.189 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadIncorrectMissingPartLLAPIAWS4 PASSED
2019-05-15T23:17:34.389 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:17:34.389 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadMultipleSizesLLAPIAWS4 STARTED
2019-05-15T23:18:00.761 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:15:28.366 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:18:00.769 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:15:28.374 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:18:00.777 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:15:28.381 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:18:00.779 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:15:28.384 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:18:00.788 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:15:28.333 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:18:00.793 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:15:28.338 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:18:01.078 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:15:28.683 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:18:01.086 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:15:28.692 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:18:01.182 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:15:28.727 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.563 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:17:34.169 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.575 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:17:34.182 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.592 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:17:34.199 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.610 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:17:34.217 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.636 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:17:34.182 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.646 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:17:34.192 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.652 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:17:34.258 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.682 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:17:34.289 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:20:06.745 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:17:34.290 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.510 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:22:23.108 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.525 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:22:23.123 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.552 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:22:23.150 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.557 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:22:23.105 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.558 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:22:23.156 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.559 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:22:23.106 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.740 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:22:23.337 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.770 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:22:23.318 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:24:55.804 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:22:23.402 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:20.635 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.608 20fc5700 -1 RGWWatcher::handle_error cookie 555289952 err (107) Transport endpoint is not connected
2019-05-15T23:28:20.637 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.627 20fc5700 -1 RGWWatcher::handle_error cookie 554968048 err (107) Transport endpoint is not connected
2019-05-15T23:28:20.663 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.640 20fc5700 -1 RGWWatcher::handle_error cookie 554916192 err (107) Transport endpoint is not connected
2019-05-15T23:28:20.683 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.645 20fc5700 -1 RGWWatcher::handle_error cookie 555142736 err (107) Transport endpoint is not connected
2019-05-15T23:28:20.700 INFO:tasks.rgw.client.0.smithi191.stdout:terminate called after throwing an instance of 'boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >'
2019-05-15T23:28:20.701 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.645 20fc5700 -1 RGWWatcher::handle_error cookie 555516832 err (107) Transport endpoint is not connected
2019-05-15T23:28:20.702 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.646 20fc5700 -1 RGWWatcher::handle_error cookie 555244704 err (107) Transport endpoint is not connected
2019-05-15T23:28:20.707 INFO:tasks.rgw.client.0.smithi191.stdout:  what():  remote_endpoint: Transport endpoint is not connected
2019-05-15T23:28:20.747 INFO:tasks.rgw.client.0.smithi191.stdout:*** Caught signal (Aborted) **
2019-05-15T23:28:20.747 INFO:tasks.rgw.client.0.smithi191.stdout: in thread aa6e0700 thread_name:memcheck-amd64-
2019-05-15T23:28:20.780 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.646 20fc5700 -1 RGWWatcher::handle_error cookie 555193680 err (107) Transport endpoint is not connected
2019-05-15T23:28:20.809 INFO:tasks.rgw.client.0.smithi191.stdout: ceph version 15.0.0-1282-g7eb6c2d (7eb6c2d557b985b59e1da1679b9f727401df3b5b) octopus (dev)
2019-05-15T23:28:20.809 INFO:tasks.rgw.client.0.smithi191.stdout: 1: (()+0xf5d0) [0x113d25d0]
2019-05-15T23:28:20.809 INFO:tasks.rgw.client.0.smithi191.stdout: 2: (gsignal()+0x37) [0x11e34207]
2019-05-15T23:28:20.810 INFO:tasks.rgw.client.0.smithi191.stdout: 3: (abort()+0x148) [0x11e358f8]
2019-05-15T23:28:20.810 INFO:tasks.rgw.client.0.smithi191.stdout: 4: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x1163f7d5]
2019-05-15T23:28:20.810 INFO:tasks.rgw.client.0.smithi191.stdout: 5: (()+0x5e746) [0x1163d746]
2019-05-15T23:28:20.810 INFO:tasks.rgw.client.0.smithi191.stdout: 6: (()+0x5e773) [0x1163d773]
2019-05-15T23:28:20.810 INFO:tasks.rgw.client.0.smithi191.stdout: 7: (()+0x5e993) [0x1163d993]
2019-05-15T23:28:20.810 INFO:tasks.rgw.client.0.smithi191.stdout: 8: (virtual thunk to boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >::rethrow() const+0x6e) [0x405ade]
2019-05-15T23:28:20.810 INFO:tasks.rgw.client.0.smithi191.stdout: 9: (boost::rethrow_exception(boost::exception_ptr const&)+0xd) [0x3fba0d]
2019-05-15T23:28:20.811 INFO:tasks.rgw.client.0.smithi191.stdout: 10: (boost::coroutines::detail::push_coroutine_impl<void>::push()+0x4e) [0x3fbafe]
2019-05-15T23:28:20.811 INFO:tasks.rgw.client.0.smithi191.stdout: 11: (boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<boost::asio::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::executor>, std::shared_lock<ceph::async::SharedMutex<boost::asio::io_context::executor_type> > >, std::tuple<boost::system::error_code, std::shared_lock<ceph::async::SharedMutex<boost::asio::io_context::executor_type> > > > >, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x16d) [0x40685d]
2019-05-15T23:28:20.811 INFO:tasks.rgw.client.0.smithi191.stdout: 12: (boost::asio::detail::executor_op<boost::asio::executor::function, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x8c) [0x3f879c]
2019-05-15T23:28:20.811 INFO:tasks.rgw.client.0.smithi191.stdout: 13: (boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>::operator()()+0x7d) [0x41e5ad]
2019-05-15T23:28:20.812 INFO:tasks.rgw.client.0.smithi191.stdout: 14: (boost::asio::detail::executor_op<boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>, boost::asio::detail::recycling_allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x94) [0x41e924]
2019-05-15T23:28:20.812 INFO:tasks.rgw.client.0.smithi191.stdout: 15: (boost::asio::detail::scheduler::run(boost::system::error_code&)+0x3f2) [0x3fd3c2]
2019-05-15T23:28:20.813 INFO:tasks.rgw.client.0.smithi191.stdout: 16: (()+0x2de2ab) [0x3e62ab]
2019-05-15T23:28:20.813 INFO:tasks.rgw.client.0.smithi191.stdout: 17: (()+0x7b8f8f) [0x642ff8f]
2019-05-15T23:28:20.813 INFO:tasks.rgw.client.0.smithi191.stdout: 18: (()+0x7dd5) [0x113cadd5]
2019-05-15T23:28:20.813 INFO:tasks.rgw.client.0.smithi191.stdout: 19: (clone()+0x6d) [0x11efbead]
2019-05-15T23:28:20.820 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.666 20fc5700 -1 RGWWatcher::handle_error cookie 555471584 err (107) Transport endpoint is not connected
2019-05-15T23:28:20.835 INFO:tasks.rgw.client.0.smithi191.stdout:2019-05-15 23:28:20.833 aa6e0700 -1 *** Caught signal (Aborted) **
2019-05-15T23:28:20.835 INFO:tasks.rgw.client.0.smithi191.stdout: in thread aa6e0700 thread_name:memcheck-amd64-
2019-05-15T23:28:20.835 INFO:tasks.rgw.client.0.smithi191.stdout:
2019-05-15T23:28:20.835 INFO:tasks.rgw.client.0.smithi191.stdout: ceph version 15.0.0-1282-g7eb6c2d (7eb6c2d557b985b59e1da1679b9f727401df3b5b) octopus (dev)
2019-05-15T23:28:20.835 INFO:tasks.rgw.client.0.smithi191.stdout: 1: (()+0xf5d0) [0x113d25d0]
2019-05-15T23:28:20.835 INFO:tasks.rgw.client.0.smithi191.stdout: 2: (gsignal()+0x37) [0x11e34207]
2019-05-15T23:28:20.836 INFO:tasks.rgw.client.0.smithi191.stdout: 3: (abort()+0x148) [0x11e358f8]
2019-05-15T23:28:20.836 INFO:tasks.rgw.client.0.smithi191.stdout: 4: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x1163f7d5]
2019-05-15T23:28:20.836 INFO:tasks.rgw.client.0.smithi191.stdout: 5: (()+0x5e746) [0x1163d746]
2019-05-15T23:28:20.836 INFO:tasks.rgw.client.0.smithi191.stdout: 6: (()+0x5e773) [0x1163d773]
2019-05-15T23:28:20.836 INFO:tasks.rgw.client.0.smithi191.stdout: 7: (()+0x5e993) [0x1163d993]
2019-05-15T23:28:20.836 INFO:tasks.rgw.client.0.smithi191.stdout: 8: (virtual thunk to boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >::rethrow() const+0x6e) [0x405ade]
2019-05-15T23:28:20.837 INFO:tasks.rgw.client.0.smithi191.stdout: 9: (boost::rethrow_exception(boost::exception_ptr const&)+0xd) [0x3fba0d]
2019-05-15T23:28:20.837 INFO:tasks.rgw.client.0.smithi191.stdout: 10: (boost::coroutines::detail::push_coroutine_impl<void>::push()+0x4e) [0x3fbafe]
2019-05-15T23:28:20.837 INFO:tasks.rgw.client.0.smithi191.stdout: 11: (boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<boost::asio::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::executor>, std::shared_lock<ceph::async::SharedMutex<boost::asio::io_context::executor_type> > >, std::tuple<boost::system::error_code, std::shared_lock<ceph::async::SharedMutex<boost::asio::io_context::executor_type> > > > >, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x16d) [0x40685d]
2019-05-15T23:28:20.837 INFO:tasks.rgw.client.0.smithi191.stdout: 12: (boost::asio::detail::executor_op<boost::asio::executor::function, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x8c) [0x3f879c]
2019-05-15T23:28:20.837 INFO:tasks.rgw.client.0.smithi191.stdout: 13: (boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>::operator()()+0x7d) [0x41e5ad]
2019-05-15T23:28:20.837 INFO:tasks.rgw.client.0.smithi191.stdout: 14: (boost::asio::detail::executor_op<boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>, boost::asio::detail::recycling_allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x94) [0x41e924]
2019-05-15T23:28:20.837 INFO:tasks.rgw.client.0.smithi191.stdout: 15: (boost::asio::detail::scheduler::run(boost::system::error_code&)+0x3f2) [0x3fd3c2]
2019-05-15T23:28:20.838 INFO:tasks.rgw.client.0.smithi191.stdout: 16: (()+0x2de2ab) [0x3e62ab]
2019-05-15T23:28:20.838 INFO:tasks.rgw.client.0.smithi191.stdout: 17: (()+0x7b8f8f) [0x642ff8f]
2019-05-15T23:28:20.838 INFO:tasks.rgw.client.0.smithi191.stdout: 18: (()+0x7dd5) [0x113cadd5]
2019-05-15T23:28:20.838 INFO:tasks.rgw.client.0.smithi191.stdout: 19: (clone()+0x6d) [0x11efbead]
2019-05-15T23:28:20.838 INFO:tasks.rgw.client.0.smithi191.stdout: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2019-05-15T23:28:20.839 INFO:tasks.rgw.client.0.smithi191.stdout:
2019-05-15T23:28:21.484 INFO:tasks.rgw.client.0.smithi191.stdout:  -105> 2019-05-15 23:28:20.608 20fc5700 -1 RGWWatcher::handle_error cookie 555289952 err (107) Transport endpoint is not connected
2019-05-15T23:28:21.485 INFO:tasks.rgw.client.0.smithi191.stdout:  -102> 2019-05-15 23:28:20.627 20fc5700 -1 RGWWatcher::handle_error cookie 554968048 err (107) Transport endpoint is not connected
2019-05-15T23:28:21.487 INFO:tasks.rgw.client.0.smithi191.stdout:   -93> 2019-05-15 23:28:20.640 20fc5700 -1 RGWWatcher::handle_error cookie 554916192 err (107) Transport endpoint is not connected
2019-05-15T23:28:21.487 INFO:tasks.rgw.client.0.smithi191.stdout:   -91> 2019-05-15 23:28:20.645 20fc5700 -1 RGWWatcher::handle_error cookie 555142736 err (107) Transport endpoint is not connected
2019-05-15T23:28:21.488 INFO:tasks.rgw.client.0.smithi191.stdout:   -89> 2019-05-15 23:28:20.645 20fc5700 -1 RGWWatcher::handle_error cookie 555516832 err (107) Transport endpoint is not connected
2019-05-15T23:28:21.489 INFO:tasks.rgw.client.0.smithi191.stdout:   -87> 2019-05-15 23:28:20.646 20fc5700 -1 RGWWatcher::handle_error cookie 555244704 err (107) Transport endpoint is not connected
2019-05-15T23:28:21.499 INFO:tasks.rgw.client.0.smithi191.stdout:   -85> 2019-05-15 23:28:20.646 20fc5700 -1 RGWWatcher::handle_error cookie 555193680 err (107) Transport endpoint is not connected
2019-05-15T23:28:21.506 INFO:tasks.rgw.client.0.smithi191.stdout:   -81> 2019-05-15 23:28:20.666 20fc5700 -1 RGWWatcher::handle_error cookie 555471584 err (107) Transport endpoint is not connected
2019-05-15T23:28:21.513 INFO:tasks.rgw.client.0.smithi191.stdout:     0> 2019-05-15 23:28:20.833 aa6e0700 -1 *** Caught signal (Aborted) **
2019-05-15T23:28:21.513 INFO:tasks.rgw.client.0.smithi191.stdout: in thread aa6e0700 thread_name:memcheck-amd64-
2019-05-15T23:28:21.513 INFO:tasks.rgw.client.0.smithi191.stdout:
2019-05-15T23:28:21.514 INFO:tasks.rgw.client.0.smithi191.stdout: ceph version 15.0.0-1282-g7eb6c2d (7eb6c2d557b985b59e1da1679b9f727401df3b5b) octopus (dev)
2019-05-15T23:28:21.514 INFO:tasks.rgw.client.0.smithi191.stdout: 1: (()+0xf5d0) [0x113d25d0]
2019-05-15T23:28:21.514 INFO:tasks.rgw.client.0.smithi191.stdout: 2: (gsignal()+0x37) [0x11e34207]
2019-05-15T23:28:21.514 INFO:tasks.rgw.client.0.smithi191.stdout: 3: (abort()+0x148) [0x11e358f8]
2019-05-15T23:28:21.514 INFO:tasks.rgw.client.0.smithi191.stdout: 4: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x1163f7d5]
2019-05-15T23:28:21.514 INFO:tasks.rgw.client.0.smithi191.stdout: 5: (()+0x5e746) [0x1163d746]
2019-05-15T23:28:21.514 INFO:tasks.rgw.client.0.smithi191.stdout: 6: (()+0x5e773) [0x1163d773]
2019-05-15T23:28:21.515 INFO:tasks.rgw.client.0.smithi191.stdout: 7: (()+0x5e993) [0x1163d993]
2019-05-15T23:28:21.515 INFO:tasks.rgw.client.0.smithi191.stdout: 8: (virtual thunk to boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >::rethrow() const+0x6e) [0x405ade]
2019-05-15T23:28:21.515 INFO:tasks.rgw.client.0.smithi191.stdout: 9: (boost::rethrow_exception(boost::exception_ptr const&)+0xd) [0x3fba0d]
2019-05-15T23:28:21.515 INFO:tasks.rgw.client.0.smithi191.stdout: 10: (boost::coroutines::detail::push_coroutine_impl<void>::push()+0x4e) [0x3fbafe]
2019-05-15T23:28:21.515 INFO:tasks.rgw.client.0.smithi191.stdout: 11: (boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<boost::asio::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::executor>, std::shared_lock<ceph::async::SharedMutex<boost::asio::io_context::executor_type> > >, std::tuple<boost::system::error_code, std::shared_lock<ceph::async::SharedMutex<boost::asio::io_context::executor_type> > > > >, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x16d) [0x40685d]
2019-05-15T23:28:21.515 INFO:tasks.rgw.client.0.smithi191.stdout: 12: (boost::asio::detail::executor_op<boost::asio::executor::function, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x8c) [0x3f879c]
2019-05-15T23:28:21.515 INFO:tasks.rgw.client.0.smithi191.stdout: 13: (boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>::operator()()+0x7d) [0x41e5ad]
2019-05-15T23:28:21.516 INFO:tasks.rgw.client.0.smithi191.stdout: 14: (boost::asio::detail::executor_op<boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>, boost::asio::detail::recycling_allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x94) [0x41e924]
2019-05-15T23:28:21.516 INFO:tasks.rgw.client.0.smithi191.stdout: 15: (boost::asio::detail::scheduler::run(boost::system::error_code&)+0x3f2) [0x3fd3c2]
2019-05-15T23:28:21.516 INFO:tasks.rgw.client.0.smithi191.stdout: 16: (()+0x2de2ab) [0x3e62ab]
2019-05-15T23:28:21.516 INFO:tasks.rgw.client.0.smithi191.stdout: 17: (()+0x7b8f8f) [0x642ff8f]
2019-05-15T23:28:21.516 INFO:tasks.rgw.client.0.smithi191.stdout: 18: (()+0x7dd5) [0x113cadd5]
2019-05-15T23:28:21.516 INFO:tasks.rgw.client.0.smithi191.stdout: 19: (clone()+0x6d) [0x11efbead]
2019-05-15T23:28:21.517 INFO:tasks.rgw.client.0.smithi191.stdout: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2019-05-15T23:28:21.517 INFO:tasks.rgw.client.0.smithi191.stdout:
2019-05-15T23:28:22.006 INFO:tasks.rgw.client.0.smithi191.stdout:  -184> 2019-05-15 23:28:20.608 20fc5700 -1 RGWWatcher::handle_error cookie 555289952 err (107) Transport endpoint is not connected
2019-05-15T23:28:22.009 INFO:tasks.rgw.client.0.smithi191.stdout:  -181> 2019-05-15 23:28:20.627 20fc5700 -1 RGWWatcher::handle_error cookie 554968048 err (107) Transport endpoint is not connected
2019-05-15T23:28:22.010 INFO:tasks.rgw.client.0.smithi191.stdout:  -172> 2019-05-15 23:28:20.640 20fc5700 -1 RGWWatcher::handle_error cookie 554916192 err (107) Transport endpoint is not connected
2019-05-15T23:28:22.010 INFO:tasks.rgw.client.0.smithi191.stdout:  -170> 2019-05-15 23:28:20.645 20fc5700 -1 RGWWatcher::handle_error cookie 555142736 err (107) Transport endpoint is not connected
2019-05-15T23:28:22.012 INFO:tasks.rgw.client.0.smithi191.stdout:  -168> 2019-05-15 23:28:20.645 20fc5700 -1 RGWWatcher::handle_error cookie 555516832 err (107) Transport endpoint is not connected
2019-05-15T23:28:22.013 INFO:tasks.rgw.client.0.smithi191.stdout:  -166> 2019-05-15 23:28:20.646 20fc5700 -1 RGWWatcher::handle_error cookie 555244704 err (107) Transport endpoint is not connected
2019-05-15T23:28:22.014 INFO:tasks.rgw.client.0.smithi191.stdout:  -164> 2019-05-15 23:28:20.646 20fc5700 -1 RGWWatcher::handle_error cookie 555193680 err (107) Transport endpoint is not connected
2019-05-15T23:28:22.014 INFO:tasks.rgw.client.0.smithi191.stdout:  -160> 2019-05-15 23:28:20.666 20fc5700 -1 RGWWatcher::handle_error cookie 555471584 err (107) Transport endpoint is not connected
2019-05-15T23:28:22.017 INFO:tasks.rgw.client.0.smithi191.stdout:   -79> 2019-05-15 23:28:20.833 aa6e0700 -1 *** Caught signal (Aborted) **
2019-05-15T23:28:22.017 INFO:tasks.rgw.client.0.smithi191.stdout: in thread aa6e0700 thread_name:memcheck-amd64-
2019-05-15T23:28:22.017 INFO:tasks.rgw.client.0.smithi191.stdout:
2019-05-15T23:28:22.017 INFO:tasks.rgw.client.0.smithi191.stdout: ceph version 15.0.0-1282-g7eb6c2d (7eb6c2d557b985b59e1da1679b9f727401df3b5b) octopus (dev)
2019-05-15T23:28:22.017 INFO:tasks.rgw.client.0.smithi191.stdout: 1: (()+0xf5d0) [0x113d25d0]
2019-05-15T23:28:22.018 INFO:tasks.rgw.client.0.smithi191.stdout: 2: (gsignal()+0x37) [0x11e34207]
2019-05-15T23:28:22.018 INFO:tasks.rgw.client.0.smithi191.stdout: 3: (abort()+0x148) [0x11e358f8]
2019-05-15T23:28:22.018 INFO:tasks.rgw.client.0.smithi191.stdout: 4: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x1163f7d5]
2019-05-15T23:28:22.018 INFO:tasks.rgw.client.0.smithi191.stdout: 5: (()+0x5e746) [0x1163d746]
2019-05-15T23:28:22.018 INFO:tasks.rgw.client.0.smithi191.stdout: 6: (()+0x5e773) [0x1163d773]
2019-05-15T23:28:22.018 INFO:tasks.rgw.client.0.smithi191.stdout: 7: (()+0x5e993) [0x1163d993]
2019-05-15T23:28:22.019 INFO:tasks.rgw.client.0.smithi191.stdout: 8: (virtual thunk to boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >::rethrow() const+0x6e) [0x405ade]
2019-05-15T23:28:22.019 INFO:tasks.rgw.client.0.smithi191.stdout: 9: (boost::rethrow_exception(boost::exception_ptr const&)+0xd) [0x3fba0d]
2019-05-15T23:28:22.019 INFO:tasks.rgw.client.0.smithi191.stdout: 10: (boost::coroutines::detail::push_coroutine_impl<void>::push()+0x4e) [0x3fbafe]
2019-05-15T23:28:22.019 INFO:tasks.rgw.client.0.smithi191.stdout: 11: (boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<boost::asio::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::executor>, std::shared_lock<ceph::async::SharedMutex<boost::asio::io_context::executor_type> > >, std::tuple<boost::system::error_code, std::shared_lock<ceph::async::SharedMutex<boost::asio::io_context::executor_type> > > > >, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x16d) [0x40685d]
2019-05-15T23:28:22.019 INFO:tasks.rgw.client.0.smithi191.stdout: 12: (boost::asio::detail::executor_op<boost::asio::executor::function, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x8c) [0x3f879c]
2019-05-15T23:28:22.019 INFO:tasks.rgw.client.0.smithi191.stdout: 13: (boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>::operator()()+0x7d) [0x41e5ad]
2019-05-15T23:28:22.019 INFO:tasks.rgw.client.0.smithi191.stdout: 14: (boost::asio::detail::executor_op<boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>, boost::asio::detail::recycling_allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x94) [0x41e924]
2019-05-15T23:28:22.020 INFO:tasks.rgw.client.0.smithi191.stdout: 15: (boost::asio::detail::scheduler::run(boost::system::error_code&)+0x3f2) [0x3fd3c2]
2019-05-15T23:28:22.020 INFO:tasks.rgw.client.0.smithi191.stdout: 16: (()+0x2de2ab) [0x3e62ab]
2019-05-15T23:28:22.020 INFO:tasks.rgw.client.0.smithi191.stdout: 17: (()+0x7b8f8f) [0x642ff8f]
2019-05-15T23:28:22.020 INFO:tasks.rgw.client.0.smithi191.stdout: 18: (()+0x7dd5) [0x113cadd5]
2019-05-15T23:28:22.020 INFO:tasks.rgw.client.0.smithi191.stdout: 19: (clone()+0x6d) [0x11efbead]
2019-05-15T23:28:22.020 INFO:tasks.rgw.client.0.smithi191.stdout: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2019-05-15T23:28:22.021 INFO:tasks.rgw.client.0.smithi191.stdout:
2019-05-15T23:28:32.314 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:25:59.907 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:32.345 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:25:59.935 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:32.355 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:25:59.949 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:32.359 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:25:59.953 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:32.397 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:25:59.944 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:32.400 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:25:59.947 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:32.461 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:26:00.054 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:32.478 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:26:00.072 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:32.561 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:26:00.108 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:28:38.413 INFO:tasks.rgw.client.0.smithi191.stderr:daemon-helper: command crashed with signal 6
2019-05-15T23:31:47.784 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:29:15.376 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:31:47.792 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:29:15.385 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:31:47.853 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:29:15.445 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:31:47.856 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:29:15.448 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:31:47.865 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:29:15.406 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:31:47.869 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:29:15.410 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:31:47.988 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:29:15.580 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:31:48.011 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:29:15.603 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:31:48.087 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:29:15.628 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:40:28.704 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:40:28.704 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadMultipleSizesLLAPIAWS4 FAILED
2019-05-15T23:40:28.704 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.http.timers.client.ClientExecutionTimeoutException at AWS4Test.java:632
2019-05-15T23:45:10.027 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:42:37.561 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:10.076 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:42:37.667 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:10.098 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:42:37.633 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:10.138 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:42:37.729 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:13.203 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:45:13.203 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadOverwriteExistingObjectLLAPIAWS4 STARTED
2019-05-15T23:45:13.203 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:45:13.203 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadOverwriteExistingObjectLLAPIAWS4 FAILED
2019-05-15T23:45:13.204 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:732
2019-05-15T23:45:13.204 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:732
2019-05-15T23:45:13.204 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:732
2019-05-15T23:45:19.571 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:42:47.162 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:19.617 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:42:47.208 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:19.621 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:42:47.211 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:19.621 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:42:47.212 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:19.634 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:42:47.169 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:19.643 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:42:47.179 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:19.670 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:42:47.261 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:19.692 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:42:47.227 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:20.143 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:42:47.734 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.133 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:43:05.724 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.143 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:43:05.734 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.163 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:43:05.753 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.171 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:43:05.762 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.172 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:43:05.707 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.204 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:43:05.740 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.220 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:43:05.811 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.227 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:43:05.817 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:45:38.303 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:43:05.838 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:46:03.203 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:46:03.203 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadSmallLLAPIAWS4 STARTED
2019-05-15T23:46:03.303 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:46:03.303 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadSmallLLAPIAWS4 FAILED
2019-05-15T23:46:03.303 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:642
2019-05-15T23:46:03.303 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:642
2019-05-15T23:46:03.304 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:642
2019-05-15T23:46:53.303 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:46:53.303 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadWithPauseAWS4 STARTED
2019-05-15T23:46:53.303 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:46:53.303 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testMultipartUploadWithPauseAWS4 FAILED
2019-05-15T23:46:53.303 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:901
2019-05-15T23:46:53.304 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:901
2019-05-15T23:46:53.304 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:901
2019-05-15T23:47:43.302 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:47:43.303 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadAuthorizationIncorrectAWS4 STARTED
2019-05-15T23:47:43.303 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:47:43.303 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadAuthorizationIncorrectAWS4 FAILED
2019-05-15T23:47:43.303 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:562
2019-05-15T23:47:43.303 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:562
2019-05-15T23:47:43.303 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:562
2019-05-15T23:48:33.402 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:48:33.403 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadAuthorizationInvalidAWS4 STARTED
2019-05-15T23:48:33.403 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:48:33.403 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadAuthorizationInvalidAWS4 FAILED
2019-05-15T23:48:33.403 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:538
2019-05-15T23:48:33.403 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:538
2019-05-15T23:48:33.403 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:538
2019-05-15T23:49:23.402 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:49:23.402 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateAfterToday4AWS4 STARTED
2019-05-15T23:49:23.403 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:49:23.403 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateAfterToday4AWS4 FAILED
2019-05-15T23:49:23.403 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:212
2019-05-15T23:49:23.403 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:212
2019-05-15T23:49:23.403 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:212
2019-05-15T23:50:13.503 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:50:13.504 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateBeforeEpochAWS4 STARTED
2019-05-15T23:50:13.504 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:50:13.504 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateBeforeEpochAWS4 FAILED
2019-05-15T23:50:13.504 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:168
2019-05-15T23:50:13.504 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:168
2019-05-15T23:50:13.504 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:168
2019-05-15T23:51:03.502 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:51:03.503 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateBeforeToday4AWS4 STARTED
2019-05-15T23:51:03.503 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:51:03.503 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateBeforeToday4AWS4 FAILED
2019-05-15T23:51:03.503 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:258
2019-05-15T23:51:03.503 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:258
2019-05-15T23:51:03.503 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:258
2019-05-15T23:51:53.603 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:51:53.603 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateEmptyAWS4 STARTED
2019-05-15T23:51:53.603 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:51:53.603 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateEmptyAWS4 FAILED
2019-05-15T23:51:53.603 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:398
2019-05-15T23:51:53.604 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:398
2019-05-15T23:51:53.604 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:398
2019-05-15T23:52:34.548 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:50:02.139 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:34.557 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:50:02.148 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:34.578 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:50:02.113 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:34.589 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:50:02.179 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:34.595 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:50:02.130 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:34.600 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:50:02.188 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:34.916 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:50:02.507 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:34.932 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:50:02.523 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:35.041 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:50:02.576 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:52:43.603 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:52:43.603 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateInvalidAWS4 STARTED
2019-05-15T23:52:43.603 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:52:43.603 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateInvalidAWS4 FAILED
2019-05-15T23:52:43.604 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:442
2019-05-15T23:52:43.604 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:442
2019-05-15T23:52:43.604 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:442
2019-05-15T23:53:33.703 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:53:33.704 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateNoneAWS4 STARTED
2019-05-15T23:53:33.704 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:53:33.704 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadDateNoneAWS4 FAILED
2019-05-15T23:53:33.704 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:304
2019-05-15T23:53:33.705 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:304
2019-05-15T23:53:33.705 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:304
2019-05-15T23:53:45.641 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:51:13.232 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:53:45.670 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:51:13.260 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:53:45.684 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:51:13.219 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:53:45.685 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:51:13.276 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:53:45.693 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:51:13.280 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:53:45.693 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:51:13.223 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:53:45.724 INFO:tasks.ceph.mon.a.smithi191.stderr:==00:00:51:13.314 10248== Warning: unimplemented fcntl command: 1036
2019-05-15T23:53:45.749 INFO:tasks.ceph.mon.c.smithi191.stderr:==00:00:51:13.339 10262== Warning: unimplemented fcntl command: 1036
2019-05-15T23:53:45.776 INFO:tasks.ceph.mon.b.smithi205.stderr:==00:00:51:13.310 10194== Warning: unimplemented fcntl command: 1036
2019-05-15T23:54:23.703 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:54:23.703 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadMd5InvalidGarbageAWS4 STARTED
2019-05-15T23:54:23.703 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:54:23.704 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testObjectCreateBadMd5InvalidGarbageAWS4 FAILED
2019-05-15T23:54:23.704 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:586
2019-05-15T23:54:23.704 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:586
2019-05-15T23:54:23.704 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:586
2019-05-15T23:55:13.703 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:55:13.703 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testUploadFileHLAPIBigFileAWS4 STARTED
2019-05-15T23:55:13.803 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:55:13.803 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testUploadFileHLAPIBigFileAWS4 FAILED
2019-05-15T23:55:13.803 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:825
2019-05-15T23:55:13.803 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:825
2019-05-15T23:55:13.804 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:825
2019-05-15T23:56:03.808 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:56:03.808 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testUploadFileListHLAPIAWS4 STARTED
2019-05-15T23:56:03.809 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:56:03.809 INFO:teuthology.orchestra.run.smithi191.stdout:suite > AWS4 tests > AWS4Test.testUploadFileListHLAPIAWS4 FAILED
2019-05-15T23:56:03.809 INFO:teuthology.orchestra.run.smithi191.stdout:    com.amazonaws.SdkClientException at AWS4Test.java:1181
2019-05-15T23:56:03.809 INFO:teuthology.orchestra.run.smithi191.stdout:        Caused by: org.apache.http.conn.HttpHostConnectException at AWS4Test.java:1181
2019-05-15T23:56:03.810 INFO:teuthology.orchestra.run.smithi191.stdout:            Caused by: java.net.ConnectException at AWS4Test.java:1181
2019-05-15T23:56:53.904 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:56:53.904 INFO:teuthology.orchestra.run.smithi191.stdout:-----------------------------------------------------------------------
2019-05-15T23:56:53.904 INFO:teuthology.orchestra.run.smithi191.stdout:|  Results: FAILURE (29 tests, 14 successes, 15 failures, 0 skipped)  |
2019-05-15T23:56:53.904 INFO:teuthology.orchestra.run.smithi191.stdout:-----------------------------------------------------------------------
2019-05-15T23:56:53.905 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:56:53.905 INFO:teuthology.orchestra.run.smithi191.stdout:29 tests completed, 15 failed
2019-05-15T23:56:54.003 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:56:54.003 INFO:teuthology.orchestra.run.smithi191.stdout:> Task :test FAILED
2019-05-15T23:56:54.156 INFO:teuthology.orchestra.run.smithi191.stderr:
2019-05-15T23:56:54.156 INFO:teuthology.orchestra.run.smithi191.stderr:FAILURE: Build failed with an exception.
2019-05-15T23:56:54.156 INFO:teuthology.orchestra.run.smithi191.stderr:
2019-05-15T23:56:54.156 INFO:teuthology.orchestra.run.smithi191.stderr:* What went wrong:
2019-05-15T23:56:54.156 INFO:teuthology.orchestra.run.smithi191.stderr:Execution failed for task ':test'.
2019-05-15T23:56:54.156 INFO:teuthology.orchestra.run.smithi191.stderr:> There were failing tests. See the report at: file:///home/ubuntu/cephtest/s3-tests-java/build/reports/tests/test/index.html
2019-05-15T23:56:54.157 INFO:teuthology.orchestra.run.smithi191.stderr:
2019-05-15T23:56:54.157 INFO:teuthology.orchestra.run.smithi191.stderr:* Try:
2019-05-15T23:56:54.157 INFO:teuthology.orchestra.run.smithi191.stderr:Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
2019-05-15T23:56:54.157 INFO:teuthology.orchestra.run.smithi191.stderr:
2019-05-15T23:56:54.157 INFO:teuthology.orchestra.run.smithi191.stderr:* Get more help at https://help.gradle.org
2019-05-15T23:56:54.157 INFO:teuthology.orchestra.run.smithi191.stdout:
2019-05-15T23:56:54.158 INFO:teuthology.orchestra.run.smithi191.stdout:BUILD FAILED in 44m 12s
2019-05-15T23:56:54.158 INFO:teuthology.orchestra.run.smithi191.stdout:5 actionable tasks: 4 executed, 1 up-to-date
2019-05-15T23:56:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1
2019-05-15T23:56:54.476 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 89, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/github.com_alimaredia_ceph_wip-s3java/qa/tasks/s3tests_java.py", line 97, in begin
    self.run_tests()
  File "/home/teuthworker/src/github.com_alimaredia_ceph_wip-s3java/qa/tasks/s3tests_java.py", line 358, in run_tests
    stdout=StringIO()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/cluster.py", line 64, in run
    return [remote.run(**kwargs) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 205, in run
    r = self._runner(client=self.ssh, name=self.shortname, **kwargs)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 437, in run
    r.wait()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 162, in wait
    self._raise_for_status()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 184, in _raise_for_status
    node=self.hostname, label=self.label
CommandFailedError: Command failed on smithi191 with status 1: 'cd /home/ubuntu/cephtest/s3-tests-java && /opt/gradle/gradle-4.7/bin/gradle clean test --rerun-tasks --no-build-cache --tests AWS4Test'
2019-05-15T23:56:54.477 DEBUG:teuthology.run_tasks:Unwinding manager s3tests-java
2019-05-15T23:56:54.491 DEBUG:tasks.s3tests_java:S3 Tests Java: END
2019-05-15T23:56:54.492 INFO:tasks.s3tests_java:S3 Tests Java: Cleaning up s3-tests-java...
2019-05-15T23:56:54.492 INFO:teuthology.orchestra.run.smithi191:Running:
2019-05-15T23:56:54.492 INFO:teuthology.orchestra.run.smithi191:> rm -rf /home/ubuntu/cephtest/s3-tests-java
2019-05-15T23:56:54.660 INFO:tasks.s3tests_java:S3 Tests Java: Deleting S3 users...
2019-05-15T23:56:54.660 INFO:teuthology.orchestra.run.smithi191:Running:
2019-05-15T23:56:54.660 INFO:teuthology.orchestra.run.smithi191:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user rm --uid tester.client.0 --purge-data --cluster ceph
2019-05-15T23:56:54.844 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:54.842 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:54.881 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:54.880 7f96538ed6c0 20 rados_obj.operate() r=-2 bl.length=0
2019-05-15T23:56:54.882 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:54.880 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:54.892 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:54.890 7f96538ed6c0 20 rados_obj.operate() r=-2 bl.length=0
2019-05-15T23:56:54.892 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:54.890 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:54.901 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:54.900 7f96538ed6c0 20 rados_obj.operate() r=-2 bl.length=0
2019-05-15T23:56:54.902 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:54.900 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:54.914 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:54.913 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=46
2019-05-15T23:56:55.043 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.041 7f96538ed6c0 20 RGWRados::pool_iterate: got zonegroup_info.84ad0c4f-e797-4ef6-8c26-a8925f2ac663
2019-05-15T23:56:55.043 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.041 7f96538ed6c0 20 RGWRados::pool_iterate: got zone_info.c1bcc155-c020-4c00-8ef0-5e50ae0d3a17
2019-05-15T23:56:55.044 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.041 7f96538ed6c0 20 RGWRados::pool_iterate: got zone_names.default
2019-05-15T23:56:55.044 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.041 7f96538ed6c0 20 RGWRados::pool_iterate: got zonegroups_names.default
2019-05-15T23:56:55.044 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.041 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.060 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.058 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=46
2019-05-15T23:56:55.061 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.058 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.073 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.071 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=951
2019-05-15T23:56:55.073 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.071 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.086 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.084 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=46
2019-05-15T23:56:55.086 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.084 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.097 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.095 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=382
2019-05-15T23:56:55.097 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.095 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.108 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.106 7f96538ed6c0 20 rados_obj.operate() r=-2 bl.length=0
2019-05-15T23:56:55.108 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.106 7f96538ed6c0 10  cannot find current period zonegroup using local zonegroup
2019-05-15T23:56:55.109 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.106 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.115 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.113 7f96538ed6c0 20 rados_obj.operate() r=-2 bl.length=0
2019-05-15T23:56:55.115 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.113 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.126 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.125 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=46
2019-05-15T23:56:55.127 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.125 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.136 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.135 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=382
2019-05-15T23:56:55.137 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.135 7f96538ed6c0 20 zonegroup default
2019-05-15T23:56:55.137 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.135 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.144 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.143 7f96538ed6c0 20 rados_obj.operate() r=-2 bl.length=0
2019-05-15T23:56:55.144 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.143 7f96538ed6c0 10 Cannot find current period zone using local zone
2019-05-15T23:56:55.144 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.143 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.152 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.151 7f96538ed6c0 20 rados_obj.operate() r=-2 bl.length=0
2019-05-15T23:56:55.153 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.151 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.168 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.166 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=46
2019-05-15T23:56:55.168 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.166 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.175 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.174 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=951
2019-05-15T23:56:55.175 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.174 7f96538ed6c0 20 zone default
2019-05-15T23:56:55.793 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.791 7f96538ed6c0 20 add_watcher() i=0
2019-05-15T23:56:55.794 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.791 7f96538ed6c0 20 add_watcher() i=1
2019-05-15T23:56:55.794 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.791 7f96538ed6c0 20 add_watcher() i=2
2019-05-15T23:56:55.794 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.791 7f96538ed6c0 20 add_watcher() i=3
2019-05-15T23:56:55.794 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.791 7f96538ed6c0 20 add_watcher() i=4
2019-05-15T23:56:55.795 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.791 7f96538ed6c0 20 add_watcher() i=5
2019-05-15T23:56:55.822 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.820 7f96538ed6c0 20 add_watcher() i=6
2019-05-15T23:56:55.837 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.835 7f96538ed6c0 20 add_watcher() i=7
2019-05-15T23:56:55.837 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.835 7f96538ed6c0  2 all 8 watchers are set, enabling cache
2019-05-15T23:56:55.854 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.852 7f9640974700  0 --  >> v2:172.21.15.205:3300/0 conn(0x561ee586b3a0 msgr2=0x561ee586b7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure
2019-05-15T23:56:55.871 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.869 7f961cff9700  2 RGWDataChangesLog::ChangesRenewThread: start
2019-05-15T23:56:55.884 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.882 7f95ff7de700 20 reqs_thread_entry: start
2019-05-15T23:56:55.884 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.882 7f95fd7da700 20 reqs_thread_entry: start
2019-05-15T23:56:55.884 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.882 7f96538ed6c0 20 init_complete bucket index max shards: 0
2019-05-15T23:56:55.884 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.883 7f95bf7fe700 20 reqs_thread_entry: start
2019-05-15T23:56:55.885 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.883 7f96538ed6c0 20 get_system_obj_state: rctx=0x7ffc2078b100 obj=default.rgw.meta:users.uid:tester.client.0 state=0x561ee58d7e90 s->prefetch_data=0
2019-05-15T23:56:55.885 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.883 7f96538ed6c0 10 cache get: name=default.rgw.meta+users.uid+tester.client.0 : miss
2019-05-15T23:56:55.899 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.897 7f96538ed6c0 10 cache put: name=default.rgw.meta+users.uid+tester.client.0 info.flags=0x16
2019-05-15T23:56:55.899 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.897 7f96538ed6c0 10 adding default.rgw.meta+users.uid+tester.client.0 to cache LRU end
2019-05-15T23:56:55.899 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.897 7f96538ed6c0 10 updating xattr: name=ceph.objclass.version bl.length()=42
2019-05-15T23:56:55.899 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.897 7f96538ed6c0 20 get_system_obj_state: s->obj_tag was set empty
2019-05-15T23:56:55.899 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.897 7f96538ed6c0 10 cache get: name=default.rgw.meta+users.uid+tester.client.0 : type miss (requested=0x11, cached=0x16)
2019-05-15T23:56:55.899 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.897 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:55.923 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.921 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=444
2019-05-15T23:56:55.924 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.922 7f96538ed6c0 10 cache put: name=default.rgw.meta+users.uid+tester.client.0 info.flags=0x11
2019-05-15T23:56:55.924 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.922 7f96538ed6c0 10 moving default.rgw.meta+users.uid+tester.client.0 to cache LRU end
2019-05-15T23:56:55.957 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.955 7f96538ed6c0 20 get_system_obj_state: rctx=0x7ffc2078b0b0 obj=default.rgw.meta:root:test-24048ed6-b4df-44f8-a706-1bf9a0f536d711 state=0x561ee58d8770 s->prefetch_data=0
2019-05-15T23:56:55.957 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.955 7f96538ed6c0 10 cache get: name=default.rgw.meta+root+test-24048ed6-b4df-44f8-a706-1bf9a0f536d711 : miss
2019-05-15T23:56:55.981 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.979 7f9640974700  0 -- 172.21.15.191:0/679785457 >> [v2:172.21.15.205:6800/11510,v1:172.21.15.205:6801/11510] conn(0x7f963c0099c0 msgr2=0x7f963c002550 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure
2019-05-15T23:56:55.981 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.979 7f96538ed6c0 10 cache put: name=default.rgw.meta+root+test-24048ed6-b4df-44f8-a706-1bf9a0f536d711 info.flags=0x16
2019-05-15T23:56:55.981 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.979 7f96538ed6c0 10 adding default.rgw.meta+root+test-24048ed6-b4df-44f8-a706-1bf9a0f536d711 to cache LRU end
2019-05-15T23:56:55.981 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.979 7f96538ed6c0 10 updating xattr: name=ceph.objclass.version bl.length()=42
2019-05-15T23:56:55.981 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.979 7f96538ed6c0 20 get_system_obj_state: s->obj_tag was set empty
2019-05-15T23:56:55.981 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.979 7f96538ed6c0 10 cache get: name=default.rgw.meta+root+test-24048ed6-b4df-44f8-a706-1bf9a0f536d711 : type miss (requested=0x11, cached=0x16)
2019-05-15T23:56:55.982 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:55.979 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:56.017 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.015 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=225
2019-05-15T23:56:56.017 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.015 7f96538ed6c0 10 cache put: name=default.rgw.meta+root+test-24048ed6-b4df-44f8-a706-1bf9a0f536d711 info.flags=0x11
2019-05-15T23:56:56.017 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.015 7f96538ed6c0 10 moving default.rgw.meta+root+test-24048ed6-b4df-44f8-a706-1bf9a0f536d711 to cache LRU end
2019-05-15T23:56:56.017 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.015 7f96538ed6c0 20 rgw_get_bucket_info: bucket instance: test-24048ed6-b4df-44f8-a706-1bf9a0f536d711[c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28]
2019-05-15T23:56:56.018 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.015 7f96538ed6c0 20 reading from default.rgw.meta:root:.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28
2019-05-15T23:56:56.018 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.015 7f96538ed6c0 20 get_system_obj_state: rctx=0x7ffc2078b0b0 obj=default.rgw.meta:root:.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 state=0x561ee58db850 s->prefetch_data=0
2019-05-15T23:56:56.018 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.015 7f96538ed6c0 10 cache get: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 : miss
2019-05-15T23:56:56.034 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 10 cache put: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 info.flags=0x16
2019-05-15T23:56:56.034 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 10 adding default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 to cache LRU end
2019-05-15T23:56:56.034 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 10 updating xattr: name=ceph.objclass.version bl.length()=42
2019-05-15T23:56:56.035 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 10 updating xattr: name=user.rgw.acl bl.length()=203
2019-05-15T23:56:56.035 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 20 get_system_obj_state: s->obj_tag was set empty
2019-05-15T23:56:56.035 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 20 Read xattr: user.rgw.acl
2019-05-15T23:56:56.035 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 20 Read xattr: user.rgw.idtag
2019-05-15T23:56:56.035 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 10 cache get: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 : type miss (requested=0x13, cached=0x16)
2019-05-15T23:56:56.035 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.032 7f96538ed6c0 20 rados->read ofs=0 len=0
2019-05-15T23:56:56.054 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.052 7f96538ed6c0 20 rados_obj.operate() r=0 bl.length=318
2019-05-15T23:56:56.054 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.052 7f96538ed6c0 10 cache put: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 info.flags=0x13
2019-05-15T23:56:56.054 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.052 7f96538ed6c0 10 moving default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 to cache LRU end
2019-05-15T23:56:56.054 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.052 7f96538ed6c0 10 updating xattr: name=ceph.objclass.version bl.length()=42
2019-05-15T23:56:56.054 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.052 7f96538ed6c0 10 updating xattr: name=user.rgw.acl bl.length()=203
2019-05-15T23:56:56.054 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.052 7f96538ed6c0 10 chain_cache_entry: cache_locator=default.rgw.meta+root+test-24048ed6-b4df-44f8-a706-1bf9a0f536d711
2019-05-15T23:56:56.055 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.052 7f96538ed6c0 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28
2019-05-15T23:56:56.079 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.077 7f96538ed6c0 10 cls_bucket_list_unordered test-24048ed6-b4df-44f8-a706-1bf9a0f536d711[c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28] start [] num_entries 1100
2019-05-15T23:56:56.104 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.102 7f96538ed6c0 10 RGWRados::cls_bucket_list_unordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.1[]
2019-05-15T23:56:56.104 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.102 7f96538ed6c0 10 RGWRados::cls_bucket_list_unordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.2[]
2019-05-15T23:56:56.104 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.102 7f96538ed6c0 10 RGWRados::cls_bucket_list_unordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.3[]
2019-05-15T23:56:56.104 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.102 7f96538ed6c0 10 RGWRados::cls_bucket_list_unordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.4[]
2019-05-15T23:56:56.104 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.102 7f96538ed6c0 10 RGWRados::cls_bucket_list_unordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.meta[]
2019-05-15T23:56:56.105 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.102 7f96538ed6c0 10 RGWRados::cls_bucket_list_unordered: got key1[]
2019-05-15T23:56:56.105 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.102 7f96538ed6c0 20 get_obj_state: rctx=0x7ffc2078aa70 obj=test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:key1 state=0x561ee58dffd0 s->prefetch_data=0
2019-05-15T23:56:56.128 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.126 7f96538ed6c0 10 manifest: total_size = 55574528
2019-05-15T23:56:56.128 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.126 7f96538ed6c0 20 get_obj_state: setting s->obj_tag to c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4349.634
2019-05-15T23:56:56.128 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.126 7f96538ed6c0 20 get_obj_state: rctx=0x7ffc2078aa70 obj=test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:key1 state=0x561ee58dffd0 s->prefetch_data=0
2019-05-15T23:56:56.128 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.126 7f96538ed6c0 20 prepare_atomic_modification: state is not atomic. state=0x561ee58dffd0
2019-05-15T23:56:56.129 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.126 7f96538ed6c0 20 reading from default.rgw.meta:root:.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28
2019-05-15T23:56:56.129 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.126 7f96538ed6c0 20 get_system_obj_state: rctx=0x7ffc20789f60 obj=default.rgw.meta:root:.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 state=0x561ee58e2000 s->prefetch_data=0
2019-05-15T23:56:56.129 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.126 7f96538ed6c0 10 cache get: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 : type miss (requested=0x16, cached=0x13)
2019-05-15T23:56:56.142 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.140 7f96538ed6c0 10 cache put: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 info.flags=0x16
2019-05-15T23:56:56.142 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.140 7f96538ed6c0 10 moving default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 to cache LRU end
2019-05-15T23:56:56.142 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.140 7f96538ed6c0 10 updating xattr: name=ceph.objclass.version bl.length()=42
2019-05-15T23:56:56.142 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.140 7f96538ed6c0 10 updating xattr: name=user.rgw.acl bl.length()=203
2019-05-15T23:56:56.142 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.140 7f96538ed6c0 20 get_system_obj_state: s->obj_tag was set empty
2019-05-15T23:56:56.142 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.140 7f96538ed6c0 10 cache get: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 : hit (requested=0x11, cached=0x17)
2019-05-15T23:56:56.143 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.140 7f96538ed6c0 20  bucket index object: .dir.c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28
2019-05-15T23:56:56.335 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.335 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=4194304 part_ofs=0 rule->part_size=11100160
2019-05-15T23:56:56.335 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=4194304 stripe_ofs=4194304 part_ofs=0 rule->part_size=11100160
2019-05-15T23:56:56.335 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.335 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=8388608 part_ofs=0 rule->part_size=11100160
2019-05-15T23:56:56.336 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=8388608 stripe_ofs=8388608 part_ofs=0 rule->part_size=11100160
2019-05-15T23:56:56.336 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.336 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=12582912 part_ofs=0 rule->part_size=11100160
2019-05-15T23:56:56.336 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=11100160 stripe_ofs=11100160 part_ofs=11100160 rule->part_size=11100160
2019-05-15T23:56:56.336 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.336 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=15294464 part_ofs=11100160 rule->part_size=11100160
2019-05-15T23:56:56.336 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=15294464 stripe_ofs=15294464 part_ofs=11100160 rule->part_size=11100160
2019-05-15T23:56:56.337 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.337 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=19488768 part_ofs=11100160 rule->part_size=11100160
2019-05-15T23:56:56.337 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=19488768 stripe_ofs=19488768 part_ofs=11100160 rule->part_size=11100160
2019-05-15T23:56:56.337 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.337 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=23683072 part_ofs=11100160 rule->part_size=11100160
2019-05-15T23:56:56.337 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=22200320 stripe_ofs=22200320 part_ofs=22200320 rule->part_size=11100160
2019-05-15T23:56:56.337 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.338 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=26394624 part_ofs=22200320 rule->part_size=11100160
2019-05-15T23:56:56.338 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=26394624 stripe_ofs=26394624 part_ofs=22200320 rule->part_size=11100160
2019-05-15T23:56:56.338 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.338 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=30588928 part_ofs=22200320 rule->part_size=11100160
2019-05-15T23:56:56.338 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=30588928 stripe_ofs=30588928 part_ofs=22200320 rule->part_size=11100160
2019-05-15T23:56:56.338 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.338 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=34783232 part_ofs=22200320 rule->part_size=11100160
2019-05-15T23:56:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=33300480 stripe_ofs=33300480 part_ofs=33300480 rule->part_size=11100160
2019-05-15T23:56:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=37494784 part_ofs=33300480 rule->part_size=11100160
2019-05-15T23:56:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=37494784 stripe_ofs=37494784 part_ofs=33300480 rule->part_size=11100160
2019-05-15T23:56:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=41689088 part_ofs=33300480 rule->part_size=11100160
2019-05-15T23:56:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=41689088 stripe_ofs=41689088 part_ofs=33300480 rule->part_size=11100160
2019-05-15T23:56:56.339 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.340 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=45883392 part_ofs=33300480 rule->part_size=11100160
2019-05-15T23:56:56.340 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=44400640 stripe_ofs=44400640 part_ofs=44400640 rule->part_size=11100160
2019-05-15T23:56:56.340 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.340 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=48594944 part_ofs=44400640 rule->part_size=11100160
2019-05-15T23:56:56.340 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=48594944 stripe_ofs=48594944 part_ofs=44400640 rule->part_size=11100160
2019-05-15T23:56:56.340 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.340 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=52789248 part_ofs=44400640 rule->part_size=11100160
2019-05-15T23:56:56.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=52789248 stripe_ofs=52789248 part_ofs=44400640 rule->part_size=11100160
2019-05-15T23:56:56.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=11100160 rules.size()=2
2019-05-15T23:56:56.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=56983552 part_ofs=44400640 rule->part_size=11100160
2019-05-15T23:56:56.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=55500800 stripe_ofs=55500800 part_ofs=55500800 rule->part_size=73728
2019-05-15T23:56:56.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=73728 rules.size()=2
2019-05-15T23:56:56.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): stripe_ofs=59695104 part_ofs=55500800 rule->part_size=73728
2019-05-15T23:56:56.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=55574528 stripe_ofs=55574528 part_ofs=55574528 rule->part_size=73728
2019-05-15T23:56:56.342 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.332 7f96538ed6c0 10 cls_bucket_list_ordered test-24048ed6-b4df-44f8-a706-1bf9a0f536d711[c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28] start _multipart_[] num_entries 1001
2019-05-15T23:56:56.428 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.426 7f96538ed6c0 10 RGWRados::cls_bucket_list_ordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.1[]
2019-05-15T23:56:56.429 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.426 7f96538ed6c0 10 RGWRados::cls_bucket_list_ordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.2[]
2019-05-15T23:56:56.429 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.426 7f96538ed6c0 10 RGWRados::cls_bucket_list_ordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.3[]
2019-05-15T23:56:56.429 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.426 7f96538ed6c0 10 RGWRados::cls_bucket_list_ordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.4[]
2019-05-15T23:56:56.429 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.426 7f96538ed6c0 10 RGWRados::cls_bucket_list_ordered: got _multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.meta[]
2019-05-15T23:56:56.462 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.462 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=4194304 stripe_ofs=4194304 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.462 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.462 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=8388608 stripe_ofs=8388608 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.462 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.462 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=10485760 stripe_ofs=10485760 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.463 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.463 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=4194304 stripe_ofs=4194304 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.463 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.463 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=8388608 stripe_ofs=8388608 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.463 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.463 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=10485760 stripe_ofs=10485760 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.463 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.464 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=4194304 stripe_ofs=4194304 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.464 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.464 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=8388608 stripe_ofs=8388608 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.464 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.464 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=10485760 stripe_ofs=10485760 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.464 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.464 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=4194304 stripe_ofs=4194304 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.465 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.465 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=8388608 stripe_ofs=8388608 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.465 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): rule->part_size=0 rules.size()=1
2019-05-15T23:56:56.465 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 RGWObjManifest::operator++(): result: ofs=10485760 stripe_ofs=10485760 part_ofs=0 rule->part_size=0
2019-05-15T23:56:56.465 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.459 7f96538ed6c0 20 get_obj_state: rctx=0x7ffc2078aa70 obj=test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:_multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.meta state=0x561ee58ecfc0 s->prefetch_data=0
2019-05-15T23:56:56.485 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 20 get_obj_state: s->obj_tag was set empty
2019-05-15T23:56:56.485 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 20 get_obj_state: rctx=0x7ffc2078aa70 obj=test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:_multipart_key1.2~WbBk69zen9Zd7rHKEx9or20SthCUm2b.meta state=0x561ee58ecfc0 s->prefetch_data=0
2019-05-15T23:56:56.485 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 20 prepare_atomic_modification: state is not atomic. state=0x561ee58ecfc0
2019-05-15T23:56:56.485 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 20 reading from default.rgw.meta:root:.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28
2019-05-15T23:56:56.485 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 20 get_system_obj_state: rctx=0x7ffc20789c60 obj=default.rgw.meta:root:.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 state=0x561ee58ec0f0 s->prefetch_data=0
2019-05-15T23:56:56.486 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 10 cache get: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 : hit (requested=0x16, cached=0x17)
2019-05-15T23:56:56.486 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 20 get_system_obj_state: s->obj_tag was set empty
2019-05-15T23:56:56.487 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 10 cache get: name=default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 : hit (requested=0x11, cached=0x17)
2019-05-15T23:56:56.487 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.483 7f96538ed6c0 20  bucket index object: .dir.c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28
2019-05-15T23:56:56.617 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.615 7f96538ed6c0  0 WARNING : aborted 1 incomplete multipart uploads
2019-05-15T23:56:56.763 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.761 7f96538ed6c0 10 removing default.rgw.meta+root+test-24048ed6-b4df-44f8-a706-1bf9a0f536d711 from cache
2019-05-15T23:56:56.763 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.761 7f96538ed6c0 10 distributing notification oid=default.rgw.control:notify.6 bl.length()=187
2019-05-15T23:56:56.774 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.772 7f9629ffb700 10 RGWWatcher::handle_notify()  notify_id 536870912009 cookie 94690699746448 notifier 4767 bl.length()=187
2019-05-15T23:56:56.920 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.918 7f96538ed6c0 10 removing default.rgw.meta+root+.bucket.meta.test-24048ed6-b4df-44f8-a706-1bf9a0f536d711:c1bcc155-c020-4c00-8ef0-5e50ae0d3a17.4355.28 from cache
2019-05-15T23:56:56.920 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.918 7f96538ed6c0 10 distributing notification oid=default.rgw.control:notify.7 bl.length()=245
2019-05-15T23:56:56.934 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:56.932 7f9629ffb700 10 RGWWatcher::handle_notify()  notify_id 536870912018 cookie 94690699747088 notifier 4767 bl.length()=245
2019-05-15T23:56:57.155 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.153 7f96538ed6c0 10 removing key index: VVYUFKYNRISBALFHUBNS
2019-05-15T23:56:57.155 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.153 7f96538ed6c0 10 distributing notification oid=default.rgw.control:notify.4 bl.length()=170
2019-05-15T23:56:57.168 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.167 7f9629ffb700 10 RGWWatcher::handle_notify()  notify_id 536870912034 cookie 94690699728464 notifier 4767 bl.length()=170
2019-05-15T23:56:57.244 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.242 7f96538ed6c0 10 removing email index: tester.client.0_test@test.test
2019-05-15T23:56:57.244 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.242 7f96538ed6c0 10 distributing notification oid=default.rgw.control:notify.3 bl.length()=181
2019-05-15T23:56:57.254 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.253 7f9629ffb700 10 RGWWatcher::handle_notify()  notify_id 536870912035 cookie 94690699724704 notifier 4767 bl.length()=181
2019-05-15T23:56:57.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.339 7f96538ed6c0 10 removing user buckets index
2019-05-15T23:56:57.341 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.339 7f96538ed6c0 10 distributing notification oid=default.rgw.control:notify.7 bl.length()=172
2019-05-15T23:56:57.359 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.357 7f9629ffb700 10 RGWWatcher::handle_notify()  notify_id 536870912019 cookie 94690699747088 notifier 4767 bl.length()=172
2019-05-15T23:56:57.435 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.433 7f96538ed6c0 10 removing user index: tester.client.0
2019-05-15T23:56:57.435 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.433 7f96538ed6c0 10 removing default.rgw.meta+users.uid+tester.client.0 from cache
2019-05-15T23:56:57.435 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.433 7f96538ed6c0 10 distributing notification oid=default.rgw.control:notify.6 bl.length()=164
2019-05-15T23:56:57.444 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.442 7f9629ffb700 10 RGWWatcher::handle_notify()  notify_id 536870912010 cookie 94690699746448 notifier 4767 bl.length()=164
2019-05-15T23:56:57.586 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.585 7f96538ed6c0 20 remove_watcher() i=0
2019-05-15T23:56:57.587 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.585 7f96538ed6c0  2 removed watcher, disabling cache
2019-05-15T23:56:57.638 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.636 7f96538ed6c0 20 remove_watcher() i=1
2019-05-15T23:56:57.685 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.684 7f96538ed6c0 20 remove_watcher() i=2
2019-05-15T23:56:57.740 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.739 7f96538ed6c0 20 remove_watcher() i=3
2019-05-15T23:56:57.781 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.779 7f96538ed6c0 20 remove_watcher() i=4
2019-05-15T23:56:57.820 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.818 7f96538ed6c0 20 remove_watcher() i=5
2019-05-15T23:56:57.882 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.880 7f96538ed6c0 20 remove_watcher() i=6
2019-05-15T23:56:57.940 INFO:teuthology.orchestra.run.smithi191.stderr:2019-05-15 23:56:57.938 7f96538ed6c0 20 remove_watcher() i=7
2019-05-15T23:56:57.949 DEBUG:teuthology.run_tasks:Unwinding manager ragweed
2019-05-15T23:56:57.969 DEBUG:teuthology.run_tasks:Unwinding manager rgw
2019-05-15T23:56:57.986 ERROR:teuthology.contextutil:Saw exception from nested tasks
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/contextutil.py", line 32, in nested
    yield vars
  File "/home/teuthworker/src/github.com_alimaredia_ceph_wip-s3java/qa/tasks/rgw.py", line 347, in task
    yield
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 89, in run_tasks
    manager.__enter__()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/__init__.py", line 123, in __enter__
    self.begin()
  File "/home/teuthworker/src/github.com_alimaredia_ceph_wip-s3java/qa/tasks/s3tests_java.py", line 97, in begin
    self.run_tests()
  File "/home/teuthworker/src/github.com_alimaredia_ceph_wip-s3java/qa/tasks/s3tests_java.py", line 358, in run_tests
    stdout=StringIO()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/cluster.py", line 64, in run
    return [remote.run(**kwargs) for remote in remotes]
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 205, in run
    r = self._runner(client=self.ssh, name=self.shortname, **kwargs)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 437, in run
    r.wait()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 162, in wait
    self._raise_for_status()
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 184, in _raise_for_status
    node=self.hostname, label=self.label
CommandFailedError: Command failed on smithi191 with status 1: 'cd /home/ubuntu/cephtest/s3-tests-java && /opt/gradle/gradle-4.7/bin/gradle clean test --rerun-tasks --no-build-cache --tests AWS4Test'
2019-05-15T23:56:57.986 DEBUG:tasks.rgw.client.0:waiting for process to exit
2019-05-15T23:56:57.986 INFO:teuthology.orchestra.run:waiting for 300
2019-05-15T23:56:57.987 INFO:tasks.rgw.client.0:Stopped