Bug #48157
Updated by Deepika Upadhyay over 3 years ago
http://qa-proxy.ceph.com/teuthology/yuriw-2020-11-09_17:43:09-rados-wip-yuri8-testing-2020-11-09-0807-octopus-distro-basic-smithi/5606004/teuthology.log http://qa-proxy.ceph.com/teuthology/yuriw-2020-11-09_17:43:09-rados-wip-yuri8-testing-2020-11-09-0807-octopus-distro-basic-smithi/5606056/teuthology.log http://qa-proxy.ceph.com/teuthology/yuriw-2020-11-09_17:43:09-rados-wip-yuri8-testing-2020-11-09-0807-octopus-distro-basic-smithi/5605996/teuthology.log <pre> 2020-11-09T19:59:45.125 INFO:tasks.workunit.client.0.smithi183.stderr:Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --net=host --entrypoint ceph -e CONTAINER_IMAGE=docker.io/ceph/daemon-base:latest-nautilus -e NODE_NAME=smithi183 docker.io/ceph/daemon-base:latest-nautilus --version 2020-11-09T19:59:45.125 INFO:tasks.workunit.client.0.smithi183.stderr:ceph:stderr Trying to pull docker.io/ceph/daemon-base:latest-nautilus... 2020-11-09T19:59:45.126 INFO:tasks.workunit.client.0.smithi183.stderr:ceph:stderr toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2020-11-09T19:59:45.126 INFO:tasks.workunit.client.0.smithi183.stderr:ceph:stderr Error: unable to pull docker.io/ceph/daemon-base:latest-nautilus: unable to pull image: Error initializing source docker://ceph/daemon-base:latest-nautilus: Error reading manifest latest-nautilus in docker.io/ceph/daemon-base: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2020-11-09T19:59:45.129 INFO:tasks.workunit.client.0.smithi183.stderr:Traceback (most recent call last): 2020-11-09T19:59:45.130 INFO:tasks.workunit.client.0.smithi183.stderr: File "/tmp/tmp.2RH4gLOl67/cephadm", line 6041, in <module> 2020-11-09T19:59:45.130 INFO:tasks.workunit.client.0.smithi183.stderr: r = args.func() 2020-11-09T19:59:45.130 INFO:tasks.workunit.client.0.smithi183.stderr: File "/tmp/tmp.2RH4gLOl67/cephadm", line 1359, in _infer_image 2020-11-09T19:59:45.131 INFO:tasks.workunit.client.0.smithi183.stderr: return func() 2020-11-09T19:59:45.131 INFO:tasks.workunit.client.0.smithi183.stderr: File "/tmp/tmp.2RH4gLOl67/cephadm", line 2641, in command_version 2020-11-09T19:59:45.131 INFO:tasks.workunit.client.0.smithi183.stderr: out = CephContainer(args.image, 'ceph', ['--version']).run() 2020-11-09T19:59:45.132 INFO:tasks.workunit.client.0.smithi183.stderr: File "/tmp/tmp.2RH4gLOl67/cephadm", line 2632, in run 2020-11-09T19:59:45.132 INFO:tasks.workunit.client.0.smithi183.stderr: self.run_cmd(), desc=self.entrypoint, timeout=timeout) 2020-11-09T19:59:45.132 INFO:tasks.workunit.client.0.smithi183.stderr: File "/tmp/tmp.2RH4gLOl67/cephadm", line 1038, in call_throws 2020-11-09T19:59:45.132 INFO:tasks.workunit.client.0.smithi183.stderr: raise RuntimeError('Failed command: %s' % ' '.join(command)) 2020-11-09T19:59:45.133 INFO:tasks.workunit.client.0.smithi183.stderr:RuntimeError: Failed command: /usr/bin/podman run --rm --ipc=host --net=host --entrypoint ceph -e CONTAINER_IMAGE=docker.io/ceph/daemon-base:latest-nautilus -e NODE_NAME=smithi183 docker.io/ceph/daemon-base:latest-nautilus --version 2020-11-09T19:59:45.149 INFO:tasks.workunit.client.0.smithi183.stderr:+ cleanup 2020-11-09T19:59:45.149 INFO:tasks.workunit.client.0.smithi183.stderr:+ '[' true = false ']' 2020-11-09T19:59:45.149 INFO:tasks.workunit.client.0.smithi183.stderr:+ dump_all_logs 00000000-0000-0000-0000-0000deadbeef 2020-11-09T19:59:45.150 INFO:tasks.workunit.client.0.smithi183.stderr:+ local fsid=00000000-0000-0000-0000-0000deadbeef 2020-11-09T19:59:45.150 INFO:tasks.workunit.client.0.smithi183.stderr:++ sudo /tmp/tmp.2RH4gLOl67/cephadm --image docker.io/ceph/daemon-base:latest-octopus ls 2020-11-09T19:59:45.150 INFO:tasks.workunit.client.0.smithi183.stderr:++ jq -r '.[] | select(.fsid == "00000000-0000-0000-0000-0000deadbeef").name' 2020-11-09T19:59:45.296 INFO:tasks.workunit.client.0.smithi183.stderr:+ local names= 2020-11-09T19:59:45.297 INFO:tasks.workunit.client.0.smithi183.stderr:+ echo 'dumping logs for daemons: ' 2020-11-09T19:59:45.297 INFO:tasks.workunit.client.0.smithi183.stderr:+ rm -rf tmp.test_cephadm.sh.4Lg3Vq 2020-11-09T19:59:45.297 INFO:tasks.workunit.client.0.smithi183.stdout:dumping logs for daemons: 2020-11-09T19:59:45.298 INFO:tasks.workunit.client.0.smithi183.stderr:+ rm -rf /tmp/tmp.2RH4gLOl67 2020-11-09T19:59:45.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2020-11-09T19:59:45.300 INFO:tasks.workunit:Stopping ['cephadm/test_cephadm.sh'] on client.0... 2020-11-09T19:59:45.301 INFO:teuthology.orchestra.run.smithi183:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2020-11-09T19:59:45.546 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 90, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 69, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_wip-yuri8-testing-2020-11-09-0807-octopus/qa/tasks/workunit.py", line 127, in task coverage_and_limits=not config.get('no_coverage_and_limits', None)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_wip-yuri8-testing-2020-11-09-0807-octopus/qa/tasks/workunit.py", line 415, in _run_tests label="workunit test {workunit}".format(workunit=workunit) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 215, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 446, in run r.wait() File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 160, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 182, in _raise_for_status node=self.hostname, label=self.label teuthology.exceptions.CommandFailedError: Command failed (workunit test cephadm/test_cephadm.sh) on smithi183 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=3b2df8c76c14ab5098113329edcfe1c0d82c9b44 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh' 2020-11-09T19:59:45.570 ERROR:teuthology.run_tasks: Sentry event: https://sentry.ceph.com/ceph/sepia/?query=b7e85beb0d9b40ba9626032d1b29528c Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 90, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 69, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_wip-yuri8-testing-2020-11-09-0807-octopus/qa/tasks/workunit.py", line 127, in task coverage_and_limits=not config.get('no_coverage_and_limits', None)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_wip-yuri8-testing-2020-11-09-0807-octopus/qa/tasks/workunit.py", line 415, in _run_tests label="workunit test {workunit}".format(workunit=workunit) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/remote.py", line 215, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 446, in run r.wait() File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 160, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/orchestra/run.py", line 182, in _raise_for_status node=self.hostname, label=self.label teuthology.exceptions.CommandFailedError: Command failed (workunit test cephadm/test_cephadm.sh) on smithi183 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=3b2df8c76c14ab5098113329edcfe1c0d82c9b44 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh </pre>