Ceph

Run: http://pulpito.ceph.com/teuthology-2019-03-20_19:32:40-upgrade:luminous-x-nautilus-distro-basic-smithi/
Jobs: all
Logs: http://qa-proxy.ceph.com/teuthology/teuthology-2019-03-20_19:32:40-upgrade:luminous-x-nautilus-distro-basic-smithi/3752702/teuthology.log

{"total":96636764160,"available":95562334208,"internally_reserved":0,"allocated":1074429952,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}]}}
2019-03-20T20:10:37.601 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 86, in run_tasks
    manager = run_one_task(taskname, ctx=ctx, config=config)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 65, in run_one_task
    return task(**kwargs)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/parallel.py", line 56, in task
    p.spawn(_run_spawned, ctx, confg, taskname)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 85, in __exit__
    for result in self:
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 99, in next
    resurrect_traceback(result)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/parallel.py", line 22, in capture_traceback
    return func(*args, **kwargs)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/parallel.py", line 64, in _run_spawned
    mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/run_tasks.py", line 65, in run_one_task
    return task(**kwargs)
  File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/task/sequential.py", line 49, in task
    mgr.__enter__()
  File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__
    return self.gen.next()
  File "/home/teuthworker/src/git.ceph.com_ceph_mimic/qa/tasks/ceph.py", line 1460, in restart
    healthy(ctx=ctx, config=dict(cluster=cluster))
  File "/home/teuthworker/src/git.ceph.com_ceph_mimic/qa/tasks/ceph.py", line 1299, in healthy
    manager.wait_for_clean()
  File "/home/teuthworker/src/git.ceph.com_ceph_mimic/qa/tasks/ceph_manager.py", line 2100, in wait_for_clean
    num_active_clean = self.get_num_active_clean()
  File "/home/teuthworker/src/git.ceph.com_ceph_mimic/qa/tasks/ceph_manager.py", line 2001, in get_num_active_clean
    pgs = self.get_pg_stats()
  File "/home/teuthworker/src/git.ceph.com_ceph_mimic/qa/tasks/ceph_manager.py", line 1790, in get_pg_stats
    return j['pg_stats']
KeyError: 'pg_stats'