Ceph

One out of 20 runs of make check, run_osd fails (the osd is not reported to be up by osd.dump). This is a transient error that does not seem to be related to low resources on the machine. There are no details about why the osd failed. At least the tail of the osd log and a ceph report would be necessary to figure that out.

./test/mon/osd-crush.sh:202: TEST_add_ruleset_failed:  grep 'Error ENOSPC'
Error ENOSPC: failed to add rule 256 because (28) No space left on device
./test/mon/osd-crush.sh:33: run:  teardown testdir/osd-crush
../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/osd-crush
../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/osd-crush
.../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
.../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
.../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/osd-crush
./test/mon/osd-crush.sh:30: run:  for func in '$funcs'
./test/mon/osd-crush.sh:31: run:  setup testdir/osd-crush
../qa/workunits/ceph-helpers.sh:92: setup:  local dir=testdir/osd-crush
../qa/workunits/ceph-helpers.sh:93: setup:  teardown testdir/osd-crush
../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/osd-crush
../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/osd-crush
.../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
.../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
.../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/osd-crush
../qa/workunits/ceph-helpers.sh:94: setup:  mkdir -p testdir/osd-crush
./test/mon/osd-crush.sh:32: run:  TEST_crush_reject_empty testdir/osd-crush
./test/mon/osd-crush.sh:220: TEST_crush_reject_empty:  local dir=testdir/osd-crush
./test/mon/osd-crush.sh:221: TEST_crush_reject_empty:  run_mon testdir/osd-crush a
../qa/workunits/ceph-helpers.sh:282: run_mon:  local dir=testdir/osd-crush
../qa/workunits/ceph-helpers.sh:283: run_mon:  shift
../qa/workunits/ceph-helpers.sh:284: run_mon:  local id=a
../qa/workunits/ceph-helpers.sh:285: run_mon:  shift
../qa/workunits/ceph-helpers.sh:286: run_mon:  local data=testdir/osd-crush/a
../qa/workunits/ceph-helpers.sh:289: run_mon:  ceph-mon --id a --mkfs --mon-data=testdir/osd-crush/a --run-dir=testdir/osd-crush
ceph-mon: mon.noname-a 127.0.0.1:7104/0 is local, renaming to mon.a
ceph-mon: set fsid to 49a65d30-da54-4f53-bbe5-0a20540fc282
ceph-mon: created monfs at testdir/osd-crush/a for mon.a
../qa/workunits/ceph-helpers.sh:296: run_mon:  ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/osd-crush/a '--log-file=testdir/osd-crush/$name.log' '--admin-socket=testdir/osd-crush/$cluster-$name.asok' --mon-cluster-log-file=testdir/osd-crush/log --run-dir=testdir/osd-crush '--pid-file=testdir/osd-crush/$name.pid'
../qa/workunits/ceph-helpers.sh:314: run_mon:  cat
.../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a fsid
.../qa/workunits/ceph-helpers.sh:651: get_config:  local daemon=mon
.../qa/workunits/ceph-helpers.sh:652: get_config:  local id=a
.../qa/workunits/ceph-helpers.sh:653: get_config:  local config=fsid
.../qa/workunits/ceph-helpers.sh:655: get_config:  CEPH_ARGS=
.../qa/workunits/ceph-helpers.sh:655: get_config:  ceph --format xml daemon testdir/osd-crush/ceph-mon.a.asok config get fsid
.../qa/workunits/ceph-helpers.sh:658: get_config:  xmlstarlet sel -t -m //fsid -v . -n
.../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a mon_host
.../qa/workunits/ceph-helpers.sh:651: get_config:  local daemon=mon
.../qa/workunits/ceph-helpers.sh:652: get_config:  local id=a
.../qa/workunits/ceph-helpers.sh:653: get_config:  local config=mon_host
.../qa/workunits/ceph-helpers.sh:655: get_config:  CEPH_ARGS=
.../qa/workunits/ceph-helpers.sh:655: get_config:  ceph --format xml daemon testdir/osd-crush/ceph-mon.a.asok config get mon_host
.../qa/workunits/ceph-helpers.sh:658: get_config:  xmlstarlet sel -t -m //mon_host -v . -n
.../qa/workunits/ceph-helpers.sh:319: run_mon:  get_config mon a mon_initial_members
.../qa/workunits/ceph-helpers.sh:651: get_config:  local daemon=mon
.../qa/workunits/ceph-helpers.sh:652: get_config:  local id=a
.../qa/workunits/ceph-helpers.sh:653: get_config:  local config=mon_initial_members
.../qa/workunits/ceph-helpers.sh:655: get_config:  CEPH_ARGS=
.../qa/workunits/ceph-helpers.sh:655: get_config:  ceph --format xml daemon testdir/osd-crush/ceph-mon.a.asok config get mon_initial_members
.../qa/workunits/ceph-helpers.sh:658: get_config:  xmlstarlet sel -t -m //mon_initial_members -v . -n
../qa/workunits/ceph-helpers.sh:319: run_mon:  test -z ''
../qa/workunits/ceph-helpers.sh:320: run_mon:  ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
pool 'rbd' removed
../qa/workunits/ceph-helpers.sh:321: run_mon:  ceph osd pool create rbd 4
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
pool 'rbd' created
./test/mon/osd-crush.sh:223: TEST_crush_reject_empty:  run_osd testdir/osd-crush 0
../qa/workunits/ceph-helpers.sh:397: run_osd:  local dir=testdir/osd-crush
../qa/workunits/ceph-helpers.sh:398: run_osd:  shift
../qa/workunits/ceph-helpers.sh:399: run_osd:  local id=0
../qa/workunits/ceph-helpers.sh:400: run_osd:  shift
../qa/workunits/ceph-helpers.sh:401: run_osd:  local osd_data=testdir/osd-crush/0
../qa/workunits/ceph-helpers.sh:403: run_osd:  local ceph_disk_args
../qa/workunits/ceph-helpers.sh:404: run_osd:  ceph_disk_args+=' --statedir=testdir/osd-crush'
../qa/workunits/ceph-helpers.sh:405: run_osd:  ceph_disk_args+=' --sysconfdir=testdir/osd-crush'
../qa/workunits/ceph-helpers.sh:406: run_osd:  ceph_disk_args+=' --prepend-to-path='
../qa/workunits/ceph-helpers.sh:408: run_osd:  mkdir -p testdir/osd-crush/0
../qa/workunits/ceph-helpers.sh:409: run_osd:  ceph-disk --statedir=testdir/osd-crush --sysconfdir=testdir/osd-crush --prepend-to-path= prepare testdir/osd-crush/0
2015-07-24 03:28:37.827889 7fc4c79bc7c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
2015-07-24 03:28:37.841755 7ffdc34f77c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
2015-07-24 03:28:37.856222 7f24220147c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
../qa/workunits/ceph-helpers.sh:412: run_osd:  activate_osd testdir/osd-crush 0
../qa/workunits/ceph-helpers.sh:478: activate_osd:  local dir=testdir/osd-crush
../qa/workunits/ceph-helpers.sh:479: activate_osd:  shift
../qa/workunits/ceph-helpers.sh:480: activate_osd:  local id=0
../qa/workunits/ceph-helpers.sh:481: activate_osd:  shift
../qa/workunits/ceph-helpers.sh:482: activate_osd:  local osd_data=testdir/osd-crush/0
../qa/workunits/ceph-helpers.sh:484: activate_osd:  local ceph_disk_args
../qa/workunits/ceph-helpers.sh:485: activate_osd:  ceph_disk_args+=' --statedir=testdir/osd-crush'
../qa/workunits/ceph-helpers.sh:486: activate_osd:  ceph_disk_args+=' --sysconfdir=testdir/osd-crush'
../qa/workunits/ceph-helpers.sh:487: activate_osd:  ceph_disk_args+=' --prepend-to-path='
../qa/workunits/ceph-helpers.sh:489: activate_osd:  local 'ceph_args=--fsid=49a65d30-da54-4f53-bbe5-0a20540fc282 --auth-supported=none --mon-host=127.0.0.1:7104 '
../qa/workunits/ceph-helpers.sh:490: activate_osd:  ceph_args+=' --osd-backfill-full-ratio=.99'
../qa/workunits/ceph-helpers.sh:491: activate_osd:  ceph_args+=' --osd-failsafe-full-ratio=.99'
../qa/workunits/ceph-helpers.sh:492: activate_osd:  ceph_args+=' --osd-journal-size=100'
../qa/workunits/ceph-helpers.sh:493: activate_osd:  ceph_args+=' --osd-data=testdir/osd-crush/0'
../qa/workunits/ceph-helpers.sh:494: activate_osd:  ceph_args+=' --chdir='
../qa/workunits/ceph-helpers.sh:495: activate_osd:  ceph_args+=' --osd-pool-default-erasure-code-directory=.libs'
../qa/workunits/ceph-helpers.sh:496: activate_osd:  ceph_args+=' --osd-class-dir=.libs'
../qa/workunits/ceph-helpers.sh:497: activate_osd:  ceph_args+=' --run-dir=testdir/osd-crush'
../qa/workunits/ceph-helpers.sh:498: activate_osd:  ceph_args+=' --debug-osd=20'
../qa/workunits/ceph-helpers.sh:499: activate_osd:  ceph_args+=' --log-file=testdir/osd-crush/$name.log'
../qa/workunits/ceph-helpers.sh:500: activate_osd:  ceph_args+=' --pid-file=testdir/osd-crush/$name.pid'
../qa/workunits/ceph-helpers.sh:501: activate_osd:  ceph_args+=' '
../qa/workunits/ceph-helpers.sh:502: activate_osd:  ceph_args+=
../qa/workunits/ceph-helpers.sh:503: activate_osd:  mkdir -p testdir/osd-crush/0
../qa/workunits/ceph-helpers.sh:504: activate_osd:  CEPH_ARGS='--fsid=49a65d30-da54-4f53-bbe5-0a20540fc282 --auth-supported=none --mon-host=127.0.0.1:7104  --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/osd-crush/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/osd-crush --debug-osd=20 --log-file=testdir/osd-crush/$name.log --pid-file=testdir/osd-crush/$name.pid  '
../qa/workunits/ceph-helpers.sh:504: activate_osd:  ceph-disk --statedir=testdir/osd-crush --sysconfdir=testdir/osd-crush --prepend-to-path= activate --mark-init=none testdir/osd-crush/0
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
got monmap epoch 1
2015-07-24 03:28:40.269384 7f626f3c07c0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
2015-07-24 03:28:40.377129 7f626f3c07c0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
2015-07-24 03:28:40.380868 7f626f3c07c0 -1 filestore(testdir/osd-crush/0) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory
2015-07-24 03:28:40.457312 7f626f3c07c0 -1 created object store testdir/osd-crush/0 journal testdir/osd-crush/0/journal for osd.0 fsid 49a65d30-da54-4f53-bbe5-0a20540fc282
2015-07-24 03:28:40.457366 7f626f3c07c0 -1 auth: error reading file: testdir/osd-crush/0/keyring: can't open testdir/osd-crush/0/keyring: (2) No such file or directory
2015-07-24 03:28:40.457499 7f626f3c07c0 -1 created new key in keyring testdir/osd-crush/0/keyring
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
added key for osd.0
starting osd.0 at :/0 osd_data testdir/osd-crush/0 testdir/osd-crush/0/journal
.../qa/workunits/ceph-helpers.sh:509: activate_osd:  cat testdir/osd-crush/0/whoami
../qa/workunits/ceph-helpers.sh:509: activate_osd:  '[' 0 = 0 ']'
../qa/workunits/ceph-helpers.sh:511: activate_osd:  ceph osd crush create-or-move 0 1 root=default host=localhost
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
create-or-move updating item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map
../qa/workunits/ceph-helpers.sh:513: activate_osd:  wait_for_osd up 0
../qa/workunits/ceph-helpers.sh:548: wait_for_osd:  local state=up
../qa/workunits/ceph-helpers.sh:549: wait_for_osd:  local id=0
../qa/workunits/ceph-helpers.sh:551: wait_for_osd:  status=1
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i=0 ))
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i < 120 ))
../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  ceph osd dump
../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  grep 'osd.0 up'
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  sleep 1
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i++ ))
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i < 120 ))
../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  ceph osd dump
../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  grep 'osd.0 up'
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  sleep 1
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i++ ))
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i < 120 ))
../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  ceph osd dump
../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  grep 'osd.0 up'
*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  sleep 1
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i++ ))
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i < 120 ))
../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  ceph osd dump
../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  grep 'osd.0 up'
...
../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  sleep 1
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i++ ))
../qa/workunits/ceph-helpers.sh:552: wait_for_osd:  (( i < 120 ))
../qa/workunits/ceph-helpers.sh:560: wait_for_osd:  return 1
../qa/workunits/ceph-helpers.sh:513: activate_osd:  return 1
./test/mon/osd-crush.sh:223: TEST_crush_reject_empty:  return 1
./test/mon/osd-crush.sh:32: run:  return 1
../qa/workunits/ceph-helpers.sh:1174: main:  code=1
../qa/workunits/ceph-helpers.sh:1176: main:  teardown testdir/osd-crush
../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/osd-crush
../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/osd-crush
.../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
.../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
.../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/osd-crush
../qa/workunits/ceph-helpers.sh:1177: main:  return 1