Bug #10769
closedcoredump in /remote/burnupi59/ in upgrade:dumpling-firefly-x-next-distro-basic-multi run
0%
Description
Could not get more info from the crash, so might FYI ticket only.
Run: http://pulpito.ceph.com/teuthology-2015-02-04_17:10:01-upgrade:dumpling-firefly-x-next-distro-basic-multi/
Job: 739855
Logs: http://qa-proxy.ceph.com/teuthology/teuthology-2015-02-04_17:10:01-upgrade:dumpling-firefly-x-next-distro-basic-multi/739855/
2015-02-04T21:23:05.429 INFO:tasks.ceph.mds.a.burnupi59.stderr:2015-02-04 21:23:05.438067 7f854bbff800 -1 mds.-1.0 *** one or more OSDs do not support TMAP2OMAP; upgrade OSDs before starting MDS (or downgrade MDS) *** 2015-02-04T21:23:11.347 INFO:tasks.ceph.osd.0:Stopped 2015-02-04T21:23:11.347 INFO:tasks.ceph.osd.0:Restarting daemon 2015-02-04T21:23:11.347 INFO:teuthology.orchestra.run.burnupi59:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 0' 2015-02-04T21:23:11.373 INFO:tasks.ceph.osd.0:Started 2015-02-04T21:23:11.374 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2015-02-04T21:23:11.374 ERROR:teuthology.parallel:Exception in parallel execution Traceback (most recent call last): File "/home/teuthworker/src/teuthology_master/teuthology/parallel.py", line 82, in __exit__ for result in self: File "/home/teuthworker/src/teuthology_master/teuthology/parallel.py", line 101, in next resurrect_traceback(result) File "/home/teuthworker/src/teuthology_master/teuthology/parallel.py", line 19, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/teuthology_master/teuthology/task/parallel.py", line 50, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/teuthology_master/teuthology/run_tasks.py", line 41, in run_one_task return fn(**kwargs) File "/home/teuthworker/src/teuthology_master/teuthology/task/sequential.py", line 48, in task mgr.__enter__() File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/var/lib/teuthworker/src/ceph-qa-suite_next/tasks/ceph.py", line 1060, in restart ctx.daemons.get_daemon(type_, id_).stop() File "/home/teuthworker/src/teuthology_master/teuthology/orchestra/daemon.py", line 45, in stop run.wait([self.proc], timeout=timeout) File "/home/teuthworker/src/teuthology_master/teuthology/orchestra/run.py", line 401, in wait proc.wait() File "/home/teuthworker/src/teuthology_master/teuthology/orchestra/run.py", line 114, in wait label=self.label) CommandFailedError: Command failed on burnupi59 with status 1: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 1' 2015-02-04T21:23:11.376 ERROR:teuthology.parallel:Exception in parallel execution Traceback (most recent call last): File "/home/teuthworker/src/teuthology_master/teuthology/parallel.py", line 82, in __exit__ for result in self: File "/home/teuthworker/src/teuthology_master/teuthology/parallel.py", line 98, in next result = self.results.get() File "/usr/lib/python2.7/dist-packages/gevent/queue.py", line 190, in get return waiter.get() File "/usr/lib/python2.7/dist-packages/gevent/hub.py", line 321, in get return get_hub().switch() File "/usr/lib/python2.7/dist-packages/gevent/hub.py", line 164, in switch return greenlet.switch(self) GreenletExit 2015-02-04T21:23:11.376 INFO:tasks.workunit:Stopping ['rados/test-upgrade-firefly.sh', 'cls'] on client.0... 2015-02-04T21:23:11.376 INFO:teuthology.orchestra.run.burnupi29:Running: 'rm -rf -- /home/ubuntu/cephtest/workunits.list /home/ubuntu/cephtest/workunit.client.0' 2015-02-04T21:23:11.395 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/teuthology_master/teuthology/run_tasks.py", line 53, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/teuthology_master/teuthology/run_tasks.py", line 41, in run_one_task return fn(**kwargs) File "/home/teuthworker/src/teuthology_master/teuthology/task/parallel.py", line 43, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/teuthology_master/teuthology/parallel.py", line 89, in __exit__ raise CommandFailedError: Command failed on burnupi59 with status 1: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 1' 2015-02-04T21:23:11.408 ERROR:sentry.errors.serializer:vars() argument must have __dict__ attribute Traceback (most recent call last): File "/home/teuthworker/src/teuthology_master/virtualenv/lib/python2.7/site-packages/raven-5.1.1-py2.7.egg/raven/utils/serializer/manager.py", line 76, in transform return repr(value) File "/home/teuthworker/src/teuthology_master/teuthology/config.py", line 87, in __repr__ return self.__str__() File "/home/teuthworker/src/teuthology_master/teuthology/config.py", line 84, in __str__ return yaml.safe_dump(self._conf, default_flow_style=False).strip() File "/usr/lib/python2.7/dist-packages/yaml/__init__.py", line 218, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/usr/lib/python2.7/dist-packages/yaml/__init__.py", line 190, in dump_all dumper.represent(data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 28, in represent node = self.represent_data(data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 223, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 41, in represent_data if self.ignore_aliases(data): File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 142, in ignore_aliases if data in [None, ()]: File "/usr/lib/python2.7/argparse.py", line 1158, in __eq__ return vars(self) == vars(other) TypeError: vars() argument must have __dict__ attribute 2015-02-04T21:23:11.429 ERROR:sentry.errors.serializer:vars() argument must have __dict__ attribute Traceback (most recent call last): File "/home/teuthworker/src/teuthology_master/virtualenv/lib/python2.7/site-packages/raven-5.1.1-py2.7.egg/raven/utils/serializer/manager.py", line 76, in transform return repr(value) File "/home/teuthworker/src/teuthology_master/teuthology/config.py", line 87, in __repr__ return self.__str__() File "/home/teuthworker/src/teuthology_master/teuthology/config.py", line 84, in __str__ return yaml.safe_dump(self._conf, default_flow_style=False).strip() File "/usr/lib/python2.7/dist-packages/yaml/__init__.py", line 218, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/usr/lib/python2.7/dist-packages/yaml/__init__.py", line 190, in dump_all dumper.represent(data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 28, in represent node = self.represent_data(data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 223, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 41, in represent_data if self.ignore_aliases(data): File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 142, in ignore_aliases if data in [None, ()]: File "/usr/lib/python2.7/argparse.py", line 1158, in __eq__ return vars(self) == vars(other) TypeError: vars() argument must have __dict__ attribute 2015-02-04T21:23:11.429 ERROR:sentry.errors.serializer:vars() argument must have __dict__ attribute Traceback (most recent call last): File "/home/teuthworker/src/teuthology_master/virtualenv/lib/python2.7/site-packages/raven-5.1.1-py2.7.egg/raven/utils/serializer/manager.py", line 76, in transform return repr(value) File "/home/teuthworker/src/teuthology_master/teuthology/config.py", line 87, in __repr__ return self.__str__() File "/home/teuthworker/src/teuthology_master/teuthology/config.py", line 84, in __str__ return yaml.safe_dump(self._conf, default_flow_style=False).strip() File "/usr/lib/python2.7/dist-packages/yaml/__init__.py", line 218, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/usr/lib/python2.7/dist-packages/yaml/__init__.py", line 190, in dump_all dumper.represent(data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 28, in represent node = self.represent_data(data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 223, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 41, in represent_data if self.ignore_aliases(data): File "/usr/lib/python2.7/dist-packages/yaml/representer.py", line 142, in ignore_aliases if data in [None, ()]: File "/usr/lib/python2.7/argparse.py", line 1158, in __eq__ return vars(self) == vars(other) TypeError: vars() argument must have __dict__ attribute 2015-02-04T21:23:11.451 ERROR:teuthology.run_tasks: Sentry event: http://sentry.ceph.com/sepia/teuthology/search?q=83d75344c8f843499502dc956dd1e714 CommandFailedError: Command failed on burnupi59 with status 1: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage
Updated by Samuel Just about 9 years ago
[New LWP 15346]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `ceph-osd -f -i 1'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f299c93bac3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4
(gdb) bt
#0 0x00007f299c93bac3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4
#1 0x00007f299c93bb7b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/libtcmalloc.so.4
#2 0x00007f299c949428 in tc_free () from /usr/lib/libtcmalloc.so.4
#3 0x00007f299c015bc9 in _cxa_demangle () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4 0x00000000007f8607 in ceph::BackTrace::print (this=this@entry=0x7f2988d89d30, out=...) at common/BackTrace.cc:45
#5 0x00000000007f62c4 in handle_fatal_signal (signum=11) at global/signal_handler.cc:92
#6 <signal handler called>
#7 0x00007f299c93bac3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4
#8 0x00007f299c93bb7b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/libtcmalloc.so.4
#9 0x00007f299c94af68 in tc_delete () from /usr/lib/libtcmalloc.so.4
#10 0x00000000007eba0c in deallocate (this=<optimized out>, __p=<optimized out>) at /usr/include/c++/4.8/ext/new_allocator.h:110
#11 _M_deallocate (this=<optimized out>, __n=<optimized out>, __p=<optimized out>) at /usr/include/c++/4.8/bits/stl_vector.h:174
#12 ~_Vector_base (this=0x7f2988d8adf0, __in_chrg=<optimized out>) at /usr/include/c++/4.8/bits/stl_vector.h:160
#13 ~vector (this=0x7f2988d8adf0, __in_chrg=<optimized out>) at /usr/include/c++/4.8/bits/stl_vector.h:416
#14 HashIndex::_lookup (this=0x32bd900, hoid=..., path=<optimized out>, mangled_name=0x7f2988d8ae80, exists_out=0x7f2988d8afdc) at os/HashIndex.cc:316
#15 0x00000000007e3a9f in LFNIndex::lookup (this=0x32bd900, hoid=..., out_path=0x7f2988d8b050, exist=0x7f2988d8afdc) at os/LFNIndex.cc:112
#16 0x000000000078f29f in FileStore::lfn_open (this=this@entry=0x3332000, cid=..., oid=..., create=create@entry=false, outfd=outfd@entry=0x7f2988d8b440,
path=0x7f2988d8b050, path@entry=0x0, index=0x7f2988d8b030, index@entry=0x0) at os/FileStore.cc:264
#17 0x0000000000794e40 in FileStore::getattr (this=this@entry=0x3332000, cid=..., oid=..., name=name@entry=0xa0f182 "", bp=...) at os/FileStore.cc:3961
#18 0x000000000060fd55 in ObjectStore::getattr (this=0x3332000, cid=..., oid=..., name=name@entry=0xa0f182 "_", value=...) at ./os/ObjectStore.h:873
#19 0x00000000005d5ca8 in ReplicatedPG::get_object_context (this=this@entry=0x358c000, soid=..., can_create=can_create@entry=true) at osd/ReplicatedPG.cc:4567
#20 0x00000000005d6965 in ReplicatedPG::find_object_context (this=this@entry=0x358c000, oid=..., pobc=pobc@entry=0x7f2988d8bee8,
can_create=can_create@entry=true, psnapid=psnapid@entry=0x7f2988d8be00) at osd/ReplicatedPG.cc:4658
#21 0x00000000005f8708 in ReplicatedPG::do_op (this=0x358c000, op=...) at osd/ReplicatedPG.cc:720
#22 0x00000000006fa334 in PG::do_request (this=0x358c000, op=..., handle=...) at osd/PG.cc:1455
#23 0x00000000006468f3 in OSD::dequeue_op (this=0x335a000, pg=..., op=..., handle=...) at osd/OSD.cc:7169
#24 0x000000000065b5ef in OSD::OpWQ::_process (this=0x335ae28, pg=..., handle=...) at osd/OSD.cc:7141
#25 0x00000000006971cc in ThreadPool::WorkQueueVal<std::pair<boost::intrusive_ptr<PG>, std::tr1::shared_ptr<OpRequest> >, boost::intrusive_ptr<PG> >::_void_process (this=0x335ae28, handle=...) at ./common/WorkQueue.h:191
#26 0x00000000008afe31 in ThreadPool::worker (this=0x335a460, wt=0x33001c0) at common/WorkQueue.cc:128
#27 0x00000000008b0d20 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:318
#28 0x00007f299d712182 in start_thread (arg=0x7f2988d8d700) at pthread_create.c:312
#29 0x00007f299b7c638d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
(gdb)
Updated by Samuel Just about 9 years ago
- Status changed from New to Can't reproduce