root@osd1:~# stat /core File: `/core' Size: 1165631488 Blocks: 1185528 IO Block: 4096 regular file Device: 801h/2049d Inode: 18081 Links: 1 Access: (0600/-rw-------) Uid: ( 0/ root) Gid: ( 0/ root) Access: 2010-06-03 13:00:14.099664317 +0200 Modify: 2010-06-03 12:58:55.568414030 +0200 Change: 2010-06-03 12:58:55.568414030 +0200 You have new mail in /var/mail/root root@osd1:~# root@osd1:~# gdb /usr/lib/debug/usr/bin/cosd /core GNU gdb (GDB) 7.1-ubuntu Copyright (C) 2010 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /usr/lib/debug/usr/bin/cosd...done. [New Thread 11134] [New Thread 11136] [New Thread 11148] [New Thread 11152] [New Thread 11036] [New Thread 11159] [New Thread 11129] [New Thread 11151] [New Thread 11158] [New Thread 11164] [New Thread 11163] [New Thread 11093] [New Thread 11135] [New Thread 11161] [New Thread 11038] [New Thread 11166] [New Thread 11126] [New Thread 11095] [New Thread 11374] [New Thread 11162] [New Thread 11089] [New Thread 11133] [New Thread 11165] [New Thread 11128] [New Thread 11132] [New Thread 11040] [New Thread 11131] [New Thread 11091] [New Thread 11130] [New Thread 11125] [New Thread 11157] [New Thread 11094] [New Thread 11127] [New Thread 11088] [New Thread 11373] [New Thread 11092] [New Thread 11090] warning: Unable to find dynamic linker breakpoint function. GDB will be unable to debug shared library initializers and track explicitly loaded dynamic code. Core was generated by `/usr/bin/cosd -i 1 -c /etc/ceph/ceph.conf'. Program terminated with signal 6, Aborted. #0 0x00007fedef5f0a75 in ?? () (gdb) bt #0 0x00007fedef5f0a75 in ?? () #1 0x00007fedef5f45c0 in ?? () #2 0x00000000005d692e in ?? () #3 0x00007fedef7043cf in ?? () #4 0x00007fedd27e31a0 in ?? () #5 0x00000000000000dc in ?? () #6 0x00007fedd27e3290 in ?? () #7 0x00007fedef6240e6 in ?? () #8 0x0000003000000018 in ?? () #9 0x00007fedd27e32a0 in ?? () #10 0x00007fedd27e31c0 in ?? () #11 0x00007fedef60d6f8 in ?? () #12 0x0000003000000030 in ?? () #13 0x00007fedd27e32b8 in ?? () #14 0x00007feda97c9cc0 in ?? () #15 0x000000000a2e6465 in ?? () #16 0x00007fedbbafe770 in ?? () #17 0x2b7472617473203e in ?? () #18 0x00007fffa5778957 in ?? () #19 0x00007fedef70247a in ?? () #20 0x00000000005d6899 in ?? () #21 0x00000000000000dc in ?? () #22 0x0000000000000020 in ?? () #23 0x0000000000000000 in ?? () (gdb) root@osd2:~# gdb /usr/lib/debug/usr/bin/cosd /core GNU gdb (GDB) 7.1-ubuntu Copyright (C) 2010 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /usr/lib/debug/usr/bin/cosd...done. [New Thread 10878] [New Thread 10897] [New Thread 10879] [New Thread 10899] [New Thread 10880] [New Thread 10898] [New Thread 10903] [New Thread 10900] [New Thread 10821] [New Thread 10904] [New Thread 10864] [New Thread 10905] [New Thread 10874] [New Thread 10906] [New Thread 10910] [New Thread 10907] [New Thread 10876] [New Thread 10908] [New Thread 10877] [New Thread 10909] [New Thread 10873] [New Thread 10966] [New Thread 10862] [New Thread 10818] [New Thread 10820] [New Thread 10858] [New Thread 10861] [New Thread 10863] [New Thread 10869] [New Thread 10872] [New Thread 10875] [New Thread 10870] [New Thread 10860] [New Thread 10857] [New Thread 10965] [New Thread 10871] [New Thread 10859] warning: Unable to find dynamic linker breakpoint function. GDB will be unable to debug shared library initializers and track explicitly loaded dynamic code. Core was generated by `/usr/bin/cosd -i 2 -c /etc/ceph/ceph.conf'. Program terminated with signal 6, Aborted. #0 0x00007f7b4b58da75 in ?? () (gdb) bt #0 0x00007f7b4b58da75 in ?? () #1 0x00007f7b4b5915c0 in ?? () #2 0x00000000005d692e in ?? () #3 0x00007f7b4b6a13cf in ?? () #4 0x00007f7b3bffe1a0 in ?? () #5 0x00000000000000dc in ?? () #6 0x00007f7b3bffe290 in ?? () #7 0x00007f7b4b5c10e6 in ?? () #8 0x0000003000000018 in ?? () #9 0x00007f7b3bffe2a0 in ?? () #10 0x00007f7b3bffe1c0 in ?? () #11 0x00007f7b4b5aa6f8 in ?? () #12 0x0000003000000030 in ?? () #13 0x00007f7b3bffe2b8 in ?? () #14 0x00007f7b205805a0 in ?? () #15 0x000000000a2e6465 in ?? () #16 0x00007f7b21c48cc0 in ?? () #17 0x2b7472617473203e in ?? () #18 0x00007fff6257a956 in ?? () #19 0x00007f7b4b69f47a in ?? () #20 0x00000000005d6899 in ?? () #21 0x00000000000000dc in ?? () #22 0x0000000000000020 in ?? () #23 0x0000000000000000 in ?? () (gdb) root@osd3:~# gdb /usr/lib/debug/usr/bin/cosd /core GNU gdb (GDB) 7.1-ubuntu Copyright (C) 2010 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /usr/lib/debug/usr/bin/cosd...done. [New Thread 10138] [New Thread 10139] [New Thread 10140] [New Thread 10156] [New Thread 10155] [New Thread 10160] [New Thread 10162] [New Thread 10159] [New Thread 10161] [New Thread 10163] [New Thread 10166] [New Thread 10164] [New Thread 10165] [New Thread 10167] [New Thread 10226] [New Thread 10168] [New Thread 10034] [New Thread 10036] [New Thread 10225] [New Thread 10037] [New Thread 10132] [New Thread 10063] [New Thread 10134] [New Thread 10067] [New Thread 10070] [New Thread 10129] [New Thread 10133] [New Thread 10064] [New Thread 10131] [New Thread 10068] [New Thread 10069] [New Thread 10137] [New Thread 10136] [New Thread 10135] [New Thread 10130] [New Thread 10066] [New Thread 10065] warning: Unable to find dynamic linker breakpoint function. GDB will be unable to debug shared library initializers and track explicitly loaded dynamic code. Core was generated by `/usr/bin/cosd -i 3 -c /etc/ceph/ceph.conf'. Program terminated with signal 6, Aborted. #0 0x00007f2c488e3a75 in ?? () (gdb) root@osd4:~# gdb /usr/lib/debug/usr/bin/cosd /core GNU gdb (GDB) 7.1-ubuntu Copyright (C) 2010 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /usr/lib/debug/usr/bin/cosd...done. [New Thread 9985] [New Thread 10001] [New Thread 9986] [New Thread 10003] [New Thread 10006] [New Thread 10002] [New Thread 9979] [New Thread 10013] [New Thread 9980] [New Thread 10004] [New Thread 10005] [New Thread 9898] [New Thread 9921] [New Thread 9976] [New Thread 9919] [New Thread 10011] [New Thread 9983] [New Thread 10009] [New Thread 9978] [New Thread 10012] [New Thread 10014] [New Thread 9984] [New Thread 10086] [New Thread 10085] [New Thread 9982] [New Thread 9918] [New Thread 9977] [New Thread 9981] [New Thread 9925] [New Thread 9902] [New Thread 9900] [New Thread 9923] [New Thread 10010] [New Thread 9987] [New Thread 9924] [New Thread 9922] [New Thread 9920] warning: Unable to find dynamic linker breakpoint function. GDB will be unable to debug shared library initializers and track explicitly loaded dynamic code. Core was generated by `/usr/bin/cosd -i 4 -c /etc/ceph/ceph.conf'. Program terminated with signal 6, Aborted. #0 0x00007f7d9205fa75 in ?? () (gdb) root@osd5:~# gdb /usr/lib/debug/usr/bin/cosd /core GNU gdb (GDB) 7.1-ubuntu Copyright (C) 2010 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /usr/lib/debug/usr/bin/cosd...done. [New Thread 11415] [New Thread 11416] [New Thread 11424] [New Thread 11417] [New Thread 11427] [New Thread 11429] [New Thread 11434] [New Thread 11432] [New Thread 11435] [New Thread 11437] [New Thread 11436] [New Thread 11438] [New Thread 11439] [New Thread 11443] [New Thread 11440] [New Thread 11444] [New Thread 11441] [New Thread 11390] [New Thread 11442] [New Thread 11401] [New Thread 11445] [New Thread 11408] [New Thread 11388] [New Thread 11409] [New Thread 11389] [New Thread 11402] [New Thread 11398] [New Thread 11403] [New Thread 11404] [New Thread 11406] [New Thread 11410] [New Thread 11411] [New Thread 11412] [New Thread 11413] [New Thread 11414] [New Thread 11400] [New Thread 11407] [New Thread 11397] [New Thread 11399] warning: Unable to find dynamic linker breakpoint function. GDB will be unable to debug shared library initializers and track explicitly loaded dynamic code. Core was generated by `/usr/bin/cosd -i 5 -c /etc/ceph/ceph.conf'. Program terminated with signal 6, Aborted. #0 0x00007fabd9d51a75 in ?? () (gdb)