Project

General

Profile

tcmalloc fail.txt

chunsong feng, 12/07/2019 08:18 AM

Download (22.2 KB)

 
1

    
2
root@node2:/etc/ceph# ps -aux | grep ceph
3
root      295591  0.0  0.0  16404 11016 ?        Ss   15:28   0:00 /usr/bin/python2.7 /usr/bin/ceph-crash
4
root      303214  0.8  0.0 816904 67516 ?        Ssl  15:53   0:07 /usr/bin/ceph-osd -f --cluster ceph --id 24 --setuser ceph --setgroup ceph
5
root      304407  0.6  0.0 816908 66844 ?        Ssl  15:53   0:06 /usr/bin/ceph-osd -f --cluster ceph --id 25 --setuser ceph --setgroup ceph
6
root      305579  0.7  0.0 816908 66520 ?        Ssl  15:54   0:06 /usr/bin/ceph-osd -f --cluster ceph --id 26 --setuser ceph --setgroup ceph
7
root      306750  0.6  0.0 816904 66512 ?        Ssl  15:54   0:05 /usr/bin/ceph-osd -f --cluster ceph --id 27 --setuser ceph --setgroup ceph
8
root      307937  0.7  0.0 816904 66048 ?        Ssl  15:54   0:06 /usr/bin/ceph-osd -f --cluster ceph --id 28 --setuser ceph --setgroup ceph
9
root      309124  0.6  0.0 815888 65612 ?        Ssl  15:55   0:05 /usr/bin/ceph-osd -f --cluster ceph --id 29 --setuser ceph --setgroup ceph
10
root      310294  0.6  0.0 816904 65948 ?        Ssl  15:55   0:05 /usr/bin/ceph-osd -f --cluster ceph --id 30 --setuser ceph --setgroup ceph
11
root      311480  0.6  0.0 815880 65812 ?        Ssl  15:55   0:05 /usr/bin/ceph-osd -f --cluster ceph --id 31 --setuser ceph --setgroup ceph
12
root      312656  0.5  0.0 815884 65544 ?        Ssl  15:56   0:04 /usr/bin/ceph-osd -f --cluster ceph --id 32 --setuser ceph --setgroup ceph
13
root      313833  0.5  0.0 815888 64464 ?        Ssl  15:56   0:04 /usr/bin/ceph-osd -f --cluster ceph --id 33 --setuser ceph --setgroup ceph
14
root      315161  0.0  0.0  25408 20872 ?        S    15:56   0:00 /usr/bin/python2.7 /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/nvme0n1p12
15
root      315454  0.0  0.0      0     0 ?        Z    15:56   0:00 [ceph-osd] <defunct>
16
root      315465  0.0  0.0 268712440 31792 ?     Ssl  15:56   0:00 /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 34 --monmap /var/lib/ceph/osd/ceph-34/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-34/ --osd-uuid 781943ae-d925-4327-a662-c04600f6b2c5 --setuser ceph --setgroup ceph
17
root      315544  1.5  0.0 449008 72464 ?        Ssl  15:58   0:10 /usr/bin/ceph-mon -f --cluster ceph --id node2 --setuser ceph --setgroup ceph
18
root      315548  1.0  0.0 598080 269652 ?       Ssl  15:58   0:06 /usr/bin/ceph-mgr -f --cluster ceph --id node2 --setuser ceph --setgroup ceph
19
root      317646  200  0.0 269280460 92232 ?     Ssl  16:01  15:06 /usr/bin/ceph-osd -f --cluster ceph --id 35 --setuser ceph --setgroup ceph
20
root      320093  200  0.0 269278352 91108 ?     Ssl  16:03  12:23 /usr/bin/ceph-osd -f --cluster ceph --id 36 --setuser ceph --setgroup ceph
21
root      320655  0.1  0.0  25540 21124 ?        S    16:03   0:00 /usr/bin/python2.7 /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/nvme1n1p4
22
root      320833  0.2  0.0 268712436 52244 ?     Sl   16:03   0:00 /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 37 --monmap /var/lib/ceph/osd/ceph-37/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-37/ --osd-uuid 87514617-ce94-47ac-9156-1d8af227fbf9 --setuser ceph --setgroup ceph
23
root      320843  0.0  0.0 268712572 35272 ?     Ssl  16:03   0:00 /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 37 --monmap /var/lib/ceph/osd/ceph-37/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-37/ --osd-uuid 87514617-ce94-47ac-9156-1d8af227fbf9 --setuser ceph --setgroup ceph
24
root      321792  199  0.0 269279436 90712 ?     Ssl  16:08   1:09 /usr/bin/ceph-osd -f --cluster ceph --id 38 --setuser ceph --setgroup ceph
25
root      322861  193  0.0 269277264 88852 ?     Ssl  16:09   0:28 /usr/bin/ceph-osd -f --cluster ceph --id 39 --setuser ceph --setgroup ceph
26
root      323436  6.6  0.0  25544 20816 ?        S    16:09   0:00 /usr/bin/python2.7 /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/nvme1n1p7
27
root      323586  0.0  0.0 703388 41552 ?        Rl   16:09   0:00 /usr/bin/python2.7 /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-40/activate.monmap
28
root      323610  0.0  0.0   4376   724 pts/2    S+   16:09   0:00 grep --color=auto ceph
29
root@node2:/etc/ceph# gdb /usr/bin/ceph-osd -p 320833
30
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
31
Copyright (C) 2018 Free Software Foundation, Inc.
32
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
33
This is free software: you are free to change and redistribute it.
34
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
35
and "show warranty" for details.
36
This GDB was configured as "aarch64-linux-gnu".
37
Type "show configuration" for configuration details.
38
For bug reporting instructions, please see:
39
<http://www.gnu.org/software/gdb/bugs/>.
40
Find the GDB manual and other documentation resources online at:
41
<http://www.gnu.org/software/gdb/documentation/>.
42
For help, type "help".
43
Type "apropos word" to search for commands related to "word"...
44
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/c1/798048d376faca23ada6b38996bb8f2e8530ba.debug...done.
45
done.
46
Attaching to program: /usr/bin/ceph-osd, process 320833
47
[New LWP 320836]
48
[New LWP 320837]
49
[New LWP 320838]
50
[New LWP 320844]
51
[Thread debugging using libthread_db enabled]
52
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
53
0x0000ffff8b636a48 in __libc_read (fd=fd@entry=113, buf=buf@entry=0xffffc05d30c8, nbytes=nbytes@entry=4) at ../sysdeps/unix/sysv/linux/read.c:27
54
27      ../sysdeps/unix/sysv/linux/read.c: No such file or directory.
55
(gdb) bt
56
#0  0x0000ffff8b636a48 in __libc_read (fd=fd@entry=113, buf=buf@entry=0xffffc05d30c8, nbytes=nbytes@entry=4) at ../sysdeps/unix/sysv/linux/read.c:27
57
#1  0x0000aaaacd2ad1a4 in read (__nbytes=4, __buf=0xffffc05d30c8, __fd=113) at /usr/include/aarch64-linux-gnu/bits/unistd.h:44
58
#2  safe_read (fd=113, buf=0xffffc05d30c8, count=count@entry=4) at ./src/common/safe_io.c:30
59
#3  0x0000aaaacd2ad21c in safe_read_exact (fd=<optimized out>, buf=<optimized out>, count=4) at ./src/common/safe_io.c:48
60
#4  0x0000aaaacc96fa40 in Preforker::parent_wait (this=0xffffc05d3768, err_msg="") at ./src/common/Preforker.h:82
61
#5  0x0000aaaacc949a68 in main (argc=<optimized out>, argv=<optimized out>) at ./src/ceph_osd.cc:203
62
(gdb) thread apply all bt
63

    
64
Thread 5 (Thread 0xffff8b0729f0 (LWP 320844)):
65
#0  0x0000ffff8b63322c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaae731ce74) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
66
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaae731cdb8, cond=0xaaaae731ce48) at pthread_cond_wait.c:502
67
#2  __pthread_cond_wait (cond=0xaaaae731ce48, mutex=0xaaaae731cdb8) at pthread_cond_wait.c:655
68
#3  0x0000ffff8b475f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
69
#4  0x0000aaaacd1911ac in ceph::logging::Log::entry (this=0xaaaae731cd80) at ./src/log/Log.cc:421
70
#5  0x0000ffff8b62d088 in start_thread (arg=0xffffc05d2fdf) at pthread_create.c:463
71
#6  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
72

    
73
Thread 4 (Thread 0xffff88ff59f0 (LWP 320838)):
74
#0  0x0000ffff8b636a48 in __libc_read (fd=<optimized out>, buf=0xffff88ff50e7, nbytes=1) at ../sysdeps/unix/sysv/linux/read.c:27
75
#1  0x0000aaaacd8395e8 in eal_thread_loop ()
76
#2  0x0000ffff8b62d088 in start_thread (arg=0xffff8a7f7b0f) at pthread_create.c:463
77
#3  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
78

    
79
Thread 3 (Thread 0xffff897f69f0 (LWP 320837)):
80
#0  0x0000ffff8b636a48 in __libc_read (fd=<optimized out>, buf=0xffff897f60e7, nbytes=1) at ../sysdeps/unix/sysv/linux/read.c:27
81
#1  0x0000aaaacd8395e8 in eal_thread_loop ()
82
#2  0x0000ffff8b62d088 in start_thread (arg=0xffff8a7f7b0f) at pthread_create.c:463
83
#3  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
84

    
85
Thread 2 (Thread 0xffff89ff79f0 (LWP 320836)):
86
#0  0x0000ffff8b24b600 in __GI_epoll_pwait (epfd=<optimized out>, events=0xffff89ff70f0, maxevents=<optimized out>, timeout=-1979748048, set=0x0)
87
    at ../sysdeps/unix/sysv/linux/epoll_pwait.c:42
88
#1  0x0000aaaacd840584 in eal_intr_thread_main ()
89
#2  0x0000ffff8b62d088 in start_thread (arg=0xffff8a7f7a9f) at pthread_create.c:463
90
#3  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
91

    
92
Thread 1 (Thread 0xffff8b0bf010 (LWP 320833)):
93
#0  0x0000ffff8b636a48 in __libc_read (fd=fd@entry=113, buf=buf@entry=0xffffc05d30c8, nbytes=nbytes@entry=4) at ../sysdeps/unix/sysv/linux/read.c:27
94
#1  0x0000aaaacd2ad1a4 in read (__nbytes=4, __buf=0xffffc05d30c8, __fd=113) at /usr/include/aarch64-linux-gnu/bits/unistd.h:44
95
#2  safe_read (fd=113, buf=0xffffc05d30c8, count=count@entry=4) at ./src/common/safe_io.c:30
96
#3  0x0000aaaacd2ad21c in safe_read_exact (fd=<optimized out>, buf=<optimized out>, count=4) at ./src/common/safe_io.c:48
97
#4  0x0000aaaacc96fa40 in Preforker::parent_wait (this=0xffffc05d3768, err_msg="") at ./src/common/Preforker.h:82
98
#5  0x0000aaaacc949a68 in main (argc=<optimized out>, argv=<optimized out>) at ./src/ceph_osd.cc:203
99

    
100

    
101

    
102

    
103

    
104

    
105

    
106
==============================================
107

    
108
root@node2:~# gdb /usr/bin/ceph-osd -p 320843
109
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
110
Copyright (C) 2018 Free Software Foundation, Inc.
111
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
112
This is free software: you are free to change and redistribute it.
113
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
114
and "show warranty" for details.
115
This GDB was configured as "aarch64-linux-gnu".
116
Type "show configuration" for configuration details.
117
For bug reporting instructions, please see:
118
<http://www.gnu.org/software/gdb/bugs/>.
119
Find the GDB manual and other documentation resources online at:
120
<http://www.gnu.org/software/gdb/documentation/>.
121
For help, type "help".
122
Type "apropos word" to search for commands related to "word"...
123
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/c1/798048d376faca23ada6b38996bb8f2e8530ba.debug...done.
124
done.
125
Attaching to program: /usr/bin/ceph-osd, process 320843
126
[New LWP 320845]
127
[New LWP 320846]
128
[New LWP 320847]
129
[New LWP 320849]
130
[New LWP 320850]
131
[New LWP 320851]
132
[New LWP 320852]
133
[Thread debugging using libthread_db enabled]
134
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
135
0x0000ffff8b900aa0 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
136
(gdb) bt
137
#0  0x0000ffff8b900aa0 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
138
#1  0x0000ffff8b900954 in SpinLock::SlowLock() () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
139
#2  0x0000ffff8b8f0e10 in tcmalloc::CentralFreeList::RemoveRange(void**, void**, int) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
140
#3  0x0000ffff8b8f3c10 in tcmalloc::ThreadCache::FetchFromCentralCache(unsigned long, unsigned long) ()
141
   from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
142
#4  0x0000ffff8b9027c4 in tc_malloc () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
143
#5  0x0000aaaaccfbfd24 in ceph::BackTrace::print (this=this@entry=0xffffc05ca140, out=...) at ./src/common/BackTrace.cc:40
144
#6  0x0000aaaaccfb5e64 in handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:169
145
#7  <signal handler called>
146
#8  0x0000ffff8b8f0950 in tcmalloc::CentralFreeList::FetchFromOneSpans(int, void**, void**) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
147
#9  0x0000ffff8b8f0c54 in tcmalloc::CentralFreeList::FetchFromOneSpansSafe(int, void**, void**) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
148
#10 0x0000ffff8b8f0d20 in tcmalloc::CentralFreeList::RemoveRange(void**, void**, int) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
149
#11 0x0000ffff8b8f3c10 in tcmalloc::ThreadCache::FetchFromCentralCache(unsigned long, unsigned long) ()
150
   from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
151
#12 0x0000ffff8b904d74 in tc_newarray () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
152
#13 0x0000ffff8b4e8bf0 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::reserve(unsigned long) ()
153
   from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
154
#14 0x0000aaaaccf6dae0 in boost::uuids::to_string[abi:cxx11](boost::uuids::uuid const&) (u=...)
155
    at ./obj-aarch64-linux-gnu/boost/include/boost/uuid/uuid_io.hpp:152
156
#15 uuid_d::print (s=0xffffc05d11c0 "\020\017w?f", this=0xffffc05d10a0) at ./src/include/uuid.h:50
157
#16 operator<< (u=..., out=...) at ./src/include/uuid.h:76
158
#17 BlueFS::mkfs (this=0xaaaae7fc6000, osd_uuid=..., layout=...) at ./src/os/bluestore/BlueFS.cc:449
159
#18 0x0000aaaacce89dcc in BlueStore::_open_bluefs (this=this@entry=0xaaaae8770d00, create=create@entry=true) at ./src/os/bluestore/BlueStore.cc:5397
160
#19 0x0000aaaacce8a214 in BlueStore::_open_db (this=this@entry=0xaaaae8770d00, create=create@entry=true, to_repair_db=to_repair_db@entry=false,
161
    read_only=read_only@entry=false) at ./src/os/bluestore/BlueStore.cc:5607
162
#20 0x0000aaaaccebe700 in BlueStore::mkfs (this=0xaaaae8770d00) at ./src/os/bluestore/BlueStore.cc:6316
163
#21 0x0000aaaacc98b740 in OSD::mkfs (cct=0xaaaae73e8000, store=0xaaaae8770d00, fsid=..., whoami=37) at ./src/osd/OSD.cc:1962
164
#22 0x0000aaaacc94a9a0 in main (argc=<optimized out>, argv=<optimized out>) at /usr/include/c++/9/bits/char_traits.h:335
165
(gdb) thread apply all bt
166

    
167
Thread 8 (Thread 0xffff87ff39f0 (LWP 320852)):
168
#0  0x0000ffff8b63322c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaae8762cc8)
169
    at ../sysdeps/unix/sysv/linux/futex-internal.h:88
170
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaae8762c70, cond=0xaaaae8762ca0) at pthread_cond_wait.c:502
171
#2  __pthread_cond_wait (cond=0xaaaae8762ca0, mutex=0xaaaae8762c70) at pthread_cond_wait.c:655
172
#3  0x0000ffff8b475f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
173
#4  0x0000aaaaccfa9124 in KernelDevice::_discard_thread (this=0xaaaae8762a80) at ./src/os/bluestore/KernelDevice.cc:627
174
#5  0x0000aaaaccfac158 in KernelDevice::DiscardThread::entry (this=<optimized out>) at ./src/os/bluestore/KernelDevice.h:74
175
#6  0x0000ffff8b62d088 in start_thread (arg=0xffffc05ceabf) at pthread_create.c:463
176
#7  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
177

    
178
Thread 7 (Thread 0xffff887f49f0 (LWP 320851)):
179
#0  0x0000ffff8b9506c0 in ?? () from /lib/aarch64-linux-gnu/libaio.so.1
180
#1  0x0000ffffc05cea20 in ?? ()
181
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
182

    
183
Thread 6 (Thread 0xffff8a7f89f0 (LWP 320850)):
184
#0  0x0000ffff8b63322c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaae8762948)
185
    at ../sysdeps/unix/sysv/linux/futex-internal.h:88
186
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaae87628f0, cond=0xaaaae8762920) at pthread_cond_wait.c:502
187
#2  __pthread_cond_wait (cond=0xaaaae8762920, mutex=0xaaaae87628f0) at pthread_cond_wait.c:655
188
#3  0x0000ffff8b475f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
189
#4  0x0000aaaaccfa9124 in KernelDevice::_discard_thread (this=0xaaaae8762700) at ./src/os/bluestore/KernelDevice.cc:627
190
#5  0x0000aaaaccfac158 in KernelDevice::DiscardThread::entry (this=<optimized out>) at ./src/os/bluestore/KernelDevice.h:74
191
#6  0x0000ffff8b62d088 in start_thread (arg=0xffffc05cf30f) at pthread_create.c:463
192
#7  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
193

    
194
Thread 5 (Thread 0xffff8b0729f0 (LWP 320849)):
195
#0  0x0000ffff8b9506c0 in ?? () from /lib/aarch64-linux-gnu/libaio.so.1
196
#1  0x0000ffffc05cf270 in ?? ()
197
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
198

    
199
Thread 4 (Thread 0xffff89ff79f0 (LWP 320847)):
200
#0  0x0000ffff8b242048 in __GI___poll (fds=fds@entry=0xffff89ff7198, nfds=281473018478292, nfds@entry=2, timeout=timeout@entry=-1)
201
    at ../sysdeps/unix/sysv/linux/poll.c:41
202
#1  0x0000aaaacd00fc00 in poll (__timeout=-1, __nfds=2, __fds=0xffff89ff7198) at /usr/include/aarch64-linux-gnu/bits/poll2.h:46
203
#2  AdminSocket::entry (this=0xaaaae7390160) at ./src/common/admin_socket.cc:240
204
#3  0x0000ffff8b47bed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
205
---Type <return> to continue, or q <return> to quit---
206
#4  0x0000ffff8b62d088 in start_thread (arg=0xffffc05d2ecf) at pthread_create.c:463
207
#5  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
208

    
209
Thread 3 (Thread 0xffff897f69f0 (LWP 320846)):
210
#0  0x0000ffff8b6335d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff897f61d8, expected=0, futex_word=0xaaaae7fe8858)
211
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
212
#1  __pthread_cond_wait_common (abstime=0xffff897f61d8, mutex=0xaaaae7fe8800, cond=0xaaaae7fe8830) at pthread_cond_wait.c:539
213
#2  __pthread_cond_timedwait (cond=0xaaaae7fe8830, mutex=0xaaaae7fe8800, abstime=0xffff897f61d8) at pthread_cond_wait.c:667
214
#3  0x0000aaaacd028ca4 in __gthread_cond_timedwait (__abs_timeout=0xffff897f61d8, __mutex=<optimized out>, __cond=0xaaaae7fe8830)
215
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
216
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>...,
217
    __lock=..., this=0xaaaae7fe8830) at /usr/include/c++/9/condition_variable:188
218
#5  std::condition_variable::wait_until<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=...,
219
    __lock=..., this=0xaaaae7fe8830) at /usr/include/c++/9/condition_variable:121
220
#6  std::condition_variable::wait_for<unsigned long, std::ratio<1l, 1000000000l> > (__rtime=<synthetic pointer>..., __lock=..., this=0xaaaae7fe8830)
221
    at /usr/include/c++/9/condition_variable:152
222
#7  CephContextServiceThread::entry (this=0xaaaae7fe87e0) at ./src/common/ceph_context.cc:206
223
#8  0x0000ffff8b62d088 in start_thread (arg=0xffffc05d2e6f) at pthread_create.c:463
224
#9  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
225

    
226
Thread 2 (Thread 0xffff88ff59f0 (LWP 320845)):
227
#0  0x0000ffff8b63322c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaae731ce70)
228
    at ../sysdeps/unix/sysv/linux/futex-internal.h:88
229
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaae731cdb8, cond=0xaaaae731ce48) at pthread_cond_wait.c:502
230
#2  __pthread_cond_wait (cond=0xaaaae731ce48, mutex=0xaaaae731cdb8) at pthread_cond_wait.c:655
231
#3  0x0000ffff8b475f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
232

    
233
#4  0x0000aaaacd1911ac in ceph::logging::Log::entry (this=0xaaaae731cd80) at ./src/log/Log.cc:421
234
#5  0x0000ffff8b62d088 in start_thread (arg=0xffffc05d2f4f) at pthread_create.c:463
235
#6  0x0000ffff8b24b4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
236

    
237
Thread 1 (Thread 0xffff8b0bf010 (LWP 320843)):
238
#0  0x0000ffff8b900aa0 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
239
#1  0x0000ffff8b900954 in SpinLock::SlowLock() () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
240
#2  0x0000ffff8b8f0e10 in tcmalloc::CentralFreeList::RemoveRange(void**, void**, int) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
241
#3  0x0000ffff8b8f3c10 in tcmalloc::ThreadCache::FetchFromCentralCache(unsigned long, unsigned long) ()
242
   from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
243
#4  0x0000ffff8b9027c4 in tc_malloc () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
244
#5  0x0000aaaaccfbfd24 in ceph::BackTrace::print (this=this@entry=0xffffc05ca140, out=...) at ./src/common/BackTrace.cc:40
245
#6  0x0000aaaaccfb5e64 in handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:169
246
---Type <return> to continue, or q <return> to quit---
247
#7  <signal handler called>
248
#8  0x0000ffff8b8f0950 in tcmalloc::CentralFreeList::FetchFromOneSpans(int, void**, void**) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
249
#9  0x0000ffff8b8f0c54 in tcmalloc::CentralFreeList::FetchFromOneSpansSafe(int, void**, void**) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
250
#10 0x0000ffff8b8f0d20 in tcmalloc::CentralFreeList::RemoveRange(void**, void**, int) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
251
#11 0x0000ffff8b8f3c10 in tcmalloc::ThreadCache::FetchFromCentralCache(unsigned long, unsigned long) ()
252
   from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
253
#12 0x0000ffff8b904d74 in tc_newarray () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
254
#13 0x0000ffff8b4e8bf0 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::reserve(unsigned long) ()
255
   from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
256
#14 0x0000aaaaccf6dae0 in boost::uuids::to_string[abi:cxx11](boost::uuids::uuid const&) (u=...)
257
    at ./obj-aarch64-linux-gnu/boost/include/boost/uuid/uuid_io.hpp:152
258
#15 uuid_d::print (s=0xffffc05d11c0 "\020\017w?f", this=0xffffc05d10a0) at ./src/include/uuid.h:50
259
#16 operator<< (u=..., out=...) at ./src/include/uuid.h:76
260
#17 BlueFS::mkfs (this=0xaaaae7fc6000, osd_uuid=..., layout=...) at ./src/os/bluestore/BlueFS.cc:449
261
#18 0x0000aaaacce89dcc in BlueStore::_open_bluefs (this=this@entry=0xaaaae8770d00, create=create@entry=true) at ./src/os/bluestore/BlueStore.cc:5397
262
#19 0x0000aaaacce8a214 in BlueStore::_open_db (this=this@entry=0xaaaae8770d00, create=create@entry=true, to_repair_db=to_repair_db@entry=false,
263
    read_only=read_only@entry=false) at ./src/os/bluestore/BlueStore.cc:5607
264
#20 0x0000aaaaccebe700 in BlueStore::mkfs (this=0xaaaae8770d00) at ./src/os/bluestore/BlueStore.cc:6316
265
#21 0x0000aaaacc98b740 in OSD::mkfs (cct=0xaaaae73e8000, store=0xaaaae8770d00, fsid=..., whoami=37) at ./src/osd/OSD.cc:1962
266
#22 0x0000aaaacc94a9a0 in main (argc=<optimized out>, argv=<optimized out>) at /usr/include/c++/9/bits/char_traits.h:335
267
(gdb)
268
(gdb) c
269
Continuing.
270

    
271
Thread 1 "ceph-osd" received signal SIGINT, Interrupt.
272
0x0000ffff8b900aa0 in base::internal::SpinLockDelay(int volatile*, int, int) () from /usr/lib/aarch64-linux-gnu/libtcmalloc.so.4
273
(gdb) quit
274