Ceph » crimson

start a crimson cluster with osd =3 by MDS=0 MGR=1 OSD=3 MON=1 ../src/vstart.sh -n --without-dashboard --bluestore -X --crimson
bin/ceph osd pool create test-pool 32 32
bin/rados bench -p test-pool 100 write -t 10 --no-cleanup

during rados bench test
kill osd.1 then then restart it:

osd.0 got crush ------
INFO 2020-09-22 23:41:37,776 [shard 0] osd - Entering state: Started
INFO 2020-09-22 23:41:37,776 [shard 0] osd - Entering state: Start
INFO 2020-09-22 23:41:37,776 [shard 0] osd - pg_epoch 20 pg[2.10( v 18'11 (0'0,18'11] local-lis/les=16/17 n=0 ec=16/16 lis/c=16/16 les/c/f=17/17/0 sis=20) [2,1,0] r=2 lpr=2 0 pi=[16,20)/1 crt=18'11 mlcod 0'0 unknown NOTIFY state<Start>: transitioning to Stray
INFO 2020-09-22 23:41:37,776 [shard 0] osd - Exiting state: Start, entered at 16008432 97.7760987, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,776 [shard 0] osd - Entering state: Started/Stray
INFO 2020-09-22 23:41:37,777 [shard 0] osd - pg_epoch 20 pg[2.1d( v 18'16 (0'0,18'16] local-lis/les=16/17 n=0 ec=16/16 lis/c=16/16 les/c/f=17/17/0 sis=19) [0,2] r=0 lpr=19 pi=[16,19)/1 crt=18'16 lcod 18'15 mlcod 0'0 peering state<Started/Primary/Peering>: Pee ring, affected_by_map, going to Reset
INFO 2020-09-22 23:41:37,777 [shard 0] osd - Exiting state: Started/Primary/Peering/Wa itUpThru, entered at 1600843297.0521634, 0.001068753 spent on 2 events
INFO 2020-09-22 23:41:37,777 [shard 0] osd - Exiting state: Started/Primary/Peering, e ntered at 1600843296.8316412, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,777 [shard 0] osd - Exiting state: Started/Primary, entered a t 1600843296.8315442, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,777 [shard 0] osd - Exiting state: Started, entered at 160084 3296.8310757, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,778 [shard 0] osd - Entering state: Reset
INFO 2020-09-22 23:41:37,778 [shard 0] osd - pg_epoch 20 pg[2.1d( v 18'16 (0'0,18'16] local-lis/les=16/17 n=0 ec=16/16 lis/c=16/16 les/c/f=17/17/0 sis=20) [0,1,2] r=0 lpr=2 0 pi=[16,20)/1 crt=18'16 lcod 18'15 mlcod 0'0 unknown start_peering_interval up {0, 2} -> {0, 1, 2}, acting {0, 2} -> {0, 1, 2}, acting_primary 0 -> 0, up_primary 0 -> 0, rol e 0 -> 0, features acting 4540138297136906239 upacting 4540138297136906239 =================================================================
86770ERROR: AddressSanitizer: heap-use-after-free on address 0x6080008db730 at pc 0 x5628d08af2ef bp 0x7ffc71d9ef70 sp 0x7ffc71d9ef60
READ of size 8 at 0x6080008db730 thread T0
Reactor stalled for 262 ms on shard 0.
Backtrace:
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000006bb2f
0x000000001801cdc4
0x0000000017ff5701
0x0000000017eaca10
0x0000000017ec0ffb
0x0000000017ebd44a
0x0000000017ebd60a
0x0000000017ec0a58
/lib/x86_64-linux-gnu/libpthread.so.0+0x000000000001289f
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013e0a9
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013e0f4
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013e16a
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013e16a
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013d9c2
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000014bfda
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000014c208
0x0000000017db3d87
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000014c45d
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013dd45
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013de71
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013163c
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013237e
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000012ce87
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000003211c
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x0000000000115cfc
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000011567e
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x0000000000116416
0x000000001353a2ee
0x0000000013554bcb
0x000000001353aace
0x0000000013ed91b8
0x0000000013ed4387
0x0000000013ec2c8c
0x0000000013bb295c
0x000000001470f37a
0x00000000147c9112
0x0000000014acd88d
0x0000000014ac8b39
0x0000000014ac3d9d
0x0000000014abe15a
0x0000000014ab86bf
0x0000000014aad1d6
0x0000000014a9fe31
0x0000000014a7faa6
0x00000000135baa66
0x0000000013ccfc12
0x0000000013c897c0
0x0000000013c89ddc
0x0000000013c4a56b
0x0000000013c19d70
0x0000000014703217
0x0000000013b97ab4
0x00000000141122d6
0x0000000014118aef
0x00000000141167ef
0x00000000141168ed
0x0000000014118dd6
0x0000000013afbedf
0x0000000013ada060
0x0000000013aacfae
0x0000000013a7947b
0x0000000013a3b37c
0x0000000013a79572
0x0000000013a3b196
0x0000000013b48486
0x0000000017ef304f
0x0000000017ef769f
0x0000000017efcbc7
0x0000000017d98296
0x0000000013715111
/lib/x86_64-linux-gnu/libc.so.6+0x0000000000021b96
0x00000000135164e9
#0 0x5628d08af2ee in seastar::future_state_base::set_exception(std::__exception_ptr ::exception_ptr&&) /home/chunmei/ceph/src/seastar/include/seastar/core/future.hh:527
#1 0x5628d08c9bcb in void seastar::internal::promise_base::set_exception_impl<std:: _exception_ptr::exception_ptr>(std::_exception_ptr::exception_ptr&&) /home/chunmei/ce ph/src/seastar/include/seastar/core/future.hh:820
#2 0x5628d08aface in seastar::internal::promise_base::set_exception(std::__exceptio n_ptr::exception_ptr&&) /home/chunmei/ceph/src/seastar/include/seastar/core/future.hh:8 39
#3 0x5628d124e1b8 in std::enable_if<!std::is_same<std::remove_reference<crimson::co mmon::actingset_changed&>::type, std::__exception_ptr::exception_ptr>::value, void>::ty pe seastar::internal::promise_base::set_exception<crimson::common::actingset_changed&>( crimson::common::actingset_changed&) /home/chunmei/ceph/src/seastar/include/seastar/cor e/future.hh:848
#4 0x5628d1249387 in std::enable_if<!std::is_same<std::remove_reference<crimson::co mmon::actingset_changed&>::type, std::__exception_ptr::exception_ptr>::value, void>::ty pe seastar::promise<>::set_exception<crimson::common::actingset_changed&>(crimson::comm on::actingset_changed&) /home/chunmei/ceph/src/seastar/include/seastar/core/future.hh:1 014
#5 0x5628d1237c8c in ReplicatedBackend::on_actingset_changed(PGBackend::peering_inf o_t) /home/chunmei/ceph/src/crimson/osd/replicated_backend.cc:108
#6 0x5628d0f2795c in crimson::osd::PG::on_change(ceph::os::Transaction&) /home/chun mei/ceph/src/crimson/osd/pg.cc:1002
#7 0x5628d1a8437a in PeeringState::start_peering_interval(boost::local_shared_ptr<O SDMap const>, std::vector<int, std::allocator<int> > const&, int, std::vector<int, std: :allocator<int> > const&, int, ceph::os::Transaction&) /home/chunmei/ceph/src/osd/Peeri ngState.cc:721
#8 0x5628d1b3e112 in PeeringState::Reset::react(PeeringState::AdvMap const&) /home/ chunmei/ceph/src/osd/PeeringState.cc:4733
#9 0x5628d1e4288d in boost::statechart::detail::reaction_result boost::statechart:: custom_reaction<PeeringState::AdvMap>::react<PeeringState::Reset, boost::statechart::ev ent_base, void const*>(PeeringState::Reset&, boost::statechart::event_base const&, void const* const&) /home/chunmei/ceph/build/boost/include/boost/statechart/custom_reaction .hpp:41

osd.2 got crush --------------------
INFO 2020-09-22 23:41:37,783 [shard 0] osd - Exiting state: Started/Primary/Pee ring/WaitUpThru, entered at 1600843297.084227, 0.0013321 spent on 3 events
INFO 2020-09-22 23:41:37,783 [shard 0] osd - Exiting state: Started/Primary/Pee ring, entered at 1600843296.7952805, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,783 [shard 0] osd - Exiting state: Started/Primary, en tered at 1600843296.7951663, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,783 [shard 0] osd - Exiting state: Started, entered at 1600843296.7946692, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,783 [shard 0] osd - Entering state: Reset
INFO 2020-09-22 23:41:37,783 [shard 0] osd - pg_epoch 20 pg[2.d( v 18'12 (0'0, 18'12] local-lis/les=16/17 n=0 ec=16/16 lis/c=16/16 les/c/f=17/17/0 sis=20) [1,2 ,0] r=1 lpr=20 pi=[16,20)/1 crt=18'12 mlcod 0'0 unknown start_peering_interval u p {2, 0} -> {1, 2, 0}, acting {2, 0} -> {1, 2, 0}, acting_primary 2 -> 1, up_pri mary 2 -> 1, role 0 -> 1, features acting 4540138297136906239 upacting 454013829 7136906239
INFO 2020-09-22 23:41:37,784 [shard 0] osd - pg_epoch 20 pg[2.1a( v 18'22 (0'0 ,18'22] local-lis/les=16/17 n=0 ec=16/16 lis/c=16/16 les/c/f=17/17/0 sis=19) [2, 0] r=0 lpr=19 pi=[16,19)/1 crt=18'22 lcod 18'21 mlcod 0'0 peering state<Started/ Primary/Peering>: Peering, affected_by_map, going to Reset
INFO 2020-09-22 23:41:37,784 [shard 0] osd - Exiting state: Started/Primary/Pee ring/WaitUpThru, entered at 1600843297.125308, 0.000942835 spent on 2 events
INFO 2020-09-22 23:41:37,784 [shard 0] osd - Exiting state: Started/Primary/Pee ring, entered at 1600843296.757347, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,784 [shard 0] osd - Exiting state: Started/Primary, en tered at 1600843296.7572174, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,784 [shard 0] osd - Exiting state: Started, entered at 1600843296.7567096, 0.0 spent on 0 events
INFO 2020-09-22 23:41:37,784 [shard 0] osd - Entering state: Reset
INFO 2020-09-22 23:41:37,784 [shard 0] osd - pg_epoch 20 pg[2.1a( v 18'22 (0'0 ,18'22] local-lis/les=16/17 n=0 ec=16/16 lis/c=16/16 les/c/f=17/17/0 sis=20) [2, 0,1] r=0 lpr=20 pi=[16,20)/1 crt=18'22 lcod 18'21 mlcod 0'0 unknown start_peerin g_interval up {2, 0} -> {2, 0, 1}, acting {2, 0} -> {2, 0, 1}, acting_primary 2 -> 2, up_primary 2 -> 2, role 0 -> 0, features acting 4540138297136906239 upacti ng 4540138297136906239 =================================================================
89522ERROR: AddressSanitizer: heap-use-after-free on address 0x6080007d8a30 at pc 0x55cea5d402ef bp 0x7ffe87a0f140 sp 0x7ffe87a0f130
READ of size 8 at 0x6080007d8a30 thread T0
Reactor stalled for 262 ms on shard 0.
Backtrace:
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000006bb2f
0x000000001801cdc4
0x0000000017ff5701
0x0000000017eaca10
0x0000000017ec0ffb
0x0000000017ebd44a
0x0000000017ebd60a
0x0000000017ec0a58
/lib/x86_64-linux-gnu/libpthread.so.0+0x000000000001289f
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013e139
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013e0f4
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013e0f4
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013e16a
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013d9c2
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000014bfda
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000014c208
0x0000000017db3d87
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000014c45d
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013dd45
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013de71
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013163c
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000013237e
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000012ce87
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000003211c
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x0000000000115cfc
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x000000000011567e
/usr/lib/x86_64-linux-gnu/libasan.so.5+0x0000000000116416
0x000000001353a2ee
0x0000000013554bcb
0x000000001353aace
0x0000000013ed91b8
0x0000000013ed4387
0x0000000013ec2c8c
0x0000000013bb295c
0x000000001470f37a
0x00000000147c9112
0x0000000014acd88d
0x0000000014ac8b39
0x0000000014ac3d9d
0x0000000014abe15a
0x0000000014ab86bf
0x0000000014aad1d6
0x0000000014a9fe31
0x0000000014a7faa6
0x00000000135baa66
0x0000000013ccfc12
0x0000000013c897c0
0x0000000013c89ddc
0x0000000013c4a56b
0x0000000013c19d70
0x0000000014703217
0x0000000013b97ab4
0x00000000141122d6
0x0000000014118aef
0x00000000141167ef
0x00000000141168ed
0x0000000014118dd6
0x0000000013afbedf
0x0000000013ada060
0x0000000013aacfae
0x0000000013a7947b
0x0000000013a3b37c
0x0000000013a79572
0x0000000013a3b196
0x0000000013b48486
0x0000000017ef304f
0x0000000017ef769f
0x0000000017efcbc7
0x0000000017d98296
0x0000000013715111
/lib/x86_64-linux-gnu/libc.so.6+0x0000000000021b96
0x00000000135164e9
#0 0x55cea5d402ee in seastar::future_state_base::set_exception(std::__except ion_ptr::exception_ptr&&) /home/chunmei/ceph/src/seastar/include/seastar/core/fu ture.hh:527
#1 0x55cea5d5abcb in void seastar::internal::promise_base::set_exception_imp l<std::__exception_ptr::exception_ptr>(std::__exception_ptr::exception_ptr&&) /h ome/chunmei/ceph/src/seastar/include/seastar/core/future.hh:820
#2 0x55cea5d40ace in seastar::internal::promise_base::set_exception(std::__e xception_ptr::exception_ptr&&) /home/chunmei/ceph/src/seastar/include/seastar/co re/future.hh:839
#3 0x55cea66df1b8 in std::enable_if<!std::is_same<std::remove_reference<crim son::common::actingset_changed&>::type, std::__exception_ptr::exception_ptr>::va lue, void>::type seastar::internal::promise_base::set_exception<crimson::common: :actingset_changed&>(crimson::common::actingset_changed&) /home/chunmei/ceph/src /seastar/include/seastar/core/future.hh:848
#4 0x55cea66da387 in std::enable_if<!std::is_same<std::remove_reference<crim son::common::actingset_changed&>::type, std::__exception_ptr::exception_ptr>::va lue, void>::type seastar::promise<>::set_exception<crimson::common::actingset_ch anged&>(crimson::common::actingset_changed&) /home/chunmei/ceph/src/seastar/incl ude/seastar/core/future.hh:1014
#5 0x55cea66c8c8c in ReplicatedBackend::on_actingset_changed(PGBackend::peer ing_info_t) /home/chunmei/ceph/src/crimson/osd/replicated_backend.cc:108
#6 0x55cea63b895c in crimson::osd::PG::on_change(ceph::os::Transaction&) /ho me/chunmei/ceph/src/crimson/osd/pg.cc:1002
#7 0x55cea6f1537a in PeeringState::start_peering_interval(boost::local_share d_ptr<OSDMap const>, std::vector<int, std::allocator<int> > const&, int, std::ve ctor<int, std::allocator<int> > const&, int, ceph::os::Transaction&) /home/chunm ei/ceph/src/osd/PeeringState.cc:721
#8 0x55cea6fcf112 in PeeringState::Reset::react(PeeringState::AdvMap const&) /home/chunmei/ceph/src/osd/PeeringState.cc:4733
#9 0x55cea72d388d in boost::statechart::detail::reaction_result boost::state chart::custom_reaction<PeeringState::AdvMap>::react<PeeringState::Reset, boost:: statechart::event_base, void const*>(PeeringState::Reset&, boost::statechart::ev ent_base const&, void const* const&) /home/chunmei/ceph/build/boost/include/boos t/statechart/custom_reaction.hpp:41

the restarted osd.1 got deadly loop crush ----------------------
INFO 2020-09-22 23:42:04,208 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.0 v2:172.25.110.24:6802/86770] execute_wait(): going to CONNECTING
INFO 2020-09-22 23:42:04,209 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.0 v2:172.25.110.24:6802/86770] write_event: delay ...
INFO 2020-09-22 23:42:04,210 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.2 v2:172.25.110.24:6811/89522] execute_wait(): going to CONNECTING
INFO 2020-09-22 23:42:04,211 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.2 v2:172.25.110.24:6811/89522] write_event: delay ...
INFO 2020-09-22 23:42:04,211 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.0 v2:172.25.110.24:6802/86770] execute_connecting(): fault at CONN ECTING, going to WAIT -- std::system_error (error system:111, Connection refused )
WARN 2020-09-22 23:42:04,212 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.0 v2:172.25.110.24:6802/86770] waiting 15.0 seconds ...
INFO 2020-09-22 23:42:04,213 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.0 v2:172.25.110.24:6802/86770] write_event: delay ...
INFO 2020-09-22 23:42:04,213 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.2 v2:172.25.110.24:6811/89522] execute_connecting(): fault at CONN ECTING, going to WAIT -- std::system_error (error system:111, Connection refused )
WARN 2020-09-22 23:42:04,213 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.2 v2:172.25.110.24:6811/89522] waiting 15.0 seconds ...
INFO 2020-09-22 23:42:04,214 [shard 0] ms - [osd.1(cluster) v2:172.25.110.24:68 07/999 >> osd.2 v2:172.25.110.24:6811/89522] write_event: delay ...
INFO 2020-09-22 23:42:05,841 [shard 0] monc - sending auth(proto 1 0 bytes epoc h 0) v1
INFO 2020-09-22 23:42:05,842 [shard 0] monc - waiting
INFO 2020-09-22 23:42:05,845 [shard 0] osd - ms_dispatch unhandled message auth _reply(proto 1 0 (0) Success) v1
INFO 2020-09-22 23:42:05,845 [shard 0] monc - handle_auth_reply mon v2:172.25.1 10.24:6806/999 => v2:172.25.110.24:40588/0 returns auth_reply(proto 1 0 (0) Succ ess) v1: 0
INFO 2020-09-22 23:42:05,846 [shard 0] monc - do_auth: mon v2:172.25.110.24:680 6/999 => v2:172.25.110.24:40588/0 returns auth_reply(proto 1 0 (0) Success) v1: 0
AddressSanitizer:DEADLYSIGNAL =================================================================
999ERROR: AddressSanitizer: stack-overflow on address 0x7ffc98b05a68 (pc 0x7 f41b9ae0f79 bp 0x7ffc98b06310 sp 0x7ffc98b05a70 T0)
#0 0x7f41b9ae0f78 (/usr/lib/x86_64-linux-gnu/libasan.so.5+0xd4f78)
#1 0x7f41b72d61f7 in _cxxabiv1::_vmi_class_type_info::__do_dyncast(long, _ cxxabiv1::_class_type_info::__sub_kind, _cxxabiv1::_class_type_info const*, void const*, _cxxabiv1::_class_type_info const*, void const*, _cxxabiv1::_cl ass_type_info::__dyncast_result&) const (/usr/lib/x86_64-linux-gnu/libstdc++.so. 6+0xa71f7)
#2 0x7f41b72d3143 in __dynamic_cast (/usr/lib/x86_64-linux-gnu/libstdc++.so. 6+0xa4143)
#3 0x7f41b7349a7d in bool std::has_facet<std::ctype<char> >(std::locale cons t&) (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0x11aa7d)
#4 0x7f41b733c183 in std::basic_ios<char, std::char_traits<char> >::_M_cache _locale(std::locale const&) (/usr/lib/x86_64-linux-gnu/libstdc++.so.6+0x10d183)
#5 0x7f41b733c5ff in std::basic_ios<char, std::char_traits<char> >::init(std ::basic_streambuf<char, std::char_traits<char> >) (/usr/lib/x86_64-linux-gnu/li bstdc++.so.6+0x10d5ff)
#6 0x7f41b7358756 in std::basic_ostream<char, std::char_traits<char> >::basi c_ostream(std::basic_streambuf<char, std::char_traits<char> >) (/usr/lib/x86_64 -linux-gnu/libstdc++.so.6+0x129756)
#7 0x559ee7913f58 in void fmt::v6::detail::format_value<char, std::chrono::t ime_point<ceph::coarse_real_clock, std::chrono::duration<unsigned long, std::rat io<1l, 1000000000l> > > >(fmt::v6::detail::buffer<char>&, std::chrono::time_poin t<ceph::coarse_real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1 000000000l> > > const&, fmt::v6::detail::locale_ref) /home/chunmei/ceph/src/fmt/ include/fmt/ostream.h:100
#8 0x559ee791248a in std::back_insert_iterator<fmt::v6::detail::buffer<char> > fmt::v6::detail::fallback_formatter<std::chrono::time_point<ceph::coarse_real _clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, ch ar, void>::format<std::back_insert_iterator<fmt::v6::detail::buffer<char> > >(st d::chrono::time_point<ceph::coarse_real_clock, std::chrono::duration<unsigned lo ng, std::ratio<1l, 1000000000l> > > const&, fmt::v6::basic_format_context<std::b ack_insert_iterator<fmt::v6::detail::buffer<char> >, char>&) /home/chunmei/ceph/ src/fmt/include/fmt/ostream.h:128
#9 0x559ee7910286 in void fmt::v6::detail::value<fmt::v6::basic_format_conte xt<std::back_insert_iterator<fmt::v6::detail::buffer<char> >, char> >::format_cu stom_arg<std::chrono::time_point<ceph::coarse_real_clock, std::chrono::duration< unsigned long, std::ratio<1l, 1000000000l> > >, fmt::v6::detail::fallback_format ter<std::chrono::time_point<ceph::coarse_real_clock, std::chrono::duration<unsig ned long, std::ratio<1l, 1000000000l> > >, char, void> >(void const*, fmt::v6::b asic_format_parse_context<char, fmt::v6::detail::error_handler>&, fmt::v6::basic _format_context<std::back_insert_iterator<fmt::v6::detail::buffer<char> >, char> &) (/home/chunmei/ceph/build/bin/crimson-osd+0x136fe286)
#10 0x559ee780c0f9 in fmt::v6::basic_format_arg<fmt::v6::basic_format_contex t<std::back_insert_iterator<fmt::v6::detail::buffer<char> >, char> >::handle::fo rmat(fmt::v6::basic_format_parse_context<char, fmt::v6::detail::error_handler>&, fmt::v6::basic_format_context<std::back_insert_iterator<fmt::v6::detail::buffer <char> >, char>&) const /home/chunmei/ceph/src/fmt/include/fmt/core.h:1131
#11 0x559eead65956 in fmt::v6::arg_formatter<fmt::v6::buffer_range<char> >:: operator()(fmt::v6::basic_format_arg<fmt::v6::basic_format_context<std::back_ins ert_iterator<fmt::v6::detail::buffer<char> >, char> >::handle) /home/chunmei/cep h/src/fmt/include/fmt/format.h:2771
#12 0x559eead65147 in decltype ({parm#1}(0)) fmt::v6::visit_format_arg<fmt:: v6::arg_formatter<fmt::v6::buffer_range<char> >, fmt::v6::basic_format_context<s td::back_insert_iterator<fmt::v6::detail::buffer<char> >, char> >(fmt::v6::arg_f ormatter<fmt::v6::buffer_range<char> >&&, fmt::v6::basic_format_arg<fmt::v6::bas ic_format_context<std::back_insert_iterator<fmt::v6::detail::buffer<char> >, cha r> > const&) /home/chunmei/ceph/src/fmt/include/fmt/core.h:1201