2019-07-10 09:16:19.500 7f9fb88e6700 1 osd.21 pg_epoch: 42875 pg[5.14e( v 41916'60364 (41401'57364,41916'60364] local-lis/les=42857/42858 n=3110 ec=40535/408 lis/c 42857/42833 les/c/f 42858/42834/0 42874/42875/42875) [22,9]/[9,21] r=1 lpr=42875 pi=[42833,42875)/1 crt=41916'60364 lcod 0'0 remapped NOTIFY mbc={}] state: transitioning to Stray 2019-07-10 09:16:22.732 7f9fba8ea700 -1 bluestore(/var/lib/ceph/osd/ceph-21) _txc_add_transaction error (39) Directory not empty not handled on operation 21 (op 1, counting from 0) 2019-07-10 09:16:22.732 7f9fba8ea700 0 bluestore(/var/lib/ceph/osd/ceph-21) _dump_transaction transaction dump: { "ops": [ { "op_num": 0, "op_name": "remove", "collection": "6.0_head", "oid": "#6:00000000::::head#" }, { "op_num": 1, "op_name": "rmcoll", "collection": "6.0_head" } ] } 2019-07-10 09:16:22.752 7f9fba8ea700 -1 /build/ceph-14.2.1/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)' thread 7f9fba8ea700 time 2019-07-10 09:16:22.737511 /build/ceph-14.2.1/src/os/bluestore/BlueStore.cc: 11089: abort() ceph version 14.2.1 (d555a9489eb35f84f2e1ef49b77e19da9d113972) nautilus (stable) 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xda) [0x850a01] 2: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x296a) [0xe44a2a] 3: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x5e6) [0xe48f96] 4: (ObjectStore::queue_transaction(boost::intrusive_ptr&, ObjectStore::Transaction&&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x7f) [0xa0099f] 5: (PG::_delete_some(ObjectStore::Transaction*)+0x710) [0xa64b30] 6: (PG::RecoveryState::Deleting::react(PG::DeleteSome const&)+0x71) [0xa658f1] 7: (boost::statechart::simple_state, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x131) [0xaaf151] 8: (boost::statechart::state_machine, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x6b) [0xa82b0b] 9: (PG::do_peering_event(std::shared_ptr, PG::RecoveryCtx*)+0x122) [0xa72052] 10: (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr, ThreadPool::TPHandle&)+0x1b4) [0x9ac6a4] 11: (OSD::dequeue_delete(OSDShard*, PG*, unsigned int, ThreadPool::TPHandle&)+0xd2) [0x9ac982] 12: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xbed) [0x9a07dd] 13: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x4ac) [0xfc37bc] 14: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xfc6970] 15: (()+0x76ba) [0x7f9fd9e0d6ba] 16: (clone()+0x6d) [0x7f9fd941441d] 2019-07-10 09:16:22.768 7f9fba8ea700 -1 *** Caught signal (Aborted) ** in thread 7f9fba8ea700 thread_name:tp_osd_tp ceph version 14.2.1 (d555a9489eb35f84f2e1ef49b77e19da9d113972) nautilus (stable) 1: (()+0x11390) [0x7f9fd9e17390] 2: (gsignal()+0x38) [0x7f9fd9342428] 3: (abort()+0x16a) [0x7f9fd934402a] 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x1a0) [0x850ac7] 5: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x296a) [0xe44a2a] 6: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x5e6) [0xe48f96] 7: (ObjectStore::queue_transaction(boost::intrusive_ptr&, ObjectStore::Transaction&&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x7f) [0xa0099f] 8: (PG::_delete_some(ObjectStore::Transaction*)+0x710) [0xa64b30] 9: (PG::RecoveryState::Deleting::react(PG::DeleteSome const&)+0x71) [0xa658f1] 10: (boost::statechart::simple_state, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x131) [0xaaf151] 11: (boost::statechart::state_machine, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x6b) [0xa82b0b] 12: (PG::do_peering_event(std::shared_ptr, PG::RecoveryCtx*)+0x122) [0xa72052] 13: (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr, ThreadPool::TPHandle&)+0x1b4) [0x9ac6a4] 14: (OSD::dequeue_delete(OSDShard*, PG*, unsigned int, ThreadPool::TPHandle&)+0xd2) [0x9ac982] 15: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xbed) [0x9a07dd] 16: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x4ac) [0xfc37bc] 17: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xfc6970] 18: (()+0x76ba) [0x7f9fd9e0d6ba] 19: (clone()+0x6d) [0x7f9fd941441d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- -5255> 2019-07-10 09:13:10.350 7f9fdc57af00 5 asok(0xbb8a000) register_command assert hook 0xb7e85b0 -5254> 2019-07-10 09:13:10.350 7f9fdc57af00 5 asok(0xbb8a000) register_command abort hook 0xb7e85b0 -5253> 2019-07-10 09:13:10.350 7f9fdc57af00 5 asok(0xbb8a000) register_command perfcounters_dump hook 0xb7e85b0 -5252> 2019-07-10 09:13:10.350 7f9fdc57af00 5 asok(0xbb8a000) register_command 1 hook 0xb7e85b0 -5251> 2019-07-10 09:13:10.350 7f9fdc57af00 5 asok(0xbb8a000) register_command perf dump hook 0xb7e85b0 -5250> 2019-07-10 09:13:10.350 7f9fdc57af00 5 asok(0xbb8a000) register_command perfcounters_schema hook 0xb7e85b0 -5249> 2019-07-10 09:13:10.350 7f9fdc57af00 5 asok(0xbb8a000) register_command perf histogram dump hook 0xb7e85b0 -5248> 2019-07-10 09:13:10.350 7f9fdc57af00 5 asok(0xbb8a000) register_command 2 hook 0xb7e85b0 -5247> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command perf schema hook 0xb7e85b0 -5246> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command perf histogram schema hook 0xb7e85b0 -5245> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command perf reset hook 0xb7e85b0 -5244> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command config show hook 0xb7e85b0 -5243> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command config help hook 0xb7e85b0 -5242> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command config set hook 0xb7e85b0 -5241> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command config unset hook 0xb7e85b0 -5240> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command config get hook 0xb7e85b0 -5239> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command config diff hook 0xb7e85b0 -5238> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command config diff get hook 0xb7e85b0 -5237> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command log flush hook 0xb7e85b0 -5236> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command log dump hook 0xb7e85b0 -5235> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command log reopen hook 0xb7e85b0 -5234> 2019-07-10 09:13:10.354 7f9fdc57af00 5 asok(0xbb8a000) register_command dump_mempools hook 0xc692068 -5233> 2019-07-10 09:13:10.370 7f9fdc57af00 10 monclient: get_monmap_and_config -5232> 2019-07-10 09:13:10.378 7f9fdc57af00 10 monclient: build_initial_monmap -5231> 2019-07-10 09:13:10.378 7f9fdc57af00 10 monclient: monmap: epoch 0 fsid d5dc1b29-606d-4f50-b0ae-1cf9c08bb14a last_changed 2019-07-10 09:13:10.380616 created 2019-07-10 09:13:10.380616 min_mon_release 0 (unknown) 0: v1:216.106.44.200:6789/0 mon.noname-a-legacy 1: v2:216.106.44.200:3300/0 mon.noname-a -5230> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding auth protocol: none -5229> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding auth protocol: none -5228> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding auth protocol: none -5227> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding con mode: crc -5226> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding con mode: crc -5225> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding con mode: crc -5224> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding con mode: crc -5223> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding con mode: crc -5222> 2019-07-10 09:13:10.378 7f9fdc57af00 5 AuthRegistry(0xc670a40) adding con mode: crc -5221> 2019-07-10 09:13:10.382 7f9fdc57af00 10 monclient: init -5220> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding auth protocol: none -5219> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding auth protocol: none -5218> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding auth protocol: none -5217> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding con mode: crc -5216> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding con mode: crc -5215> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding con mode: crc -5214> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding con mode: crc -5213> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding con mode: crc -5212> 2019-07-10 09:13:10.382 7f9fdc57af00 5 AuthRegistry(0x7fffb14737c8) adding con mode: crc -5211> 2019-07-10 09:13:10.382 7f9fdc57af00 10 monclient: _reopen_session rank -1 -5210> 2019-07-10 09:13:10.382 7f9fdc57af00 10 monclient(hunting): picked mon.noname-a con 0xba7fa80 addr v2:216.106.44.200:3300/0 -5209> 2019-07-10 09:13:10.382 7f9fdc57af00 10 monclient(hunting): picked mon.noname-a-legacy con 0xc776000 addr v1:216.106.44.200:6789/0 -5208> 2019-07-10 09:13:10.382 7f9fdc57af00 10 monclient(hunting): start opening mon connection -5207> 2019-07-10 09:13:10.382 7f9fdc57af00 10 monclient(hunting): _renew_subs -5206> 2019-07-10 09:13:10.382 7f9fdc57af00 10 monclient(hunting): authenticate will time out at 2019-07-10 09:18:10.385333 -5205> 2019-07-10 09:13:10.382 7f9fd6f40700 10 monclient(hunting): get_auth_request con 0xba7fa80 auth_method 0 -5204> 2019-07-10 09:13:10.382 7f9fd6f40700 10 monclient(hunting): get_auth_request method 1 preferred_modes [1] -5203> 2019-07-10 09:13:10.382 7f9fd6f40700 10 monclient(hunting): _init_auth method 1 -5202> 2019-07-10 09:13:10.382 7f9fd6f40700 10 monclient(hunting): handle_auth_done global_id 666642 payload 0 -5201> 2019-07-10 09:13:10.386 7f9fd6f40700 10 monclient: _finish_hunting 0 -5200> 2019-07-10 09:13:10.386 7f9fd6f40700 1 monclient: found mon.noname-a -5199> 2019-07-10 09:13:10.386 7f9fd6f40700 10 monclient: _send_mon_message to mon.noname-a at v2:216.106.44.200:3300/0 -5198> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: handle_monmap mon_map magic: 0 v1 -5197> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: got monmap 5 from mon.noname-a (according to old e5) -5196> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: dump: epoch 5 fsid d5dc1b29-606d-4f50-b0ae-1cf9c08bb14a last_changed 2019-06-11 16:39:28.479042 created 2019-04-29 16:40:19.247816 min_mon_release 14 (nautilus) 0: [v2:216.106.44.200:3300/0,v1:216.106.44.200:6789/0] mon.synergy0 1: [v2:216.106.44.201:3300/0,v1:216.106.44.201:6789/0] mon.synergy1 2: [v2:216.106.44.202:3300/0,v1:216.106.44.202:6789/0] mon.synergy2 -5195> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: _finish_auth 0 -5194> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 09:12:40.389654) -5193> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: handle_config config(14 keys) v1 -5192> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: handle_monmap mon_map magic: 0 v1 -5191> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: got monmap 5 from mon.synergy0 (according to old e5) -5190> 2019-07-10 09:13:10.386 7f9fd473b700 4 set_mon_vals no callback set -5189> 2019-07-10 09:13:10.386 7f9fd473b700 10 set_mon_vals device_failure_prediction_mode = local -5188> 2019-07-10 09:13:10.386 7f9fd473b700 10 set_mon_vals osd_max_backfills = 30 -5187> 2019-07-10 09:13:10.386 7f9fd473b700 10 set_mon_vals osd_memory_target = 2147483648 -5186> 2019-07-10 09:13:10.386 7f9fd473b700 10 set_mon_vals osd_recovery_max_active = 30 -5185> 2019-07-10 09:13:10.386 7f9fd473b700 10 set_mon_vals osd_recovery_max_single_start = 30 -5184> 2019-07-10 09:13:10.386 7f9fd473b700 10 set_mon_vals osd_recovery_sleep = 0.000000 -5183> 2019-07-10 09:13:10.386 7f9fd5f3e700 10 monclient: dump: epoch 5 fsid d5dc1b29-606d-4f50-b0ae-1cf9c08bb14a last_changed 2019-06-11 16:39:28.479042 created 2019-04-29 16:40:19.247816 min_mon_release 14 (nautilus) 0: [v2:216.106.44.200:3300/0,v1:216.106.44.200:6789/0] mon.synergy0 1: [v2:216.106.44.201:3300/0,v1:216.106.44.201:6789/0] mon.synergy1 2: [v2:216.106.44.202:3300/0,v1:216.106.44.202:6789/0] mon.synergy2 -5182> 2019-07-10 09:13:10.386 7f9fdc57af00 5 monclient: authenticate success, global_id 666642 -5181> 2019-07-10 09:13:10.386 7f9fdc57af00 10 monclient: get_monmap_and_config success -5180> 2019-07-10 09:13:10.386 7f9fdc57af00 10 monclient: shutdown -5179> 2019-07-10 09:13:10.394 7f9fdc57af00 0 set uid:gid to 64045:64045 (ceph:ceph) -5178> 2019-07-10 09:13:10.394 7f9fdc57af00 0 ceph version 14.2.1 (d555a9489eb35f84f2e1ef49b77e19da9d113972) nautilus (stable), process ceph-osd, pid 18189 -5177> 2019-07-10 09:13:10.394 7f9fdc57af00 0 pidfile_write: ignore empty --pid-file -5176> 2019-07-10 09:13:10.398 7f9fdc57af00 5 asok(0xbb8a000) init /var/run/ceph/ceph-osd.21.asok -5175> 2019-07-10 09:13:10.398 7f9fdc57af00 5 asok(0xbb8a000) bind_and_listen /var/run/ceph/ceph-osd.21.asok -5174> 2019-07-10 09:13:10.402 7f9fdc57af00 5 asok(0xbb8a000) register_command 0 hook 0xb7e4ed8 -5173> 2019-07-10 09:13:10.402 7f9fdc57af00 5 asok(0xbb8a000) register_command version hook 0xb7e4ed8 -5172> 2019-07-10 09:13:10.402 7f9fdc57af00 5 asok(0xbb8a000) register_command git_version hook 0xb7e4ed8 -5171> 2019-07-10 09:13:10.402 7f9fdc57af00 5 asok(0xbb8a000) register_command help hook 0xb7e8450 -5170> 2019-07-10 09:13:10.402 7f9fdc57af00 5 asok(0xbb8a000) register_command get_command_descriptions hook 0xb7e83f0 -5169> 2019-07-10 09:13:10.402 7f9fd4f3c700 5 asok(0xbb8a000) entry start -5168> 2019-07-10 09:13:10.402 7f9fdc57af00 5 object store type is bluestore -5167> 2019-07-10 09:13:10.402 7f9fdc57af00 1 bdev create path /var/lib/ceph/osd/ceph-21/block type kernel -5166> 2019-07-10 09:13:10.402 7f9fdc57af00 1 bdev(0xc67e000 /var/lib/ceph/osd/ceph-21/block) open path /var/lib/ceph/osd/ceph-21/block -5165> 2019-07-10 09:13:10.402 7f9fdc57af00 0 bdev(0xc67e000 /var/lib/ceph/osd/ceph-21/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-21/block failed: (22) Invalid argument -5164> 2019-07-10 09:13:10.402 7f9fdc57af00 1 bdev(0xc67e000 /var/lib/ceph/osd/ceph-21/block) open size 2000397795328 (0x1d1c1000000, 1.8 TiB) block_size 4096 (4 KiB) rotational discard not supported -5163> 2019-07-10 09:13:10.406 7f9fdc57af00 1 bluestore(/var/lib/ceph/osd/ceph-21) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2 -5162> 2019-07-10 09:13:10.406 7f9fdc57af00 1 bdev create path /var/lib/ceph/osd/ceph-21/block type kernel -5161> 2019-07-10 09:13:10.406 7f9fdc57af00 1 bdev(0xc67e380 /var/lib/ceph/osd/ceph-21/block) open path /var/lib/ceph/osd/ceph-21/block -5160> 2019-07-10 09:13:10.406 7f9fdc57af00 0 bdev(0xc67e380 /var/lib/ceph/osd/ceph-21/block) ioctl(F_SET_FILE_RW_HINT) on /var/lib/ceph/osd/ceph-21/block failed: (22) Invalid argument -5159> 2019-07-10 09:13:10.406 7f9fdc57af00 1 bdev(0xc67e380 /var/lib/ceph/osd/ceph-21/block) open size 2000397795328 (0x1d1c1000000, 1.8 TiB) block_size 4096 (4 KiB) rotational discard not supported -5158> 2019-07-10 09:13:10.406 7f9fdc57af00 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-21/block size 1.8 TiB -5157> 2019-07-10 09:13:10.406 7f9fdc57af00 1 bdev(0xc67e380 /var/lib/ceph/osd/ceph-21/block) close -5156> 2019-07-10 09:13:10.694 7f9fdc57af00 1 bdev(0xc67e000 /var/lib/ceph/osd/ceph-21/block) close -5155> 2019-07-10 09:13:10.942 7f9fdc57af00 5 AuthRegistry(0xc671340) adding auth protocol: none -5154> 2019-07-10 09:13:10.942 7f9fdc57af00 5 AuthRegistry(0xc671340) adding auth protocol: none -5153> 2019-07-10 09:13:10.942 7f9fdc57af00 5 AuthRegistry(0xc671340) adding auth protocol: none -5152> 2019-07-10 09:13:10.942 7f9fdc57af00 5 AuthRegistry(0xc671340) adding con mode: crc -5151> 2019-07-10 09:13:10.942 7f9fdc57af00 5 AuthRegistry(0xc671340) adding con mode: crc -5150> 2019-07-10 09:13:10.942 7f9fdc57af00 5 AuthRegistry(0xc671340) adding con mode: crc