Project

General

Profile

ceph-osd.5.clippedtoupload.log

OSD 5 crashed - Edward Kalk, 07/10/2019 02:46 PM

Download (77.4 KB)

 
1
2019-07-10 09:16:24.744 7f73b6c49700  1 osd.5 pg_epoch: 42880 pg[5.1ed( v 41916'56469 (41401'53469,41916'56469] lb 5:b7862273:::rbd_data.fc3c6b8b4567.000000000010e222:head (bitwise) local-lis/les=42855/42856 n=128 ec=40535/408 lis/c 42857/42821 les/c/f 42858/42822/0 42874/42880/42875) [22,13]/[13] r=-1 lpr=42880 pi=[42821,42880)/2 crt=41916'56469 remapped NOTIFY mbc={}] state<Start>: transitioning to Stray
2
2019-07-10 09:16:24.744 7f73b6448700  1 osd.5 pg_epoch: 42880 pg[5.c2( v 42727'41898 (41401'38898,42727'41898] lb 5:4305f6c7:::rbd_data.fc3c6b8b4567.0000000000128d6e:head (bitwise) local-lis/les=42855/42856 n=146 ec=408/408 lis/c 42857/42840 les/c/f 42858/42841/0 42874/42880/42875) [22,9]/[9] r=-1 lpr=42880 pi=[42840,42880)/2 crt=42727'41898 remapped NOTIFY mbc={}] state<Start>: transitioning to Stray
3
2019-07-10 09:16:27.728 7f73b844c700 -1 bluestore(/var/lib/ceph/osd/ceph-5) _txc_add_transaction error (39) Directory not empty not handled on operation 21 (op 1, counting from 0)
4
2019-07-10 09:16:27.728 7f73b844c700  0 bluestore(/var/lib/ceph/osd/ceph-5) _dump_transaction transaction dump:
5
{
6
    "ops": [
7
        {
8
            "op_num": 0,
9
            "op_name": "remove",
10
            "collection": "6.0_head",
11
            "oid": "#6:00000000::::head#"
12
        },
13
        {
14
            "op_num": 1,
15
            "op_name": "rmcoll",
16
            "collection": "6.0_head"
17
        }
18
    ]
19
}
20

    
21
2019-07-10 09:16:27.760 7f73b844c700 -1 /build/ceph-14.2.1/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)' thread 7f73b844c700 time 2019-07-10 09:16:27.735166
22
/build/ceph-14.2.1/src/os/bluestore/BlueStore.cc: 11089: abort()
23

    
24
 ceph version 14.2.1 (d555a9489eb35f84f2e1ef49b77e19da9d113972) nautilus (stable)
25
 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xda) [0x850a01]
26
 2: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x296a) [0xe44a2a]
27
 3: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x5e6) [0xe48f96]
28
 4: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x7f) [0xa0099f]
29
 5: (PG::_delete_some(ObjectStore::Transaction*)+0x710) [0xa64b30]
30
 6: (PG::RecoveryState::Deleting::react(PG::DeleteSome const&)+0x71) [0xa658f1]
31
 7: (boost::statechart::simple_state<PG::RecoveryState::Deleting, PG::RecoveryState::ToDelete, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x131) [0xaaf151]
32
 8: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x6b) [0xa82b0b]
33
 9: (PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, PG::RecoveryCtx*)+0x122) [0xa72052]
34
 10: (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x1b4) [0x9ac6a4]
35
 11: (OSD::dequeue_delete(OSDShard*, PG*, unsigned int, ThreadPool::TPHandle&)+0xd2) [0x9ac982]
36
 12: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xbed) [0x9a07dd]
37
 13: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x4ac) [0xfc37bc]
38
 14: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xfc6970]
39
 15: (()+0x76ba) [0x7f73d796f6ba]
40
 16: (clone()+0x6d) [0x7f73d6f7641d]
41

    
42
2019-07-10 09:16:27.788 7f73b844c700 -1 *** Caught signal (Aborted) **
43
 in thread 7f73b844c700 thread_name:tp_osd_tp
44

    
45
 ceph version 14.2.1 (d555a9489eb35f84f2e1ef49b77e19da9d113972) nautilus (stable)
46
 1: (()+0x11390) [0x7f73d7979390]
47
 2: (gsignal()+0x38) [0x7f73d6ea4428]
48
 3: (abort()+0x16a) [0x7f73d6ea602a]
49
 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1a0) [0x850ac7]
50
 5: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x296a) [0xe44a2a]
51
 6: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x5e6) [0xe48f96]
52
 7: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x7f) [0xa0099f]
53
 8: (PG::_delete_some(ObjectStore::Transaction*)+0x710) [0xa64b30]
54
 9: (PG::RecoveryState::Deleting::react(PG::DeleteSome const&)+0x71) [0xa658f1]
55
 10: (boost::statechart::simple_state<PG::RecoveryState::Deleting, PG::RecoveryState::ToDelete, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x131) [0xaaf151]
56
 11: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x6b) [0xa82b0b]
57
 12: (PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, PG::RecoveryCtx*)+0x122) [0xa72052]
58
 13: (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x1b4) [0x9ac6a4]
59
 14: (OSD::dequeue_delete(OSDShard*, PG*, unsigned int, ThreadPool::TPHandle&)+0xd2) [0x9ac982]
60
 15: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xbed) [0x9a07dd]
61
 16: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x4ac) [0xfc37bc]
62
 17: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xfc6970]
63
 18: (()+0x76ba) [0x7f73d796f6ba]
64
 19: (clone()+0x6d) [0x7f73d6f7641d]
65
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
66

    
67
--- begin dump of recent events ---
68
 -9999> 2019-07-10 08:02:10.298 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb8fbb20:::rbd_data.fc3c6b8b4567.0000000000099bc2:head (bitwise) local-lis/les=42719/42720 n=365 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1537605632 local 1537605632
69
 -9998> 2019-07-10 08:02:10.398 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 384726395 data_alloc: 180355072 data_used: 109445120
70
 -9997> 2019-07-10 08:02:10.526 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060a333b:::rbd_data.fc3c6b8b4567.0000000000132ac5:head (bitwise) local-lis/les=42719/42720 n=215 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 906735616 local 915124224
71
 -9996> 2019-07-10 08:02:10.798 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 581099520 mapped: 1775812608 old cache_size: 1020054728 new cache size: 1020054728
72
 -9995> 2019-07-10 08:02:12.218 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798c62d8:::rbd_data.fc3c6b8b4567.00000000001516b0:head (bitwise) local-lis/les=42719/42720 n=283 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1243742208 local 1164050432
73
 -9994> 2019-07-10 08:02:12.394 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb8fc48c:::rbd_data.fc3c6b8b4567.000000000014c830:head (bitwise) local-lis/les=42719/42720 n=367 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1545994240 local 1545994240
74
 -9993> 2019-07-10 08:02:13.942 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x130ae350000/0x67000000/0x1d1c1000000, data 0xa0a6a8e44e/0xa0abca0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
75
 -9992> 2019-07-10 08:02:15.406 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 384933993 data_alloc: 180355072 data_used: 109445120
76
 -9991> 2019-07-10 08:02:15.810 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 572612608 mapped: 1784299520 old cache_size: 1020054728 new cache size: 1020054728
77
 -9990> 2019-07-10 08:02:16.910 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc88b2ee:::rbd_data.fc3c6b8b4567.00000000000b6ac9:head (bitwise) local-lis/les=42719/42720 n=215 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 954335232 local 941752320
78
 -9989> 2019-07-10 08:02:16.934 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060a8c73:::rbd_data.fc3c6b8b4567.0000000000137a4c:head (bitwise) local-lis/les=42719/42720 n=217 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 936095744 local 940290048
79
 -9988> 2019-07-10 08:02:17.442 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x130ab350000/0x67000000/0x1d1c1000000, data 0xa0a9a8e44e/0xa0aeca0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
80
 -9987> 2019-07-10 08:02:17.902 7f73c2460700 10 monclient: tick
81
 -9986> 2019-07-10 08:02:17.902 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:01:47.904654)
82
 -9985> 2019-07-10 08:02:18.862 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc88c5cb:::rbd_data.fc3c6b8b4567.000000000000a832:head (bitwise) local-lis/les=42719/42720 n=227 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 958529536 local 945946624
83
 -9984> 2019-07-10 08:02:19.190 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060aa56e:::rbd_data.6d3ad6ffff417.0000000000008331:head (bitwise) local-lis/les=42719/42720 n=224 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 940290048 local 944484352
84
 -9983> 2019-07-10 08:02:20.419 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 385075327 data_alloc: 180355072 data_used: 109445120
85
 -9982> 2019-07-10 08:02:20.819 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 572506112 mapped: 1784406016 old cache_size: 1020054728 new cache size: 1020054728
86
 -9981> 2019-07-10 08:02:21.043 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798cb924:::rbd_data.fc3c6b8b4567.00000000000f1ce6:head (bitwise) local-lis/les=42719/42720 n=291 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1277296640 local 1197604864
87
 -9980> 2019-07-10 08:02:21.539 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc88c910:::rbd_data.fc3c6b8b4567.0000000000092ad8:head (bitwise) local-lis/les=42719/42720 n=228 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 962723840 local 954335232
88
 -9979> 2019-07-10 08:02:22.291 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060abfda:::rbd_data.fc3c6b8b4567.00000000000dfb2c:head (bitwise) local-lis/les=42719/42720 n=225 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 944484352 local 944484352
89
 -9978> 2019-07-10 08:02:22.743 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x130a5f50000/0x67000000/0x1d1c1000000, data 0xa0aee8e44e/0xa0b40a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [0,0,0,0,0,1])
90
 -9977> 2019-07-10 08:02:23.847 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x130a4f50000/0x67000000/0x1d1c1000000, data 0xa0afe8e44e/0xa0b50a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
91
 -9976> 2019-07-10 08:02:24.335 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798ce33c:::rbd_data.fc3c6b8b4567.000000000013288f:head (bitwise) local-lis/les=42719/42720 n=299 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1289879552 local 1210187776
92
 -9975> 2019-07-10 08:02:24.947 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x130a3750000/0x67000000/0x1d1c1000000, data 0xa0b168e44e/0xa0b68a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
93
 -9974> 2019-07-10 08:02:25.431 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 385218471 data_alloc: 180355072 data_used: 109445120
94
 -9973> 2019-07-10 08:02:25.771 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb908bca:::rbd_data.fc3c6b8b4567.0000000000139eeb:head (bitwise) local-lis/les=42719/42720 n=369 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1613103104 local 1613103104
95
 -9972> 2019-07-10 08:02:25.831 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 559906816 mapped: 1797005312 old cache_size: 1020054728 new cache size: 1020054728
96
 -9971> 2019-07-10 08:02:27.027 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc88cd24:::rbd_data.fc3c6b8b4567.00000000000a82cc:head (bitwise) local-lis/les=42719/42720 n=229 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 971112448 local 962723840
97
 -9970> 2019-07-10 08:02:27.903 7f73c2460700 10 monclient: tick
98
 -9969> 2019-07-10 08:02:27.903 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:01:57.904972)
99
 -9968> 2019-07-10 08:02:29.883 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb9097c8:::rbd_data.fc3c6b8b4567.00000000000cde4d:head (bitwise) local-lis/les=42719/42720 n=385 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1625686016 local 1625686016
100
 -9967> 2019-07-10 08:02:30.439 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 385395295 data_alloc: 180355072 data_used: 109445120
101
 -9966> 2019-07-10 08:02:30.843 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 578699264 mapped: 1778212864 old cache_size: 1020054728 new cache size: 1020054728
102
 -9965> 2019-07-10 08:02:30.847 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1309d350000/0x67000000/0x1d1c1000000, data 0xa0b7a8e44e/0xa0bcca0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
103
 -9964> 2019-07-10 08:02:32.735 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118b18f6:::rbd_data.fc3c6b8b4567.0000000000065420:head (bitwise) local-lis/les=42719/42720 n=275 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1224736768 local 1199570944
104
 -9963> 2019-07-10 08:02:33.427 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118b19ee:::rbd_data.fc3c6b8b4567.00000000000b3142:head (bitwise) local-lis/les=42719/42720 n=290 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1228931072 local 1199570944
105
 -9962> 2019-07-10 08:02:33.855 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060b4046:::rbd_data.fc3c6b8b4567.0000000000014031:head (bitwise) local-lis/les=42719/42720 n=226 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1003204608 local 1011593216
106
 -9961> 2019-07-10 08:02:34.947 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13099150000/0x67000000/0x1d1c1000000, data 0xa0bbc8e44e/0xa0c0ea0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
107
 -9960> 2019-07-10 08:02:35.447 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13099150000/0x67000000/0x1d1c1000000, data 0xa0bbc8e44e/0xa0c0ea0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
108
 -9959> 2019-07-10 08:02:35.451 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 385561199 data_alloc: 180355072 data_used: 109445120
109
 -9958> 2019-07-10 08:02:35.851 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 580788224 mapped: 1776123904 old cache_size: 1020054728 new cache size: 1020054728
110
 -9957> 2019-07-10 08:02:36.215 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798d2249:::rbd_data.fc3c6b8b4567.00000000001680ba:head (bitwise) local-lis/les=42719/42720 n=302 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1323433984 local 1243742208
111
 -9956> 2019-07-10 08:02:36.807 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb9118c5:::rbd_data.fc3c6b8b4567.00000000000df3e7:head (bitwise) local-lis/les=42719/42720 n=388 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1684406272 local 1676017664
112
 -9955> 2019-07-10 08:02:36.843 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118b2b7f:::rbd_data.fc3c6b8b4567.0000000000027d17:head (bitwise) local-lis/les=42719/42720 n=291 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1241513984 local 1220542464
113
 -9954> 2019-07-10 08:02:37.383 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060b7e92:::rbd_data.6d3ad6ffff417.000000000000a867:head (bitwise) local-lis/les=42719/42720 n=240 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1019981824 local 1024176128
114
 -9953> 2019-07-10 08:02:37.903 7f73c2460700 10 monclient: tick
115
 -9952> 2019-07-10 08:02:37.903 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:02:07.905261)
116
 -9951> 2019-07-10 08:02:38.155 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc88e3d4:::rbd_data.fc3c6b8b4567.0000000000033929:head (bitwise) local-lis/les=42719/42720 n=231 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 992083968 local 983695360
117
 -9950> 2019-07-10 08:02:39.275 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798d405a:::rbd_data.fc3c6b8b4567.0000000000053663:head (bitwise) local-lis/les=42719/42720 n=310 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1348599808 local 1268908032
118
 -9949> 2019-07-10 08:02:39.959 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060b7ee9:::rbd_data.fc3c6b8b4567.000000000010e44d:head (bitwise) local-lis/les=42719/42720 n=244 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1024176128 local 1032564736
119
 -9948> 2019-07-10 08:02:40.147 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13092150000/0x67000000/0x1d1c1000000, data 0xa0c2c8e44e/0xa0c7ea0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [0,1])
120
 -9947> 2019-07-10 08:02:40.467 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 385779915 data_alloc: 180355072 data_used: 109445120
121
 -9946> 2019-07-10 08:02:40.791 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118b7463:::rbd_data.fc3c6b8b4567.000000000016a956:head (bitwise) local-lis/les=42719/42720 n=294 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1266679808 local 1237319680
122
 -9945> 2019-07-10 08:02:40.867 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 570138624 mapped: 1786773504 old cache_size: 1020054728 new cache size: 1020054728
123
 -9944> 2019-07-10 08:02:40.951 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798d62c2:::rbd_data.fc3c6b8b4567.000000000007045b:head (bitwise) local-lis/les=42719/42720 n=316 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1361182720 local 1281490944
124
 -9943> 2019-07-10 08:02:41.443 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798d7d25:::rbd_data.fc3c6b8b4567.000000000011b5a7:head (bitwise) local-lis/les=42719/42720 n=319 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1369571328 local 1281490944
125
 -9942> 2019-07-10 08:02:41.543 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc88e789:::rbd_data.fc3c6b8b4567.000000000004aedf:head (bitwise) local-lis/les=42719/42720 n=236 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1000472576 local 992083968
126
 -9941> 2019-07-10 08:02:41.847 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1308e750000/0x67000000/0x1d1c1000000, data 0xa0c668e44e/0xa0cb8a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
127
 -9940> 2019-07-10 08:02:42.031 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118b7a35:::rbd_data.fc3c6b8b4567.0000000000054001:head (bitwise) local-lis/les=42719/42720 n=300 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1270874112 local 1241513984
128
 -9939> 2019-07-10 08:02:43.383 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb91a34d:::rbd_data.fc3c6b8b4567.000000000010e020:head (bitwise) local-lis/les=42719/42720 n=402 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1717960704 local 1713766400
129
 -9938> 2019-07-10 08:02:44.251 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060bcf23:::rbd_data.fc3c6b8b4567.000000000006245d:head (bitwise) local-lis/les=42719/42720 n=245 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1049341952 local 1053536256
130
 -9937> 2019-07-10 08:02:44.747 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1308bf50000/0x67000000/0x1d1c1000000, data 0xa0c8e8e44e/0xa0ce0a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
131
 -9936> 2019-07-10 08:02:44.843 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc88f4ae:::rbd_data.fc3c6b8b4567.000000000003e751:head (bitwise) local-lis/les=42719/42720 n=238 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1008861184 local 1000472576
132
 -9935> 2019-07-10 08:02:45.179 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb91aa2e:::rbd_data.6d3ad6ffff417.0000000000003ff5:head (bitwise) local-lis/les=42719/42720 n=410 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1722155008 local 1722155008
133
 -9934> 2019-07-10 08:02:45.479 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 385932153 data_alloc: 180355072 data_used: 109445120
134
 -9933> 2019-07-10 08:02:45.879 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 576421888 mapped: 1780490240 old cache_size: 1020054728 new cache size: 1020054728
135
 -9932> 2019-07-10 08:02:46.283 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798de2fa:::rbd_data.fc3c6b8b4567.000000000006f335:head (bitwise) local-lis/les=42719/42720 n=321 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1390542848 local 1306656768
136
 -9931> 2019-07-10 08:02:46.583 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060bd953:::rbd_data.fc3c6b8b4567.00000000000e33b7:head (bitwise) local-lis/les=42719/42720 n=251 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1053536256 local 1057730560
137
 -9930> 2019-07-10 08:02:47.347 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb91ca6d:::rbd_data.6d3ad6ffff417.000000000001d1db:head (bitwise) local-lis/les=42719/42720 n=411 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1730543616 local 1730543616
138
 -9929> 2019-07-10 08:02:47.899 7f73c2460700 10 monclient: tick
139
 -9928> 2019-07-10 08:02:47.899 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:02:17.905511)
140
 -9927> 2019-07-10 08:02:48.079 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060be441:::rbd_data.fc3c6b8b4567.000000000015e136:head (bitwise) local-lis/les=42719/42720 n=252 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1057730560 local 1061924864
141
 -9926> 2019-07-10 08:02:48.495 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798dfd26:::rbd_data.fc3c6b8b4567.0000000000113d02:head (bitwise) local-lis/les=42719/42720 n=326 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1394737152 local 1315045376
142
 -9925> 2019-07-10 08:02:48.847 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13088750000/0x67000000/0x1d1c1000000, data 0xa0cc68e44e/0xa0d18a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
143
 -9924> 2019-07-10 08:02:50.063 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060bf4a1:::rbd_data.fc3c6b8b4567.0000000000145ade:head (bitwise) local-lis/les=42719/42720 n=253 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1061924864 local 1070313472
144
 -9923> 2019-07-10 08:02:50.491 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 386052275 data_alloc: 180355072 data_used: 109445120
145
 -9922> 2019-07-10 08:02:50.551 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13087f50000/0x67000000/0x1d1c1000000, data 0xa0cce8e44e/0xa0d20a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
146
 -9921> 2019-07-10 08:02:50.891 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 580542464 mapped: 1776369664 old cache_size: 1020054728 new cache size: 1020054728
147
 -9920> 2019-07-10 08:02:51.671 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c2db6:::rbd_data.fc3c6b8b4567.00000000000fc1ac:head (bitwise) local-lis/les=42719/42720 n=254 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1070313472 local 1070313472
148
 -9919> 2019-07-10 08:02:51.843 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118b9e9e:::rbd_data.fc3c6b8b4567.000000000008e067:head (bitwise) local-lis/les=42719/42720 n=301 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1275068416 local 1254096896
149
 -9918> 2019-07-10 08:02:53.451 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13084b50000/0x67000000/0x1d1c1000000, data 0xa0d028e44e/0xa0d54a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
150
 -9917> 2019-07-10 08:02:53.767 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb920381:::rbd_data.fc3c6b8b4567.0000000000128702:head (bitwise) local-lis/les=42719/42720 n=413 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1755709440 local 1751515136
151
 -9916> 2019-07-10 08:02:53.891 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798e48c9:::rbd_data.fc3c6b8b4567.0000000000159932:head (bitwise) local-lis/les=42719/42720 n=327 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1424097280 local 1344405504
152
 -9915> 2019-07-10 08:02:55.095 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c4b62:::rbd_data.fc3c6b8b4567.00000000000e8882:head (bitwise) local-lis/les=42719/42720 n=256 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1082896384 local 1087090688
153
 -9914> 2019-07-10 08:02:55.503 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 386151827 data_alloc: 180355072 data_used: 109445120
154
 -9913> 2019-07-10 08:02:55.859 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb9217fb:::rbd_data.fc3c6b8b4567.00000000000f40d8:head (bitwise) local-lis/les=42719/42720 n=419 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1759903744 local 1759903744
155
 -9912> 2019-07-10 08:02:55.903 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 551043072 mapped: 1805869056 old cache_size: 1020054728 new cache size: 1020054728
156
 -9911> 2019-07-10 08:02:56.351 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13082f50000/0x67000000/0x1d1c1000000, data 0xa0d1e8e44e/0xa0d70a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
157
 -9910> 2019-07-10 08:02:56.447 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c590b:::rbd_data.fc3c6b8b4567.000000000002c30d:head (bitwise) local-lis/les=42719/42720 n=259 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1087090688 local 1087090688
158
 -9909> 2019-07-10 08:02:57.900 7f73c2460700 10 monclient: tick
159
 -9908> 2019-07-10 08:02:57.900 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:02:27.905837)
160
 -9907> 2019-07-10 08:02:57.988 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c5fec:::rbd_data.fc3c6b8b4567.00000000000e8aa5:head (bitwise) local-lis/les=42719/42720 n=260 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1095479296 local 1099673600
161
 -9906> 2019-07-10 08:02:58.052 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1307fb50000/0x67000000/0x1d1c1000000, data 0xa0d528e44e/0xa0da4a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
162
 -9905> 2019-07-10 08:02:58.200 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb9252f6:::rbd_data.fc3c6b8b4567.00000000000230d2:head (bitwise) local-lis/les=42719/42720 n=420 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1776680960 local 1768292352
163
 -9904> 2019-07-10 08:02:58.312 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c6c54:::rbd_data.fc3c6b8b4567.0000000000162203:head (bitwise) local-lis/les=42719/42720 n=262 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1099673600 local 1099673600
164
 -9903> 2019-07-10 08:02:58.592 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118bbf0e:::rbd_data.fc3c6b8b4567.0000000000038fbc:head (bitwise) local-lis/les=42719/42720 n=302 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1296039936 local 1270874112
165
 -9902> 2019-07-10 08:02:59.156 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1307e750000/0x67000000/0x1d1c1000000, data 0xa0d668e44e/0xa0db8a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
166
 -9901> 2019-07-10 08:02:59.656 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1307e750000/0x67000000/0x1d1c1000000, data 0xa0d668e44e/0xa0db8a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
167
 -9900> 2019-07-10 08:02:59.928 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c78e0:::rbd_data.fc3c6b8b4567.0000000000155165:head (bitwise) local-lis/les=42719/42720 n=263 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1103867904 local 1108062208
168
 -9899> 2019-07-10 08:03:00.512 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 386340839 data_alloc: 180355072 data_used: 109445120
169
 -9898> 2019-07-10 08:03:00.916 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 592969728 mapped: 1763942400 old cache_size: 1020054728 new cache size: 1020054728
170
 -9897> 2019-07-10 08:03:02.244 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118bc30f:::rbd_data.fc3c6b8b4567.000000000006cfd4:head (bitwise) local-lis/les=42719/42720 n=307 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1300234240 local 1279262720
171
 -9896> 2019-07-10 08:03:02.500 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc893e87:::rbd_data.fc3c6b8b4567.00000000000342ff:head (bitwise) local-lis/les=42719/42720 n=240 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1046609920 local 1034027008
172
 -9895> 2019-07-10 08:03:03.096 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c7cbc:::rbd_data.fc3c6b8b4567.000000000015fece:head (bitwise) local-lis/les=42719/42720 n=264 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1108062208 local 1112256512
173
 -9894> 2019-07-10 08:03:03.140 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798ead52:::rbd_data.fc3c6b8b4567.00000000000f5e6e:head (bitwise) local-lis/les=42719/42720 n=334 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1470234624 local 1390542848
174
 -9893> 2019-07-10 08:03:03.156 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1307ab50000/0x67000000/0x1d1c1000000, data 0xa0da28e44e/0xa0df4a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
175
 -9892> 2019-07-10 08:03:03.328 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118bd551:::rbd_data.fc3c6b8b4567.00000000000ef4e1:head (bitwise) local-lis/les=42719/42720 n=308 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1308622848 local 1283457024
176
 -9891> 2019-07-10 08:03:04.300 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118bed99:::rbd_data.fc3c6b8b4567.00000000000a294d:head (bitwise) local-lis/les=42719/42720 n=310 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1312817152 local 1283457024
177
 -9890> 2019-07-10 08:03:04.364 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c7d89:::rbd_data.fc3c6b8b4567.00000000000f86c8:head (bitwise) local-lis/les=42719/42720 n=265 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1112256512 local 1116450816
178
 -9889> 2019-07-10 08:03:05.284 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc894dc4:::rbd_data.fc3c6b8b4567.000000000006cc4a:head (bitwise) local-lis/les=42719/42720 n=249 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1050804224 local 1042415616
179
 -9888> 2019-07-10 08:03:05.524 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 386462211 data_alloc: 180355072 data_used: 109445120
180
 -9887> 2019-07-10 08:03:05.924 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 580435968 mapped: 1776476160 old cache_size: 1020054728 new cache size: 1020054728
181
 -9886> 2019-07-10 08:03:06.056 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13076b50000/0x67000000/0x1d1c1000000, data 0xa0de28e44e/0xa0e34a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
182
 -9885> 2019-07-10 08:03:06.088 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c81e0:::rbd_data.fc3c6b8b4567.0000000000110a83:head (bitwise) local-lis/les=42719/42720 n=266 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1116450816 local 1120645120
183
 -9884> 2019-07-10 08:03:07.788 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798ef091:::rbd_data.fc3c6b8b4567.00000000000039c2:head (bitwise) local-lis/les=42719/42720 n=345 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1495400448 local 1415708672
184
 -9883> 2019-07-10 08:03:07.900 7f73c2460700 10 monclient: tick
185
 -9882> 2019-07-10 08:03:07.900 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:02:37.906186)
186
 -9881> 2019-07-10 08:03:08.724 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8971e1:::rbd_data.6d3ad6ffff417.000000000000f6ea:head (bitwise) local-lis/les=42719/42720 n=250 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1063387136 local 1054998528
187
 -9880> 2019-07-10 08:03:09.352 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060c95f0:::rbd_data.fc3c6b8b4567.0000000000040a88:head (bitwise) local-lis/les=42719/42720 n=267 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1120645120 local 1129033728
188
 -9879> 2019-07-10 08:03:10.156 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13072f50000/0x67000000/0x1d1c1000000, data 0xa0e1a8e44e/0xa0e6ca0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
189
 -9878> 2019-07-10 08:03:10.536 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 386632153 data_alloc: 180355072 data_used: 109445120
190
 -9877> 2019-07-10 08:03:10.936 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 565641216 mapped: 1791270912 old cache_size: 1020054728 new cache size: 1020054728
191
 -9876> 2019-07-10 08:03:12.228 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798f0944:::rbd_data.fc3c6b8b4567.0000000000060c5e:head (bitwise) local-lis/les=42719/42720 n=351 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1516371968 local 1436680192
192
 -9875> 2019-07-10 08:03:12.460 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1306ff50000/0x67000000/0x1d1c1000000, data 0xa0e4e8e44e/0xa0ea0a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
193
 -9874> 2019-07-10 08:03:13.504 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb927338:::rbd_data.6d3ad6ffff417.000000000000417f:head (bitwise) local-lis/les=42719/42720 n=424 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1789263872 local 1831206912
194
 -9873> 2019-07-10 08:03:15.072 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc89934f:::rbd_data.fc3c6b8b4567.00000000000eb153:head (bitwise) local-lis/les=42719/42720 n=253 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1080164352 local 1071775744
195
 -9872> 2019-07-10 08:03:15.552 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 386797939 data_alloc: 180355072 data_used: 109445120
196
 -9871> 2019-07-10 08:03:15.956 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 555704320 mapped: 1801207808 old cache_size: 1020054728 new cache size: 1020054728
197
 -9870> 2019-07-10 08:03:16.220 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060cb123:::rbd_data.fc3c6b8b4567.00000000000a1543:head (bitwise) local-lis/les=42719/42720 n=268 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1154199552 local 1162588160
198
 -9869> 2019-07-10 08:03:17.004 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118c49f7:::rbd_data.fc3c6b8b4567.00000000000f5a9a:head (bitwise) local-lis/les=42719/42720 n=311 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1342177280 local 1363148800
199
 -9868> 2019-07-10 08:03:17.316 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118cfda9:::rbd_data.fc3c6b8b4567.000000000010fb3e:head (bitwise) local-lis/les=42719/42720 n=318 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1392508928 local 1363148800
200
 -9867> 2019-07-10 08:03:17.656 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb930a19:::rbd_data.fc3c6b8b4567.000000000009f1ee:head (bitwise) local-lis/les=42719/42720 n=427 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1839595520 local 1839595520
201
 -9866> 2019-07-10 08:03:17.900 7f73c2460700 10 monclient: tick
202
 -9865> 2019-07-10 08:03:17.900 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:02:47.906860)
203
 -9864> 2019-07-10 08:03:18.360 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13068350000/0x67000000/0x1d1c1000000, data 0xa0eca8e44e/0xa0f1ca0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
204
 -9863> 2019-07-10 08:03:19.676 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798f6b9c:::rbd_data.fc3c6b8b4567.00000000000a7724:head (bitwise) local-lis/les=42719/42720 n=356 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1554120704 local 1470234624
205
 -9862> 2019-07-10 08:03:20.020 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060ce5fe:::rbd_data.fc3c6b8b4567.000000000004fa52:head (bitwise) local-lis/les=42719/42720 n=276 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1166782464 local 1175171072
206
 -9861> 2019-07-10 08:03:20.180 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb9317c5:::rbd_data.fc3c6b8b4567.000000000008ce93:head (bitwise) local-lis/les=42719/42720 n=439 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1847984128 local 1847984128
207
 -9860> 2019-07-10 08:03:20.212 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798f6f57:::rbd_data.fc3c6b8b4567.000000000007c170:head (bitwise) local-lis/les=42719/42720 n=365 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1558315008 local 1470234624
208
 -9859> 2019-07-10 08:03:20.568 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 386936829 data_alloc: 180355072 data_used: 109445120
209
 -9858> 2019-07-10 08:03:20.968 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 571883520 mapped: 1785028608 old cache_size: 1020054728 new cache size: 1020054728
210
 -9857> 2019-07-10 08:03:21.420 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118cfe30:::rbd_data.6d3ad6ffff417.00000000000110bc:head (bitwise) local-lis/les=42719/42720 n=330 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1396703232 local 1375731712
211
 -9856> 2019-07-10 08:03:22.460 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13064f50000/0x67000000/0x1d1c1000000, data 0xa0efe8e44e/0xa0f50a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
212
 -9855> 2019-07-10 08:03:22.808 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc89c737:::rbd_data.fc3c6b8b4567.0000000000160bdb:head (bitwise) local-lis/les=42719/42720 n=257 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1105330176 local 1092747264
213
 -9854> 2019-07-10 08:03:22.904 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060ce9f2:::rbd_data.fc3c6b8b4567.0000000000070f18:head (bitwise) local-lis/les=42719/42720 n=279 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1175171072 local 1179365376
214
 -9853> 2019-07-10 08:03:22.964 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13063b50000/0x67000000/0x1d1c1000000, data 0xa0f128e44e/0xa0f64a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
215
 -9852> 2019-07-10 08:03:23.672 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060cf82d:::rbd_data.fc3c6b8b4567.00000000000bada4:head (bitwise) local-lis/les=42719/42720 n=281 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1179365376 local 1179365376
216
 -9851> 2019-07-10 08:03:25.576 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 387080973 data_alloc: 180355072 data_used: 109445120
217
 -9850> 2019-07-10 08:03:25.840 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb93438e:::rbd_data.fc3c6b8b4567.00000000000a4203:head (bitwise) local-lis/les=42719/42720 n=441 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1868955648 local 1868955648
218
 -9849> 2019-07-10 08:03:25.864 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13060b50000/0x67000000/0x1d1c1000000, data 0xa0f428e44e/0xa0f94a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
219
 -9848> 2019-07-10 08:03:25.980 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 555057152 mapped: 1801854976 old cache_size: 1020054728 new cache size: 1020054728
220
 -9847> 2019-07-10 08:03:27.900 7f73c2460700 10 monclient: tick
221
 -9846> 2019-07-10 08:03:27.900 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:02:57.907068)
222
 -9845> 2019-07-10 08:03:28.676 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc89c85a:::rbd_data.fc3c6b8b4567.00000000000433f6:head (bitwise) local-lis/les=42719/42720 n=263 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1109524480 local 1101135872
223
 -9844> 2019-07-10 08:03:29.696 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:798fb744:::rbd_data.fc3c6b8b4567.00000000000b6cf6:head (bitwise) local-lis/les=42719/42720 n=366 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1591869440 local 1503789056
224
 -9843> 2019-07-10 08:03:30.588 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 387231425 data_alloc: 180355072 data_used: 109445120
225
 -9842> 2019-07-10 08:03:30.988 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 567582720 mapped: 1789329408 old cache_size: 1020054728 new cache size: 1020054728
226
 -9841> 2019-07-10 08:03:31.564 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118d768e:::rbd_data.fc3c6b8b4567.0000000000110b82:head (bitwise) local-lis/les=42719/42720 n=331 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1438646272 local 1417674752
227
 -9840> 2019-07-10 08:03:31.764 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1305b350000/0x67000000/0x1d1c1000000, data 0xa0f9a8e44e/0xa0feca0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
228
 -9839> 2019-07-10 08:03:31.960 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060d4706:::rbd_data.fc3c6b8b4567.0000000000141bdd:head (bitwise) local-lis/les=42719/42720 n=282 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1204531200 local 1208725504
229
 -9838> 2019-07-10 08:03:33.465 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13058750000/0x67000000/0x1d1c1000000, data 0xa0fc68e44e/0xa1018a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
230
 -9837> 2019-07-10 08:03:33.681 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118d7bd1:::rbd_data.fc3c6b8b4567.00000000000e1fc0:head (bitwise) local-lis/les=42719/42720 n=341 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1447034880 local 1426063360
231
 -9836> 2019-07-10 08:03:34.965 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060d5009:::rbd_data.fc3c6b8b4567.0000000000132506:head (bitwise) local-lis/les=42719/42720 n=288 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1208725504 local 1212919808
232
 -9835> 2019-07-10 08:03:35.449 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118da997:::rbd_data.fc3c6b8b4567.00000000000bc94c:head (bitwise) local-lis/les=42719/42720 n=343 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1455423488 local 1434451968
233
 -9834> 2019-07-10 08:03:35.601 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 387372357 data_alloc: 180355072 data_used: 109445120
234
 -9833> 2019-07-10 08:03:36.005 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 580100096 mapped: 1776812032 old cache_size: 1020054728 new cache size: 1020054728
235
 -9832> 2019-07-10 08:03:36.709 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8a1ad1:::rbd_data.fc3c6b8b4567.000000000005a1aa:head (bitwise) local-lis/les=42719/42720 n=264 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1138884608 local 1130496000
236
 -9831> 2019-07-10 08:03:37.565 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13055350000/0x67000000/0x1d1c1000000, data 0xa0ff68e44e/0xa1048a0000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
237
 -9830> 2019-07-10 08:03:37.885 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb939b5b:::rbd_data.fc3c6b8b4567.00000000000ea7d1:head (bitwise) local-lis/les=42719/42720 n=446 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1902510080 local 1902510080
238
 -9829> 2019-07-10 08:03:37.901 7f73c2460700 10 monclient: tick
239
 -9828> 2019-07-10 08:03:37.901 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:03:07.907307)
240
 -9827> 2019-07-10 08:03:38.201 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8a1d71:::rbd_data.fc3c6b8b4567.000000000000e06d:head (bitwise) local-lis/les=42719/42720 n=271 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1147273216 local 1134690304
241
 -9826> 2019-07-10 08:03:38.469 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118dae2f:::rbd_data.fc3c6b8b4567.00000000000953a8:head (bitwise) local-lis/les=42719/42720 n=345 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1463812096 local 1442840576
242
 -9825> 2019-07-10 08:03:39.069 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8a22b7:::rbd_data.6d3ad6ffff417.000000000000d2ce:head (bitwise) local-lis/les=42719/42720 n=273 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1151467520 local 1138884608
243
 -9824> 2019-07-10 08:03:39.869 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:79903aec:::rbd_data.fc3c6b8b4567.00000000000dcf9c:head (bitwise) local-lis/les=42719/42720 n=374 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1650589696 local 1570897920
244
 -9823> 2019-07-10 08:03:39.881 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060d55f9:::rbd_data.fc3c6b8b4567.00000000000cc57a:head (bitwise) local-lis/les=42719/42720 n=289 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1212919808 local 1217114112
245
 -9822> 2019-07-10 08:03:40.613 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 387502211 data_alloc: 180355072 data_used: 109445120
246
 -9821> 2019-07-10 08:03:41.017 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 550748160 mapped: 1806163968 old cache_size: 1020054728 new cache size: 1020054728
247
 -9820> 2019-07-10 08:03:41.189 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8a2866:::rbd_data.fc3c6b8b4567.0000000000010b37:head (bitwise) local-lis/les=42719/42720 n=274 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1155661824 local 1147273216
248
 -9819> 2019-07-10 08:03:42.437 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:79905d77:::rbd_data.fc3c6b8b4567.00000000000d13ca:head (bitwise) local-lis/les=42719/42720 n=388 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1658978304 local 1579286528
249
 -9818> 2019-07-10 08:03:42.609 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060d5798:::rbd_data.6d3ad6ffff417.000000000000710c:head (bitwise) local-lis/les=42719/42720 n=290 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1217114112 local 1229697024
250
 -9817> 2019-07-10 08:03:43.469 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1304cf60000/0x67000000/0x1d1c1000000, data 0xa107e7e44e/0xa10d090000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
251
 -9816> 2019-07-10 08:03:43.661 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8a7491:::rbd_data.fc3c6b8b4567.00000000000a27d7:head (bitwise) local-lis/les=42719/42720 n=275 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1168244736 local 1151467520
252
 -9815> 2019-07-10 08:03:44.069 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060d7468:::rbd_data.fc3c6b8b4567.00000000000d1a29:head (bitwise) local-lis/les=42719/42720 n=291 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1229697024 local 1233891328
253
 -9814> 2019-07-10 08:03:45.405 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8a8962:::rbd_data.fc3c6b8b4567.000000000010c56e:head (bitwise) local-lis/les=42719/42720 n=278 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1172439040 local 1159856128
254
 -9813> 2019-07-10 08:03:45.629 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 387688709 data_alloc: 180355072 data_used: 109445120
255
 -9812> 2019-07-10 08:03:45.833 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060d74e8:::rbd_data.fc3c6b8b4567.00000000001037fd:head (bitwise) local-lis/les=42719/42720 n=294 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1233891328 local 1238085632
256
 -9811> 2019-07-10 08:03:45.965 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118e6b2c:::rbd_data.fc3c6b8b4567.00000000000e12f9:head (bitwise) local-lis/les=42719/42720 n=347 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1518338048 local 1514143744
257
 -9810> 2019-07-10 08:03:46.033 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 561217536 mapped: 1795694592 old cache_size: 1020054728 new cache size: 1020054728
258
 -9809> 2019-07-10 08:03:46.909 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8a8f51:::rbd_data.fc3c6b8b4567.0000000000122b62:head (bitwise) local-lis/les=42719/42720 n=279 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1176633344 local 1168244736
259
 -9808> 2019-07-10 08:03:47.905 7f73c2460700 10 monclient: tick
260
 -9807> 2019-07-10 08:03:47.905 7f73c2460700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2019-07-10 08:03:17.907572)
261
 -9806> 2019-07-10 08:03:49.201 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060d757f:::rbd_data.fc3c6b8b4567.000000000005fe5f:head (bitwise) local-lis/les=42719/42720 n=295 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1238085632 local 1246474240
262
 -9805> 2019-07-10 08:03:49.253 7f73b6448700  0 osd.5 pg_epoch: 42727 pg[5.19e( v 42727'52231 (41401'49227,42727'52231] lb 5:7990ca3f:::rbd_data.fc3c6b8b4567.00000000000e48e7:head (bitwise) local-lis/les=42719/42720 n=390 ec=40535/408 lis/c 42719/42682 les/c/f 42720/42683/0 42718/42725/42718) [10,5]/[10] r=-1 lpr=42725 pi=[42682,42725)/1 luod=0'0 lua=41916'52227 crt=42727'52231 active+remapped mbc={}] do_backfill primary 1696727040 local 1617035264
263
 -9804> 2019-07-10 08:03:49.369 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13046d60000/0x67000000/0x1d1c1000000, data 0xa10e07e44e/0xa113290000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
264
 -9803> 2019-07-10 08:03:49.765 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118ec6c6:::rbd_data.fc3c6b8b4567.00000000000033bb:head (bitwise) local-lis/les=42719/42720 n=360 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1551892480 local 1526726656
265
 -9802> 2019-07-10 08:03:49.933 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8ab0d2:::rbd_data.fc3c6b8b4567.00000000000ceaf9:head (bitwise) local-lis/les=42719/42720 n=280 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1193410560 local 1185021952
266
 -9801> 2019-07-10 08:03:50.469 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x13044160000/0x67000000/0x1d1c1000000, data 0xa110c7e44e/0xa115e90000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
267
 -9800> 2019-07-10 08:03:50.517 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118ed19a:::rbd_data.fc3c6b8b4567.000000000000ab68:head (bitwise) local-lis/les=42719/42720 n=368 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1556086784 local 1530920960
268
 -9799> 2019-07-10 08:03:50.641 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 387882053 data_alloc: 180355072 data_used: 109445120
269
 -9798> 2019-07-10 08:03:51.041 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 580001792 mapped: 1776910336 old cache_size: 1020054728 new cache size: 1020054728
270
 -9797> 2019-07-10 08:03:51.657 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118ed75b:::rbd_data.fc3c6b8b4567.00000000001450c0:head (bitwise) local-lis/les=42719/42720 n=369 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1560281088 local 1539309568
271
 -9796> 2019-07-10 08:03:52.357 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060dcd34:::rbd_data.6d3ad6ffff417.00000000000197cc:head (bitwise) local-lis/les=42719/42720 n=296 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1271640064 local 1271640064
272
 -9795> 2019-07-10 08:03:52.609 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.188( v 42717'50242 (41401'47242,42717'50242] lb 5:118f03bc:::rbd_data.fc3c6b8b4567.00000000000509b3:head (bitwise) local-lis/les=42719/42720 n=370 ec=40535/408 lis/c 42725/41880 les/c/f 42726/41881/0 42718/42725/41880) [6,5]/[6] r=-1 lpr=42725 pi=[41880,42725)/1 luod=0'0 crt=42717'50242 active+remapped mbc={}] do_backfill primary 1572864000 local 1543503872
273
 -9794> 2019-07-10 08:03:52.833 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb93c8e0:::rbd_data.fc3c6b8b4567.0000000000053a43:head (bitwise) local-lis/les=42719/42720 n=454 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1940258816 local 1936064512
274
 -9793> 2019-07-10 08:03:54.973 7f73b844c700  0 osd.5 pg_epoch: 42727 pg[5.13b( v 41916'36924 (41401'33924,41916'36924] lb 5:dc8acab8:::rbd_data.6d3ad6ffff417.0000000000009712:head (bitwise) local-lis/les=42719/42720 n=284 ec=40535/408 lis/c 42725/41911 les/c/f 42726/41912/0 42718/42725/42719) [5,16]/[16] r=-1 lpr=42725 pi=[41911,42725)/1 luod=0'0 crt=41916'36924 active+remapped mbc={}] do_backfill primary 1201799168 local 1193410560
275
 -9792> 2019-07-10 08:03:55.653 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _trim_shards cache_size: 1020054728 kv_alloc: 419430400 kv_used: 409694963 meta_alloc: 419430400 meta_used: 388035979 data_alloc: 180355072 data_used: 109445120
276
 -9791> 2019-07-10 08:03:56.053 7f73c946e700  5 bluestore.MempoolThread(0xd527b60) _tune_cache_size target: 2147483648 heap: 2356912128 unmapped: 571580416 mapped: 1785331712 old cache_size: 1020054728 new cache size: 1020054728
277
 -9790> 2019-07-10 08:03:56.265 7f73b7c4b700  0 osd.5 pg_epoch: 42727 pg[5.60( v 41916'43095 (41401'40095,41916'43095] lb 5:060df7ed:::rbd_data.fc3c6b8b4567.000000000016c75b:head (bitwise) local-lis/les=42719/42720 n=304 ec=408/408 lis/c 42725/41888 les/c/f 42726/41889/0 42718/42725/42719) [5,18]/[18] r=-1 lpr=42725 pi=[41888,42725)/1 luod=0'0 crt=41916'43095 active+remapped mbc={}] do_backfill primary 1288417280 local 1288417280
278
 -9789> 2019-07-10 08:03:56.369 7f73b5446700  5 osd.5 42727 heartbeat osd_stat(store_statfs(0x1303d960000/0x67000000/0x1d1c1000000, data 0xa11747e44e/0xa11c690000, compress 0x0/0x0/0x0, omap 0x15114, meta 0x66feaeec), peers [0,1,2,3,4,6,7,8,9,10,12,13,14,15,16,17,18,19,22] op hist [])
279
 -9788> 2019-07-10 08:03:56.529 7f73b744a700  0 osd.5 pg_epoch: 42727 pg[5.1dd( v 42727'90513 (41401'87510,42727'90513] lb 5:bb93dbf0:::rbd_data.fc3c6b8b4567.0000000000096e4a:head (bitwise) local-lis/les=42719/42720 n=463 ec=40535/408 lis/c 42719/41884 les/c/f 42720/41885/0 42718/42725/42718) [3,5]/[3] r=-1 lpr=42725 pi=[41884,42725)/1 luod=0'0 lua=42717'90510 crt=42727'90513 active+remapped mbc={}] do_backfill primary 1944453120 local 1936064512