2018-12-06 09:44:35.913272 7f39327d1700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:0/1387855531 conn(0x55d0978dc800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer 2018-12-06 09:44:36.075806 7f391aaf2700 0 log_channel(audit) log [DBG] : from='client.4461 172.17.0.7:0/1387855531' entity='client.admin' cmd=[{"prefix": "balancer optimize", "plan": "plan1", "target": ["mgr", ""]}]: dispatch 2018-12-06 09:44:36.102382 7f391b2f3700 30 calc_pg_upmaps pool 2 ruleno 1 2018-12-06 09:44:36.102392 7f391b2f3700 20 osd.0 weight 0.0625 pgs 27 2018-12-06 09:44:36.102413 7f391b2f3700 20 osd.1 weight 0.0625 pgs 34 2018-12-06 09:44:36.102414 7f391b2f3700 20 osd.2 weight 0.0625 pgs 32 2018-12-06 09:44:36.102416 7f391b2f3700 20 osd.3 weight 0.0625 pgs 29 2018-12-06 09:44:36.102417 7f391b2f3700 20 osd.4 weight 0.0625 pgs 36 2018-12-06 09:44:36.102418 7f391b2f3700 20 osd.5 weight 0.0625 pgs 36 2018-12-06 09:44:36.102419 7f391b2f3700 20 osd.6 weight 0.0625 pgs 33 2018-12-06 09:44:36.102420 7f391b2f3700 20 osd.7 weight 0.0625 pgs 36 2018-12-06 09:44:36.102422 7f391b2f3700 20 osd.8 weight 0.0625 pgs 32 2018-12-06 09:44:36.102423 7f391b2f3700 20 osd.9 weight 0.0625 pgs 31 2018-12-06 09:44:36.102424 7f391b2f3700 20 osd.10 weight 0.0625 pgs 30 2018-12-06 09:44:36.102425 7f391b2f3700 20 osd.11 weight 0.0625 pgs 28 2018-12-06 09:44:36.102427 7f391b2f3700 10 osd_weight_total 0.75 2018-12-06 09:44:36.102429 7f391b2f3700 10 pgs_per_weight 512 2018-12-06 09:44:36.102431 7f391b2f3700 20 osd.0 pgs 27 target 32 deviation -5 2018-12-06 09:44:36.102434 7f391b2f3700 20 osd.1 pgs 34 target 32 deviation 2 2018-12-06 09:44:36.102436 7f391b2f3700 20 osd.2 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102439 7f391b2f3700 20 osd.3 pgs 29 target 32 deviation -3 2018-12-06 09:44:36.102441 7f391b2f3700 20 osd.4 pgs 36 target 32 deviation 4 2018-12-06 09:44:36.102443 7f391b2f3700 20 osd.5 pgs 36 target 32 deviation 4 2018-12-06 09:44:36.102445 7f391b2f3700 20 osd.6 pgs 33 target 32 deviation 1 2018-12-06 09:44:36.102447 7f391b2f3700 20 osd.7 pgs 36 target 32 deviation 4 2018-12-06 09:44:36.102449 7f391b2f3700 20 osd.8 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102451 7f391b2f3700 20 osd.9 pgs 31 target 32 deviation -1 2018-12-06 09:44:36.102453 7f391b2f3700 20 osd.10 pgs 30 target 32 deviation -2 2018-12-06 09:44:36.102454 7f391b2f3700 20 osd.11 pgs 28 target 32 deviation -4 2018-12-06 09:44:36.102457 7f391b2f3700 10 total_deviation 30 overfull 1,4,5,6,7 underfull [0,11,3,10,9] 2018-12-06 09:44:36.102460 7f391b2f3700 10 osd.7 move 4 2018-12-06 09:44:36.102464 7f391b2f3700 10 trying 2.d 2018-12-06 09:44:36.102498 7f391b2f3700 10 2.d [8,3,7] -> [8,3,11] 2018-12-06 09:44:36.102503 7f391b2f3700 10 2.d pg_upmap_items [7,11] 2018-12-06 09:44:36.102506 7f391b2f3700 20 osd.0 pgs 27 target 32 deviation -5 2018-12-06 09:44:36.102508 7f391b2f3700 20 osd.1 pgs 34 target 32 deviation 2 2018-12-06 09:44:36.102510 7f391b2f3700 20 osd.2 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102512 7f391b2f3700 20 osd.3 pgs 29 target 32 deviation -3 2018-12-06 09:44:36.102513 7f391b2f3700 20 osd.4 pgs 36 target 32 deviation 4 2018-12-06 09:44:36.102515 7f391b2f3700 20 osd.5 pgs 36 target 32 deviation 4 2018-12-06 09:44:36.102517 7f391b2f3700 20 osd.6 pgs 33 target 32 deviation 1 2018-12-06 09:44:36.102519 7f391b2f3700 20 osd.7 pgs 35 target 32 deviation 3 2018-12-06 09:44:36.102521 7f391b2f3700 20 osd.8 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102523 7f391b2f3700 20 osd.9 pgs 31 target 32 deviation -1 2018-12-06 09:44:36.102525 7f391b2f3700 20 osd.10 pgs 30 target 32 deviation -2 2018-12-06 09:44:36.102526 7f391b2f3700 20 osd.11 pgs 29 target 32 deviation -3 2018-12-06 09:44:36.102529 7f391b2f3700 10 total_deviation 28 overfull 1,4,5,6,7 underfull [0,3,11,10,9] 2018-12-06 09:44:36.102531 7f391b2f3700 10 osd.5 move 4 2018-12-06 09:44:36.102534 7f391b2f3700 10 trying 2.3 2018-12-06 09:44:36.102558 7f391b2f3700 10 2.3 [11,1,5] -> [11,0,9] 2018-12-06 09:44:36.102561 7f391b2f3700 10 2.3 pg_upmap_items [1,0,5,9] 2018-12-06 09:44:36.102563 7f391b2f3700 20 osd.0 pgs 28 target 32 deviation -4 2018-12-06 09:44:36.102565 7f391b2f3700 20 osd.1 pgs 33 target 32 deviation 1 2018-12-06 09:44:36.102567 7f391b2f3700 20 osd.2 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102569 7f391b2f3700 20 osd.3 pgs 29 target 32 deviation -3 2018-12-06 09:44:36.102570 7f391b2f3700 20 osd.4 pgs 36 target 32 deviation 4 2018-12-06 09:44:36.102572 7f391b2f3700 20 osd.5 pgs 35 target 32 deviation 3 2018-12-06 09:44:36.102574 7f391b2f3700 20 osd.6 pgs 33 target 32 deviation 1 2018-12-06 09:44:36.102576 7f391b2f3700 20 osd.7 pgs 35 target 32 deviation 3 2018-12-06 09:44:36.102578 7f391b2f3700 20 osd.8 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102579 7f391b2f3700 20 osd.9 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102581 7f391b2f3700 20 osd.10 pgs 30 target 32 deviation -2 2018-12-06 09:44:36.102583 7f391b2f3700 20 osd.11 pgs 29 target 32 deviation -3 2018-12-06 09:44:36.102585 7f391b2f3700 10 total_deviation 24 overfull 1,4,5,6,7 underfull [0,3,11,10] 2018-12-06 09:44:36.102587 7f391b2f3700 10 osd.4 move 4 2018-12-06 09:44:36.102588 7f391b2f3700 10 dropping pg_upmap_items 2.0 [0,8,5,4,6,12] 2018-12-06 09:44:36.102592 7f391b2f3700 20 osd.0 pgs 29 target 32 deviation -3 2018-12-06 09:44:36.102593 7f391b2f3700 20 osd.1 pgs 33 target 32 deviation 1 2018-12-06 09:44:36.102595 7f391b2f3700 20 osd.2 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102597 7f391b2f3700 20 osd.3 pgs 29 target 32 deviation -3 2018-12-06 09:44:36.102598 7f391b2f3700 20 osd.4 pgs 35 target 32 deviation 3 2018-12-06 09:44:36.102600 7f391b2f3700 20 osd.5 pgs 36 target 32 deviation 4 2018-12-06 09:44:36.102602 7f391b2f3700 20 osd.6 pgs 33 target 32 deviation 1 2018-12-06 09:44:36.102603 7f391b2f3700 20 osd.7 pgs 35 target 32 deviation 3 2018-12-06 09:44:36.102605 7f391b2f3700 20 osd.8 pgs 31 target 32 deviation -1 2018-12-06 09:44:36.102607 7f391b2f3700 20 osd.9 pgs 32 target 32 deviation 0 2018-12-06 09:44:36.102608 7f391b2f3700 20 osd.10 pgs 30 target 32 deviation -2 2018-12-06 09:44:36.102610 7f391b2f3700 20 osd.11 pgs 29 target 32 deviation -3 2018-12-06 09:44:36.104533 7f391b2f3700 -1 /sds_env/ningtao/ceph.sds/src/osd/OSDMap.cc: In function 'int OSDMap::calc_pg_upmaps(CephContext*, float, int, const std::set&, OSDMap::Incremental*)' thread 7f391b2f3700 time 2018-12-06 09:44:36.102612 /sds_env/ningtao/ceph.sds/src/osd/OSDMap.cc: 4153: FAILED assert(osd_weight.count(i.first)) ceph version 12.2.7-326-gd81683f (d81683f7d90b936d6bee42ffb24ca86d6add17ee) luminous (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x110) [0x55d08baa6260] 2: (OSDMap::calc_pg_upmaps(CephContext*, float, int, std::set, std::allocator > const&, OSDMap::Incremental*)+0xf3f) [0x55d08bb9dcbf] 3: (()+0x2ef434) [0x55d08b94b434] 4: (PyEval_EvalFrameEx()+0x6df0) [0x7f393ca62cf0] 5: (PyEval_EvalCodeEx()+0x7ed) [0x7f393ca6503d] 6: (PyEval_EvalFrameEx()+0x663c) [0x7f393ca6253c] 7: (PyEval_EvalFrameEx()+0x67bd) [0x7f393ca626bd] 8: (PyEval_EvalFrameEx()+0x67bd) [0x7f393ca626bd] 9: (PyEval_EvalCodeEx()+0x7ed) [0x7f393ca6503d] 10: (()+0x70978) [0x7f393c9ee978] 11: (PyObject_Call()+0x43) [0x7f393c9c9a63] 12: (()+0x5aa55) [0x7f393c9d8a55] 13: (PyObject_Call()+0x43) [0x7f393c9c9a63] 14: (()+0x4bb45) [0x7f393c9c9b45] 15: (PyObject_CallMethod()+0xbb) [0x7f393c9c9e7b] 16: (ActivePyModule::handle_command(std::map >, std::vector >, std::vector > > >, std::less, std::allocator >, std::vector >, std::vector > > > > > const&, std::basic_stringstream, std::allocator >*, std::basic_stringstream, std::allocator >)+0x2ed) [0x55d08b9568cd] 17: (()+0x2b0174) [0x55d08b90c174] 18: (FunctionContext::finish(int)+0x2a) [0x55d08b91fc6a] 19: (Context::complete(int)+0x9) [0x55d08b91b009] 20: (Finisher::finisher_thread_entry()+0x198) [0x55d08baa51f8] 21: (()+0x7e25) [0x7f3938ff5e25] 22: (clone()+0x6d) [0x7f39380d5bad] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- -301> 2018-12-06 09:44:20.775194 7f393cf49bc0 5 asok(0x55d096510700) register_command perfcounters_dump hook 0x55d09635bf20 -300> 2018-12-06 09:44:20.775209 7f393cf49bc0 5 asok(0x55d096510700) register_command 1 hook 0x55d09635bf20 -299> 2018-12-06 09:44:20.775214 7f393cf49bc0 5 asok(0x55d096510700) register_command perf dump hook 0x55d09635bf20 -298> 2018-12-06 09:44:20.775219 7f393cf49bc0 5 asok(0x55d096510700) register_command perfcounters_schema hook 0x55d09635bf20 -297> 2018-12-06 09:44:20.775223 7f393cf49bc0 5 asok(0x55d096510700) register_command perf histogram dump hook 0x55d09635bf20 -296> 2018-12-06 09:44:20.775227 7f393cf49bc0 5 asok(0x55d096510700) register_command 2 hook 0x55d09635bf20 -295> 2018-12-06 09:44:20.775230 7f393cf49bc0 5 asok(0x55d096510700) register_command perf schema hook 0x55d09635bf20 -294> 2018-12-06 09:44:20.775234 7f393cf49bc0 5 asok(0x55d096510700) register_command perf histogram schema hook 0x55d09635bf20 -293> 2018-12-06 09:44:20.775241 7f393cf49bc0 5 asok(0x55d096510700) register_command perf reset hook 0x55d09635bf20 -292> 2018-12-06 09:44:20.775245 7f393cf49bc0 5 asok(0x55d096510700) register_command config show hook 0x55d09635bf20 -291> 2018-12-06 09:44:20.775250 7f393cf49bc0 5 asok(0x55d096510700) register_command config help hook 0x55d09635bf20 -290> 2018-12-06 09:44:20.775255 7f393cf49bc0 5 asok(0x55d096510700) register_command config set hook 0x55d09635bf20 -289> 2018-12-06 09:44:20.775260 7f393cf49bc0 5 asok(0x55d096510700) register_command config get hook 0x55d09635bf20 -288> 2018-12-06 09:44:20.775264 7f393cf49bc0 5 asok(0x55d096510700) register_command config diff hook 0x55d09635bf20 -287> 2018-12-06 09:44:20.775268 7f393cf49bc0 5 asok(0x55d096510700) register_command config diff get hook 0x55d09635bf20 -286> 2018-12-06 09:44:20.775270 7f393cf49bc0 5 asok(0x55d096510700) register_command log flush hook 0x55d09635bf20 -285> 2018-12-06 09:44:20.775275 7f393cf49bc0 5 asok(0x55d096510700) register_command log dump hook 0x55d09635bf20 -284> 2018-12-06 09:44:20.775278 7f393cf49bc0 5 asok(0x55d096510700) register_command log reopen hook 0x55d09635bf20 -283> 2018-12-06 09:44:20.775300 7f393cf49bc0 5 asok(0x55d096510700) register_command dump_mempools hook 0x55d0966742c8 -282> 2018-12-06 09:44:20.780749 7f393cf49bc0 1 lockdep start -281> 2018-12-06 09:44:20.780940 7f393cf49bc0 0 ceph version 12.2.7-326-gd81683f (d81683f7d90b936d6bee42ffb24ca86d6add17ee) luminous (stable), process ceph-mgr, pid 688365 -280> 2018-12-06 09:44:20.781683 7f393cf49bc0 1 finished global_init_daemonize -279> 2018-12-06 09:44:20.808324 7f393cf49bc0 1 lockdep using id 0 -278> 2018-12-06 09:44:20.808655 7f393cf49bc0 5 asok(0x55d096510700) init /tmp/ceph-asok.PhRv63/mgr.x.asok -277> 2018-12-06 09:44:20.808668 7f393cf49bc0 5 asok(0x55d096510700) bind_and_listen /tmp/ceph-asok.PhRv63/mgr.x.asok -276> 2018-12-06 09:44:20.808717 7f393cf49bc0 1 lockdep using id 1 -275> 2018-12-06 09:44:20.808723 7f393cf49bc0 5 asok(0x55d096510700) register_command 0 hook 0x55d096358550 -274> 2018-12-06 09:44:20.808731 7f393cf49bc0 5 asok(0x55d096510700) register_command version hook 0x55d096358550 -273> 2018-12-06 09:44:20.808740 7f393cf49bc0 5 asok(0x55d096510700) register_command git_version hook 0x55d096358550 -272> 2018-12-06 09:44:20.808749 7f393cf49bc0 5 asok(0x55d096510700) register_command help hook 0x55d096806000 -271> 2018-12-06 09:44:20.808754 7f393cf49bc0 5 asok(0x55d096510700) register_command get_command_descriptions hook 0x55d096806010 -270> 2018-12-06 09:44:20.808785 7f3932fd2700 5 asok(0x55d096510700) entry start -269> 2018-12-06 09:44:20.808790 7f393cf49bc0 1 lockdep using id 2 -268> 2018-12-06 09:44:20.808794 7f393cf49bc0 1 lockdep using id 3 -267> 2018-12-06 09:44:20.808819 7f393cf49bc0 1 lockdep using id 4 -266> 2018-12-06 09:44:20.808824 7f393cf49bc0 1 lockdep using id 5 -265> 2018-12-06 09:44:20.808826 7f393cf49bc0 1 lockdep using id 6 -264> 2018-12-06 09:44:20.808829 7f393cf49bc0 1 lockdep using id 7 -263> 2018-12-06 09:44:20.808833 7f393cf49bc0 1 lockdep using id 8 -262> 2018-12-06 09:44:20.808838 7f393cf49bc0 1 lockdep using id 9 -261> 2018-12-06 09:44:20.808851 7f393cf49bc0 1 lockdep using id 10 -260> 2018-12-06 09:44:20.808858 7f393cf49bc0 1 lockdep using id 11 -259> 2018-12-06 09:44:20.808875 7f393cf49bc0 1 lockdep using id 12 -258> 2018-12-06 09:44:20.808964 7f393cf49bc0 1 lockdep using id 13 -257> 2018-12-06 09:44:20.809094 7f393cf49bc0 1 lockdep using id 14 -256> 2018-12-06 09:44:20.809269 7f393cf49bc0 1 lockdep using id 15 -255> 2018-12-06 09:44:20.809324 7f393cf49bc0 1 lockdep using id 16 -254> 2018-12-06 09:44:20.809339 7f393cf49bc0 1 lockdep using id 17 -253> 2018-12-06 09:44:20.809370 7f393cf49bc0 1 lockdep using id 18 -252> 2018-12-06 09:44:20.809374 7f393cf49bc0 1 lockdep using id 19 -251> 2018-12-06 09:44:20.809376 7f393cf49bc0 1 lockdep using id 20 -250> 2018-12-06 09:44:20.809409 7f393cf49bc0 1 lockdep using id 21 -249> 2018-12-06 09:44:20.809471 7f393cf49bc0 1 lockdep using id 22 -248> 2018-12-06 09:44:20.809477 7f393cf49bc0 1 lockdep using id 23 -247> 2018-12-06 09:44:20.809483 7f393cf49bc0 1 lockdep using id 24 -246> 2018-12-06 09:44:20.809487 7f393cf49bc0 1 lockdep using id 25 -245> 2018-12-06 09:44:20.809724 7f393cf49bc0 10 monclient: build_initial_monmap -244> 2018-12-06 09:44:20.809765 7f393cf49bc0 10 monclient: init -243> 2018-12-06 09:44:20.809773 7f393cf49bc0 5 adding auth protocol: cephx -242> 2018-12-06 09:44:20.809776 7f393cf49bc0 10 monclient: auth_supported 2 method cephx -241> 2018-12-06 09:44:20.809886 7f393cf49bc0 2 auth: KeyRing::load: loaded key file /sds_env/ningtao/ceph.sds/build/dev/mgr.x/keyring -240> 2018-12-06 09:44:20.809892 7f393cf49bc0 1 lockdep using id 26 -239> 2018-12-06 09:44:20.809953 7f393cf49bc0 10 monclient: _reopen_session rank -1 -238> 2018-12-06 09:44:20.809978 7f393cf49bc0 10 monclient(hunting): picked mon.a con 0x55d096897000 addr 172.17.0.7:40803/0 -237> 2018-12-06 09:44:20.809997 7f393cf49bc0 10 monclient(hunting): _renew_subs -236> 2018-12-06 09:44:20.810357 7f392f7cb700 10 monclient(hunting): handle_monmap mon_map magic: 0 v1 -235> 2018-12-06 09:44:20.810371 7f392f7cb700 10 monclient(hunting): got monmap 1, mon.a is now rank 0 -234> 2018-12-06 09:44:20.810375 7f392f7cb700 10 monclient(hunting): dump: epoch 1 fsid 834bf3db-13fd-4354-8f4e-0fad6ab7ec0f last_changed 2018-12-06 05:52:52.712252 created 2018-12-06 05:52:52.712252 0: 172.17.0.7:40803/0 mon.a -233> 2018-12-06 09:44:20.810400 7f392f7cb700 1 lockdep using id 27 -232> 2018-12-06 09:44:20.810412 7f392f7cb700 10 monclient(hunting): my global_id is 4460 -231> 2018-12-06 09:44:20.810951 7f392f7cb700 1 monclient: found mon.a -230> 2018-12-06 09:44:20.810962 7f392f7cb700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -229> 2018-12-06 09:44:20.810975 7f392f7cb700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2018-12-06 09:43:50.810974) -228> 2018-12-06 09:44:20.810981 7f392f7cb700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -227> 2018-12-06 09:44:20.810998 7f393cf49bc0 5 monclient: authenticate success, global_id 4460 -226> 2018-12-06 09:44:20.811029 7f393cf49bc0 10 log_channel(cluster) update_config to_monitors: true to_syslog: false syslog_facility: daemon prio: info to_graylog: false graylog_host: 127.0.0.1 graylog_port: 12201) -225> 2018-12-06 09:44:20.811035 7f393cf49bc0 10 log_channel(audit) update_config to_monitors: true to_syslog: false syslog_facility: local0 prio: info to_graylog: false graylog_host: 127.0.0.1 graylog_port: 12201) -224> 2018-12-06 09:44:20.811042 7f393cf49bc0 1 lockdep using id 28 -223> 2018-12-06 09:44:20.811116 7f393cf49bc0 5 asok(0x55d096510700) register_command objecter_requests hook 0x55d096806140 -222> 2018-12-06 09:44:20.811135 7f393cf49bc0 10 monclient: _renew_subs -221> 2018-12-06 09:44:20.811139 7f393cf49bc0 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -220> 2018-12-06 09:44:20.811197 7f393cf49bc0 1 lockdep using id 29 -219> 2018-12-06 09:44:20.811231 7f393cf49bc0 5 asok(0x55d096510700) register_command mds_requests hook 0x7ffc96b93c50 -218> 2018-12-06 09:44:20.811240 7f393cf49bc0 5 asok(0x55d096510700) register_command mds_sessions hook 0x7ffc96b93c50 -217> 2018-12-06 09:44:20.811248 7f393cf49bc0 5 asok(0x55d096510700) register_command dump_cache hook 0x7ffc96b93c50 -216> 2018-12-06 09:44:20.811253 7f393cf49bc0 5 asok(0x55d096510700) register_command kick_stale_sessions hook 0x7ffc96b93c50 -215> 2018-12-06 09:44:20.811257 7f393cf49bc0 5 asok(0x55d096510700) register_command status hook 0x7ffc96b93c50 -214> 2018-12-06 09:44:20.811261 7f393cf49bc0 5 asok(0x55d096510700) register_command snap create hook 0x7ffc96b93c50 -213> 2018-12-06 09:44:20.811265 7f393cf49bc0 5 asok(0x55d096510700) register_command snap remove hook 0x7ffc96b93c50 -212> 2018-12-06 09:44:20.811269 7f393cf49bc0 5 asok(0x55d096510700) register_command snap list hook 0x7ffc96b93c50 -211> 2018-12-06 09:44:20.811273 7f393cf49bc0 5 asok(0x55d096510700) register_command snap rename hook 0x7ffc96b93c50 -210> 2018-12-06 09:44:20.811278 7f393cf49bc0 5 asok(0x55d096510700) register_command xattr list hook 0x7ffc96b93c50 -209> 2018-12-06 09:44:20.811281 7f393cf49bc0 5 asok(0x55d096510700) register_command xattr listall hook 0x7ffc96b93c50 -208> 2018-12-06 09:44:20.811285 7f393cf49bc0 5 asok(0x55d096510700) register_command xattr get hook 0x7ffc96b93c50 -207> 2018-12-06 09:44:20.811290 7f393cf49bc0 5 asok(0x55d096510700) register_command xattr set hook 0x7ffc96b93c50 -206> 2018-12-06 09:44:20.811295 7f393cf49bc0 5 asok(0x55d096510700) register_command xattr remove hook 0x7ffc96b93c50 -205> 2018-12-06 09:44:20.811300 7f393cf49bc0 5 asok(0x55d096510700) register_command quota set hook 0x7ffc96b93c50 -204> 2018-12-06 09:44:20.811305 7f393cf49bc0 5 asok(0x55d096510700) register_command quota get hook 0x7ffc96b93c50 -203> 2018-12-06 09:44:20.811309 7f393cf49bc0 5 asok(0x55d096510700) register_command rstat get hook 0x7ffc96b93c50 -202> 2018-12-06 09:44:20.811311 7f393cf49bc0 5 asok(0x55d096510700) register_command lookup inosnap hook 0x7ffc96b93c50 -201> 2018-12-06 09:44:20.811547 7f393cf49bc0 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -200> 2018-12-06 09:44:20.811568 7f393cf49bc0 1 lockdep using id 30 -199> 2018-12-06 09:44:20.811577 7f393cf49bc0 -1 SignalHandler: pipe_cloexec19,20 -198> 2018-12-06 09:44:20.811619 7f393cf49bc0 -1 register_handler:pipe_cloexec21,22 -197> 2018-12-06 09:44:20.811637 7f393cf49bc0 -1 register_handler:pipe_cloexec23,24 -196> 2018-12-06 09:44:20.811645 7f393cf49bc0 -1 register_handler:pipe_cloexec25,26 -195> 2018-12-06 09:44:20.923343 7f392f7cb700 -1 mgr load Module not found: 'dashboard' -194> 2018-12-06 09:44:20.923355 7f392f7cb700 -1 mgr load Traceback (most recent call last): File "/sds_env/ningtao/ceph.sds/src/pybind/mgr/dashboard/__init__.py", line 2, in from module import * # NOQA File "/sds_env/ningtao/ceph.sds/src/pybind/mgr/dashboard/module.py", line 37, in import rbd_iscsi File "/sds_env/ningtao/ceph.sds/src/pybind/mgr/dashboard/rbd_iscsi.py", line 3, in import rbd ImportError: No module named rbd -193> 2018-12-06 09:44:20.923761 7f392f7cb700 -1 mgr init Error loading module 'dashboard': (2) No such file or directory -192> 2018-12-06 09:44:21.264116 7f392f7cb700 -1 log_channel(cluster) log [ERR] : Failed to load ceph-mgr modules: dashboard -191> 2018-12-06 09:44:21.264160 7f392f7cb700 10 monclient: handle_monmap mon_map magic: 0 v1 -190> 2018-12-06 09:44:21.264175 7f392f7cb700 10 monclient: got monmap 1, mon.a is now rank 0 -189> 2018-12-06 09:44:21.264179 7f392f7cb700 10 monclient: dump: epoch 1 fsid 834bf3db-13fd-4354-8f4e-0fad6ab7ec0f last_changed 2018-12-06 05:52:52.712252 created 2018-12-06 05:52:52.712252 0: 172.17.0.7:40803/0 mon.a -188> 2018-12-06 09:44:21.264284 7f392f7cb700 10 monclient: _check_auth_rotating have uptodate secrets (they expire after 2018-12-06 09:43:51.264284) -187> 2018-12-06 09:44:21.264470 7f392f7cb700 1 lockdep using id 31 -186> 2018-12-06 09:44:21.264481 7f392f7cb700 1 lockdep using id 32 -185> 2018-12-06 09:44:21.264486 7f392f7cb700 1 lockdep using id 33 -184> 2018-12-06 09:44:21.264500 7f392f7cb700 1 lockdep using id 34 -183> 2018-12-06 09:44:21.264508 7f392f7cb700 1 lockdep using id 35 -182> 2018-12-06 09:44:21.264528 7f392f7cb700 1 lockdep using id 36 -181> 2018-12-06 09:44:21.264549 7f392f7cb700 1 lockdep using id 37 -180> 2018-12-06 09:44:21.264567 7f392f7cb700 1 lockdep using id 38 -179> 2018-12-06 09:44:21.264584 7f392f7cb700 1 lockdep using id 39 -178> 2018-12-06 09:44:21.264599 7f392f7cb700 1 lockdep using id 40 -177> 2018-12-06 09:44:21.264619 7f392f7cb700 1 lockdep using id 41 -176> 2018-12-06 09:44:21.264636 7f392f7cb700 1 lockdep using id 42 -175> 2018-12-06 09:44:21.264653 7f392f7cb700 1 lockdep using id 43 -174> 2018-12-06 09:44:21.264665 7f392f7cb700 1 lockdep using id 44 -173> 2018-12-06 09:44:21.264671 7f392f7cb700 5 adding auth protocol: cephx -172> 2018-12-06 09:44:21.264675 7f392f7cb700 5 adding auth protocol: cephx -171> 2018-12-06 09:44:21.264676 7f392f7cb700 1 lockdep using id 45 -170> 2018-12-06 09:44:21.264979 7f391b2f3700 1 lockdep using id 46 -169> 2018-12-06 09:44:21.264997 7f391b2f3700 10 monclient: _send_command 1 [{"prefix": "mds metadata"}] -168> 2018-12-06 09:44:21.265003 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -167> 2018-12-06 09:44:21.265018 7f391b2f3700 10 monclient: _send_command 2 [{"prefix": "osd metadata"}] -166> 2018-12-06 09:44:21.265021 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -165> 2018-12-06 09:44:21.265028 7f391b2f3700 10 monclient: _send_command 3 [{"prefix": "mon metadata"}] -164> 2018-12-06 09:44:21.265031 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -163> 2018-12-06 09:44:21.265324 7f392f7cb700 10 monclient: handle_mon_command_ack 1 [{"prefix": "mds metadata"}] -162> 2018-12-06 09:44:21.265330 7f392f7cb700 10 monclient: _finish_command 1 = 0 -161> 2018-12-06 09:44:21.266231 7f392f7cb700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd metadata"}] -160> 2018-12-06 09:44:21.266234 7f392f7cb700 10 monclient: _finish_command 2 = 0 -159> 2018-12-06 09:44:21.266291 7f392f7cb700 10 monclient: handle_mon_command_ack 3 [{"prefix": "mon metadata"}] -158> 2018-12-06 09:44:21.266294 7f392f7cb700 10 monclient: _finish_command 3 = 0 -157> 2018-12-06 09:44:21.267291 7f391b2f3700 1 lockdep using id 47 -156> 2018-12-06 09:44:21.267779 7f391b2f3700 10 monclient: _renew_subs -155> 2018-12-06 09:44:21.267785 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -154> 2018-12-06 09:44:21.267798 7f391b2f3700 10 monclient: _reopen_session rank -1 -153> 2018-12-06 09:44:21.267823 7f391b2f3700 10 monclient(hunting): picked mon.a con 0x55d097751800 addr 172.17.0.7:40803/0 -152> 2018-12-06 09:44:21.267836 7f391b2f3700 10 monclient(hunting): _renew_subs -151> 2018-12-06 09:44:21.268734 7f392f7cb700 1 monclient: found mon.a -150> 2018-12-06 09:44:21.268745 7f392f7cb700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -149> 2018-12-06 09:44:21.268759 7f392f7cb700 10 monclient: _check_auth_rotating have uptodate secrets (they expire after 2018-12-06 09:43:51.268758) -148> 2018-12-06 09:44:21.268880 7f391b2f3700 1 lockdep reusing last freed id 46 -147> 2018-12-06 09:44:21.268890 7f391b2f3700 10 monclient: _send_command 4 [{"prefix": "config-key ls"}] -146> 2018-12-06 09:44:21.268894 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -145> 2018-12-06 09:44:21.269088 7f392f7cb700 10 monclient: handle_mon_command_ack 4 [{"prefix": "config-key ls"}] -144> 2018-12-06 09:44:21.269092 7f392f7cb700 10 monclient: _finish_command 4 = 0 -143> 2018-12-06 09:44:21.269123 7f391b2f3700 10 monclient: _send_command 5 [{"prefix": "config-key get", "key": "mgr/balancer/active"}] -142> 2018-12-06 09:44:21.269143 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -141> 2018-12-06 09:44:21.269284 7f392f7cb700 10 monclient: handle_mon_command_ack 5 [{"prefix": "config-key get", "key": "mgr/balancer/active"}] -140> 2018-12-06 09:44:21.269288 7f392f7cb700 10 monclient: _finish_command 5 = 0 obtained 'mgr/balancer/active' -139> 2018-12-06 09:44:21.269305 7f391b2f3700 10 monclient: _send_command 6 [{"prefix": "config-key get", "key": "mgr/balancer/mode"}] -138> 2018-12-06 09:44:21.269308 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -137> 2018-12-06 09:44:21.269440 7f392f7cb700 10 monclient: handle_mon_command_ack 6 [{"prefix": "config-key get", "key": "mgr/balancer/mode"}] -136> 2018-12-06 09:44:21.269443 7f392f7cb700 10 monclient: _finish_command 6 = 0 obtained 'mgr/balancer/mode' -135> 2018-12-06 09:44:21.269457 7f391b2f3700 10 monclient: _send_command 7 [{"prefix": "config-key get", "key": "mgr/dashboard/x/server_port"}] -134> 2018-12-06 09:44:21.269461 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -133> 2018-12-06 09:44:21.269585 7f392f7cb700 10 monclient: handle_mon_command_ack 7 [{"prefix": "config-key get", "key": "mgr/dashboard/x/server_port"}] -132> 2018-12-06 09:44:21.269588 7f392f7cb700 10 monclient: _finish_command 7 = 0 obtained 'mgr/dashboard/x/server_port' -131> 2018-12-06 09:44:21.269603 7f391b2f3700 10 monclient: _send_command 8 [{"prefix": "config-key get", "key": "mgr/restful/keys/admin"}] -130> 2018-12-06 09:44:21.269605 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -129> 2018-12-06 09:44:21.269732 7f392f7cb700 10 monclient: handle_mon_command_ack 8 [{"prefix": "config-key get", "key": "mgr/restful/keys/admin"}] -128> 2018-12-06 09:44:21.269735 7f392f7cb700 10 monclient: _finish_command 8 = 0 obtained 'mgr/restful/keys/admin' -127> 2018-12-06 09:44:21.269749 7f391b2f3700 10 monclient: _send_command 9 [{"prefix": "config-key get", "key": "mgr/restful/x/crt"}] -126> 2018-12-06 09:44:21.269755 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -125> 2018-12-06 09:44:21.269889 7f392f7cb700 10 monclient: handle_mon_command_ack 9 [{"prefix": "config-key get", "key": "mgr/restful/x/crt"}] -124> 2018-12-06 09:44:21.269892 7f392f7cb700 10 monclient: _finish_command 9 = 0 obtained 'mgr/restful/x/crt' -123> 2018-12-06 09:44:21.269906 7f391b2f3700 10 monclient: _send_command 10 [{"prefix": "config-key get", "key": "mgr/restful/x/key"}] -122> 2018-12-06 09:44:21.269909 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -121> 2018-12-06 09:44:21.270037 7f392f7cb700 10 monclient: handle_mon_command_ack 10 [{"prefix": "config-key get", "key": "mgr/restful/x/key"}] -120> 2018-12-06 09:44:21.270040 7f392f7cb700 10 monclient: _finish_command 10 = 0 obtained 'mgr/restful/x/key' -119> 2018-12-06 09:44:21.270054 7f391b2f3700 10 monclient: _send_command 11 [{"prefix": "config-key get", "key": "mgr/restful/x/server_port"}] -118> 2018-12-06 09:44:21.270056 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -117> 2018-12-06 09:44:21.270180 7f392f7cb700 10 monclient: handle_mon_command_ack 11 [{"prefix": "config-key get", "key": "mgr/restful/x/server_port"}] -116> 2018-12-06 09:44:21.270183 7f392f7cb700 10 monclient: _finish_command 11 = 0 obtained 'mgr/restful/x/server_port' -115> 2018-12-06 09:44:21.270200 7f391b2f3700 1 lockdep reusing last freed id 46 -114> 2018-12-06 09:44:21.271274 7f391b2f3700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -113> 2018-12-06 09:44:21.864370 7f3931fd0700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6809/674719 conn(0x55d097885800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -112> 2018-12-06 09:44:21.864391 7f39317cf700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6801/674095 conn(0x55d097887000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -111> 2018-12-06 09:44:21.864509 7f39317cf700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6825/675967 conn(0x55d0978a0000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -110> 2018-12-06 09:44:21.864539 7f39327d1700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:0/673688 conn(0x55d097884000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -109> 2018-12-06 09:44:21.864574 7f39317cf700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6813/675031 conn(0x55d09788f800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -108> 2018-12-06 09:44:21.864608 7f39327d1700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6805/674407 conn(0x55d097888800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -107> 2018-12-06 09:44:21.864687 7f39327d1700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6821/675655 conn(0x55d097891000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -106> 2018-12-06 09:44:21.864746 7f39327d1700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6853/678151 conn(0x55d0978a1800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -105> 2018-12-06 09:44:21.864768 7f39317cf700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6841/677215 conn(0x55d0978a4800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -104> 2018-12-06 09:44:21.864796 7f39327d1700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6849/677839 conn(0x55d0978b2000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -103> 2018-12-06 09:44:21.864824 7f39317cf700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6837/676903 conn(0x55d0978b5000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -102> 2018-12-06 09:44:21.864930 7f39327d1700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6857/678463 conn(0x55d0978b6800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -101> 2018-12-06 09:44:21.865084 7f3931fd0700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6833/676591 conn(0x55d097892800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -100> 2018-12-06 09:44:21.865171 7f3931fd0700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6817/675343 conn(0x55d09788e000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -99> 2018-12-06 09:44:21.865234 7f3931fd0700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6829/676279 conn(0x55d0978a3000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -98> 2018-12-06 09:44:21.865313 7f3931fd0700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6861/678775 conn(0x55d0978b3800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -97> 2018-12-06 09:44:21.865696 7f3931fd0700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:6845/677527 conn(0x55d0978c6000 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -96> 2018-12-06 09:44:21.869978 7f391aaf2700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -95> 2018-12-06 09:44:22.811810 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -94> 2018-12-06 09:44:22.811934 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -93> 2018-12-06 09:44:24.812180 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -92> 2018-12-06 09:44:24.812305 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -91> 2018-12-06 09:44:25.808457 7f39337d3700 1 lockdep using id 48 -90> 2018-12-06 09:44:26.812575 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -89> 2018-12-06 09:44:26.812734 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -88> 2018-12-06 09:44:28.812994 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -87> 2018-12-06 09:44:28.814086 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -86> 2018-12-06 09:44:30.809988 7f392e7c9700 10 monclient: tick -85> 2018-12-06 09:44:30.810002 7f392e7c9700 10 monclient: _check_auth_rotating have uptodate secrets (they expire after 2018-12-06 09:44:00.810000) -84> 2018-12-06 09:44:30.814332 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -83> 2018-12-06 09:44:30.814456 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -82> 2018-12-06 09:44:32.814724 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -81> 2018-12-06 09:44:32.815795 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -80> 2018-12-06 09:44:34.816045 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -79> 2018-12-06 09:44:34.816168 7f392c7c5700 10 monclient: _send_mon_message to mon.a at 172.17.0.7:40803/0 -78> 2018-12-06 09:44:35.913272 7f39327d1700 0 -- 172.17.0.7:6800/688367 >> 172.17.0.7:0/1387855531 conn(0x55d0978dc800 :6800 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=1).handle_connect_msg: challenging authorizer -77> 2018-12-06 09:44:36.075806 7f391aaf2700 0 log_channel(audit) log [DBG] : from='client.4461 172.17.0.7:0/1387855531' entity='client.admin' cmd=[{"prefix": "balancer optimize", "plan": "plan1", "target": ["mgr", ""]}]: dispatch -76> 2018-12-06 09:44:36.102382 7f391b2f3700 30 calc_pg_upmaps pool 2 ruleno 1 -75> 2018-12-06 09:44:36.102392 7f391b2f3700 20 osd.0 weight 0.0625 pgs 27 -74> 2018-12-06 09:44:36.102413 7f391b2f3700 20 osd.1 weight 0.0625 pgs 34 -73> 2018-12-06 09:44:36.102414 7f391b2f3700 20 osd.2 weight 0.0625 pgs 32 -72> 2018-12-06 09:44:36.102416 7f391b2f3700 20 osd.3 weight 0.0625 pgs 29 -71> 2018-12-06 09:44:36.102417 7f391b2f3700 20 osd.4 weight 0.0625 pgs 36 -70> 2018-12-06 09:44:36.102418 7f391b2f3700 20 osd.5 weight 0.0625 pgs 36 -69> 2018-12-06 09:44:36.102419 7f391b2f3700 20 osd.6 weight 0.0625 pgs 33 -68> 2018-12-06 09:44:36.102420 7f391b2f3700 20 osd.7 weight 0.0625 pgs 36 -67> 2018-12-06 09:44:36.102422 7f391b2f3700 20 osd.8 weight 0.0625 pgs 32 -66> 2018-12-06 09:44:36.102423 7f391b2f3700 20 osd.9 weight 0.0625 pgs 31 -65> 2018-12-06 09:44:36.102424 7f391b2f3700 20 osd.10 weight 0.0625 pgs 30 -64> 2018-12-06 09:44:36.102425 7f391b2f3700 20 osd.11 weight 0.0625 pgs 28 -63> 2018-12-06 09:44:36.102427 7f391b2f3700 10 osd_weight_total 0.75 -62> 2018-12-06 09:44:36.102429 7f391b2f3700 10 pgs_per_weight 512 -61> 2018-12-06 09:44:36.102431 7f391b2f3700 20 osd.0 pgs 27 target 32 deviation -5 -60> 2018-12-06 09:44:36.102434 7f391b2f3700 20 osd.1 pgs 34 target 32 deviation 2 -59> 2018-12-06 09:44:36.102436 7f391b2f3700 20 osd.2 pgs 32 target 32 deviation 0 -58> 2018-12-06 09:44:36.102439 7f391b2f3700 20 osd.3 pgs 29 target 32 deviation -3 -57> 2018-12-06 09:44:36.102441 7f391b2f3700 20 osd.4 pgs 36 target 32 deviation 4 -56> 2018-12-06 09:44:36.102443 7f391b2f3700 20 osd.5 pgs 36 target 32 deviation 4 -55> 2018-12-06 09:44:36.102445 7f391b2f3700 20 osd.6 pgs 33 target 32 deviation 1 -54> 2018-12-06 09:44:36.102447 7f391b2f3700 20 osd.7 pgs 36 target 32 deviation 4 -53> 2018-12-06 09:44:36.102449 7f391b2f3700 20 osd.8 pgs 32 target 32 deviation 0 -52> 2018-12-06 09:44:36.102451 7f391b2f3700 20 osd.9 pgs 31 target 32 deviation -1 -51> 2018-12-06 09:44:36.102453 7f391b2f3700 20 osd.10 pgs 30 target 32 deviation -2 -50> 2018-12-06 09:44:36.102454 7f391b2f3700 20 osd.11 pgs 28 target 32 deviation -4 -49> 2018-12-06 09:44:36.102457 7f391b2f3700 10 total_deviation 30 overfull 1,4,5,6,7 underfull [0,11,3,10,9] -48> 2018-12-06 09:44:36.102460 7f391b2f3700 10 osd.7 move 4 -47> 2018-12-06 09:44:36.102464 7f391b2f3700 10 trying 2.d -46> 2018-12-06 09:44:36.102498 7f391b2f3700 10 2.d [8,3,7] -> [8,3,11] -45> 2018-12-06 09:44:36.102503 7f391b2f3700 10 2.d pg_upmap_items [7,11] -44> 2018-12-06 09:44:36.102506 7f391b2f3700 20 osd.0 pgs 27 target 32 deviation -5 -43> 2018-12-06 09:44:36.102508 7f391b2f3700 20 osd.1 pgs 34 target 32 deviation 2 -42> 2018-12-06 09:44:36.102510 7f391b2f3700 20 osd.2 pgs 32 target 32 deviation 0 -41> 2018-12-06 09:44:36.102512 7f391b2f3700 20 osd.3 pgs 29 target 32 deviation -3 -40> 2018-12-06 09:44:36.102513 7f391b2f3700 20 osd.4 pgs 36 target 32 deviation 4 -39> 2018-12-06 09:44:36.102515 7f391b2f3700 20 osd.5 pgs 36 target 32 deviation 4 -38> 2018-12-06 09:44:36.102517 7f391b2f3700 20 osd.6 pgs 33 target 32 deviation 1 -37> 2018-12-06 09:44:36.102519 7f391b2f3700 20 osd.7 pgs 35 target 32 deviation 3 -36> 2018-12-06 09:44:36.102521 7f391b2f3700 20 osd.8 pgs 32 target 32 deviation 0 -35> 2018-12-06 09:44:36.102523 7f391b2f3700 20 osd.9 pgs 31 target 32 deviation -1 -34> 2018-12-06 09:44:36.102525 7f391b2f3700 20 osd.10 pgs 30 target 32 deviation -2 -33> 2018-12-06 09:44:36.102526 7f391b2f3700 20 osd.11 pgs 29 target 32 deviation -3 -32> 2018-12-06 09:44:36.102529 7f391b2f3700 10 total_deviation 28 overfull 1,4,5,6,7 underfull [0,3,11,10,9] -31> 2018-12-06 09:44:36.102531 7f391b2f3700 10 osd.5 move 4 -30> 2018-12-06 09:44:36.102534 7f391b2f3700 10 trying 2.3 -29> 2018-12-06 09:44:36.102558 7f391b2f3700 10 2.3 [11,1,5] -> [11,0,9] -28> 2018-12-06 09:44:36.102561 7f391b2f3700 10 2.3 pg_upmap_items [1,0,5,9] -27> 2018-12-06 09:44:36.102563 7f391b2f3700 20 osd.0 pgs 28 target 32 deviation -4 -26> 2018-12-06 09:44:36.102565 7f391b2f3700 20 osd.1 pgs 33 target 32 deviation 1 -25> 2018-12-06 09:44:36.102567 7f391b2f3700 20 osd.2 pgs 32 target 32 deviation 0 -24> 2018-12-06 09:44:36.102569 7f391b2f3700 20 osd.3 pgs 29 target 32 deviation -3 -23> 2018-12-06 09:44:36.102570 7f391b2f3700 20 osd.4 pgs 36 target 32 deviation 4 -22> 2018-12-06 09:44:36.102572 7f391b2f3700 20 osd.5 pgs 35 target 32 deviation 3 -21> 2018-12-06 09:44:36.102574 7f391b2f3700 20 osd.6 pgs 33 target 32 deviation 1 -20> 2018-12-06 09:44:36.102576 7f391b2f3700 20 osd.7 pgs 35 target 32 deviation 3 -19> 2018-12-06 09:44:36.102578 7f391b2f3700 20 osd.8 pgs 32 target 32 deviation 0 -18> 2018-12-06 09:44:36.102579 7f391b2f3700 20 osd.9 pgs 32 target 32 deviation 0 -17> 2018-12-06 09:44:36.102581 7f391b2f3700 20 osd.10 pgs 30 target 32 deviation -2 -16> 2018-12-06 09:44:36.102583 7f391b2f3700 20 osd.11 pgs 29 target 32 deviation -3 -15> 2018-12-06 09:44:36.102585 7f391b2f3700 10 total_deviation 24 overfull 1,4,5,6,7 underfull [0,3,11,10] -14> 2018-12-06 09:44:36.102587 7f391b2f3700 10 osd.4 move 4 -13> 2018-12-06 09:44:36.102588 7f391b2f3700 10 dropping pg_upmap_items 2.0 [0,8,5,4,6,12] -12> 2018-12-06 09:44:36.102592 7f391b2f3700 20 osd.0 pgs 29 target 32 deviation -3 -11> 2018-12-06 09:44:36.102593 7f391b2f3700 20 osd.1 pgs 33 target 32 deviation 1 -10> 2018-12-06 09:44:36.102595 7f391b2f3700 20 osd.2 pgs 32 target 32 deviation 0 -9> 2018-12-06 09:44:36.102597 7f391b2f3700 20 osd.3 pgs 29 target 32 deviation -3 -8> 2018-12-06 09:44:36.102598 7f391b2f3700 20 osd.4 pgs 35 target 32 deviation 3 -7> 2018-12-06 09:44:36.102600 7f391b2f3700 20 osd.5 pgs 36 target 32 deviation 4 -6> 2018-12-06 09:44:36.102602 7f391b2f3700 20 osd.6 pgs 33 target 32 deviation 1 -5> 2018-12-06 09:44:36.102603 7f391b2f3700 20 osd.7 pgs 35 target 32 deviation 3 -4> 2018-12-06 09:44:36.102605 7f391b2f3700 20 osd.8 pgs 31 target 32 deviation -1 -3> 2018-12-06 09:44:36.102607 7f391b2f3700 20 osd.9 pgs 32 target 32 deviation 0 -2> 2018-12-06 09:44:36.102608 7f391b2f3700 20 osd.10 pgs 30 target 32 deviation -2 -1> 2018-12-06 09:44:36.102610 7f391b2f3700 20 osd.11 pgs 29 target 32 deviation -3 0> 2018-12-06 09:44:36.104533 7f391b2f3700 -1 /sds_env/ningtao/ceph.sds/src/osd/OSDMap.cc: In function 'int OSDMap::calc_pg_upmaps(CephContext*, float, int, const std::set&, OSDMap::Incremental*)' thread 7f391b2f3700 time 2018-12-06 09:44:36.102612 /sds_env/ningtao/ceph.sds/src/osd/OSDMap.cc: 4153: FAILED assert(osd_weight.count(i.first)) ceph version 12.2.7-326-gd81683f (d81683f7d90b936d6bee42ffb24ca86d6add17ee) luminous (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x110) [0x55d08baa6260] 2: (OSDMap::calc_pg_upmaps(CephContext*, float, int, std::set, std::allocator > const&, OSDMap::Incremental*)+0xf3f) [0x55d08bb9dcbf] 3: (()+0x2ef434) [0x55d08b94b434] 4: (PyEval_EvalFrameEx()+0x6df0) [0x7f393ca62cf0] 5: (PyEval_EvalCodeEx()+0x7ed) [0x7f393ca6503d] 6: (PyEval_EvalFrameEx()+0x663c) [0x7f393ca6253c] 7: (PyEval_EvalFrameEx()+0x67bd) [0x7f393ca626bd] 8: (PyEval_EvalFrameEx()+0x67bd) [0x7f393ca626bd] 9: (PyEval_EvalCodeEx()+0x7ed) [0x7f393ca6503d] 10: (()+0x70978) [0x7f393c9ee978] 11: (PyObject_Call()+0x43) [0x7f393c9c9a63] 12: (()+0x5aa55) [0x7f393c9d8a55] 13: (PyObject_Call()+0x43) [0x7f393c9c9a63] 14: (()+0x4bb45) [0x7f393c9c9b45] 15: (PyObject_CallMethod()+0xbb) [0x7f393c9c9e7b] 16: (ActivePyModule::handle_command(std::map >, std::vector >, std::vector > > >, std::less, std::allocator >, std::vector >, std::vector > > > > > const&, std::basic_stringstream, std::allocator >*, std::basic_stringstream, std::allocator >)+0x2ed) [0x55d08b9568cd] 17: (()+0x2b0174) [0x55d08b90c174] 18: (FunctionContext::finish(int)+0x2a) [0x55d08b91fc6a] 19: (Context::complete(int)+0x9) [0x55d08b91b009] 20: (Finisher::finisher_thread_entry()+0x198) [0x55d08baa51f8] 21: (()+0x7e25) [0x7f3938ff5e25] 22: (clone()+0x6d) [0x7f39380d5bad] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 ocs 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 30/30 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 0 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 1 reserver 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 xio 1/ 5 compressor 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 4/ 5 memdb 1/ 5 kinetic 1/ 5 fuse 0/ 0 mgr 1/ 5 mgrc 1/ 5 dpdk 1/ 5 eventtrace -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /sds_env/ningtao/ceph.sds/build/out/mgr.x.log --- end dump of recent events --- 2018-12-06 09:44:36.107638 7f391b2f3700 -1 *** Caught signal (Aborted) ** in thread 7f391b2f3700 thread_name:mgr-fin ceph version 12.2.7-326-gd81683f (d81683f7d90b936d6bee42ffb24ca86d6add17ee) luminous (stable) 1: (()+0x40d201) [0x55d08ba69201] 2: (()+0xf6d0) [0x7f3938ffd6d0] 3: (gsignal()+0x37) [0x7f393800d277] 4: (abort()+0x148) [0x7f393800e968] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x284) [0x55d08baa63d4] 6: (OSDMap::calc_pg_upmaps(CephContext*, float, int, std::set, std::allocator > const&, OSDMap::Incremental*)+0xf3f) [0x55d08bb9dcbf] 7: (()+0x2ef434) [0x55d08b94b434] 8: (PyEval_EvalFrameEx()+0x6df0) [0x7f393ca62cf0] 9: (PyEval_EvalCodeEx()+0x7ed) [0x7f393ca6503d] 10: (PyEval_EvalFrameEx()+0x663c) [0x7f393ca6253c] 11: (PyEval_EvalFrameEx()+0x67bd) [0x7f393ca626bd] 12: (PyEval_EvalFrameEx()+0x67bd) [0x7f393ca626bd] 13: (PyEval_EvalCodeEx()+0x7ed) [0x7f393ca6503d] 14: (()+0x70978) [0x7f393c9ee978] 15: (PyObject_Call()+0x43) [0x7f393c9c9a63] 16: (()+0x5aa55) [0x7f393c9d8a55] 17: (PyObject_Call()+0x43) [0x7f393c9c9a63] 18: (()+0x4bb45) [0x7f393c9c9b45] 19: (PyObject_CallMethod()+0xbb) [0x7f393c9c9e7b] 20: (ActivePyModule::handle_command(std::map >, std::vector >, std::vector > > >, std::less, std::allocator >, std::vector >, std::vector > > > > > const&, std::basic_stringstream, std::allocator >*, std::basic_stringstream, std::allocator >)+0x2ed) [0x55d08b9568cd] 21: (()+0x2b0174) [0x55d08b90c174] 22: (FunctionContext::finish(int)+0x2a) [0x55d08b91fc6a] 23: (Context::complete(int)+0x9) [0x55d08b91b009] 24: (Finisher::finisher_thread_entry()+0x198) [0x55d08baa51f8] 25: (()+0x7e25) [0x7f3938ff5e25] 26: (clone()+0x6d) [0x7f39380d5bad] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- 0> 2018-12-06 09:44:36.107638 7f391b2f3700 -1 *** Caught signal (Aborted) ** in thread 7f391b2f3700 thread_name:mgr-fin ceph version 12.2.7-326-gd81683f (d81683f7d90b936d6bee42ffb24ca86d6add17ee) luminous (stable) 1: (()+0x40d201) [0x55d08ba69201] 2: (()+0xf6d0) [0x7f3938ffd6d0] 3: (gsignal()+0x37) [0x7f393800d277] 4: (abort()+0x148) [0x7f393800e968] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x284) [0x55d08baa63d4] 6: (OSDMap::calc_pg_upmaps(CephContext*, float, int, std::set, std::allocator > const&, OSDMap::Incremental*)+0xf3f) [0x55d08bb9dcbf] 7: (()+0x2ef434) [0x55d08b94b434] 8: (PyEval_EvalFrameEx()+0x6df0) [0x7f393ca62cf0] 9: (PyEval_EvalCodeEx()+0x7ed) [0x7f393ca6503d] 10: (PyEval_EvalFrameEx()+0x663c) [0x7f393ca6253c] 11: (PyEval_EvalFrameEx()+0x67bd) [0x7f393ca626bd] 12: (PyEval_EvalFrameEx()+0x67bd) [0x7f393ca626bd] 13: (PyEval_EvalCodeEx()+0x7ed) [0x7f393ca6503d] 14: (()+0x70978) [0x7f393c9ee978] 15: (PyObject_Call()+0x43) [0x7f393c9c9a63] 16: (()+0x5aa55) [0x7f393c9d8a55] 17: (PyObject_Call()+0x43) [0x7f393c9c9a63] 18: (()+0x4bb45) [0x7f393c9c9b45] 19: (PyObject_CallMethod()+0xbb) [0x7f393c9c9e7b] 20: (ActivePyModule::handle_command(std::map >, std::vector >, std::vector > > >, std::less, std::allocator >, std::vector >, std::vector > > > > > const&, std::basic_stringstream, std::allocator >*, std::basic_stringstream, std::allocator >)+0x2ed) [0x55d08b9568cd] 21: (()+0x2b0174) [0x55d08b90c174] 22: (FunctionContext::finish(int)+0x2a) [0x55d08b91fc6a] 23: (Context::complete(int)+0x9) [0x55d08b91b009] 24: (Finisher::finisher_thread_entry()+0x198) [0x55d08baa51f8] 25: (()+0x7e25) [0x7f3938ff5e25] 26: (clone()+0x6d) [0x7f39380d5bad] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 ocs 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 30/30 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 0 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 1 reserver 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 xio 1/ 5 compressor 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 4/ 5 memdb 1/ 5 kinetic 1/ 5 fuse 0/ 0 mgr 1/ 5 mgrc 1/ 5 dpdk 1/ 5 eventtrace -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /sds_env/ningtao/ceph.sds/build/out/mgr.x.log --- end dump of recent events ---