Bug #52744
Updated by J. Eric Ivancich over 2 years ago
This is initiated by a `radosgw-admin bucket reshard ...` run. Despite the crash the process seems to recover and the reshard continues. <pre> 2021-09-27T13:57:52.526-0400 7f6072067980 1 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, const RGWCacheNotifyInfo&, optional_yield):394 Notify failed on object default.rgw.meta:root:.bucket.meta.bkt1:6ce73ef2-22ae-4cdf-bbd2-2b14a08036d9.9004114.1: (110) Connection timed out 2021-09-27T13:57:52.526-0400 7f6072067980 1 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, const RGWCacheNotifyInfo&, optional_yield):397 Backtrace: : ceph version Development (no_version) octopus (stable) 1: (RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, RGWCacheNotifyInfo const&, optional_yield)+0xcc) [0x5563e87075d8] 2: (RGWSI_Notify::distribute(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWCacheNotifyInfo const&, optional_yield)+0x1c2) [0x5563e87073e4] 3: (RGWSI_SysObj_Cache::distribute_cache(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rgw_raw_obj const&, ObjectCacheInfo&, int, optional_yield)+0xc2) [0x5563e87113b8] 4: (RGWSI_SysObj_Cache::write(rgw_raw_obj const&, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >*, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, bool, ceph::buffer::v15_2_0::list const&, RGWObjVersionTracker*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, optional_yield)+0x267) [0x5563e87108dd] 5: (RGWSI_SysObj::Obj::WOp::write(ceph::buffer::v15_2_0::list&, optional_yield)+0x8f) [0x5563e8084631] 6: (rgw_put_system_obj(RGWSysObjectCtx&, rgw_pool const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list&, bool, RGWObjVersionTracker*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, optional_yield, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >*)+0x13d) [0x5563e85d6f61] 7: (RGWSI_MetaBackend_SObj::put_entry(RGWSI_MetaBackend::Context*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWSI_MetaBackend::PutParams&, RGWObjVersionTracker*, optional_yield)+0x102) [0x5563e8078784] 8: (()+0x127e29a) [0x5563e870329a] 9: (()+0x127f5fe) [0x5563e87045fe] 10: (()+0x127f118) [0x5563e8704118] 11: (()+0x127ec6e) [0x5563e8703c6e] 12: (std::function<int ()>::operator()() const+0x32) [0x5563e8704e4e] 13: (RGWSI_MetaBackend::do_mutate(RGWSI_MetaBackend::Context*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const&, RGWObjVersionTracker*, RGWMDLogStatus, optional_yield, std::function<int ()>, bool)+0x12d) [0x5563e8703143] 14: (RGWSI_MetaBackend::put(RGWSI_MetaBackend::Context*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWSI_MetaBackend::PutParams&, RGWObjVersionTracker*, optional_yield)+0x10e) [0x5563e87033ae] 15: (RGWSI_Bucket_SObj::store_bucket_instance_info(ptr_wrapper<RGWSI_MetaBackend::Context, 4>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWBucketInfo&, std::optional<RGWBucketInfo*>, bool, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >*, optional_yield)+0x543) [0x5563e86e3f17] 16: (RGWBucketCtl::do_store_bucket_instance_info(ptr_wrapper<RGWSI_MetaBackend::Context, 4>&, rgw_bucket const&, RGWBucketInfo&, optional_yield, RGWBucketCtl::BucketInstance::PutParams const&)+0xfb) [0x5563e80dd58d] 17: (()+0xc58636) [0x5563e80dd636] 18: (()+0xc6148d) [0x5563e80e648d] 19: (()+0xc5fe76) [0x5563e80e4e76] 20: (()+0xc5e7ee) [0x5563e80e37ee] 21: (std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>::operator()(ptr_wrapper<RGWSI_MetaBackend::Context, 4>&) const+0x49) [0x5563e80f3623] 22: (RGWBucketInstanceMetadataHandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)::{lambda(RGWSI_MetaBackend_Handler::Op*)#1}::operator()(RGWSI_MetaBackend_Handler::Op*) const+0x53) [0x5563e80eea07] 23: (int std::__invoke_impl<int, RGWBucketInstanceMetadataHandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)::{lambda(RGWSI_MetaBackend_Handler::Op*)#1}&, RGWSI_MetaBackend_Handler::Op*>(std::__invoke_other, RGWBucketInstanceMetadataHandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)::{lambda(RGWSI_MetaBackend_Handler::Op*)#1}&, RGWSI_MetaBackend_Handler::Op*&&)+0x3a) [0x5563e80ff2ba] 24: (std::enable_if<is_invocable_r_v<int, RGWBucketInstanceMetadataHandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)::{lambda(RGWSI_MetaBackend_Handler::Op*)#1}&, RGWSI_MetaBackend_Handler::Op*>, std::enable_if>::type std::__invoke_r<int, RGWBucketInstanceMetadataHandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)::{lambda(RGWSI_MetaBackend_Handler::Op*)#1}&, RGWSI_MetaBackend_Handler::Op*>(int&&, (RGWBucketInstanceMetadataHandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)::{lambda(RGWSI_MetaBackend_Handler::Op*)#1}&)...)+0x37) [0x5563e80fc026] 25: (std::_Function_handler<int (RGWSI_MetaBackend_Handler::Op*), RGWBucketInstanceMetadataHandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)::{lambda(RGWSI_MetaBackend_Handler::Op*)#1}>::_M_invoke(std::_Any_data const&, RGWSI_MetaBackend_Handler::Op*&&)+0x37) [0x5563e80f76e0] 26: (std::function<int (RGWSI_MetaBackend_Handler::Op*)>::operator()(RGWSI_MetaBackend_Handler::Op*) const+0x49) [0x5563e8704e99] 27: (()+0x127e733) [0x5563e8703733] 28: (()+0x127f812) [0x5563e8704812] 29: (()+0x127f3f1) [0x5563e87043f1] 30: (()+0x127ee5b) [0x5563e8703e5b] 31: (std::function<int (RGWSI_MetaBackend::Context*)>::operator()(RGWSI_MetaBackend::Context*) const+0x49) [0x5563e8079a0f] 32: (RGWSI_MetaBackend_SObj::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (RGWSI_MetaBackend::Context*)>)+0x76) [0x5563e8078076] 33: (RGWSI_MetaBackend_Handler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (RGWSI_MetaBackend_Handler::Op*)>)+0x79) [0x5563e87037c3] 34: (RGWBucketInstanceMetadataHandler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)+0x69) [0x5563e80eea87] 35: (RGWBucketInstanceMetadataHandler::call(std::function<int (ptr_wrapper<RGWSI_MetaBackend::Context, 4>&)>)+0x5f) [0x5563e80ee969] 36: (RGWBucketCtl::store_bucket_instance_info(rgw_bucket const&, RGWBucketInfo&, optional_yield, RGWBucketCtl::BucketInstance::PutParams const&)+0xb3) [0x5563e80dd6ef] 37: (RGWRados::put_bucket_instance_info(RGWBucketInfo&, bool, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >*)+0xb2) [0x5563e84e538c] 38: (()+0xf5a78b) [0x5563e83df78b] 39: (RGWBucketReshard::create_new_bucket_instance(int, RGWBucketInfo&)+0x3f) [0x5563e83df88b] 40: (RGWBucketReshard::execute(int, int, bool, std::ostream*, ceph::Formatter*, RGWReshard*)+0xab) [0x5563e83e20a7] 41: (main()+0x189ba) [0x5563e7f8004b] 42: (__libc_start_main()+0xf2) [0x7f607265b1e2] 43: (_start()+0x2e) [0x5563e7f565ae] 2021-09-27T13:57:52.526-0400 7f6072067980 1 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, const RGWCacheNotifyInfo&, optional_yield):413 Invalidating obj=default.rgw.meta:root:.bucket.meta.bkt1:6ce73ef2-22ae-4cdf-bbd2-2b14a08036d9.9004114.1 tries=0 2021-09-27T13:58:02.529-0400 7f6072067980 1 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, const RGWCacheNotifyInfo&, optional_yield):418 invalidation attempt 0 failed: (110) Connection timed out 2021-09-27T13:58:02.529-0400 7f6072067980 1 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, const RGWCacheNotifyInfo&, optional_yield):413 Invalidating obj=default.rgw.meta:root:.bucket.meta.bkt1:6ce73ef2-22ae-4cdf-bbd2-2b14a08036d9.9004114.1 tries=1 tenant: bucket name: bkt1 old bucket instance id: 6ce73ef2-22ae-4cdf-bbd2-2b14a08036d9.8984114.1 new bucket instance id: 6ce73ef2-22ae-4cdf-bbd2-2b14a08036d9.9004114.1 total entries: 1000 2000 3000 4000 5000 6000 7000 8000 9000 10000 11000 12000 13000 14000 15000 16000 17000 18000 19000 20000 21000 22000 23000 24000 25000 26000 27000 28000 29000 30000 31000 32000 33000 34000 35000 36000 37000 38000 39000 40000 41000 42000 43000 44000 45000 46000 47000 48000 49000 50000 51000 52000 53000 54000 55000 56000 57000 58000 59000 60000 61000 62000 63000 64000 65000 66000 67000 68000 69000 70000 71000 72000 73000 74000 75000 76000 77000 78000 79000 80000 81000 </pre> [and so on...]