Ceph » rgw

Hi:

I am creating new Ceph clusters with Ceph 15.2.10. Currently we notice objects that created before we set up secondary cluster will not be synced. Only new objects will be synced. Follow the documentation https://docs.ceph.com/en/latest/radosgw/multisite-sync-policy/, we create symmetrical sync flow and it still does not sync old objects.

Also the radosgw crashed on secondary cluster, here is the report:

{
    "archived": "2021-04-20 08:30:09.690933",
    "backtrace": [
        "(()+0x3f040) [0x7f40e13b6040]",
        "(RGWDataChangesLog::filter_bucket(rgw_bucket const&, optional_yield) const+0) [0x7f40e1cd0460]",
        "(RGWDataChangesLog::add_entry(RGWBucketInfo const&, int)+0x39) [0x7f40e1ce6cd9]",
        "(RGWRados::Bucket::UpdateIndex::complete(long, unsigned long, unsigned long, unsigned long, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list*, RGWObjCategory, std::__cxx11::list<cls_rgw_obj_key, std::allocator<cls_rgw_obj_key> >*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const*, bool)+0x2ed) [0x7f40e1f4fc1d]",
        "(RGWRados::Object::Write::_do_write_meta(unsigned long, unsigned long, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, bool, bool, void*, optional_yield)+0xc1d) [0x7f40e1f62ddd]",
        "(RGWRados::Object::Write::write_meta(unsigned long, unsigned long, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, optional_yield)+0x225) [0x7f40e1f63ee5]",
        "(rgw::putobj::AtomicObjectProcessor::complete(unsigned long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, char const*, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const*, rgw_zone_set*, bool*, optional_yield)+0x2aa) [0x7f40e1f2a02a]",
        "(RGWRados::fetch_remote_obj(RGWObjectCtx&, rgw_user const&, req_info*, rgw_zone_id const&, rgw_obj const&, rgw_obj const&, RGWBucketInfo const&, RGWBucketInfo const*, std::optional<rgw_placement_rule>, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const*, bool, char const*, char const*, RGWRados::AttrsMod, bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, RGWObjCategory, std::optional<unsigned long>, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*, void (*)(long, void*), void*, DoutPrefixProvider const*, RGWFetchObjFilter*, rgw_zone_set*, std::optional<unsigned long>*)+0x16d2) [0x7f40e1f6ec02]",
        "(RGWAsyncFetchRemoteObj::_send_request()+0x3d6) [0x7f40e1eb8376]",
        "(RGWAsyncRadosProcessor::handle_request(RGWAsyncRadosRequest*)+0x20) [0x7f40e1eb2170]",
        "(RGWAsyncRadosProcessor::RGWWQ::_process(RGWAsyncRadosRequest*, ThreadPool::TPHandle&)+0xd) [0x7f40e1eba6bd]",
        "(ThreadPool::worker(ThreadPool::WorkThread*)+0x9fa) [0x7f40d7c95aaa]",
        "(ThreadPool::WorkThread::entry()+0x11) [0x7f40d7c96991]",
        "(()+0x76db) [0x7f40d6c506db]",
        "(clone()+0x3f) [0x7f40e149871f]" 
    ],
    "ceph_version": "15.2.10",
    "crash_id": "2021-04-20T08:27:59.141259Z_e2271182-b329-4b99-a092-3ebca4e55f68",
    "entity_name": "client.rgw.rgw.rgw0",
    "os_id": "ubuntu",
    "os_name": "Ubuntu",
    "os_version": "18.04.5 LTS (Bionic Beaver)",
    "os_version_id": "18.04",
    "process_name": "radosgw",
    "stack_sig": "83f21166a837650e67954ae8d09579e5344c9dc091e8afac3e03afd7974e5d07",
    "timestamp": "2021-04-20T08:27:59.141259Z",
    "utsname_hostname": "rgw",
    "utsname_machine": "x86_64",
    "utsname_release": "4.15.0-135-generic",
    "utsname_sysname": "Linux",
    "utsname_version": "#139-Ubuntu SMP Mon Jan 18 17:38:24 UTC 2021" 
}

Could anyone help us on this? Is this a miss-config problem or bug?

Thanks