Ceph » bluestore

[ RUN ] ObjectStore/StoreTestSpecificAUSize.SyntheticMatrixSharding/1
---------------------- 1 / 1 ----------------------
bluestore_min_alloc_size = 4096
num_ops = 50000
max_write = 65536
max_size = 262144
alignment = 4096
bluestore_max_blob_size = 65536
bluestore_extent_map_shard_min_size = 60
bluestore_extent_map_shard_max_size = 300
bluestore_extent_map_shard_target_size = 150
bluestore_default_buffered_read = true
bluestore_default_buffered_write = true
seeding object 0
seeding object 500
Op 0
available_objects: 1000 in_flight_objects: 0 total objects: 1000 in_flight 0
Op 1000
available_objects: 1014 in_flight_objects: 0 total objects: 1014 in_flight 0
Op 2000
available_objects: 1013 in_flight_objects: 0 total objects: 1013 in_flight 0
Op 3000
available_objects: 1003 in_flight_objects: 0 total objects: 1003 in_flight 0
Op 4000
available_objects: 1014 in_flight_objects: 0 total objects: 1014 in_flight 0
Op 5000
available_objects: 1020 in_flight_objects: 0 total objects: 1020 in_flight 0
Op 6000
available_objects: 1023 in_flight_objects: 0 total objects: 1023 in_flight 0
Op 7000
available_objects: 1033 in_flight_objects: 0 total objects: 1033 in_flight 0
Op 8000
available_objects: 1045 in_flight_objects: 0 total objects: 1045 in_flight 0
Op 9000
available_objects: 1057 in_flight_objects: 0 total objects: 1057 in_flight 0
Op 10000
available_objects: 1055 in_flight_objects: 0 total objects: 1055 in_flight 0
Op 11000
available_objects: 1051 in_flight_objects: 0 total objects: 1051 in_flight 0
/home/if/ceph.3/src/os/bluestore/BlueStore.cc: In function 'BlueStore::Blob::copy_extents(ceph::common::CephContext*, const BlueStore::Blob&, uint32_t, uint32_t, uint32_t, uint32_t)::<lambda(const PExtentVector&, uint32_t, uint32_t)>' thread 7efc1fc4fa40 time 2023-12-08T14:44:27.011081+0300
/home/if/ceph.3/src/os/bluestore/BlueStore.cc: 2550: FAILED ceph_assert(pos + len <= it->length)
ceph version Development (no_version) reef (dev)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x132) [0x7efc206aba37]
2: /home/if/ceph.3/build/lib/libceph-common.so.2(+0x2abc17) [0x7efc206abc17]
3: (BlueStore::Blob::copy_extents(ceph::common::CephContext*, BlueStore::Blob const&, unsigned int, unsigned int, unsigned int, unsigned int)+0x480) [0x55628cb448a0]
4: (BlueStore::Blob::copy_from(ceph::common::CephContext*, BlueStore::Blob const&, unsigned int, unsigned int, unsigned int)+0x13e) [0x55628cb44a2e]
5: (BlueStore::ExtentMap::dup_esb(BlueStore*, BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long&, unsigned long&, unsigned long&)+0x89f) [0x55628cbd8def]
6: (BlueStore::_do_clone_range(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, unsigned long, unsigned long)+0xe6) [0x55628cbda036]
7: (BlueStore::_clone_range(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, unsigned long, unsigned long)+0x22e) [0x55628cbdb89e]
8: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x1536) [0x55628cbdd286]
9: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2e1) [0x55628cbde991]
10: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ceph::os::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0xa1) [0x55628ca6e3d1]
11: ../build/bin/ceph_test_objectstore(+0x43390e) [0x55628c9fd90e]
12: (StoreTest::doSyntheticTest(int, int, unsigned long, unsigned long, unsigned long)+0x829) [0x55628ca27c99]
13: (StoreTestSpecificAUSize::do_matrix_choose(char const* () [10], int, int, int, void (StoreTestSpecificAUSize::)())+0x147) [0x55628ca88be7]
14: (StoreTestSpecificAUSize::do_matrix_choose(char const* () [10], int, int, int, void (StoreTestSpecificAUSize::)())+0x147) [0x55628ca88be7]
15: (StoreTestSpecificAUSize::do_matrix_choose(char const* () [10], int, int, int, void (StoreTestSpecificAUSize::)())+0x147) [0x55628ca88be7]
16: (StoreTestSpecificAUSize::do_matrix_choose(char const* () [10], int, int, int, void (StoreTestSpecificAUSize::)())+0x147) [0x55628ca88be7]
17: (StoreTestSpecificAUSize::do_matrix_choose(char const* () [10], int, int, int, void (StoreTestSpecificAUSize::)())+0x147) [0x55628ca88be7]
18: (StoreTestSpecificAUSize::do_matrix_choose(char const* () [10], int, int, int, void (StoreTestSpecificAUSize::)())+0x147) [0x55628ca88be7]
19: (StoreTestSpecificAUSize::do_matrix_choose(char const* () [10], int, int, int, void (StoreTestSpecificAUSize::)())+0x147) [0x55628ca88be7]
20: ../build/bin/ceph_test_objectstore(+0x4bf1e3) [0x55628ca891e3]
21: (StoreTestSpecificAUSize::do_matrix(char const* () [10], void (StoreTestSpecificAUSize::)())+0x29d) [0x55628ca89b7d]
22: ../build/bin/ceph_test_objectstore(+0x3ccbbd) [0x55628c996bbd]
23: (void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*)+0x4d) [0x55628ccf45bd]
24: (testing::Test::Run()+0xce) [0x55628cceb0fe]
25: (testing::TestInfo::Run()+0x135) [0x55628cceb255]
26: (testing::TestSuite::Run()+0xc9) [0x55628cceb349]
27: (testing::internal::UnitTestImpl::RunAllTests()+0x48c) [0x55628cceb8bc]
28: (testing::UnitTest::Run()+0x84) [0x55628ccebb74]
29: main()
30: /lib64/libc.so.6(+0x281b0) [0x7efc1f6281b0]
31: __libc_start_main()

Replicated, verified.

Problem is replicated by:
./bin/ceph_test_objectstore --gtest_filter=\*SyntheticMatrixSharding/1 --bluestore_allocator=stupid

The bug is in Blob::copy_extents function that assumes it operates on single allocation units,
while it might be multiple ones.