Ceph » crimson

osd.3: https://pulpito.ceph.com/matan-2024-03-27_13:02:57-crimson-rados-main-distro-crimson-smithi/7626294

After adding a restart OSDs to the thrash tests: https://github.com/ceph/ceph/pull/56511

DEBUG 2024-03-27 13:26:06,805 [shard 0:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): starting start_pg_operation
DEBUG 2024-03-27 13:26:06,805 [shard 0:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): start_pg_operation in await_active stage
DEBUG 2024-03-27 13:26:06,805 [shard 0:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): start_pg_operation active, entering await_map
DEBUG 2024-03-27 13:26:06,805 [shard 0:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): start_pg_operation await_map stage
DEBUG 2024-03-27 13:26:06,806 [shard 0:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): got map 26, entering get_pg_mapping
DEBUG 2024-03-27 13:26:06,806 [shard 0:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): can_create=false, target-core=2
DEBUG 2024-03-27 13:26:06,806 [shard 0:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): send 37 to the remote pg core 2
DEBUG 2024-03-27 13:26:06,806 [shard 2:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): entering create_or_wait_pg
DEBUG 2024-03-27 13:26:06,806 [shard 2:main] osd - background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))})): have_pg
DEBUG 2024-03-27 13:26:06,806 [shard 2:main] osd - 0x603000429b00 RecoverySubRequest::with_pg: RecoverySubRequest::with_pg: background_recovery_sub(id=362, detail=MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))}))
DEBUG 2024-03-27 13:26:06,806 [shard 2:main] osd - handle_pull_response: MOSDPGPush(3.d 26/25 {PushOp(3:bd1211d5:::smithi05531420-40:1, version: 18'16, data_included: [655473~716476,2099033~332100], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))}) v4
DEBUG 2024-03-27 13:26:06,806 [shard 2:main] osd - handle_pull_response ObjectRecoveryInfo(3:bd1211d5:::smithi05531420-40:1@0'0, size: 2655473, copy_subset: [(0, 2655473)], clone_subset: {}, snapset: 1=[]:{1: [1]}, object_exist: false) ObjectRecoveryProgress(!first, data_recovered_to: 2431133, data_complete: false, omap_recovered_to: , omap_complete: true, error: false) data.size() is 1048576 data_included: [(655473, 716476), (2099033, 332100)]
DEBUG 2024-03-27 13:26:06,807 [shard 2:main] osd -  pg_epoch 26 pg[3.d( v 26'20 lc 17'15 (0'0,26'20] local-lis/les=25/26 n=0 ec=14/14 lis/c=25/14 les/c/f=26/15/0 sis=25) [3,0] r=0 lpr=25 pi=[14,25)/1 luod=26'21 lua=21'18 crt=26'21 mlcod 17'15 active+recovering+undersized+degraded  ObjectContextLoader::with_head_obc: object 3:bd1211d5:::smithi05531420-40:head
DEBUG 2024-03-27 13:26:06,807 [shard 2:main] osd -  pg_epoch 26 pg[3.d( v 26'20 lc 17'15 (0'0,26'20] local-lis/les=25/26 n=0 ec=14/14 lis/c=25/14 les/c/f=26/15/0 sis=25) [3,0] r=0 lpr=25 pi=[14,25)/1 luod=26'21 lua=21'18 crt=26'21 mlcod 17'15 active+recovering+undersized+degraded  ObjectContextLoader::get_or_load_obc: cache hit on 3:bd1211d5:::smithi05531420-40:head
DEBUG 2024-03-27 13:26:06,807 [shard 2:main] osd - resolve_oid oid.snap=1,head snapset.seq=1
DEBUG 2024-03-27 13:26:06,807 [shard 2:main] osd -  pg_epoch 26 pg[3.d( v 26'20 lc 17'15 (0'0,26'20] local-lis/les=25/26 n=0 ec=14/14 lis/c=25/14 les/c/f=26/15/0 sis=25) [3,0] r=0 lpr=25 pi=[14,25)/1 luod=26'21 lua=21'18 crt=26'21 mlcod 17'15 active+recovering+undersized+degraded  ObjectContextLoader::get_or_load_obc: cache miss on 3:bd1211d5:::smithi05531420-40:1
DEBUG 2024-03-27 13:26:06,807 [shard 2:main] osd - load_metadata: object 3:bd1211d5:::smithi05531420-40:1 doesn't exist, returning empty metadata
DEBUG 2024-03-27 13:26:06,807 [shard 2:main] osd -  pg_epoch 26 pg[3.d( v 26'20 lc 17'15 (0'0,26'20] local-lis/les=25/26 n=0 ec=14/14 lis/c=25/14 les/c/f=26/15/0 sis=25) [3,0] r=0 lpr=25 pi=[14,25)/1 luod=26'21 lua=21'18 crt=26'21 mlcod 17'15 active+recovering+undersized+degraded  ObjectContextLoader::load_obc: loaded obs 3:bd1211d5:::smithi05531420-40:1(0'0 unknown.0.0:0  s 0 uv 0 alloc_hint [0 0 0]) for 3:bd1211d5:::smithi05531420-40:1
DEBUG 2024-03-27 13:26:06,807 [shard 2:main] osd -  pg_epoch 26 pg[3.d( v 26'20 lc 17'15 (0'0,26'20] local-lis/les=25/26 n=0 ec=14/14 lis/c=25/14 les/c/f=26/15/0 sis=25) [3,0] r=0 lpr=25 pi=[14,25)/1 luod=26'21 lua=21'18 crt=26'21 mlcod 17'15 active+recovering+undersized+degraded  ObjectContextLoader::load_obc: returning obc 3:bd1211d5:::smithi05531420-40:1(0'0 unknown.0.0:0  s 0 uv 0 alloc_hint [0 0 0]) for 3:bd1211d5:::smithi05531420-40:1

ceph-osd: /home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.0.0-2476-g56e21662/rpm/el9/BUILD/ceph-19.0.0-2476-g56e21662/src/crimson/osd/replicated_recovery_backend.cc:886: void ReplicatedRecoveryBackend::recalc_subsets(ObjectRecoveryInfo&, crimson::osd::SnapSetContextRef): Assertion `ssc' failed.
Aborting on shard 2.
Backtrace:
 0# 0x00007F182BAA154C in /lib64/libc.so.6
 1# raise in /lib64/libc.so.6
 2# abort in /lib64/libc.so.6
 3# 0x00007F182BA2871B in /lib64/libc.so.6
 4# 0x00007F182BA4DCA6 in /lib64/libc.so.6
 5# ReplicatedRecoveryBackend::recalc_subsets(ObjectRecoveryInfo&, boost::intrusive_ptr<crimson::osd::SnapSetContext>) in ceph-osd