Actions
Bug #65532
openosd crashes due to invalid clone_range ops
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Description
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _txc_create osr 0x4d34b00 = 0x4f78900 seq 9
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1).collection(3.0_head 0x4d092c0) get_onode oid #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# key 0x7F8000000000000003005A1FEE'!scephqa02.cpp.bjat94996-4!='0x0000000000000028FFFFFFFFFFFFFFFF6F
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1).collection(3.0_head 0x4d092c0) r 0 v.len 470
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore.OnodeSpace(0x4d09400 in 0x300000b1c800) add_onode #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# 0x4f9cb00
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - _add 0x300000b1c800 #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# added, num=73
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _rmattrs 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:28#
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _rmattrs 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# = 0
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _truncate 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# 0x3c85d3
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _do_truncate 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# 0x3c85d3
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _truncate 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# 0x3c85d3 = 0
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _setattrs 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# 2 keys
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _setattrs 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# 2 keys = 0
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1).collection(3.0_head 0x4d092c0) get_onode oid #3:005a1fee:::scephqa02.cpp.bjat94996-4:2d# key 0x7F8000000000000003005A1FEE'!scephqa02.cpp.bjat94996-4!='0x000000000000002DFFFFFFFFFFFFFFFF6F
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluefs - bluefs _read_random h 0x4d2e900 0x2c8d~a05 from file(ino 25 size 0x54bb1 mtime 2024-04-16T10:34:32.964782+0000 allocated 60000 alloc_commit 60000 extents [1:0x5b8000~60000])
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluefs - bluefs _read_random read random 0x2c8d~a05 of 1:0x5b8000~60000
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluefs - bluefs _read_random got 0xa05
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1).collection(3.0_head 0x4d092c0) r 0 v.len 451
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore.OnodeSpace(0x4d09400 in 0x300000b1c800) add_onode #3:005a1fee:::scephqa02.cpp.bjat94996-4:2d# 0x4f9cdc0
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - _add 0x300000b1c800 #3:005a1fee:::scephqa02.cpp.bjat94996-4:2d# added, num=74
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _clone_range 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:2d# -> #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# from 0x0~3c85d3 to offset 0x0
DEBUG 2024-04-16 10:34:35,354 [shard 0:main] bluestore - bluestore(/da1/var/lib/ceph/osd/ceph-1) _clone_range 3.0_head #3:005a1fee:::scephqa02.cpp.bjat94996-4:2d# -> #3:005a1fee:::scephqa02.cpp.bjat94996-4:28# from 0x0~3c85d3 to offset 0x0 = -22
ERROR 2024-04-16 10:34:35,354 [shard 0:main] none - bluestore(/da1/var/lib/ceph/osd/ceph-1) _txc_add_transaction error (22) Invalid argument not handled on operation 30 (op 3, counting from 0)
ERROR 2024-04-16 10:34:35,354 [shard 0:main] none - bluestore(/da1/var/lib/ceph/osd/ceph-1) unexpected error code
WARN 2024-04-16 10:34:35,354 [shard 0:main] bluestore - _dump_transaction transaction dump:
{
"ops": [
{
"op_num": 0,
"op_name": "rmattrs",
"collection": "3.0_head",
"oid": "#3:005a1fee:::scephqa02.cpp.bjat94996-4:28#"
},
{
"op_num": 1,
"op_name": "truncate",
"collection": "3.0_head",
"oid": "#3:005a1fee:::scephqa02.cpp.bjat94996-4:28#",
"offset": 3966419
},
{
"op_num": 2,
"op_name": "setattrs",
"collection": "3.0_head",
"oid": "#3:005a1fee:::scephqa02.cpp.bjat94996-4:28#",
"attr_lens": {
"_": 232,
"__header": 80
}
},
{
"op_num": 3,
"op_name": "clonerange2",
"collection": "3.0_head",
"src_oid": "#3:005a1fee:::scephqa02.cpp.bjat94996-4:2d#",
"dst_oid": "#3:005a1fee:::scephqa02.cpp.bjat94996-4:28#",
"src_offset": 0,
"len": 3966419,
"dst_offset": 0
},
{
"op_num": 4,
"op_name": "omap_rmkeys",
"collection": "3.0_head",
"oid": "#3:00000000:::snapmapper:0#",
"attrs": [
"OBJ_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4.."
]
},
{
"op_num": 5,
"op_name": "omap_rmkeys",
"collection": "3.0_head",
"oid": "#3:00000000:::snapmapper:0#",
"attrs": [
"SNA_3_0000000000000021_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4..",
"SNA_3_0000000000000026_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4..",
"SNA_3_0000000000000027_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4..",
"SNA_3_0000000000000028_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4.."
]
},
{
"op_num": 6,
"op_name": "omap_setkeys",
"collection": "3.0_head",
"oid": "#3:00000000:::snapmapper:0#",
"attr_lens": {
"OBJ_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4..": 113
}
},
{
"op_num": 7,
"op_name": "omap_setkeys",
"collection": "3.0_head",
"oid": "#3:00000000:::snapmapper:0#",
"attr_lens": {
"SNA_3_0000000000000021_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4..": 93,
"SNA_3_0000000000000027_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4..": 93,
"SNA_3_0000000000000028_0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4..": 93
}
},
{
"op_num": 8,
"op_name": "omap_rmkeys",
"collection": "3.0_head",
"oid": "#3:00000000::::head#",
"attrs": [
"missing/0000000000000003.00A58F77.28.scephqa02%ecpp%ebjat94996-4.."
]
},
{
"op_num": 9,
"op_name": "omap_setkeys",
"collection": "3.0_head",
"oid": "#3:00000000::::head#",
"attr_lens": {
"_info": 1017
}
},
{
"op_num": 10,
"op_name": "omap_rmkeys",
"collection": "3.0_head",
"oid": "#3:00000000::::head#",
"attrs": [
"_fastinfo"
]
}
]
}
ERROR 2024-04-16 10:34:35,354 none - /da1/xxh/rpmbuild/BUILD/ceph-19.0.0-2545-g6d9d2613e90/src/os/bluestore/BlueStore.cc:15141 : In function 'void BlueStore::_txc_add_transaction(TransContext*, ObjectStore::Transaction*)', abort(%s)
unexpected error
Aborting.
Backtrace:
0# gsignal in /lib64/libc.so.6
1# abort in /lib64/libc.so.6
2# ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) in ceph-osd
3# BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*) in ceph-osd
4# BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*) in ceph-osd
5# 0x0000000001FBF74E in ceph-osd
6# crimson::os::ThreadPool::loop(std::chrono::duration<long, std::ratio<1l, 1000l> >, unsigned long) in ceph-osd
7# 0x00002B1C4DD3BBA3 in /lib64/libstdc++.so.6
8# 0x00002B1C50FC91CF in /lib64/libpthread.so.0
9# clone in /lib64/libc.so.6
Content of /proc/self/maps:
00400000-0118f000 r--p 00000000 fd:00 5919297 /usr/bin/ceph-osd
0118f000-03100000 r-xp 00d8f000 fd:00 5919297 /usr/bin/ceph-osd
03100000-03f43000 r--p 02d00000 fd:00 5919297 /usr/bin/ceph-osd
03f43000-0400e000 r--p 03b42000 fd:00 5919297 /usr/bin/ceph-osd
0400e000-04048000 rw-p 03c0d000 fd:00 5919297 /usr/bin/ceph-osd
04048000-04097000 rw-p 00000000 00:00 0
044dc000-0dfa0000 rw-p 00000000 00:00 0 [heap]
Updated by Matan Breizman 9 days ago
- Status changed from New to Fix Under Review
- Pull request ID changed from 56938 to 56606
Actions