Ceph » bluestore

After a power cut, OSD refuse to start, the output showing as OSD.png
The corrupted files allocated with the following extents:

2023-10-08T03:33:15.614+0000 7f688d480880 20 bluefs _replay 0x16fea:  op_dir_link  db.wal/053463.log to 53459
...
2023-10-08T03:33:22.135+0000 7f688d480880 10 bluefs _read h 0x556cb98c9880 0x10e80000~8000 from file(ino 53459 size 0x114c2cd0 mtime 2023-10-06T19:18:20.272274+0000 allocated 114d0000 alloc_commit 114d0000 extents [1:0x347be110000~10000,1:0x347be1c0000~190000,1:0x347be360000~20000,1:0x851b800000~20000,1:0x347be380000~170000,1:0x7ba48e0000~80000,1:0x5620640000~40000,1:0x347be4f0000~10000,1:0x851b820000~20000,1:0x347be500000~110000,1:0x851b840000~20000,1:0x347be610000~120000,1:0x851b860000~20000,1:0x347be730000~10000,1:0x851b880000~20000,1:0x347be740000~b0000,1:0x851b8a0000~20000,1:0x347be7f0000~f0000,1:0x851b900000~80000,1:0x347be8e0000~20000,1:0x851b980000~60000,1:0x347be910000~40000,1:0x851bac0000~20000,1:0x347be950000~30000,1:0x851bae0000~20000,1:0x347be980000~20000,1:0x347be9b0000~f0000,1:0x851bb00000~20000,1:0x347beaa0000~50000,1:0x347beb40000~90000,1:0x851bb20000~20000,1:0x347bebd0000~80000,1:0x851bb40000~20000,1:0x347bec50000~70000,1:0x347becd0000~d0000,1:0x347bee50000~20000,1:0x347bf270000~f0000,1:0x851bb60000~60000,1:0x347bf360000~10000,1:0x851bbc0000~20000,1:0x347bf370000~70000,1:0x347bf3f0000~20000,1:0x851bbe0000~20000,1:0x347bf410000~60000,1:0x851bc00000~20000,1:0x347bf470000~70000,1:0x851bc40000~20000,1:0x347bf4e0000~40000,1:0x851bc60000~20000,1:0x347bf520000~a0000,1:0x5620680000~40000,1:0x347bf6f0000~50000,1:0x851bc80000~20000,1:0x347bf740000~f0000,1:0x347bfc50000~200000,1:0x347bfe80000~230000,1:0x347c00c0000~1d0000,1:0x347c02a0000~40000,1:0x851bcc0000~20000,1:0x347c02e0000~280000,1:0x851bd20000~20000,1:0x347c0560000~70000,1:0x851bd40000~20000,1:0x347c05d0000~c0000,1:0x347c06a0000~90000,1:0x851bd60000~60000,1:0x347c0730000~150000,1:0x851bdc0000~20000,1:0x347c0880000~10000,1:0x56206c0000~40000,1:0x347c0890000~f0000,1:0x851bde0000~20000,1:0x347c0980000~70000,1:0x347c0a00000~e0000,1:0x347c0af0000~1c0000,1:0x347c0cc0000~100000,1:0x347c11c0000~a0000,1:0x347c1270000~10000,1:0x851be00000~20000,1:0x347c1290000~e0000,1:0x851be20000~20000,1:0x347c1370000~30000,1:0x7ba49f0000~80000,1:0x851be40000~20000,1:0x347c13a0000~130000,1:0x851be60000~20000,1:0x347c14d0000~70000,1:0x851be80000~20000,1:0x347c1540000~60000,1:0x851bea0000~20000,1:0x347c15a0000~10000,1:0x851bec0000~20000,1:0x347c15b0000~c0000,1:0x5620700000~40000,1:0x851bee0000~20000,1:0x347c1670000~30000,1:0x5620740000~40000,1:0x347c16a0000~130000,1:0x851bf20000~20000,1:0x347c17d0000~50000,1:0x347c1830000~30000,1:0x851bf60000~20000,1:0x347c1860000~c0000,1:0x851bf80000~20000,1:0x347c1920000~230000,1:0x851bfc0000~20000,1:0x347c1b50000~50000,1:0x851bfe0000~20000,1:0x347c1ba0000~c0000,1:0x851c000000~20000,1:0x347c1c60000~160000,1:0x347c1f00000~d0000,1:0x851c020000~20000,1:0x347c1fd0000~240000,1:0x851c040000~40000,1:0x56207a0000~40000,1:0x347c2210000~d0000,1:0x851c080000~20000,1:0x347c22e0000~110000,1:0x851c0a0000~20000,1:0x347c23f0000~50000,1:0x347c2470000~c0000,1:0x851c0c0000~20000,1:0x347c2530000~a0000,1:0x851c0e0000~20000,1:0x347c25d0000~30000,1:0x347c2650000~70000,1:0x851c100000~40000,1:0x347c26c0000~70000,1:0x56207e0000~40000,1:0x347c2730000~20000,1:0x851c140000~20000,1:0x347c2750000~20000,1:0x851c160000~40000,1:0x347c2770000~140000,1:0x347c2cb0000~10000,1:0x851c1a0000~20000,1:0x347c2cc0000~80000,1:0x851c1c0000~20000,1:0x347c2d40000~90000,1:0x851c1e0000~20000,1:0x347c2dd0000~20000,1:0x347c2e00000~1c0000,1:0x347c3130000~40000,1:0x851c240000~20000,1:0x347c3170000~150000,1:0x347c3380000~20000,1:0x851c260000~20000,1:0x347c33a0000~f0000,1:0x5620820000~40000,1:0x851c280000~60000,1:0x347c3490000~b0000,1:0x851c2e0000~60000,1:0x347c3540000~30000,1:0x5620860000~40000,1:0x347c3570000~30000,1:0x851c340000~20000,1:0x347c35a0000~190000,1:0x851c360000~20000,1:0x347c3730000~b0000,1:0x851c380000~80000,1:0x347c37f0000~40000,1:0x851c400000~20000,1:0x347c3830000~100000,1:0x851c420000~20000,1:0x347c39a0000~20000,1:0x851c440000~20000,1:0x347c39c0000~40000,1:0x347c3a10000~30000,1:0x851c460000~20000,1:0x347c3a40000~10000,1:0x851c480000~40000,1:0x347c3a50000~70000,1:0x347c3ad0000~50000,1:0x347c3c50000~b0000,1:0x851c4c0000~20000,1:0x347c3d00000~d0000,1:0x851c4e0000~20000,1:0x347c3dd0000~10000,1:0x851c500000~20000,1:0x347c3de0000~70000,1:0x347c3e60000~80000,1:0x347c3ef0000~a0000,1:0x851c520000~40000,1:0x347c3f90000~70000,1:0x851c560000~20000,1:0x347c4000000~310000,1:0x851c6a0000~20000,1:0x347c4310000~10000,1:0x851c6c0000~20000,1:0x347c4320000~a0000,1:0x851c6e0000~20000,1:0x347c47c0000~70000,1:0x851c700000~20000,1:0x347c4830000~70000,1:0x851c720000~20000,1:0x347c48a0000~170000,1:0x347c4a40000~b0000,1:0x851c740000~20000,1:0x347c4af0000~8a0000,1:0x56208a0000~40000,1:0x347c5390000~10000,1:0x851c900000~20000,1:0x56208e0000~40000,1:0x347c53a0000~f0000,1:0x7ba4a70000~80000,1:0x851c920000~20000,1:0x347c5490000~120000,1:0x347c55c0000~200000,1:0x347c57d0000~40000,1:0x851c960000~20000,1:0x5620920000~40000,1:0x347c5810000~200000,1:0x851c980000~20000,1:0x347c5a10000~50000,1:0x347c5e60000~80000,1:0x5620960000~40000,1:0x347c5ee0000~10000,1:0x851c9a0000~60000,1:0x347c5ef0000~10000,1:0x851ca00000~20000,1:0x347c5f00000~80000,1:0x851ca20000~40000,1:0x347c5f80000~50000,1:0x851ca60000~20000,1:0x347c5fd0000~60000,1:0x347c60c0000~60000,1:0x851ca80000~20000,1:0x347c6120000~30000,1:0x851caa0000~20000,1:0x347c6150000~30000,1:0x347c65a0000~f0000,1:0x851cac0000~40000,1:0x347c6690000~80000,1:0x851cb00000~20000,1:0x347c6710000~80000,1:0x851cb20000~20000,1:0x7ba4af0000~80000,1:0x347c6790000~10000,1:0x56209a0000~40000,1:0x347c67a0000~c0000,1:0x347c6870000~e0000,1:0x7ba4b70000~80000,1:0x851cb40000~40000,1:0x347c6950000~50000,1:0x851cb80000~20000,1:0x347c69a0000~10000,1:0x851cba0000~20000,1:0x347c69b0000~10000,1:0x851cbc0000~20000,1:0x347c69c0000~90000,1:0x851cbe0000~60000,1:0x347c6a50000~120000,1:0x851cca0000~20000,1:0x347c6b70000~130000,1:0x851ccc0000~20000,1:0x347c6ca0000~30000,1:0x347c6d00000~60000,1:0x851cce0000~60000,1:0x347c6d60000~b0000,1:0x851cd40000~20000,1:0x347c6e10000~70000,1:0x851cd60000~20000,1:0x347c6e80000~120000,1:0x347c7030000~a0000,1:0x851ce60000~60000,1:0x347c70d0000~10000,1:0x851cec0000~20000,1:0x347c70e0000~30000,1:0x851cee0000~20000,1:0x347c7110000~3f0000,1:0x347c7690000~20000,1:0x851cf00000~20000,1:0x347c76b0000~a0000,1:0x347c7b50000~210000,1:0x56209e0000~40000,1:0x347c7d60000~30000,1:0x851cf40000~40000,1:0x347c7d90000~10000,1:0x851cf80000~20000,1:0x347c7da0000~60000,1:0x851cfa0000~20000,1:0x347c7e90000~b0000,1:0x851cfc0000~20000,1:0x347c7f40000~10000,1:0x5620a20000~40000,1:0x347c7f50000~20000,1:0x851cfe0000~40000,1:0x347c7f70000~1d0000,1:0x851d0c0000~20000,1:0x347c8150000~240000,1:0x851d0e0000~20000,1:0x347c8390000~e0000,1:0x851d100000~20000,1:0x347c8490000~10000,1:0x851d120000~20000,1:0x347c84a0000~20000,1:0x851d140000~20000,1:0x347c84d0000~10000,1:0x851d160000~20000,1:0x347c84e0000~40000,1:0x851d180000~20000,1:0x347c8520000~10000,1:0x347c8540000~90000,1:0x851d280000~20000,1:0x347c85d0000~e0000,1:0x851d2a0000~20000,1:0x347c86b0000~20000,1:0x851d2c0000~20000,1:0x347c86d0000~80000,1:0x851d3c0000~20000,1:0x347c8750000~1c0000,1:0x347c8960000~1a0000,1:0x851d400000~20000,1:0x347c8b00000~90000,1:0x851d440000~20000,1:0x347c8b90000~e0000,1:0x7ba4c70000~80000,1:0x5620a60000~40000,1:0x347c8c70000~10000,1:0x851d460000~20000,1:0x347c8c80000~20000,1:0x851d480000~20000,1:0x347c8ca0000~20000,1:0x851d4a0000~20000,1:0x5620aa0000~40000,1:0x851d4c0000~20000,1:0x347c8d30000~50000,1:0x851d4e0000~20000,1:0x347c8d80000~30000,1:0x851d500000~20000,1:0x347c8db0000~20000,1:0x347c8de0000~10000,1:0x851d520000~20000,1:0x5620ae0000~40000,1:0x851d540000~40000,1:0x347c8df0000~10000,1:0x851d580000~20000,1:0x347c8e00000~50000,1:0x851d5a0000~20000,1:0x347c8e60000~f0000,1:0x5620b20000~40000,1:0x347c8f50000~110000,1:0x851d720000~60000,1:0x347c9060000~10000,1:0x5620d20000~40000,1:0x347c9070000~140000,1:0x851d780000~20000,1:0x347c91b0000~90000,1:0x5620d60000~40000,1:0x347c9240000~10000,1:0x851d7a0000~20000,1:0x347c9250000~10000,1:0x851d7c0000~40000,1:0x347c9260000~60000,1:0x347c92d0000~30000,1:0x347c9700000~20000,1:0x851d800000~20000,1:0x347c9720000~90000,1:0x347c97e0000~80000,1:0x347c9870000~1b0000,1:0x851d900000~20000,1:0x347c9a20000~10000,1:0x347c9a40000~60000,1:0x347c9c30000~50000,1:0x347c9c90000~20000,1:0x347c9cc0000~100000,1:0x851d920000~20000,1:0x5620da0000~40000,1:0x347c9dc0000~150000,1:0x5620de0000~40000,1:0x851d940000~40000,1:0x347ca040000~10000,1:0x851d980000~20000,1:0x347ca050000~110000,1:0x851d9a0000~20000,1:0x347ca160000~20000,1:0x347ca250000~50000,1:0x851d9c0000~80000,1:0x347ca2a0000~50000,1:0x347ca300000~90000,1:0x851da40000~20000,1:0x347ca390000~10000,1:0x851da60000~20000,1:0x347ca3a0000~30000,1:0x851da80000~20000,1:0x347ca3d0000~90000,1:0x347ca610000~c0000,1:0x5620e20000~40000,1:0x347ca6d0000~10000,1:0x851dc00000~20000,1:0x347ca6e0000~10000,1:0x851dc20000~60000,1:0x347ca6f0000~80000,1:0x347ca780000~30000,1:0x347ca7c0000~70000,1:0x851dc80000~20000,1:0x347ca830000~70000,1:0x851dcc0000~40000,1:0x347ca920000~30000,1:0x347ca960000~10000,1:0x851dd00000~60000,1:0x347ca970000~20000,1:0x851dd60000~20000,1:0x347ca990000~20000,1:0x851dd80000~20000,1:0x347ca9b0000~c0000,1:0x851dda0000~20000,1:0x347caa70000~30000,1:0x851ddc0000~20000,1:0x347caaa0000~90000,1:0x851dde0000~20000,1:0x347cab30000~30000,1:0x851de00000~20000,1:0x347cab60000~a0000,1:0x851de20000~20000,1:0x347cac00000~60000,1:0x851de40000~20000,1:0x347cac60000~90000,1:0x851de80000~20000,1:0x347cacf0000~140000,1:0x851df80000~20000,1:0x347cae30000~240000,1:0x851dfa0000~20000,1:0x347cb070000~30000,1:0x347cb0b0000~60000,1:0x347cb510000~b0000,1:0x851dfc0000~60000,1:0x347cb5c0000~50000,1:0x851e020000~20000,1:0x347cb610000~40000,1:0x851e040000~20000,1:0x347cb650000~30000,1:0x851e060000~20000,1:0x347cb770000~30000,1:0x851e080000~20000,1:0x347cb7a0000~a0000,1:0x851e0a0000~20000,1:0x347cb840000~80000,1:0x851e0c0000~60000,1:0x347cb8d0000~170000,1:0x851e120000~20000,1:0x347cba40000~10000,1:0x347cba60000~160000,1:0x851e140000~20000,1:0x347cbbc0000~30000,1:0x851e160000~20000,1:0x347cbbf0000~10000,1:0x7ba4d80000~80000,1:0x347cbc00000~70000,1:0x5620ea0000~40000,1:0x851e180000~20000,1:0x347cbc70000~40000,1:0x851e1a0000~20000,1:0x347cbcb0000~1d0000,1:0x851e1c0000~20000,1:0x347cbe90000~70000,1:0x851e1e0000~20000,1:0x347cbf70000~30000,1:0x851e200000~60000,1:0x347cbfa0000~10000,1:0x851e260000~40000,1:0x347cbfb0000~c0000,1:0x851e2a0000~60000,1:0x347cc070000~40000,1:0x5620fa0000~40000,1:0x347cc0b0000~80000,1:0x851e300000~20000,1:0x347cc130000~90000,1:0x347cc1d0000~150000,1:0x347cc3b0000~e0000,1:0x851e340000~20000,1:0x347cc490000~20000,1:0x851e360000~20000,1:0x347cc4b0000~30000,1:0x851e380000~20000,1:0x347cc4e0000~130000,1:0x851e3a0000~80000,1:0x347cc610000~10000,1:0x851e420000~20000,1:0x347cc620000~50000,1:0x347cc6a0000~40000,1:0x851e440000~20000,1:0x347cc6f0000~40000,1:0x347cc860000~40000,1:0x851e460000~20000,1:0x347cc8a0000~140000,1:0x851e480000~20000,1:0x347cc9e0000~40000,1:0x851e4a0000~20000,1:0x347cca20000~20000,1:0x347cca50000~50000,1:0x851e4c0000~60000,1:0x56212e0000~40000,1:0x347ccaa0000~80000,1:0x851e520000~20000,1:0x347ccb20000~130000,1:0x347ccca0000~40000,1:0x347ccd90000~60000,1:0x347cce00000~70000,1:0x347cd270000~60000,1:0x5621320000~40000,1:0x851e540000~40000,1:0x347cd2d0000~50000,1:0x851e580000~20000,1:0x347cd320000~60000,1:0x851e5a0000~20000,1:0x347cd380000~20000,1:0x851e5c0000~20000,1:0x347cd3a0000~10000,1:0x5621360000~40000,1:0x851e5e0000~40000,1:0x347cd3b0000~20000,1:0x851e620000~20000,1:0x347cd3d0000~140000,1:0x347cd520000~60000,1:0x347cd6b0000~30000,1:0x851e640000~20000,1:0x347cd6e0000~f0000,1:0x851e660000~20000,1:0x347cd7d0000~50000,1:0x851e680000~20000,1:0x347cd820000~20000,1:0x56213a0000~40000,1:0x347cd840000~20000,1:0x851e6a0000~20000,1:0x347cd860000~10000,1:0x851e6c0000~20000,1:0x347cdc90000~20000,1:0x851e6e0000~20000,1:0x347cdcb0000~10000,1:0x851e700000~20000,1:0x347cdcc0000~d0000,1:0x56213e0000~40000,1:0x347cdd90000~c0000,1:0x347cdee0000~10000,1:0x851e720000~20000,1:0x347ce000000~90000,1:0x851e740000~20000,1:0x347ce090000~70000,1:0x347ce110000~10000,1:0x347ce1b0000~100000,1:0x56214e0000~40000,1:0x347ce2b0000~110000,1:0x851e760000~40000,1:0x347ce3c0000~30000,1:0x5621520000~40000,1:0x347ce3f0000~60000,1:0x851e7a0000~20000,1:0x347ce450000~140000,1:0x851e7c0000~20000,1:0x5621560000~40000,1:0x347ce590000~50000,1:0x56215a0000~40000,1:0x347ce5e0000~120000,1:0x851e800000~20000,1:0x347ce7f0000~60000,1:0x851e820000~20000,1:0x347ce850000~40000,1:0x851e840000~20000,1:0x347ce890000~e0000,1:0x851ea00000~20000,1:0x347ce970000~90000,1:0x851ea80000~20000,1:0x347cea00000~10000,1:0x851eaa0000~20000,1:0x347cea10000~100000,1:0x851eac0000~20000,1:0x347ceb20000~40000,1:0x851eae0000~60000,1:0x347ceb60000~b0000,1:0x851eb40000~20000,1:0x347cec10000~30000,1:0x851eb60000~40000,1:0x347cec40000~60000,1:0x347ced10000~50000,1:0x851eba0000~20000,1:0x347ced60000~50000,1:0x347cede0000~80000,1:0x347ceeb0000~f0000,1:0x347cefb0000~50000,1:0x347cf400000~20000,1:0x851ece0000~20000,1:0x56215e0000~40000,1:0x347cf420000~30000,1:0x5621620000~40000,1:0x347cf450000~110000,1:0x5621660000~40000,1:0x347cf560000~70000,1:0x7ba4ee0000~80000,1:0x347cf5d0000~10000,1:0x56216a0000~40000,1:0x347cf5e0000~10000,1:0x851ed20000~20000,1:0x347cf5f0000~30000,1:0x56216e0000~40000,1:0x851ed40000~40000,1:0x347cf620000~60000,1:0x5621720000~40000,1:0x347cf680000~60000,1:0x851ed80000~20000,1:0x347cf6e0000~140000,1:0x851eda0000~40000,1:0x347cf830000~200000,1:0x851ede0000~20000,1:0x347cfa30000~30000,1:0x5621760000~40000,1:0x851ee00000~20000,1:0x347cfa60000~60000,1:0x851ee20000~20000,1:0x347cfaf0000~10000,1:0x347cfbb0000~50000,1:0x851ee40000~20000,1:0x56217a0000~40000,1:0x347cfc00000~50000,1:0x347cfd20000~120000,1:0x851ee60000~20000,1:0x347cfe40000~30000,1:0x851ee80000~20000,1:0x347cfe70000~10000,1:0x851eea0000~20000,1:0x347cfe80000~100000,1:0x851eec0000~60000,1:0x347cff80000~110000,1:0x851ef20000~80000,1:0x347d0090000~10000,1:0x851efa0000~20000,1:0x347d00a0000~10000,1:0x851efc0000~20000,1:0x347d00b0000~80000,1:0x851efe0000~20000,1:0x347d0130000~10000,1:0x56217e0000~40000,1:0x347d0140000~40000,1:0x851f000000~20000,1:0x347d0180000~10000,1:0x851f020000~20000,1:0x347d0190000~250000,1:0x851f040000~20000,1:0x347d03e0000~f0000,1:0x851f060000~20000,1:0x347d04d0000~c0000,1:0x347d05a0000~80000,1:0x851f080000~20000,1:0x347d0650000~10000,1:0x347d0670000~70000,1:0x851f0c0000~20000,1:0x347d06e0000~50000,1:0x851f0e0000~40000,1:0x347d0730000~30000,1:0x851f120000~20000,1:0x347d0760000~a0000,1:0x851f140000~20000,1:0x347d0800000~70000,1:0x347d0880000~190000,1:0x347d0e10000~d0000,1:0x851f160000~60000,1:0x347d0ee0000~70000,1:0x347d0f60000~40000,1:0x347d1170000~e0000,1:0x851f1c0000~20000,1:0x347d1250000~1a0000,1:0x56219c0000~40000,1:0x851f300000~20000,1:0x347d13f0000~270000,1:0x347d1690000~180000,1:0x851f4c0000~20000,1:0x347d1810000~20000,1:0x851f4e0000~20000,1:0x347d19e0000~100000,1:0x5621a00000~40000,1:0x347d1ae0000~30000,1:0x851f520000~20000,1:0x347d1b10000~10000])

By investigating the corrupted log file, we confirmed two extents are overwritten, which is 0x56219c0000~40000 and 0x5621a00000~40000, the contents is highly similar to 053414.sst offset 0xc60000, seems like part of a new SST file, although unable to be found in BlueFS journal log.

The BlueFS has the following files:

/:
db  db.slow  db.wal  manifest  sharding

db/:
000050.sst  053406.sst  053413.sst  053430.sst  053435.sst  053449.sst  053454.sst  053460.sst  053466.sst  LOCK             OPTIONS-000041
053386.sst  053408.sst  053414.sst  053431.sst  053442.sst  053450.sst  053455.sst  053462.sst  CURRENT     MANIFEST-000038
053390.sst  053409.sst  053415.sst  053432.sst  053447.sst  053451.sst  053457.sst  053464.sst  IDENTITY    OPTIONS-000033

db.wal:
053448.log  053452.log  053456.log  053458.log  053461.log  053463.log  053465.log

sharding:
def

Sadly we're unable to dump all data, so we have no idea about the exactly range of that missing SST file extent, all we got is dumped RocksDB files

We struggled to locate the reason this happen, but it seems too difficult to us, it seems related to this issue https://tracker.ceph.com/issues/62282

Due to file size limit I'm unable to upload everything