grep 16.1e /var/log/ceph/ceph-osd.131.log 2022-06-29T12:33:49.213+0300 7f37cefad700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T12:49:29.272+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T12:50:46.629+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T12:54:00.683+0300 7f37cefad700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:04:05.758+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:12:52.468+0300 7f37cefad700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:13:13.188+0300 7f37cafa5700 1 osd.131 pg_epoch: 723033 pg[16.1e( v 723032'1044565479 lc 0'0 (723031'1044555441,723032'1044565479] local-lis/les=722726/722727 n=7259 ec=6813/6813 lis/c=722726/713864 les/c/f=722727/713865/76013 sis=723033 pruub=8.442323685s) [41,131]/[131] backfill=[41,141] r=0 lpr=723033 pi=[713864,723033)/4 crt=723032'1044565479 mlcod 0'0 active+remapped pruub 160422.625000000s@ MUST_REPAIR MUST_DEEP_SCRUB MUST_SCRUB planned REQ_SCRUB m=1 u=1 mbc={}] start_peering_interval up [41,141] -> [41,131], acting [131] -> [131], acting_primary 131 -> 131, up_primary 41 -> 41, role 0 -> 0, features acting 4540138297136906239 upacting 4540138297136906239 2022-06-29T13:13:13.192+0300 7f37cafa5700 1 osd.131 pg_epoch: 723033 pg[16.1e( v 723032'1044565479 lc 0'0 (723031'1044555441,723032'1044565479] local-lis/les=722726/722727 n=7259 ec=6813/6813 lis/c=722726/713864 les/c/f=722727/713865/76013 sis=723033 pruub=8.442323685s) [41,131]/[131] r=0 lpr=723033 pi=[713864,723033)/4 crt=723032'1044565479 mlcod 0'0 remapped pruub 160422.625000000s@ MUST_REPAIR MUST_DEEP_SCRUB MUST_SCRUB planned REQ_SCRUB m=1 mbc={}] state: transitioning to Primary 2022-06-29T13:13:14.712+0300 7f37cefad700 0 log_channel(cluster) log [INF] : 16.1e continuing backfill to osd.41 from (723031'1044555441,723032'1044565479] MIN to 723032'1044565479 2022-06-29T13:18:01.451+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:27:10.766+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:34:42.955+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:37:06.973+0300 7f37cefad700 1 osd.131 pg_epoch: 723038 pg[16.1e( v 723037'1044611140 lc 0'0 (723036'1044601041,723037'1044611140] local-lis/les=723033/723034 n=7254 ec=6813/6813 lis/c=723033/713864 les/c/f=723034/713865/76013 sis=723038 pruub=15.828340530s) [42,131]/[131] backfill=[41] r=0 lpr=723038 pi=[713864,723038)/4 crt=723037'1044611140 mlcod 0'0 active+remapped pruub 161863.812500000s@ MUST_REPAIR MUST_DEEP_SCRUB MUST_SCRUB planned REQ_SCRUB m=1 u=1 mbc={}] start_peering_interval up [41,131] -> [42,131], acting [131] -> [131], acting_primary 131 -> 131, up_primary 41 -> 42, role 0 -> 0, features acting 4540138297136906239 upacting 4540138297136906239 2022-06-29T13:37:06.973+0300 7f37cefad700 1 osd.131 pg_epoch: 723038 pg[16.1e( v 723037'1044611140 lc 0'0 (723036'1044601041,723037'1044611140] local-lis/les=723033/723034 n=7254 ec=6813/6813 lis/c=723033/713864 les/c/f=723034/713865/76013 sis=723038 pruub=15.828340530s) [42,131]/[131] r=0 lpr=723038 pi=[713864,723038)/4 crt=723037'1044611140 mlcod 0'0 remapped pruub 161863.812500000s@ MUST_REPAIR MUST_DEEP_SCRUB MUST_SCRUB planned REQ_SCRUB m=1 mbc={}] state: transitioning to Primary 2022-06-29T13:37:08.005+0300 7f37cefad700 0 log_channel(cluster) log [DBG] : 16.1e starting backfill to osd.42 from (722712'1041612441,722712'1041622482] MIN to 723037'1044611140 2022-06-29T13:40:12.043+0300 7f37cefad700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:41:09.416+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:42:27.073+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:42:34.633+0300 7f37cefad700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:42:39.753+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:44:36.027+0300 7f37cafa5700 1 osd.131 pg_epoch: 723045 pg[16.1e( v 723044'1044624668 lc 0'0 (723039'1044614641,723044'1044624668] local-lis/les=723038/723039 n=7265 ec=6813/6813 lis/c=723038/713864 les/c/f=723039/713865/76013 sis=723045 pruub=8.584972382s) [41,141]/[131] backfill=[42] r=0 lpr=723045 pi=[713864,723045)/4 crt=723044'1044624668 mlcod 0'0 active+remapped pruub 162305.609375000s@ MUST_REPAIR MUST_DEEP_SCRUB MUST_SCRUB planned REQ_SCRUB m=1 u=1 mbc={}] start_peering_interval up [42,131] -> [41,141], acting [131] -> [131], acting_primary 131 -> 131, up_primary 42 -> 41, role 0 -> 0, features acting 4540138297136906239 upacting 4540138297136906239 2022-06-29T13:44:36.027+0300 7f37cafa5700 1 osd.131 pg_epoch: 723045 pg[16.1e( v 723044'1044624668 lc 0'0 (723039'1044614641,723044'1044624668] local-lis/les=723038/723039 n=7265 ec=6813/6813 lis/c=723038/713864 les/c/f=723039/713865/76013 sis=723045 pruub=8.584972382s) [41,141]/[131] r=0 lpr=723045 pi=[713864,723045)/4 crt=723044'1044624668 mlcod 0'0 remapped pruub 162305.609375000s@ MUST_REPAIR MUST_DEEP_SCRUB MUST_SCRUB planned REQ_SCRUB m=1 mbc={}] state: transitioning to Primary 2022-06-29T13:44:37.071+0300 7f37cefad700 0 log_channel(cluster) log [DBG] : 16.1e starting backfill to osd.41 from (723036'1044601041,723037'1044611140] MIN to 723044'1044624668 2022-06-29T13:44:37.099+0300 7f37cefad700 0 log_channel(cluster) log [DBG] : 16.1e starting backfill to osd.141 from (723031'1044555441,723032'1044565479] MIN to 723044'1044624668 2022-06-29T13:46:59.696+0300 7f37cefad700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:47:01.196+0300 7f37cafa5700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost 2022-06-29T13:48:47.026+0300 7f37cefad700 -1 log_channel(cluster) log [ERR] : 16.1e has 1 objects unfound and apparently lost grep 16.1e /var/log/ceph/ceph-osd.41.log 2022-06-29T13:13:13.191+0300 7f61c4c26700 1 osd.41 pg_epoch: 723033 pg[16.1e( v 723032'1044565479 (723031'1044555441,723032'1044565479] lb MIN local-lis/les=722713/722714 n=0 ec=6813/6813 lis/c=722713/713864 les/c/f=722714/713865/76013 sis=723033) [41,131]/[131] r=-1 lpr=723033 pi=[713864,723033)/4 luod=0'0 lua=722929'1043551464 crt=723032'1044565479 lcod 723032'1044565478 mlcod 0'0 active+remapped mbc={}] start_peering_interval up [41,141] -> [41,131], acting [131] -> [131], acting_primary 131 -> 131, up_primary 41 -> 41, role -1 -> -1, features acting 4540138297136906239 upacting 4540138297136906239 2022-06-29T13:13:13.195+0300 7f61c4c26700 1 osd.41 pg_epoch: 723033 pg[16.1e( v 723032'1044565479 (723031'1044555441,723032'1044565479] lb MIN local-lis/les=722713/722714 n=0 ec=6813/6813 lis/c=722713/713864 les/c/f=722714/713865/76013 sis=723033) [41,131]/[131] r=-1 lpr=723033 pi=[713864,723033)/4 crt=723032'1044565479 lcod 723032'1044565478 mlcod 0'0 remapped NOTIFY mbc={}] state: transitioning to Stray 2022-06-29T13:37:06.989+0300 7f61c4c26700 1 osd.41 pg_epoch: 723038 pg[16.1e( v 723037'1044611140 (723036'1044601041,723037'1044611140] lb MIN local-lis/les=722713/722714 n=0 ec=6813/6813 lis/c=722713/713864 les/c/f=722714/713865/76013 sis=723038) [42,131]/[131] r=-1 lpr=723038 pi=[713864,723038)/4 luod=0'0 lua=723032'1044565479 crt=723037'1044611140 lcod 723037'1044611139 mlcod 0'0 active+remapped mbc={}] start_peering_interval up [41,131] -> [42,131], acting [131] -> [131], acting_primary 131 -> 131, up_primary 41 -> 42, role -1 -> -1, features acting 4540138297136906239 upacting 4540138297136906239 2022-06-29T13:37:06.989+0300 7f61c4c26700 1 osd.41 pg_epoch: 723038 pg[16.1e( v 723037'1044611140 (723036'1044601041,723037'1044611140] lb MIN local-lis/les=722713/722714 n=0 ec=6813/6813 lis/c=722713/713864 les/c/f=722714/713865/76013 sis=723038) [42,131]/[131] r=-1 lpr=723038 pi=[713864,723038)/4 crt=723037'1044611140 lcod 723037'1044611139 mlcod 0'0 remapped NOTIFY mbc={}] state: transitioning to Stray 2022-06-29T13:44:36.023+0300 7f61c4c26700 1 osd.41 pg_epoch: 723045 pg[16.1e( v 723037'1044611140 (723036'1044601041,723037'1044611140] lb MIN local-lis/les=722713/722714 n=0 ec=6813/6813 lis/c=723033/713864 les/c/f=723034/713865/76013 sis=723045) [41,141]/[131] r=-1 lpr=723045 pi=[713864,723045)/4 crt=723037'1044611140 lcod 723037'1044611139 mlcod 0'0 remapped NOTIFY mbc={}] start_peering_interval up [42,131] -> [41,141], acting [131] -> [131], acting_primary 131 -> 131, up_primary 42 -> 41, role -1 -> -1, features acting 4540138297136906239 upacting 4540138297136906239 2022-06-29T13:44:36.023+0300 7f61c4c26700 1 osd.41 pg_epoch: 723045 pg[16.1e( v 723037'1044611140 (723036'1044601041,723037'1044611140] lb MIN local-lis/les=722713/722714 n=0 ec=6813/6813 lis/c=723033/713864 les/c/f=723034/713865/76013 sis=723045) [41,141]/[131] r=-1 lpr=723045 pi=[713864,723045)/4 crt=723037'1044611140 lcod 723037'1044611139 mlcod 0'0 remapped NOTIFY mbc={}] state: transitioning to Stray