|
grep -w 16.1e /var/log/ceph/ceph.log
|
|
|
|
2022-06-29T11:40:00.000887+0300 mon.ceph-nimbus-05 (mon.0) 1023041 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T11:40:00.000908+0300 mon.ceph-nimbus-05 (mon.0) 1023043 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T11:48:32.755122+0300 osd.131 (osd.131) 554 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T11:50:00.000336+0300 mon.ceph-nimbus-05 (mon.0) 1023350 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T11:50:00.000356+0300 mon.ceph-nimbus-05 (mon.0) 1023352 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T11:50:00.000383+0300 mon.ceph-nimbus-05 (mon.0) 1023354 : cluster [ERR] pg 16.1e is stuck undersized for 24h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T11:50:00.000405+0300 mon.ceph-nimbus-05 (mon.0) 1023356 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T11:50:00.000429+0300 mon.ceph-nimbus-05 (mon.0) 1023358 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T11:50:39.961350+0300 osd.131 (osd.131) 555 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T11:56:04.370136+0300 osd.131 (osd.131) 556 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:00:00.000467+0300 mon.ceph-nimbus-05 (mon.0) 1023653 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T12:00:00.000509+0300 mon.ceph-nimbus-05 (mon.0) 1023655 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T12:00:00.000537+0300 mon.ceph-nimbus-05 (mon.0) 1023657 : cluster [ERR] pg 16.1e is stuck undersized for 24h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T12:00:00.000663+0300 mon.ceph-nimbus-05 (mon.0) 1023659 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T12:00:00.000695+0300 mon.ceph-nimbus-05 (mon.0) 1023661 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T12:04:28.946537+0300 osd.131 (osd.131) 557 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:09:05.789427+0300 osd.131 (osd.131) 558 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:10:00.000406+0300 mon.ceph-nimbus-05 (mon.0) 1023955 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T12:10:00.000433+0300 mon.ceph-nimbus-05 (mon.0) 1023957 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T12:10:00.000454+0300 mon.ceph-nimbus-05 (mon.0) 1023959 : cluster [ERR] pg 16.1e is stuck undersized for 25h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T12:10:00.000478+0300 mon.ceph-nimbus-05 (mon.0) 1023961 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T12:10:00.000500+0300 mon.ceph-nimbus-05 (mon.0) 1023963 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T12:13:33.139030+0300 osd.131 (osd.131) 559 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:20:00.000376+0300 mon.ceph-nimbus-05 (mon.0) 1024251 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T12:20:00.000407+0300 mon.ceph-nimbus-05 (mon.0) 1024253 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T12:20:00.000432+0300 mon.ceph-nimbus-05 (mon.0) 1024255 : cluster [ERR] pg 16.1e is stuck undersized for 25h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T12:20:00.000460+0300 mon.ceph-nimbus-05 (mon.0) 1024257 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T12:20:00.000513+0300 mon.ceph-nimbus-05 (mon.0) 1024259 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T12:27:58.826058+0300 osd.131 (osd.131) 560 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:28:43.300823+0300 osd.131 (osd.131) 561 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:30:00.000288+0300 mon.ceph-nimbus-05 (mon.0) 1024561 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T12:30:00.000311+0300 mon.ceph-nimbus-05 (mon.0) 1024563 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T12:30:00.000334+0300 mon.ceph-nimbus-05 (mon.0) 1024565 : cluster [ERR] pg 16.1e is stuck undersized for 25h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T12:30:00.000355+0300 mon.ceph-nimbus-05 (mon.0) 1024567 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T12:30:00.000385+0300 mon.ceph-nimbus-05 (mon.0) 1024569 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T12:33:23.373298+0300 osd.131 (osd.131) 562 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:33:49.216370+0300 osd.131 (osd.131) 563 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:40:00.000336+0300 mon.ceph-nimbus-05 (mon.0) 1024869 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T12:40:00.000369+0300 mon.ceph-nimbus-05 (mon.0) 1024871 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T12:40:00.000399+0300 mon.ceph-nimbus-05 (mon.0) 1024873 : cluster [ERR] pg 16.1e is stuck undersized for 25h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T12:40:00.000429+0300 mon.ceph-nimbus-05 (mon.0) 1024875 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T12:40:00.000465+0300 mon.ceph-nimbus-05 (mon.0) 1024877 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T12:49:29.277633+0300 osd.131 (osd.131) 565 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:50:00.000402+0300 mon.ceph-nimbus-05 (mon.0) 1025177 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T12:50:00.000434+0300 mon.ceph-nimbus-05 (mon.0) 1025179 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T12:50:00.000464+0300 mon.ceph-nimbus-05 (mon.0) 1025181 : cluster [ERR] pg 16.1e is stuck undersized for 25h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T12:50:00.000496+0300 mon.ceph-nimbus-05 (mon.0) 1025183 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T12:50:00.000526+0300 mon.ceph-nimbus-05 (mon.0) 1025185 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T12:50:46.632326+0300 osd.131 (osd.131) 566 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T12:54:00.688822+0300 osd.131 (osd.131) 567 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:00:00.000350+0300 mon.ceph-nimbus-05 (mon.0) 1025477 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T13:00:00.000371+0300 mon.ceph-nimbus-05 (mon.0) 1025479 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T13:00:00.000401+0300 mon.ceph-nimbus-05 (mon.0) 1025481 : cluster [ERR] pg 16.1e is stuck undersized for 25h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T13:00:00.000435+0300 mon.ceph-nimbus-05 (mon.0) 1025483 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T13:00:00.000466+0300 mon.ceph-nimbus-05 (mon.0) 1025485 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T13:04:05.760884+0300 osd.131 (osd.131) 568 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:10:00.000364+0300 mon.ceph-nimbus-05 (mon.0) 1025789 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T13:10:00.000389+0300 mon.ceph-nimbus-05 (mon.0) 1025791 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T13:10:00.000414+0300 mon.ceph-nimbus-05 (mon.0) 1025793 : cluster [ERR] pg 16.1e is stuck undersized for 26h, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T13:10:00.000435+0300 mon.ceph-nimbus-05 (mon.0) 1025795 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T13:10:00.000464+0300 mon.ceph-nimbus-05 (mon.0) 1025797 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T13:12:52.472457+0300 osd.131 (osd.131) 569 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:13:14.718778+0300 osd.131 (osd.131) 570 : cluster [INF] 16.1e continuing backfill to osd.41 from (723031'1044555441,723032'1044565479] MIN to 723032'1044565479
|
|
2022-06-29T13:18:01.458724+0300 osd.131 (osd.131) 571 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:20:00.000327+0300 mon.ceph-nimbus-05 (mon.0) 1026102 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T13:20:00.000358+0300 mon.ceph-nimbus-05 (mon.0) 1026104 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T13:20:00.000383+0300 mon.ceph-nimbus-05 (mon.0) 1026106 : cluster [ERR] pg 16.1e is stuck undersized for 6m, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T13:20:00.000405+0300 mon.ceph-nimbus-05 (mon.0) 1026108 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T13:20:00.000428+0300 mon.ceph-nimbus-05 (mon.0) 1026110 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T13:27:10.770987+0300 osd.131 (osd.131) 572 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:30:00.000300+0300 mon.ceph-nimbus-05 (mon.0) 1026410 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T13:30:00.000325+0300 mon.ceph-nimbus-05 (mon.0) 1026412 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T13:30:00.000349+0300 mon.ceph-nimbus-05 (mon.0) 1026414 : cluster [ERR] pg 16.1e is stuck undersized for 16m, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T13:30:00.000388+0300 mon.ceph-nimbus-05 (mon.0) 1026416 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T13:30:00.000410+0300 mon.ceph-nimbus-05 (mon.0) 1026418 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T13:34:42.961359+0300 osd.131 (osd.131) 573 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:37:08.007290+0300 osd.131 (osd.131) 574 : cluster [DBG] 16.1e starting backfill to osd.42 from (722712'1041612441,722712'1041622482] MIN to 723037'1044611140
|
|
2022-06-29T13:40:00.000306+0300 mon.ceph-nimbus-05 (mon.0) 1026733 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T13:40:00.000324+0300 mon.ceph-nimbus-05 (mon.0) 1026735 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T13:40:00.000342+0300 mon.ceph-nimbus-05 (mon.0) 1026737 : cluster [ERR] pg 16.1e is stuck undersized for 2m, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T13:40:00.000377+0300 mon.ceph-nimbus-05 (mon.0) 1026739 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T13:40:00.000400+0300 mon.ceph-nimbus-05 (mon.0) 1026741 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
2022-06-29T13:40:12.048711+0300 osd.131 (osd.131) 575 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:41:09.419580+0300 osd.131 (osd.131) 576 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:42:27.078924+0300 osd.131 (osd.131) 577 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:42:34.636260+0300 osd.131 (osd.131) 578 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:42:39.756345+0300 osd.131 (osd.131) 579 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:44:37.073096+0300 osd.131 (osd.131) 580 : cluster [DBG] 16.1e starting backfill to osd.41 from (723036'1044601041,723037'1044611140] MIN to 723044'1044624668
|
|
2022-06-29T13:44:37.103043+0300 osd.131 (osd.131) 581 : cluster [DBG] 16.1e starting backfill to osd.141 from (723031'1044555441,723032'1044565479] MIN to 723044'1044624668
|
|
2022-06-29T13:46:59.702124+0300 osd.131 (osd.131) 582 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:47:01.201601+0300 osd.131 (osd.131) 583 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:48:47.029613+0300 osd.131 (osd.131) 584 : cluster [ERR] 16.1e has 1 objects unfound and apparently lost
|
|
2022-06-29T13:50:00.000384+0300 mon.ceph-nimbus-05 (mon.0) 1027052 : cluster [ERR] pg 16.1e has 1 unfound objects
|
|
2022-06-29T13:50:00.000414+0300 mon.ceph-nimbus-05 (mon.0) 1027054 : cluster [ERR] pg 16.1e is active+recovery_unfound+undersized+degraded+remapped, acting [131], 1 unfound
|
|
2022-06-29T13:50:00.000443+0300 mon.ceph-nimbus-05 (mon.0) 1027056 : cluster [ERR] pg 16.1e is stuck undersized for 5m, current state active+recovery_unfound+undersized+degraded+remapped, last acting [131]
|
|
2022-06-29T13:50:00.000474+0300 mon.ceph-nimbus-05 (mon.0) 1027058 : cluster [ERR] pg 16.1e not deep-scrubbed since 2022-06-03T01:20:13.786232+0300
|
|
2022-06-29T13:50:00.000506+0300 mon.ceph-nimbus-05 (mon.0) 1027060 : cluster [ERR] pg 16.1e not scrubbed since 2022-06-09T03:27:36.771392+0300
|
|
|