Actions
Bug #39087
openec_lost_unfound: a EC shard has missing object after `osd lost`
Status:
New
Priority:
Normal
Assignee:
-
Category:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
16729 2019-04-01T14:49:52.035 INFO:tasks.ec_lost_unfound:listing missing/lost in 2.4 state active+recovery_unfound+undersized+degraded
16730 2019-04-01T14:49:52.035 INFO:teuthology.orchestra.run.mira111:Running:
16731 2019-04-01T14:49:52.035 INFO:teuthology.orchestra.run.mira111:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- pg 2.4 list_unfound '{}'
16732 2019-04-01T14:49:52.309 INFO:tasks.ec_lost_unfound.radosbench.client.admin.mira111.stdout: 13 20 6191 6171 3.70606 1.88281 0.0225818 0.0421165
16733 2019-04-01T14:49:52.505 INFO:teuthology.orchestra.run.mira111.stdout:{
16734 2019-04-01T14:49:52.506 INFO:teuthology.orchestra.run.mira111.stdout: "offset": {
16735 2019-04-01T14:49:52.506 INFO:teuthology.orchestra.run.mira111.stdout: "oid": "",
16736 2019-04-01T14:49:52.506 INFO:teuthology.orchestra.run.mira111.stdout: "key": "",
16737 2019-04-01T14:49:52.506 INFO:teuthology.orchestra.run.mira111.stdout: "snapid": 0,
16738 2019-04-01T14:49:52.506 INFO:teuthology.orchestra.run.mira111.stdout: "hash": 0,
16739 2019-04-01T14:49:52.506 INFO:teuthology.orchestra.run.mira111.stdout: "max": 0,
16740 2019-04-01T14:49:52.506 INFO:teuthology.orchestra.run.mira111.stdout: "pool": -9223372036854775808,
16741 2019-04-01T14:49:52.507 INFO:teuthology.orchestra.run.mira111.stdout: "namespace": ""
16742 2019-04-01T14:49:52.507 INFO:teuthology.orchestra.run.mira111.stdout: },
16743 2019-04-01T14:49:52.507 INFO:teuthology.orchestra.run.mira111.stdout: "num_missing": 3,
16744 2019-04-01T14:49:52.507 INFO:teuthology.orchestra.run.mira111.stdout: "num_unfound": 3,
16745 2019-04-01T14:49:52.507 INFO:teuthology.orchestra.run.mira111.stdout: "objects": [
16746 2019-04-01T14:49:52.507 INFO:teuthology.orchestra.run.mira111.stdout: {
16747 2019-04-01T14:49:52.507 INFO:teuthology.orchestra.run.mira111.stdout: "oid": {
16748 2019-04-01T14:49:52.508 INFO:teuthology.orchestra.run.mira111.stdout: "oid": "existed_4",
16749 2019-04-01T14:49:52.508 INFO:teuthology.orchestra.run.mira111.stdout: "key": "",
16750 2019-04-01T14:49:52.508 INFO:teuthology.orchestra.run.mira111.stdout: "snapid": -2,
16751 2019-04-01T14:49:52.508 INFO:teuthology.orchestra.run.mira111.stdout: "hash": 873891716,
16752 2019-04-01T14:49:52.508 INFO:teuthology.orchestra.run.mira111.stdout: "max": 0,
16753 2019-04-01T14:49:52.508 INFO:teuthology.orchestra.run.mira111.stdout: "pool": 2,
16754 2019-04-01T14:49:52.508 INFO:teuthology.orchestra.run.mira111.stdout: "namespace": ""
16755 2019-04-01T14:49:52.509 INFO:teuthology.orchestra.run.mira111.stdout: },
16756 2019-04-01T14:49:52.509 INFO:teuthology.orchestra.run.mira111.stdout: "need": "21'8",
16757 2019-04-01T14:49:52.509 INFO:teuthology.orchestra.run.mira111.stdout: "have": "0'0",
16758 2019-04-01T14:49:52.509 INFO:teuthology.orchestra.run.mira111.stdout: "flags": "none",
16759 2019-04-01T14:49:52.509 INFO:teuthology.orchestra.run.mira111.stdout: "locations": [
16760 2019-04-01T14:49:52.509 INFO:teuthology.orchestra.run.mira111.stdout: "2(2)"
16761 2019-04-01T14:49:52.509 INFO:teuthology.orchestra.run.mira111.stdout: ]
16762 2019-04-01T14:49:52.509 INFO:teuthology.orchestra.run.mira111.stdout: },
16763 2019-04-01T14:49:52.510 INFO:teuthology.orchestra.run.mira111.stdout: {
16764 2019-04-01T14:49:52.510 INFO:teuthology.orchestra.run.mira111.stdout: "oid": {
16765 2019-04-01T14:49:52.510 INFO:teuthology.orchestra.run.mira111.stdout: "oid": "existing_7",
16766 2019-04-01T14:49:52.510 INFO:teuthology.orchestra.run.mira111.stdout: "key": "",
16767 2019-04-01T14:49:52.510 INFO:teuthology.orchestra.run.mira111.stdout: "snapid": -2,
16768 2019-04-01T14:49:52.510 INFO:teuthology.orchestra.run.mira111.stdout: "hash": 3996155460,
16769 2019-04-01T14:49:52.510 INFO:teuthology.orchestra.run.mira111.stdout: "max": 0,
16770 2019-04-01T14:49:52.510 INFO:teuthology.orchestra.run.mira111.stdout: "pool": 2,
16771 2019-04-01T14:49:52.511 INFO:teuthology.orchestra.run.mira111.stdout: "namespace": ""
16772 2019-04-01T14:49:52.511 INFO:teuthology.orchestra.run.mira111.stdout: },
16773 2019-04-01T14:49:52.511 INFO:teuthology.orchestra.run.mira111.stdout: "need": "21'9",
16774 2019-04-01T14:49:52.511 INFO:teuthology.orchestra.run.mira111.stdout: "have": "18'6",
16775 2019-04-01T14:49:52.511 INFO:teuthology.orchestra.run.mira111.stdout: "flags": "none",
16776 2019-04-01T14:49:52.511 INFO:teuthology.orchestra.run.mira111.stdout: "locations": [
16777 2019-04-01T14:49:52.511 INFO:teuthology.orchestra.run.mira111.stdout: "2(2)"
16778 2019-04-01T14:49:52.511 INFO:teuthology.orchestra.run.mira111.stdout: ]
16779 2019-04-01T14:49:52.512 INFO:teuthology.orchestra.run.mira111.stdout: },
16780 2019-04-01T14:49:52.512 INFO:teuthology.orchestra.run.mira111.stdout: {
16781 2019-04-01T14:49:52.512 INFO:teuthology.orchestra.run.mira111.stdout: "oid": {
16782 2019-04-01T14:49:52.512 INFO:teuthology.orchestra.run.mira111.stdout: "oid": "existed_2",
16783 2019-04-01T14:49:52.512 INFO:teuthology.orchestra.run.mira111.stdout: "key": "",
16784 2019-04-01T14:49:52.512 INFO:teuthology.orchestra.run.mira111.stdout: "snapid": -2,
16785 2019-04-01T14:49:52.512 INFO:teuthology.orchestra.run.mira111.stdout: "hash": 1767667508,
16786 2019-04-01T14:49:52.513 INFO:teuthology.orchestra.run.mira111.stdout: "max": 0,
16787 2019-04-01T14:49:52.513 INFO:teuthology.orchestra.run.mira111.stdout: "pool": 2,
16788 2019-04-01T14:49:52.513 INFO:teuthology.orchestra.run.mira111.stdout: "namespace": ""
16789 2019-04-01T14:49:52.513 INFO:teuthology.orchestra.run.mira111.stdout: },
16790 2019-04-01T14:49:52.513 INFO:teuthology.orchestra.run.mira111.stdout: "need": "21'7",
16791 2019-04-01T14:49:52.513 INFO:teuthology.orchestra.run.mira111.stdout: "have": "0'0",
16792 2019-04-01T14:49:52.513 INFO:teuthology.orchestra.run.mira111.stdout: "flags": "none",
16793 2019-04-01T14:49:52.513 INFO:teuthology.orchestra.run.mira111.stdout: "locations": [
16794 2019-04-01T14:49:52.514 INFO:teuthology.orchestra.run.mira111.stdout: "2(2)"
16795 2019-04-01T14:49:52.514 INFO:teuthology.orchestra.run.mira111.stdout: ]
16796 2019-04-01T14:49:52.514 INFO:teuthology.orchestra.run.mira111.stdout: }
16797 2019-04-01T14:49:52.514 INFO:teuthology.orchestra.run.mira111.stdout: ],
16798 2019-04-01T14:49:52.514 INFO:teuthology.orchestra.run.mira111.stdout: "more": false
16799 2019-04-01T14:49:52.514 INFO:teuthology.orchestra.run.mira111.stdout:}
16800 2019-04-01T14:49:52.519 INFO:tasks.ec_lost_unfound:{u'num_unfound': 3, u'objects': [{u'need': u"21'8", u'oid': {u'hash': 873891716, u'max': 0, u'oid': u'existed_4', u'namespace': u'', u'snapid': - 2, u'key': u'', u'pool': 2}, u'flags': u'none', u'locations': [u'2(2)'], u'have': u"0'0"}, {u'need': u"21'9", u'oid': {u'hash': 3996155460, u'max': 0, u'oid': u'existing_7', u'namespace': u'', u's napid': -2, u'key': u'', u'pool': 2}, u'flags': u'none', u'locations': [u'2(2)'], u'have': u"18'6"}, {u'need': u"21'7", u'oid': {u'hash': 1767667508, u'max': 0, u'oid': u'existed_2', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 2}, u'flags': u'none', u'locations': [u'2(2)'], u'have': u"0'0"}], u'num_missing': 3, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -9223372036854775808}}
16801 2019-04-01T14:49:52.519 INFO:tasks.ec_lost_unfound:reverting unfound in 2.4
1924724 2019-04-01T15:40:40.758 INFO:tasks.ceph.osd.1.mira111.stderr:2019-04-01 15:40:40.758 7fbcb881f700 -1 log_channel(cluster) log [ERR] : 2.4s0 shard 0(1) 2:29be9562:::dummy:head : missing
1924725 2019-04-01T15:40:40.758 INFO:tasks.ceph.osd.1.mira111.stderr:2019-04-01 15:40:40.758 7fbcb881f700 -1 log_channel(cluster) log [ERR] : 2.4s0 scrub 1 missing, 0 inconsistent objects
{"pgid":"2.4","version":"30'5134","report ed_seq":"5352","reported_epoch":"61","state":"active+clean","last_fresh":"2019-04-01 15:40:33.883225","last_change":"2019-04-01 15:12:49.501106","last_active":"2019-04-01 15:40:33.883225","last_pe ered":"2019-04-01 15:40:33.883225","last_clean":"2019-04-01 15:40:33.883225","last_became_active":"2019-04-01 15:12:49.437988","last_became_peered":"2019-04-01 15:12:49.437988","last_unstale":"201 9-04-01 15:40:33.883225","last_undegraded":"2019-04-01 15:40:33.883225","last_fullsized":"2019-04-01 15:40:33.883225","mapping_epoch":41,"log_start":"30'2100","ondisk_log_start":"30'2100","created ":15,"last_epoch_clean":42,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2019-04-01 14:48:53.857396","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2019-04-01 14:48 :53.857396","last_clean_scrub_stamp":"2019-04-01 14:48:53.857396","log_size":3034,"ondisk_log_size":3034,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_i nvalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":12676,"num_objects":4,"num_object_clones":0,"num _object_copies":16,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"nu m_read":0,"num_read_kb":0,"num_write":5131,"num_write_kb":20516,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1578,"num_bytes_recovered":0,"nu m_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode _high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes": 0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2,3],"acting":[1,0,2,3],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}
Updated by Chang Liu about 5 years ago
is this `scrub error` we expect? what we should do is to find out why ceph doesn't recovery PG 2.4s0 ?
Updated by Neha Ojha about 5 years ago
/a/yuriw-2019-04-02_20:09:55-rados-wip-yuri3-testing-2019-04-02-1623-mimic-distro-basic-smithi/3801955/ - looks like a similar failure in mimic
Actions