Actions
Bug #52948
openosd: fails to come up: "teuthology.misc:7 of 8 OSDs are up"
% Done:
0%
Source:
Q/A
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
fs
Component(RADOS):
OSD
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
2021-10-15T03:22:32.089 INFO:teuthology.misc.health.smithi089.stdout:{"epoch":17,"fsid":"dcc6a9bc-1936-44b9-aa05-4bd710856c95","created":"2021-10-15T03:11:55.278461+0000","modified":"2021-10-15T03:22:05.151198+0000","last_up_change":"2021-10-15T03:12:06.037577+0000","last_in_change":"2021-10-15T03:22:05.151198+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2021-10-15T03:12:06.169415+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cd5c8454-0a20-454e-a59b-51cb3c3e9a78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6823","nonce":36889},{"type":"v1","addr":"172.21.15.89:6825","nonce":36889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":36889},{"type":"v1","addr":"172.21.15.89:6827","nonce":36889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":36889},{"type":"v1","addr":"172.21.15.89:6831","nonce":36889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":36889},{"type":"v1","addr":"172.21.15.89:6829","nonce":36889}]},"public_addr":"172.21.15.89:6825/36889","cluster_addr":"172.21.15.89:6827/36889","heartbeat_back_addr":"172.21.15.89:6831/36889","heartbeat_front_addr":"172.21.15.89:6829/36889","state":["exists","up"]},{"osd":1,"uuid":"23b82270-88f9-4b1e-954d-609d5b4f7f3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":36887},{"type":"v1","addr":"172.21.15.89:6809","nonce":36887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":36887},{"type":"v1","addr":"172.21.15.89:6811","nonce":36887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":36887},{"type":"v1","addr":"172.21.15.89:6815","nonce":36887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":36887},{"type":"v1","addr":"172.21.15.89:6813","nonce":36887}]},"public_addr":"172.21.15.89:6809/36887","cluster_addr":"172.21.15.89:6811/36887","heartbeat_back_addr":"172.21.15.89:6815/36887","heartbeat_front_addr":"172.21.15.89:6813/36887","state":["exists","up"]},{"osd":2,"uuid":"166f20d4-cc82-4f6e-ba0a-1700c16cdea4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":36886},{"type":"v1","addr":"172.21.15.89:6801","nonce":36886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":36886},{"type":"v1","addr":"172.21.15.89:6803","nonce":36886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":36886},{"type":"v1","addr":"172.21.15.89:6807","nonce":36886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":36886},{"type":"v1","addr":"172.21.15.89:6805","nonce":36886}]},"public_addr":"172.21.15.89:6801/36886","cluster_addr":"172.21.15.89:6803/36886","heartbeat_back_addr":"172.21.15.89:6807/36886","heartbeat_front_addr":"172.21.15.89:6805/36886","state":["exists","up"]},{"osd":3,"uuid":"cbe05e41-eb54-4d93-b06b-ef8adffd6261","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["autoout","exists","new"]},{"osd":4,"uuid":"8389a217-b195-4550-9874-ddea95eb0eab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6808","nonce":37119},{"type":"v1","addr":"172.21.15.25:6809","nonce":37119}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6810","nonce":37119},{"type":"v1","addr":"172.21.15.25:6811","nonce":37119}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6814","nonce":37119},{"type":"v1","addr":"172.21.15.25:6815","nonce":37119}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6812","nonce":37119},{"type":"v1","addr":"172.21.15.25:6813","nonce":37119}]},"public_addr":"172.21.15.25:6809/37119","cluster_addr":"172.21.15.25:6811/37119","heartbeat_back_addr":"172.21.15.25:6815/37119","heartbeat_front_addr":"172.21.15.25:6813/37119","state":["exists","up"]},{"osd":5,"uuid":"4d1f724e-4fc9-4862-a8a6-041554d0586f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6816","nonce":37135},{"type":"v1","addr":"172.21.15.25:6817","nonce":37135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6818","nonce":37135},{"type":"v1","addr":"172.21.15.25:6819","nonce":37135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6822","nonce":37135},{"type":"v1","addr":"172.21.15.25:6823","nonce":37135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6820","nonce":37135},{"type":"v1","addr":"172.21.15.25:6821","nonce":37135}]},"public_addr":"172.21.15.25:6817/37135","cluster_addr":"172.21.15.25:6819/37135","heartbeat_back_addr":"172.21.15.25:6823/37135","heartbeat_front_addr":"172.21.15.25:6821/37135","state":["exists","up"]},{"osd":6,"uuid":"adf42516-390e-46ac-9ed4-f70e4642c5c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6800","nonce":37118},{"type":"v1","addr":"172.21.15.25:6801","nonce":37118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6802","nonce":37118},{"type":"v1","addr":"172.21.15.25:6803","nonce":37118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6806","nonce":37118},{"type":"v1","addr":"172.21.15.25:6807","nonce":37118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6804","nonce":37118},{"type":"v1","addr":"172.21.15.25:6805","nonce":37118}]},"public_addr":"172.21.15.25:6801/37118","cluster_addr":"172.21.15.25:6803/37118","heartbeat_back_addr":"172.21.15.25:6807/37118","heartbeat_front_addr":"172.21.15.25:6805/37118","state":["exists","up"]},{"osd":7,"uuid":"ca84ff07-273e-4a21-9ac7-e8761c0e1264","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6824","nonce":37122},{"type":"v1","addr":"172.21.15.25:6825","nonce":37122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6826","nonce":37122},{"type":"v1","addr":"172.21.15.25:6827","nonce":37122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6830","nonce":37122},{"type":"v1","addr":"172.21.15.25:6831","nonce":37122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.25:6828","nonce":37122},{"type":"v1","addr":"172.21.15.25:6829","nonce":37122}]},"public_addr":"172.21.15.25:6825/37122","cluster_addr":"172.21.15.25:6827/37122","heartbeat_back_addr":"172.21.15.25:6831/37122","heartbeat_front_addr":"172.21.15.25:6829/37122","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-15T03:12:02.877193+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-15T03:12:02.903567+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-15T03:12:02.901545+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-15T03:12:02.855567+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-15T03:12:02.657142+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-15T03:12:02.826109+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-15T03:12:02.930599+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
...
2021-10-15T03:22:32.101 DEBUG:teuthology.misc:7 of 8 OSDs are up
2021-10-15T03:22:32.102 ERROR:teuthology.contextutil:Saw exception from nested tasks
Traceback (most recent call last):
File "/home/teuthworker/src/git.ceph.com_git_teuthology_423bd94291582cb44cbfac84a53ec84580eb3f08/teuthology/contextutil.py", line 31, in nested
vars.append(enter())
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/teuthworker/src/git.ceph.com_ceph-c_9bcded95726562b598e8b1357a01cf828b42f563/qa/tasks/ceph.py", line 403, in create_rbd_pool
ceph_cluster=cluster_name,
File "/home/teuthworker/src/git.ceph.com_git_teuthology_423bd94291582cb44cbfac84a53ec84580eb3f08/teuthology/misc.py", line 858, in wait_until_osds_up
while proceed():
File "/home/teuthworker/src/git.ceph.com_git_teuthology_423bd94291582cb44cbfac84a53ec84580eb3f08/teuthology/contextutil.py", line 133, in __call__
raise MaxWhileTries(error_msg)
teuthology.exceptions.MaxWhileTries: reached maximum tries (90) after waiting for 540 seconds
From: /ceph/teuthology-archive/pdonnell-2021-10-15_02:49:27-fs-wip-pdonnell-testing-20211012.192211-distro-basic-smithi/6443932/teuthology.log
OSD log:
/ceph/teuthology-archive/pdonnell-2021-10-15_02:49:27-fs-wip-pdonnell-testing-20211012.192211-distro-basic-smithi/6443932/remote/smithi089/log/ceph-osd.3.log.gz
Other jobs affected:
Failure: reached maximum tries (90) after waiting for 540 seconds
4 jobs: ['6443932', '6443950', '6443942', '6443924']
suites intersection: ['2-workunit/suites/ffsb}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/thrash/workloads/{begin', 'mds', 'mon', 'msgr-failures/osd-mds-delay', 'osd}', 'overrides/{frag', 'ranks/3', 'session_timeout', 'thrashosds-health', 'whitelist_health', 'whitelist_wrongly_marked_down}']
suites union: ['2-workunit/suites/ffsb}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_8.stream}', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/thrash/workloads/{begin', 'k-testing}', 'mds', 'mon', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{frag', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health', 'whitelist_health', 'whitelist_wrongly_marked_down}']
Actions