Bug #20891
Updated by Greg Farnum over 6 years ago
From: /ceph/teuthology-archive/pdonnell-2017-08-02_17:25:29-fs-wip-pdonnell-testing-20170802-distro-basic-smithi/1474672/teuthology.log
<pre>
2017-08-02T19:46:30.014 ERROR:teuthology.contextutil:Saw exception from nested tasks
Traceback (most recent call last):
File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/contextutil.py", line 32, in nested
yield vars
File "/home/teuthworker/src/git.ceph.com_ceph-c_wip-pdonnell-testing-20170802/qa/tasks/ceph.py", line 1654, in task
healthy(ctx=ctx, config=dict(cluster=config['cluster']))
File "/home/teuthworker/src/git.ceph.com_ceph-c_wip-pdonnell-testing-20170802/qa/tasks/ceph.py", line 1251, in healthy
ceph_cluster=cluster_name,
File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/misc.py", line 903, in wait_until_healthy
while proceed():
File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/teuthology/contextutil.py", line 132, in __call__
raise MaxWhileTries(error_msg)
MaxWhileTries: 'wait_until_healthy' reached maximum tries (150) after waiting for 900 seconds
</pre>
and
<pre>
2017-08-02T19:46:05.284 INFO:teuthology.misc.health.smithi193.stdout:HEALTH_WARN application not enabled on 1 pool(s)
2017-08-02T19:46:05.286 DEBUG:teuthology.misc:Ceph health: HEALTH_WARN application not enabled on 1 pool(s)
</pre>
This is mysterious for two reasons: (a) there are three pools at the time of the error and all of them have no application set; (b) the `fs new` command did not set the application. According to the `osd dump`:
<pre>
"pools":[
{
"pool":1,
"pool_name":"rbd",
"flags":1,
"flags_names":"hashpspool",
"type":1,
"size":2,
"min_size":1,
"crush_rule":0,
"object_hash":2,
"pg_num":8,
"pg_placement_num":8,
"crash_replay_interval":0,
"last_change":"10",
"last_force_op_resend":"0",
"last_force_op_resend_preluminous":"0",
"auid":0,
"snap_mode":"selfmanaged",
"snap_seq":0,
"snap_epoch":0,
"pool_snaps":[
],
"removed_snaps":"[]",
"quota_max_bytes":0,
"quota_max_objects":0,
"tiers":[
],
"tier_of":-1,
"read_tier":-1,
"write_tier":-1,
"cache_mode":"none",
"target_max_bytes":0,
"target_max_objects":0,
"cache_target_dirty_ratio_micro":400000,
"cache_target_dirty_high_ratio_micro":600000,
"cache_target_full_ratio_micro":800000,
"cache_min_flush_age":0,
"cache_min_evict_age":0,
"erasure_code_profile":"",
"hit_set_params":{
"type":"none"
},
"hit_set_period":0,
"hit_set_count":0,
"use_gmt_hitset":true,
"min_read_recency_for_promote":0,
"min_write_recency_for_promote":0,
"hit_set_grade_decay_rate":0,
"hit_set_search_last_n":0,
"grade_table":[
],
"stripe_width":0,
"expected_num_objects":0,
"fast_read":false,
"options":{
},
"application_metadata":{
}
},
{
"pool":2,
"pool_name":"cephfs_metadata",
"flags":1,
"flags_names":"hashpspool",
"type":1,
"size":2,
"min_size":1,
"crush_rule":0,
"object_hash":2,
"pg_num":4,
"pg_placement_num":4,
"crash_replay_interval":0,
"last_change":"12",
"last_force_op_resend":"0",
"last_force_op_resend_preluminous":"0",
"auid":0,
"snap_mode":"selfmanaged",
"snap_seq":0,
"snap_epoch":0,
"pool_snaps":[
],
"removed_snaps":"[]",
"quota_max_bytes":0,
"quota_max_objects":0,
"tiers":[
],
"tier_of":-1,
"read_tier":-1,
"write_tier":-1,
"cache_mode":"none",
"target_max_bytes":0,
"target_max_objects":0,
"cache_target_dirty_ratio_micro":400000,
"cache_target_dirty_high_ratio_micro":600000,
"cache_target_full_ratio_micro":800000,
"cache_min_flush_age":0,
"cache_min_evict_age":0,
"erasure_code_profile":"",
"hit_set_params":{
"type":"none"
},
"hit_set_period":0,
"hit_set_count":0,
"use_gmt_hitset":true,
"min_read_recency_for_promote":0,
"min_write_recency_for_promote":0,
"hit_set_grade_decay_rate":0,
"hit_set_search_last_n":0,
"grade_table":[
],
"stripe_width":0,
"expected_num_objects":0,
"fast_read":false,
"options":{
},
"application_metadata":{
}
},
{
"pool":3,
"pool_name":"cephfs_data",
"flags":1,
"flags_names":"hashpspool",
"type":1,
"size":2,
"min_size":1,
"crush_rule":0,
"object_hash":2,
"pg_num":4,
"pg_placement_num":4,
"crash_replay_interval":0,
"last_change":"13",
"last_force_op_resend":"0",
"last_force_op_resend_preluminous":"0",
"auid":0,
"snap_mode":"selfmanaged",
"snap_seq":0,
"snap_epoch":0,
"pool_snaps":[
],
"removed_snaps":"[]",
"quota_max_bytes":0,
"quota_max_objects":0,
"tiers":[
],
"tier_of":-1,
"read_tier":-1,
"write_tier":-1,
"cache_mode":"none",
"target_max_bytes":0,
"target_max_objects":0,
"cache_target_dirty_ratio_micro":400000,
"cache_target_dirty_high_ratio_micro":600000,
"cache_target_full_ratio_micro":800000,
"cache_min_flush_age":0,
"cache_min_evict_age":0,
"erasure_code_profile":"",
"hit_set_params":{
"type":"none"
},
"hit_set_period":0,
"hit_set_count":0,
"use_gmt_hitset":true,
"min_read_recency_for_promote":0,
"min_write_recency_for_promote":0,
"hit_set_grade_decay_rate":0,
"hit_set_search_last_n":0,
"grade_table":[
],
"stripe_width":0,
"expected_num_objects":0,
"fast_read":false,
"options":{
},
"application_metadata":{
}
}
] "pools":[{"pool":1,"pool_name":"rbd","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"object_hash":2,"pg_num":8,"pg_placement_num":8,"crash_replay_interval":0,"last_change":"10","last_force_op_resend":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}},{"pool":2,"pool_name":"cephfs_metadata","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"object_hash":2,"pg_num":4,"pg_placement_num":4,"crash_replay_interval":0,"last_change":"12","last_force_op_resend":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}},{"pool":3,"pool_name":"cephfs_data","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"object_hash":2,"pg_num":4,"pg_placement_num":4,"crash_replay_interval":0,"last_change":"13","last_force_op_resend":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}]
</pre>