Project

General

Profile

Sepia » History » Revision 143

Revision 142 (Yuri Weinstein, 07/20/2017 06:00 AM) → Revision 143/220 (Yuri Weinstein, 08/08/2017 04:53 PM)

h3. Crontab<pre>MAILTO="ceph-infra@redhat.com;yweinste@redhat.com" 
 #MAILTO="tmuthami@redhat.com;yweinste@redhat.com" 
 CEPH_QA_EMAIL="ceph-qa@ceph.com" 
 # Edit this file to introduce tasks to be run by cron. 
 #  

 # Each task to run has to be defined through a single line 
 # indicating with different fields when the task will be run 
 # and what command to run for the task 
 #  
 # To define the time you can provide concrete values for 
 # minute (m), hour (h), day of month (dom), month (mon), 
 # and day of week (dow) or use '*' in these fields (for 'any').#  
 # Notice that tasks will be started based on the cron's system 
 # daemon's notion of time and timezones. 
 #  
 # Output of the crontab jobs (including errors) is sent through 
 # email to the user the crontab file belongs to (unless redirected). 
 #  
 # For example, you can run a backup of all your user accounts 
 # at 5 a.m every week with: 
 # 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ 
 #  
 # For more information see the manual EPH_BRANCH=kraken; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/hammer-jewel-x ~/vps.yaml -e $CEPH_QA_EMAIL --filter ubuntu -d centos -D "7.2" 
 #  
 # m h    dom mon dow     command 
 PATH=/home/teuthology/src/teuthology_master/virtualenv/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 


 #Publish this crontab to the Tracker page http://tracker.ceph.com/projects/ceph-releases/wiki/Crontab 
 00 6 * * *    crontab=$(crontab -l | perl -p -e 's/&lt;/&lt;/g; s/&gt;/&gt;/g; s/&/&amp;/g') ; header=$(echo h3. Crontab ; echo) ; curl --verbose -X PUT --header 'Content-type: application/xml' --data-binary '&lt;?xml version="1.0"?&gt;&lt;wiki_page&gt;&lt;text&gt;'"$header"'&lt;pre&gt;'"$crontab"'&lt;/pre&gt;&lt;/text&gt;&lt;/wiki_page&gt;' http://tracker.ceph.com/projects/ceph-releases/wiki/sepia.xml?key=$(cat /etc/redmine-key) 

 ## Disabled - dgalloway - ~/src/ceph-qa-suite_master/suites/upgrade/infernalis does not exist 
 #Publish this suite descriptions to the Tracker page http://tracker.ceph.com/projects/ceph-qa-suite/infernalis-x 
 #10 8 * * *    SUITE_NAME=~/src/ceph-qa-suite_master/suites/upgrade/infernalis-x; crontab=$(teuthology-describe-tests --show-facet no $SUITE_NAME | perl -p -e 's/&lt;/&lt;/g; s/&gt;/&gt;/g; s/&/&amp;/g') ; header=$(echo h4. $SUITE_NAME ; echo " "; echo " ") ; curl --verbose -X PUT --header 'Content-type: application/xml' --data-binary '&lt;?xml version="1.0"?&gt;&lt;wiki_page&gt;&lt;text&gt;'"$header"'&lt;pre&gt;'"$crontab"'&lt;/pre&gt;&lt;/text&gt;&lt;/wiki_page&gt;' http://tracker.ceph.com/projects/ceph-qa-suite/wiki/infernalis-x.xml?key=$(cat /etc/redmine-key) 

 ## Disabled - dgalloway - ~/src/ceph-qa-suite_master/suites/upgrade/infernalis does not exist 
 #20 8 * * *    SUITE_NAME=~/src/ceph-qa-suite_master/suites/upgrade/infernalis; crontab=$(teuthology-describe-tests --show-facet no $SUITE_NAME | perl -p -e 's/&lt;/&lt;/g; s/&gt;/&gt;/g; s/&/&amp;/g') ; header=$(echo h4. $SUITE_NAME ; echo " "; echo " ") ; curl --verbose -X PUT --header 'Content-type: application/xml' --data-binary '&lt;?xml version="1.0"?&gt;&lt;wiki_page&gt;&lt;text&gt;'"$header"'&lt;pre&gt;'"$crontab"'&lt;/pre&gt;&lt;/text&gt;&lt;/wiki_page&gt;' http://tracker.ceph.com/projects/ceph-qa-suite/wiki/infernalis.xml?key=$(cat /etc/redmine-key) 

 30 8 * * *    SUITE_NAME=~/src/ceph-qa-suite_master/suites/ceph-ansible; crontab=$(teuthology-describe-tests --show-facet no $SUITE_NAME | perl -p -e 's/&lt;/&lt;/g; s/&gt;/&gt;/g; s/&/&amp;/g') ; header=$(echo h4. $SUITE_NAME ; echo " "; echo " ") ; curl --verbose -X PUT --header 'Content-type: application/xml' --data-binary '&lt;?xml version="1.0"?&gt;&lt;wiki_page&gt;&lt;text&gt;'"$header"'&lt;pre&gt;'"$crontab"'&lt;/pre&gt;&lt;/text&gt;&lt;/wiki_page&gt;' http://tracker.ceph.com/projects/ceph-qa-suite/wiki/ceph-ansible.xml?key=$(cat /etc/redmine-key) 

 # Ensure teuthology is up-to-date 
 00 * * * *     cd /home/teuthology/src/teuthology_master && /home/teuthology/bin/cron_wrapper git pull 
 55 16 * * *    cd /home/teuthology/src/git.ceph.com_ceph_master && /home/teuthology/bin/cron_wrapper git pull 
 # Ensure ceph-sepia-secrets is up-to-date 
 */5 * * * *    cd /home/teuthology/ceph-sepia-secrets && /home/teuthology/bin/cron_wrapper git pull 

 #Powercycle tests - lets try it manually and get a green before we add it to the nightlies 
 #02 01 * * 1,3,5,7 CEPH_BRANCH=jewel; MACHINE_NAME=mira; teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -k testing OR DISTRO -s powercycle -e $CEPH_QA_EMAIL 

 #********** master branch 
 0 5    * * *     CEPH_BRANCH=master; MACHINE_NAME=vps;      /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s smoke -k testing -p 70 -e $CEPH_QA_EMAIL ~/vps.yaml 

 ### OLD LINE BEFOE SUBSET 01 02 * * 7     CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s rbd -k distro -e $CEPH_QA_EMAIL ~/shaman.yaml 
 01 02 * * 1     CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 0 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 01 02 * * 2     CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 01 02 * * 3     CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 2 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 01 02 * * 4     CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 3 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 01 02 * * 5     CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 01 02 * * 6     CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 5 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 01 02 * * 7     CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 6 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 


 05 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s rgw -k distro -e $CEPH_QA_EMAIL 
 15 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s fs -k distro -e $CEPH_QA_EMAIL 
 20 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=smithi;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s krbd -k testing -e $CEPH_QA_EMAIL 
 25 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s kcephfs -k testing -e $CEPH_QA_EMAIL 
 35 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s knfs -k testing -e $CEPH_QA_EMAIL 
 #45 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=mira;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s hadoop -e $CEPH_QA_EMAIL 
 #50 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s samba -e $CEPH_QA_EMAIL 
 55 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s rest -k distro -e $CEPH_QA_EMAIL 
 59 03 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=vps;      /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s ceph-deploy -k distro -e $CEPH_QA_EMAIL 
 05 04 * * 1,6 CEPH_BRANCH=master; MACHINE_NAME=vps;      /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 7 -m $MACHINE_NAME -s ceph-ansible -k distro -e $CEPH_QA_EMAIL 


 #********** jewel branch 

 00 02 * * 1 CEPH_BRANCH=jewel; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 0 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 02 * * 2 CEPH_BRANCH=jewel; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 02 * * 3 CEPH_BRANCH=jewel; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 2 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 02 * * 4 CEPH_BRANCH=jewel; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 3 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 02 * * 6 CEPH_BRANCH=jewel; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 02 * * 6 CEPH_BRANCH=jewel; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 5 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 02 * * 7 CEPH_BRANCH=jewel; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 6 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 

 ########################## 

 #********** kraken branch PLACEHOLDER 

 30 02 * * 1 CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 0 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 2 CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 3 CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 2 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 4 CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 3 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 5 CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 6 CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 5 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 7 CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 6 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 


 #********** luminous branch PLACEHOLDER 

 30 02 * * 1 CEPH_BRANCH=luminous; MACHINE_NAME=ovh; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 0 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 2 CEPH_BRANCH=luminous; MACHINE_NAME=ovh; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 3 CEPH_BRANCH=luminous; MACHINE_NAME=ovh; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 2 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 4 CEPH_BRANCH=luminous; MACHINE_NAME=ovh; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 3 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 5 CEPH_BRANCH=luminous; MACHINE_NAME=ovh; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 6 CEPH_BRANCH=luminous; MACHINE_NAME=ovh; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 5 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 30 02 * * 7 CEPH_BRANCH=luminous; MACHINE_NAME=ovh; SUITE_NAME=rados; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 6 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 




 ########################## hammer ################## 
 ### rados suite divided into 14 parts 1 part every day of the 2 weeks, the bc part does 0-6 on even weeks and 7-13 on odd 
 ### temporary running hammer rados on smithi per Sam's request as we observe many ENOSPC failures in ovh lab 


 #00 13 * * 1 CEPH_BRANCH=hammer; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 0 $CEPH_BRANCH $MACHINE_NAME $CEPH_QA_EMAIL 
 #00 13 * * 2 CEPH_BRANCH=hammer; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 1 $CEPH_BRANCH $MACHINE_NAME $CEPH_QA_EMAIL 
 #00 13 * * 3 CEPH_BRANCH=hammer; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 2 $CEPH_BRANCH $MACHINE_NAME $CEPH_QA_EMAIL 
 #00 13 * * 4 CEPH_BRANCH=hammer; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 3 $CEPH_BRANCH $MACHINE_NAME $CEPH_QA_EMAIL 
 #00 13 * * 5 CEPH_BRANCH=hammer; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 4 $CEPH_BRANCH $MACHINE_NAME $CEPH_QA_EMAIL 
 #00 13 * * 6 CEPH_BRANCH=hammer; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 5 $CEPH_BRANCH $MACHINE_NAME $CEPH_QA_EMAIL 
 #00 13 * * 7 CEPH_BRANCH=hammer; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 6 $CEPH_BRANCH $MACHINE_NAME $CEPH_QA_EMAIL 



 ########################## 

 ########################## jewel ################## 

 00 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s rbd -k distro -e $CEPH_QA_EMAIL 
 05 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s rgw -k distro -e $CEPH_QA_EMAIL 
 10 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s fs -k distro -e $CEPH_QA_EMAIL 
 15 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s krbd -k testing -e $CEPH_QA_EMAIL 
 20 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s kcephfs -k testing -e $CEPH_QA_EMAIL 
 25 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s knfs -k testing -e $CEPH_QA_EMAIL 
 #30 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=mira;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s hadoop -e $CEPH_QA_EMAIL 
 #35 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=mira;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s samba -e $CEPH_QA_EMAIL 
 40 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=mira;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s rest -k distro -e $CEPH_QA_EMAIL 
 #removed per Greg's request 18 23 * * 4    CEPH_BRANCH=jewel; MACHINE_NAME=smithi; teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -k testing -s multimds -e $CEPH_QA_EMAIL 
 45 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s upgrade/client-upgrade -k distro -e $CEPH_QA_EMAIL 
 50 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=vps;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s ceph-deploy -k distro -e $CEPH_QA_EMAIL 
 10 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=vps;      /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s ceph-disk -k distro -e $CEPH_QA_EMAIL 
 15 04 * * 3,7    CEPH_BRANCH=jewel; MACHINE_NAME=vps;      /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s ceph-ansible -k distro -e $CEPH_QA_EMAIL 


 ########################## kraken    ################## 

 ### OLD LINE BEFOE SUBSET    00 11 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s rbd -k distro -e $CEPH_QA_EMAIL ~/shaman.yaml 
 00 05    * * 1     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 0 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 2     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 3     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 2 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 4     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 3 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 5     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 6     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 5 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 7     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 6 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 


 05 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s rgw -k distro -e $CEPH_QA_EMAIL 
 10 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s fs -k distro -e $CEPH_QA_EMAIL 
 15 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s krbd -k testing -e $CEPH_QA_EMAIL 
 20 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s kcephfs -k testing -e $CEPH_QA_EMAIL 
 25 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s knfs -k testing -e $CEPH_QA_EMAIL 
 #30 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=mira;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s hadoop -e $CEPH_QA_EMAIL 
 #35 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=mira;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s samba -e $CEPH_QA_EMAIL 
 40 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=mira;     /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s rest -k distro -e $CEPH_QA_EMAIL 
 45 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=ovh; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s upgrade/client-upgrade -k distro -e $CEPH_QA_EMAIL 
 55 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=vps;      /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s ceph-deploy -k distro ~/vps.yaml -e $CEPH_QA_EMAIL 
 10 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=vps;      /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s ceph-disk -k distro ~/vps.yaml -e $CEPH_QA_EMAIL 
 15 05 * * 2,4,6    CEPH_BRANCH=kraken; MACHINE_NAME=vps;      /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s ceph-ansible -k distro -e $CEPH_QA_EMAIL 

 ########################## 
 ######################### 


 ########################## luminous    ################## #### COPY of kraken on ovh ###################### 

 00 05    * * 1     CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 0 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 2     CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 3     CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 2 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 4     CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 3 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 5     CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 6     CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 5 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 
 00 05    * * 7     CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; SUITE_NAME=rbd; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 6 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL 


 05 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s rgw -k distro -e $CEPH_QA_EMAIL 
 10 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s fs -k distro -e $CEPH_QA_EMAIL 
 15 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s krbd -k testing -e $CEPH_QA_EMAIL 
 20 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s kcephfs -k testing -e $CEPH_QA_EMAIL 
 25 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s knfs -k testing -e $CEPH_QA_EMAIL 
 #30 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=smithi;     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s hadoop -e $CEPH_QA_EMAIL 
 #35 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=smithi;     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s samba -e $CEPH_QA_EMAIL 
 40 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=mira;     CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s rest -k distro -e $CEPH_QA_EMAIL 
 45 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=smithi; CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s upgrade/client-upgrade -k distro -e $CEPH_QA_EMAIL 
 55 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=vps;      CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s ceph-deploy -k distro ~/vps.yaml -e $CEPH_QA_EMAIL 
 10 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=vps;      CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s ceph-disk -k distro ~/vps.yaml -e $CEPH_QA_EMAIL 
 15 05 * * 2,4,6    CEPH_BRANCH=luminous; MACHINE_NAME=vps;      CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH    -n 7 -m $MACHINE_NAME -s ceph-ansible -k distro -e $CEPH_QA_EMAIL 

 ########################## 


 ######################### 




 # disabled pre Sage 05 18 * * 2,4,6     CEPH_BRANCH=infernalis; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -m $MACHINE_NAME -s upgrade/hammer-x ~/vps.yaml -e $CEPH_QA_EMAIL 
 #NO SUPPORT 10 18 * * 1,7       CEPH_BRANCH=infernalis; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -m $MACHINE_NAME -s upgrade/infernalis ~/vps.yaml -e $CEPH_QA_EMAIL 
 15 01 * * 1,5      CEPH_BRANCH=jewel; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -m $MACHINE_NAME -s upgrade/hammer-x ~/vps.yaml -e $CEPH_QA_EMAIL 
 #NO SUPPORT 20 17 * * *         CEPH_BRANCH=jewel; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -m $MACHINE_NAME -s upgrade/infernalis-x ~/vps.yaml -e $CEPH_QA_EMAIL 

 ###### on vps 
 20 04 * * *        CEPH_BRANCH=kraken; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/jewel-x ~/vps.yaml -e $CEPH_QA_EMAIL 
 23 04 * * *        CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/jewel-x ~/vps.yaml -e $CEPH_QA_EMAIL 
 25 03 * * 1,4      CEPH_BRANCH=kraken; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/hammer-jewel-x ~/vps.yaml -e $CEPH_QA_EMAIL 
 29 03 * * *        CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/hammer-jewel-x ~/vps.yaml -e $CEPH_QA_EMAIL 
 25 02 * * *        CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=vps; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/kraken-x ~/vps.yaml -e $CEPH_QA_EMAIL 

 ###### on smithi 
 20 05 * * *         CEPH_BRANCH=kraken; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/jewel-x -e $CEPH_QA_EMAIL --filter ubuntu 
 23 04 * * *         CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/jewel-x -e $CEPH_QA_EMAIL 
 25 02 * * 1,4       CEPH_BRANCH=kraken; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/hammer-jewel-x -e $CEPH_QA_EMAIL --filter ubuntu 
 29 03 * * *         CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/hammer-jewel-x ~/vps.yaml -e $CEPH_QA_EMAIL 
 25 03 * * *         CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/kraken-x -e $CEPH_QA_EMAIL --filter ubuntu 


 ##### on ovh 
 20 05 * * *         CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/jewel-x -e $CEPH_QA_EMAIL --filter ubuntu 
 23 04 * * *         CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/jewel-x -e $CEPH_QA_EMAIL 
 25 02 * * 1,4       CEPH_BRANCH=kraken; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/hammer-jewel-x -e $CEPH_QA_EMAIL --filter ubuntu 
 29 03 * * *         CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/hammer-jewel-x ~/vps.yaml -e $CEPH_QA_EMAIL 
 25 03 * * *         CEPH_BRANCH=luminous; CEPH_BRANCH=master; MACHINE_NAME=ovh; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 7 -m $MACHINE_NAME -s upgrade/kraken-x -e $CEPH_QA_EMAIL --filter ubuntu</pre>