Ceph

Version grep (never got to jewel):

2015-11-13 12:54:17.204648 7fa815761900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 7248
2015-11-13 12:54:17.307211 7f14b381d900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 7271
2015-11-13 12:54:17.429785 7f90375c8900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 7294
2015-11-13 12:54:25.983999 7fc8d20a5900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 4345
2015-11-13 12:54:26.723619 7ff8ebb32900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 4369
2015-11-13 12:54:27.455920 7f31738d0900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 4390
2015-11-13 12:54:41.561663 7f4665f35900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 7470
2015-11-13 12:54:41.566609 7fbfe275e900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 7472
2015-11-13 12:54:41.568608 7f0e721fa900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 7471
2015-11-13 12:54:41.576829 7fc29cd5f900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 4547
2015-11-13 12:54:41.578491 7f0f3d3f5900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 4548
2015-11-13 12:54:41.584790 7fa3cd24b900 0 ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a), process ceph-osd, pid 4551
2015-11-13 12:54:41.672077 7fbfc7b70700 10 osd.2 0 _collect_metadata {arch=x86_64,back_addr=10.214.134.132:6805/7472,ceph_version=ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a),cpu=Intel(R) Xeon(R) CPU X3440 2.53GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.134.132:6804/7472,hb_back_addr=10.214.134.132:6806/7472,hb_front_addr=10.214.134.132:6807/7472,hostname=mira059,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=16425252,os=Linux,osd_data=/var/lib/ceph/osd/ceph-2,osd_journal=/var/lib/ceph/osd/ceph-2/journal,osd_objectstore=filestore}
2015-11-13 12:54:41.673569 7f0e5760c700 10 osd.1 0 _collect_metadata {arch=x86_64,back_addr=10.214.134.132:6809/7471,ceph_version=ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a),cpu=Intel(R) Xeon(R) CPU X3440 2.53GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.134.132:6808/7471,hb_back_addr=10.214.134.132:6810/7471,hb_front_addr=10.214.134.132:6811/7471,hostname=mira059,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=16425252,os=Linux,osd_data=/var/lib/ceph/osd/ceph-1,osd_journal=/var/lib/ceph/osd/ceph-1/journal,osd_objectstore=filestore}
2015-11-13 12:54:41.729197 7f464b347700 10 osd.0 0 _collect_metadata {arch=x86_64,back_addr=10.214.134.132:6801/7470,ceph_version=ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a),cpu=Intel(R) Xeon(R) CPU X3440 2.53GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.134.132:6800/7470,hb_back_addr=10.214.134.132:6802/7470,hb_front_addr=10.214.134.132:6803/7470,hostname=mira059,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=16425252,os=Linux,osd_data=/var/lib/ceph/osd/ceph-0,osd_journal=/var/lib/ceph/osd/ceph-0/journal,osd_objectstore=filestore}
2015-11-13 12:54:41.861677 7f0f22807700 10 osd.4 0 _collect_metadata {arch=x86_64,back_addr=10.214.131.20:6805/4548,ceph_version=ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a),cpu=Intel(R) Xeon(R) CPU E5620 2.40GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.131.20:6804/4548,hb_back_addr=10.214.131.20:6806/4548,hb_front_addr=10.214.131.20:6807/4548,hostname=plana20,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=8160988,os=Linux,osd_data=/var/lib/ceph/osd/ceph-4,osd_journal=/var/lib/ceph/osd/ceph-4/journal,osd_objectstore=filestore}
2015-11-13 12:54:41.879377 7fc282171700 10 osd.3 0 _collect_metadata {arch=x86_64,back_addr=10.214.131.20:6801/4547,ceph_version=ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a),cpu=Intel(R) Xeon(R) CPU E5620 2.40GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.131.20:6800/4547,hb_back_addr=10.214.131.20:6802/4547,hb_front_addr=10.214.131.20:6803/4547,hostname=plana20,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=8160988,os=Linux,osd_data=/var/lib/ceph/osd/ceph-3,osd_journal=/var/lib/ceph/osd/ceph-3/journal,osd_objectstore=filestore}
2015-11-13 12:54:41.920279 7fa3b265d700 10 osd.5 0 _collect_metadata {arch=x86_64,back_addr=10.214.131.20:6809/4551,ceph_version=ceph version 0.94 (e61c4f093f88e44961d157f65091733580cea79a),cpu=Intel(R) Xeon(R) CPU E5620 2.40GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.131.20:6808/4551,hb_back_addr=10.214.131.20:6810/4551,hb_front_addr=10.214.131.20:6811/4551,hostname=plana20,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=8160988,os=Linux,osd_data=/var/lib/ceph/osd/ceph-5,osd_journal=/var/lib/ceph/osd/ceph-5/journal,osd_objectstore=filestore}
2015-11-13 13:14:16.995260 7ff221cbb900 0 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3), process ceph-osd, pid 10416
2015-11-13 13:14:17.231787 7ff2070cd700 10 osd.0 6 _collect_metadata {arch=x86_64,back_addr=10.214.134.132:6814/10416,ceph_version=ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3),cpu=Intel(R) Xeon(R) CPU X3440 2.53GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.134.132:6813/10416,hb_back_addr=10.214.134.132:6815/10416,hb_front_addr=10.214.134.132:6816/10416,hostname=mira059,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=16425252,os=Linux,osd_data=/var/lib/ceph/osd/ceph-0,osd_journal=/var/lib/ceph/osd/ceph-0/journal,osd_objectstore=filestore}
2015-11-13 13:14:54.713836 7fe9ac4fb900 0 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3), process ceph-osd, pid 10809
2015-11-13 13:14:55.089629 7fe99190d700 10 osd.1 9 _collect_metadata {arch=x86_64,back_addr=10.214.134.132:6818/10809,ceph_version=ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3),cpu=Intel(R) Xeon(R) CPU X3440 2.53GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.134.132:6817/10809,hb_back_addr=10.214.134.132:6819/10809,hb_front_addr=10.214.134.132:6820/10809,hostname=mira059,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=16425252,os=Linux,osd_data=/var/lib/ceph/osd/ceph-1,osd_journal=/var/lib/ceph/osd/ceph-1/journal,osd_objectstore=filestore}
2015-11-13 13:15:32.333626 7fa98dbee900 0 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3), process ceph-osd, pid 11205
2015-11-13 13:15:32.554115 7fa973000700 10 osd.2 12 _collect_metadata {arch=x86_64,back_addr=10.214.134.132:6801/11205,ceph_version=ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3),cpu=Intel(R) Xeon(R) CPU X3440 2.53GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.134.132:6800/11205,hb_back_addr=10.214.134.132:6802/11205,hb_front_addr=10.214.134.132:6803/11205,hostname=mira059,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=16425252,os=Linux,osd_data=/var/lib/ceph/osd/ceph-2,osd_journal=/var/lib/ceph/osd/ceph-2/journal,osd_objectstore=filestore}
2015-11-13 13:16:10.190953 7f6dec2f3900 0 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3), process ceph-osd, pid 8975
2015-11-13 13:16:10.775937 7f6dd1705700 10 osd.3 15 _collect_metadata {arch=x86_64,back_addr=10.214.131.20:6813/8975,ceph_version=ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3),cpu=Intel(R) Xeon(R) CPU E5620 2.40GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.131.20:6812/8975,hb_back_addr=10.214.131.20:6814/8975,hb_front_addr=10.214.131.20:6815/8975,hostname=plana20,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=8160988,os=Linux,osd_data=/var/lib/ceph/osd/ceph-3,osd_journal=/var/lib/ceph/osd/ceph-3/journal,osd_objectstore=filestore}
2015-11-13 13:16:47.964594 7fcfd4340900 0 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3), process ceph-osd, pid 9129
2015-11-13 13:16:48.478726 7fcfb9752700 10 osd.4 18 _collect_metadata {arch=x86_64,back_addr=10.214.131.20:6817/9129,ceph_version=ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3),cpu=Intel(R) Xeon(R) CPU E5620 2.40GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.131.20:6816/9129,hb_back_addr=10.214.131.20:6818/9129,hb_front_addr=10.214.131.20:6819/9129,hostname=plana20,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=8160988,os=Linux,osd_data=/var/lib/ceph/osd/ceph-4,osd_journal=/var/lib/ceph/osd/ceph-4/journal,osd_objectstore=filestore}
2015-11-13 13:17:25.720117 7f4272c31900 0 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3), process ceph-osd, pid 9302
2015-11-13 13:17:26.247622 7f4258043700 10 osd.5 21 _collect_metadata {arch=x86_64,back_addr=10.214.131.20:6801/9302,ceph_version=ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3),cpu=Intel(R) Xeon(R) CPU E5620 2.40GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.131.20:6800/9302,hb_back_addr=10.214.131.20:6802/9302,hb_front_addr=10.214.131.20:6803/9302,hostname=plana20,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=8160988,os=Linux,osd_data=/var/lib/ceph/osd/ceph-5,osd_journal=/var/lib/ceph/osd/ceph-5/journal,osd_objectstore=filestore}
2015-11-13 13:56:30.709821 7fa04fbbe900 0 ceph version 0.94.5-86-ga79acd4 (a79acd41187e6b049432bdc314f192e3fbb560a3), process ceph-osd, pid 14631
2015-11-13 13:56:31.082093 7fa034fc5700 10 osd.0 1310 _collect_metadata {arch=x86_64,back_addr=10.214.134.132:6806/14631,ceph_version=ceph version 0.94.5-86-ga79acd4 (a79acd41187e6b049432bdc314f192e3fbb560a3),cpu=Intel(R) Xeon(R) CPU X3440 @ 2.53GHz,distro=Ubuntu,distro_codename=trusty,distro_description=Ubuntu 14.04.3 LTS,distro_version=14.04,filestore_backend=xfs,filestore_f_type=0x58465342,front_addr=10.214.134.132:6805/14631,hb_back_addr=10.214.134.132:6807/14631,hb_front_addr=10.214.134.132:6808/14631,hostname=mira059,kernel_description=#111-Ubuntu SMP Fri Nov 6 18:17:06 UTC 2015,kernel_version=3.13.0-68-generic,mem_swap_kb=0,mem_total_kb=16425252,os=Linux,osd_data=/var/lib/ceph/osd/ceph-0,osd_journal=/var/lib/ceph/osd/ceph-0/journal,osd_objectstore=filestore}

More details at http://tracker.ceph.com/issues/13356#note-27

One of these four commits is responsible for the regression:

• https://github.com/ceph/ceph/pull/6336 PASS
• https://github.com/ceph/ceph/pull/6391 PASS
• https://github.com/ceph/ceph/pull/6199 PASS
• https://github.com/ceph/ceph/pull/5825 FAIL

Failed job: upgrade:hammer/older/{0-cluster/start.yaml 1-install/v0.94.yaml 2-workload/blogbench.yaml 3-upgrade-sequence/upgrade-osd-mon-mds.yaml 4-final/{monthrash.yaml osdthrash.yaml testrados.yaml} distros/ubuntu_14.04.yaml}

Reverting OSDMap: fill in known encode_features where possible does not help.

osd.0 waits for a new map (pg_notify(0.17(3),2.8(2) epoch 9) currently wait for new map ) after upgrading and discards it when it is sent by the mon.

• osd.0 is upgraded from v0.94 to the latest hammer and restarts
• osd.0 receives an incremental osd map update which it cannot apply and asks for a full osdmap
  

  2015-11-16 22:19:09.054300 7f8910e9e700  2 osd.0 8 got incremental 9 but failed to encode full with correct crc; requesting
  2015-11-16 22:19:09.054309 7f8910e9e700  0 log_channel(cluster) log [WRN] : failed to encode map e9 with expected crc
  2015-11-16 22:19:09.054336 7f8910e9e700  1 -- 10.214.130.159:6800/12338 --> 10.214.130.159:6789/0 -- mon_get_osdmap(full 9-9) v1 -- ?+0 0x5677e00 con 0x5117600
  

• mon.b is the leader but mon.a handles the request from osd.0, does not forward it and sends an incremental update instead of the requested full map.
  

  2015-11-16 22:19:09.052692 7f7d2d2aa700  1 -- 10.214.130.159:6789/0 <== osd.0 10.214.130.159:6800/12338 22 ==== mon_get_osdmap(full 9-9) v1 ==== 34+0+0 (3059409895 0 0) 0x453f600 con 0x40adb20
  ...
  2015-11-16 22:19:09.052805 7f7d2d2aa700 10 mon.a@1(peon).paxosservice(osdmap 1..9) dispatch mon_get_osdmap(full 9-9) v1 from osd.0 10.214.130.159:6800/12338
  ...
  2015-11-16 22:19:09.054013 7f7d2d2aa700  5 mon.a@1(peon).osd e9 send_incremental [9..9] to osd.0 10.214.130.159:6800/12338
  2015-11-16 22:19:09.054037 7f7d2d2aa700 10 mon.a@1(peon).osd e9 build_incremental [9..9]
  2015-11-16 22:19:09.054078 7f7d2d2aa700 20 mon.a@1(peon).osd e9 build_incremental    inc 9 222 bytes
  

• mon.a sends a full osd_map shortly after
  

  2015-11-16 22:19:09.066501 7f7d2d2aa700 10 mon.a@1(peon).osd e9 preprocess_get_osdmap mon_get_osdmap(full 9-9) v1
  2015-11-16 22:19:09.066504 7f7d2d2aa700  1 -- 10.214.130.159:6789/0 --> 10.214.130.159:6800/12338 -- osd_map(0..0 src has 1..9) v3 -- ?+0 0x43c5600 con 0x40adb20
  

• osd.0 receives the full map but think there is nothing there for it
  

  2015-11-16 22:19:09.057208 7f8910e9e700  3 osd.0 8 handle_osd_map epochs [0,0], i have 8, src has [1,9]
  2015-11-16 22:19:09.057210 7f8910e9e700 10 osd.0 8  no new maps here, dropping
  

The problem appeared after v0.94.5 ( http://pulpito.ceph.com/teuthology-2015-11-02_16:05:02-upgrade:hammer-hammer-distro-basic-vps/ ) and before November 13 (bad upgrade run http://pulpito.ceph.com/teuthology-2015-11-13_16:05:09-upgrade:hammer-hammer-distro-basic-vps/). Double checking v0.94.5 passes at http://pulpito.ceph.com/loic-2015-11-17_12:45:04-upgrade:hammer-v0.94.5---basic-vps/

• Merge pull request #6213 from SUSE/wip-13425-hammer
  • CephxServiceHandler.cc: fix get_auth conditional
• Merge pull request #6336 from Abhishekvrshny/wip-13535-hammer
  • osdc/Objecter: distinguish between multiple notify completions
  • osd: reply to notify request with our unique notify_id
• Merge pull request #6391 from SUSE/wip-13590-hammer
  • mon: should not set isvalid = true when cephx_verify_authorizer return false
• Merge pull request #6398 from SUSE/wip-13588-hammer
  • OSD:shall reset primary and up_primary fields when beginning a new past_interval.
• Merge pull request #6199 from liewegas/wip-randomize-scrub-hammer
  • osd: use another name for randomize scrub option
  • osd: randomize scrub times to avoid scrub wave
  • osd: use func in log messages
  • osd: simplify OSD::scrub_load_below_threshold() a little bit
• Merge pull request #5825 from tchaikov/wip-12848-hammer
  • ReplicatedPG::hit_set_setup: fix hit_set_remove_all call
  • osd/: assert in HitSet constructor if type is TYPE_NONE as well
  • osd: only remove all the hit set objects on primary osd
  • osd: implement hit_set_remove_all
  • mon: disable gmt_hitset if not supported
  • osd: do not let OSD_HITSET_GMT reuse the feature bit
  • osd: Decode use_gmt_hitset with a unique version
  • "mon: print use_gmt_hitset in "ceph osd pool get"":https://github.com/ceph/ceph/commit/26c7e968523ac09f608c4ed7c50681fbb741ca1f
  • "mon: add "ceph osd pool set $pool use_gmt_hitset true cmd
  • osd: use GMT time for the object name of hitsets
• Merge pull request #6157 from dachary/wip-13040-hammer
  • Common/Thread: pthread_attr_destroy(thread_attr) when done with it When a thread attributes object is no longer required, it should be destroyed using the pthread_attr_destroy() function. Destroying a thread attributes object has no effect on threads that were created using that object.
  • Thread.cc: remove malloc/free pair
• Merge pull request #6155 from dachary/wip-13171-hammer
  • osdc/Objecter: optimize Objecter::tick.
  • osdc/Objecter: remove the unuseful code.
  • osdc/Objecter: Don't forget call _op_cancel_map_check when cancel linger op.
  • osdc/Objecter: In _cancel_linger_op, it should make num_unacked/num_committed decrease.
  • osdc/Objeter: When cancel op, decrease num_unacked/num_uncommitted.
  • osdc/Objecter: For func op_cancel_writes it can directly call op_cancel.
  • Objecter: Take RLocker when call is_active.
• Merge pull request #6152 from dachary/wip-13233-hammer
  • osd: print min_last_epoch_clean along with pg dump
• Merge pull request #6146 from dachary/wip-13337-hammer
  • OSDService::agent_entry: don't use PG::operator<< without pg lock
• Merge pull request #6145 from dachary/wip-13338-hammer
  • filestore: fix peek_queue for OpSequencer
• Merge pull request #6144 from dachary/wip-13339-hammer
  • mon/Elector: do a trivial write on every election cycle
  • mon/MonitorDBStore: assert/crash if there is a write error
• Merge pull request #5891 from Abhishekvrshny/wip-13035-hammer
  • bug fix: osd: requeue_scrub when kick_object_context_blocked
• Merge pull request #5890 from Abhishekvrshny/wip-13037-hammer
  • PG::handle_advance_map: on_pool_change after handling the map change
• Merge pull request #6322 from sponce/WIP-13210-hammer
  • radosstriper : Fixed broken Makefiles after integration of lttng
• Merge pull request #5810 from Abhishekvrshny/wip-12948-hammer
  • tools: Don't delete, recreate and re-fill buffers in rados bench.
  • bufferlist: implement bufferlist::invalidate_crc()
• Merge pull request #5530 from SUSE/wip-12587-hammer
  • common/syncfs: fall back to sync(2) if syncfs(2) not available
• Merge pull request #5889 from Abhishekvrshny/wip-13042-hammer
  • WorkQueue: add/remove_work_queue methods now thread safe
• Merge pull request #6151 from dachary/wip-13245-hammer
  • use simplifed messenger constructor for clients
  • msg: add simplified messenger constructor
• Merge pull request #6159 from dachary/wip-13341-hammer
  • init-rbdmap: fix CMDPARAMS
• Merge pull request #6154 from dachary/wip-13195-hammer
  • mon/PGMap: calc min_last_epoch_clean when decode
• Merge pull request #6132 from SUSE/wip-13307-hammer
  • OSDMap: fill in known encode_features where possible
• Merge pull request #6097 from dillaman/wip-13045-hammer
  • librbd: invoking RBD::open twice will leak memory
  • librbd: diff_iterate needs to handle holes in parent images
  • tests: new test case for librbd diff_iterate over discard extents
• Merge pull request #6060 from XinzeChi/wip-hammer-outdata-set
  • bug fix: osd: avoid multi set osd_op.outdata in tier pool
• Merge pull request #5897 from tchaikov/wip-12940-hammer
  • ceph_test_rados_api_aio: add a test for aio_sparse_read
  • ceph_test_rados_api_io: add tests for sparse_read
  • ceph_test_rados: also send sparse_read in ReadOp
  • osd: should use ec_pool() when checking for an ecpool
  • osd: translate sparse_read to read for ecpool
• Merge pull request #5787 from SUSE/wip-12923-hammer
  • logrotate.conf: prefer service over invoke-rc.d
• Merge pull request #5766 from dachary/wip-12590-hammer
  • mon: fix checks on mds add_data_pool
• Merge pull request #5361 from ceph/wip-11786.hammer
  • mon: MonitorDBStore: make get_next_key() work properly
  • mon: MonitorDBStore: get_next_key() only if prefix matches

The hammer-backports-bisect-try branch is the hammer branch with

git log --before '2015-11-14' --merges --format='%H' --grep 'Merge pull request #\(6213\|6336\|6391\|6199\|5825\|6157\|6155\|6152\|6146\|6145\|6144\|5891\|6322\|5810\|5530\|5889\|6151\|6159\|6097\|6060\|5897\|5787\|5766\|5361\)' tags/v0.94.5..ceph/hammer | tac | xargs git cherry-pick -m1

which is which is v0.94.5 with all the pull requests above except for the pull requests 6398 5890 6154 6132 is still bad

git log --before '2015-11-14' --merges --format='%H' --grep 'Merge pull request #\(6213\|6336\|6391\|6199\|5825\|6157\|6152\|6146\|6145\|6144\|5891\|6322\|5810\|5530\|5889\|6151\|6159\|6097\|6060\|5897\|5787\|5766\|5361\|5890\|6154\|6132\)' tags/v0.94.5..ceph/hammer | tac | xargs git cherry-pick -m1 # 6398 6155

which is v0.94.5 with all the pull requests above except for the pull requests 6398 6155 (https://github.com/dachary/ceph/commit/8f76f4c467dc69cd2af75d9cbb134c26f2d04a84 is in 6155 and is about osd map).

The above fails.

git log --before '2015-11-14' --merges --format='%H' --grep 'Merge pull request #\(6213\|6336\|6391\|6199\|5825\|6157\|6152\|6146\|6145\|6144\|5891\|6322\|5810\|5530\|6151\|6159\|6097\|5897\|5787\|5766\)' tags/v0.94.5..ceph/hammer | tac | xargs git cherry-pick -m1 # 6398 6155 5890 6154 6132 5889 6060 5361

The above fails.

git log --before '2015-11-14' --merges --format='%H' --grep 'Merge pull request #\(6213\|6157\|6152\|6146\|6145\|6144\|5891\|6322\|5810\|5530\|6151\|6159\|6097\|5897\|5787\|5766\)' tags/v0.94.5..ceph/hammer | tac | xargs git cherry-pick -m1 # 6336 6391 6199 5825 # 6398 6155 5890 6154 6132 5889 6060 5361

The above passes.

git log --before '2015-11-14' --merges --format='%H' --grep 'Merge pull request #\(6336\|6391\|6199\|5825\)' tags/v0.94.5..ceph/hammer | tac | xargs git cherry-pick -m1

teuthology-openstack --verbose --key-name myself  --key-filename ~/Downloads/myself --ceph-git-url https://github.com/dachary/ceph.git --ceph-qa-suite-git-url https://github.com/dachary/ceph-qa-suite.git --filter='upgrade:hammer/older/{0-cluster/start.yaml 1-install/v0.94.yaml 2-workload/blogbench.yaml 3-upgrade-sequence/upgrade-osd-mon-mds.yaml 4-final/{monthrash.yaml osdthrash.yaml testrados.yaml} distros/ubuntu_14.04.yaml}' --verbose --suite upgrade/hammer --suite-branch wip-buildpackages-refactor-hammer --ceph hammer-backport-bisect-bad

• running http://167.114.233.32:8081/ubuntu-2015-11-18_10:51:44-upgrade:hammer-hammer-backport-bisect-bad---basic-openstack/