Ceph

   -42> 2015-03-20 09:31:52.104655 7f0e8c1cc700 10 osd.1 pg_epoch: 242 pg[1.9s3( v 212'121 (0'0,212'121] local-les=215 n=3 ec=9 les/c 223/223 213/241/214) [2,4,3,2147483647]/[2,4,3,1] r=3 lpr=241 pi=211-240/14 crt=211'117 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 242 epoch_requested: 242 MLogRec from 2(0)
   -41> 2015-03-20 09:31:52.104668 7f0e8c1cc700 10 osd.1 pg_epoch: 242 pg[1.9s3( v 212'121 (0'0,212'121] local-les=215 n=3 ec=9 les/c 223/223 213/241/214) [2,4,3,2147483647]/[2,4,3,1] r=3 lpr=241 pi=211-240/14 crt=211'117 lcod 0'0 remapped NOTIFY] state<Started/Stray>: got info+log from osd.2(0) 1.9s0( v 223'123 (0'0,223'123] local-les=242 n=2 ec=9 les/c 223/223 222/241/222) log((206'113,223'123], crt=212'121)
   -40> 2015-03-20 09:31:52.104680 7f0e8c1cc700 10 merge_log log((206'113,223'123], crt=212'121) from osd.2(0) into log((0'0,212'121], crt=211'117)
   -39> 2015-03-20 09:31:52.104683 7f0e8c1cc700 10 merge_log extending head to 223'123
   -38> 2015-03-20 09:31:52.104685 7f0e8c1cc700 20   ? 223'123 (206'113) modify   b48b1589/burnupi3533489-10/head//1 by client.4122.0:1509 2015-03-20 09:30:39.148621
   -37> 2015-03-20 09:31:52.104689 7f0e8c1cc700 20   ? 223'122 (206'113) clone    b48b1589/burnupi3533489-10/8d//1 by unknown.0.0:0 2015-03-20 09:29:24.621077 snaps [8d]
   -36> 2015-03-20 09:31:52.104694 7f0e8c1cc700 20 merge_log 223'122 (206'113) clone    b48b1589/burnupi3533489-10/8d//1 by unknown.0.0:0 2015-03-20 09:29:24.621077 snaps [8d]
   -35> 2015-03-20 09:31:52.104701 7f0e8c1cc700 20 merge_log 223'123 (206'113) modify   b48b1589/burnupi3533489-10/head//1 by client.4122.0:1509 2015-03-20 09:30:39.148621
   -34> 2015-03-20 09:31:52.104709 7f0e8c1cc700 10 merge_log divergent 212'121 (183'99) delete   b1c51e99/burnupi3533489-4/head//1 by client.4122.0:1482 2015-03-20 09:30:37.117966
   -33> 2015-03-20 09:31:52.104713 7f0e8c1cc700 10 merge_log divergent 212'120 (0'0) modify   b1c51e99/burnupi3533489-4/snapdir//1 by unknown.0.0:0 2015-03-20 09:30:37.117966
   -32> 2015-03-20 09:31:52.104717 7f0e8c1cc700 10 merge_log divergent 212'119 (183'99) clone    b1c51e99/burnupi3533489-4/8d//1 by unknown.0.0:0 2015-03-20 09:29:22.473599 snaps [8d]
   -31> 2015-03-20 09:31:52.104721 7f0e8c1cc700 10 merge_log divergent 211'118 (211'117) modify   79b35479/burnupi3533489-23/head//1 by client.4122.0:1472 2015-03-20 09:30:36.026161
   -30> 2015-03-20 09:31:52.104724 7f0e8c1cc700 10 merge_log divergent 211'117 (211'116) modify   79b35479/burnupi3533489-23/head//1 by client.4122.0:1471 2015-03-20 09:30:36.026106
   -29> 2015-03-20 09:31:52.104727 7f0e8c1cc700 10 merge_log divergent 211'116 (211'115) modify   79b35479/burnupi3533489-23/head//1 by client.4122.0:1470 2015-03-20 09:30:36.021861
   -28> 2015-03-20 09:31:52.104731 7f0e8c1cc700 10 merge_log divergent 211'115 (167'97) modify   79b35479/burnupi3533489-23/head//1 by client.4122.0:1469 2015-03-20 09:30:36.001902
   -27> 2015-03-20 09:31:52.104734 7f0e8c1cc700 10 merge_log divergent 211'114 (167'97) clone    79b35479/burnupi3533489-23/8d//1 by unknown.0.0:0 2015-03-20 09:28:59.401246 snaps [8d,82]
   -26> 2015-03-20 09:31:52.104809 7f0e8c1cc700 10 _merge_object_divergent_entries: merging hoid 79b35479/burnupi3533489-23/8d//1 entries: 211'114 (167'97) clone    79b35479/burnupi3533489-23/8d//1 by unknown.0.0:0 2015-03-20 09:28:59.401246 snaps [8d,82]
   -25> 2015-03-20 09:31:52.104816 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid 79b35479/burnupi3533489-23/8d//1 prior_version: 167'97 first_divergent_update: 211'114 last_divergent_update: 211'114
   -24> 2015-03-20 09:31:52.104818 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid 79b35479/burnupi3533489-23/8d//1 has no more recent entries in log
   -23> 2015-03-20 09:31:52.104820 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid 79b35479/burnupi3533489-23/8d//1 prior_version or op type indicates creation, deleting
   -22> 2015-03-20 09:31:52.104822 7f0e8c1cc700 10 _merge_object_divergent_entries: merging hoid 79b35479/burnupi3533489-23/head//1 entries: 211'115 (167'97) modify   79b35479/burnupi3533489-23/head//1 by client.4122.0:1469 2015-03-20 09:30:36.001902,211'116 (211'115) modify   79b35479/burnupi3533489-23/head//1 by client.4122.0:1470 2015-03-20 09:30:36.021861,211'117 (211'116) modify   79b35479/burnupi3533489-23/head//
1 by client.4122.0:1471 2015-03-20 09:30:36.026106,211'118 (211'117) modify   79b35479/burnupi3533489-23/head//1 by client.4122.0:1472 2015-03-20 09:30:36.026161
   -21> 2015-03-20 09:31:52.104834 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid 79b35479/burnupi3533489-23/head//1 prior_version: 167'97 first_divergent_update: 211'115 last_divergent_update: 211'118
   -20> 2015-03-20 09:31:52.104836 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid 79b35479/burnupi3533489-23/head//1 has no more recent entries in log
   -19> 2015-03-20 09:31:52.104838 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid 79b35479/burnupi3533489-23/head//1 must be rolled back or recovered, attempting to rollback
   -18> 2015-03-20 09:31:52.104839 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid 79b35479/burnupi3533489-23/head//1 cannot rollback 211'117 (211'116) modify   79b35479/burnupi3533489-23/head//1 by client.4122.0:1471 2015-03-20 09:30:36.026106
   -17> 2015-03-20 09:31:52.104843 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid 79b35479/burnupi3533489-23/head//1 cannot roll back, removing and adding to missing
   -16> 2015-03-20 09:31:52.104846 7f0e8c1cc700 10 _merge_object_divergent_entries: merging hoid b1c51e99/burnupi3533489-4/8d//1 entries: 212'119 (183'99) clone    b1c51e99/burnupi3533489-4/8d//1 by unknown.0.0:0 2015-03-20 09:29:22.473599 snaps [8d]
   -15> 2015-03-20 09:31:52.104850 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/8d//1 prior_version: 183'99 first_divergent_update: 212'119 last_divergent_update: 212'119
   -14> 2015-03-20 09:31:52.104852 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/8d//1 has no more recent entries in log
   -13> 2015-03-20 09:31:52.104853 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/8d//1 prior_version or op type indicates creation, deleting
   -12> 2015-03-20 09:31:52.104855 7f0e8c1cc700 10 _merge_object_divergent_entries: merging hoid b1c51e99/burnupi3533489-4/head//1 entries: 212'121 (183'99) delete   b1c51e99/burnupi3533489-4/head//1 by client.4122.0:1482 2015-03-20 09:30:37.117966
   -11> 2015-03-20 09:31:52.104860 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/head//1 prior_version: 183'99 first_divergent_update: 212'121 last_divergent_update: 212'121
   -10> 2015-03-20 09:31:52.104861 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/head//1 has no more recent entries in log
    -9> 2015-03-20 09:31:52.104863 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/head//1 must be rolled back or recovered, attempting to rollback
    -8> 2015-03-20 09:31:52.104864 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/head//1 rolling back 212'121 (183'99) delete   b1c51e99/burnupi3533489-4/head//1 by client.4122.0:1482 2015-03-20 09:30:37.117966
    -7> 2015-03-20 09:31:52.104868 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/head//1 rolled back
    -6> 2015-03-20 09:31:52.104869 7f0e8c1cc700 10 _merge_object_divergent_entries: merging hoid b1c51e99/burnupi3533489-4/snapdir//1 entries: 212'120 (0'0) modify   b1c51e99/burnupi3533489-4/snapdir//1 by unknown.0.0:0 2015-03-20 09:30:37.117966
    -5> 2015-03-20 09:31:52.104873 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/snapdir//1 prior_version: 0'0 first_divergent_update: 212'120 last_divergent_update: 212'120
    -4> 2015-03-20 09:31:52.104875 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/snapdir//1 has no more recent entries in log
    -3> 2015-03-20 09:31:52.104876 7f0e8c1cc700 10 _merge_object_divergent_entries: hoid b1c51e99/burnupi3533489-4/snapdir//1 prior_version or op type indicates creation, deleting
    -2> 2015-03-20 09:31:52.104880 7f0e8c1cc700 10 merge_log result log((0'0,223'123], crt=223'123) missing(3) changed=1
    -1> 2015-03-20 09:31:52.104897 7f0e8c1cc700 20 snap_mapper.remove_oid 79b35479/burnupi3533489-23/8d//1
     0> 2015-03-20 09:31:52.108444 7f0e8c1cc700 -1 osd/SnapMapper.cc: In function 'int SnapMapper::remove_oid(const hobject_t&, MapCacher::Transaction<std::basic_string<char>, ceph::buffer::list>*)' thread 7f0e8c1cc700 time 2015-03-20 09:31:52.104906
osd/SnapMapper.cc: 282: FAILED assert(check(oid))

 ceph version 0.93-652-ga2f20ed (a2f20ed36fba09da7671684c6b083ea919081cdb)
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0xbc768b]
 2: (SnapMapper::remove_oid(hobject_t const&, MapCacher::Transaction<std::string, ceph::buffer::list>*)+0x1ed) [0x72a42d]
 3: (PG::clear_object_snap_mapping(ObjectStore::Transaction*, hobject_t const&)+0x94) [0x7c40b4]
 4: (PG::remove_snap_mapped_object(ObjectStore::Transaction&, hobject_t const&)+0xbd) [0x7d745d]
 5: (PG::PGLogEntryHandler::apply(PG*, ObjectStore::Transaction*)+0x101) [0x8268f1]
 6: (PG::merge_log(ObjectStore::Transaction&, pg_info_t&, pg_log_t&, pg_shard_t)+0xaa) [0x7d758a]
 7: (PG::RecoveryState::Stray::react(PG::MLogRec const&)+0x437) [0x7f9177]
 8: (boost::statechart::simple_state<PG::RecoveryState::Stray, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x1f4) [0x8356
34]
 9: (boost::statechart::stateubuntu@teuthology:/var/lib/teuthworker/archive/sage-2015-03-20_06:54:22-rados-wip-sage-testing---basic-multi/812894_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x820e2b]
 10: (PG::handle_peering_event(std::tr1::shared_ptr<PG::CephPeeringEvt>, PG::RecoveryCtx*)+0x1ce) [0x7ce16e]
 11: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x2c0) [0x6b2720]
 12: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x18) [0x709bc8]
 13: (ThreadPool::worker(ThreadPool::WorkThread*)+0xa5e) [0xbb881e]
 14: (ThreadPool::WorkThread::entry()+0x10) [0xbb98c0]
 15: (()+0x8182) [0x7f0ea8aa8182]
 16: (clone()+0x6d) [0x7f0ea701438d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

ubuntu@teuthology:/var/lib/teuthworker/archive/sage-2015-03-20_06:54:22-rados-wip-sage-testing---basic-multi/812894
(basically master; no suspicious osd changes)