Ceph

The kernel configuration ( 3.10.11 sources ) is here : config.gz
The general ceph setup is described here : http://dachary.org/?p=2087
The following happened today on a machine heavily loaded with recovering OSDs and a lot of I/O.

root@miltonduff:~# uptime
 11:59:47 up 1 day, 19:37,  2 users,  load average: 546,05, 546,08, 534,52

root@miltonduff:~# uname -a
Linux miltonduff 3.10.11-dsiun-130711 #44 SMP Mon Sep 9 16:59:12 CEST 2013 x86_64 GNU/Linux
NFO: task ceph-osd:17047 blocked for more than 120 seconds.
[153972.073476] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[153972.076322] ceph-osd        D ffff880869a28a60     0 17047   5423 0x00000000
[153972.076324]  ffff880869a28750 0000000000000002 ffff880867788ee8 ffff8807e4e47500
[153972.079259]  ffff880517addfd8 ffff880517addfd8 ffff880517addfd8 ffff880869a28750
[153972.082244]  0000000000000004 ffff880517addd48 ffff880517addd50 7fffffffffffffff
[153972.085278] Call Trace:
[153972.088310]  [<ffffffff81410f4a>] ? schedule_timeout+0x1ca/0x270
[153972.091371]  [<ffffffff8100abf1>] ? native_sched_clock+0x11/0x70
[153972.094386]  [<ffffffff81070cda>] ? try_to_wake_up+0x1ea/0x270
[153972.097358]  [<ffffffff81412623>] ? wait_for_completion+0xa3/0x120
[153972.100322]  [<ffffffff81070d60>] ? try_to_wake_up+0x270/0x270
[153972.103292]  [<ffffffff811a3702>] ? do_coredump+0x1b2/0xee0
[153972.106212]  [<ffffffff811a3899>] ? do_coredump+0x349/0xee0
[153972.109085]  [<ffffffff8134e0c4>] ? skb_queue_tail+0x24/0x60
[153972.111954]  [<ffffffff813eb73a>] ? unix_dgram_sendmsg+0x5aa/0x640
[153972.114839]  [<ffffffff81053049>] ? get_signal_to_deliver+0x199/0x5a0
[153972.117743]  [<ffffffff81002353>] ? do_signal+0x63/0x8c0
[153972.120632]  [<ffffffff81052030>] ? do_send_sig_info+0x60/0x90
[153972.123521]  [<ffffffff81002c38>] ? do_notify_resume+0x88/0xa0
[153972.126400]  [<ffffffff81414d6a>] ? int_signal+0x12/0x17
[153972.299643] XFS: possible memory allocation deadlock in kmem_alloc (mode:0x250)
[153972.868782] XFS: possible memory allocation deadlock in kmem_alloc (mode:0x250)
[153973.038189] XFS: possible memory allocation deadlock in kmem_alloc (mode:0x250)
[153974.309978] XFS: possible memory allocation deadlock in kmem_alloc (mode:0x250)
[153974.763236] INPUT miltonduff IN=eth2.2092 OUT= MAC=01:00:5e:00:00:01:00:1b:21:b9:99:3c:08:00 SRC=0.0.0.0 DST=224.0.0.1 LEN=32 TOS=0x00 PREC=0xC0 TTL=1 ID=0 DF PROTO=2 
[153974.870473] XFS: possible memory allocation deadlock in kmem_alloc (mode:0x250)
[153975.039848] XFS: possible memory allocation deadlock in kmem_alloc (mode:0x250)

root@miltonduff:~# cat /proc/mounts
rootfs / rootfs rw 0 0
sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
udev /dev devtmpfs rw,relatime,size=10240k,nr_inodes=4116954,mode=755 0 0
devpts /dev/pts devpts rw,nosuid,noexec,relatime,mode=600,ptmxmode=000 0 0
tmpfs /run tmpfs rw,nosuid,noexec,relatime,size=3301508k,mode=755 0 0
/dev/disk/by-uuid/37dd603c-168c-49de-830d-ef1b5c6982f8 / xfs rw,relatime,attr2,inode64,noquota 0 0
tmpfs /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0
tmpfs /run/shm tmpfs rw,nosuid,nodev,noexec,relatime,size=6709040k 0 0
/dev/sdd1 /boot ext2 rw,relatime,errors=continue 0 0
none /var/local/cgroup cgroup rw,relatime,hugetlb,perf_event,blkio,net_cls,freezer,devices,memory,cpuacct,cpu,debug,cpuset,clone_children 0 0
/dev/mapper/LocalDisk-root--ceph--osd--loi--A--3--1 /var/lib/lxc/01-ceph-osd-loi-A-3-1 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/osd--loi--3--1-A24 /var/lib/lxc/01-ceph-osd-loi-A-3-1/rootfs/CEPH/A.24 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-journal--ceph--osd--loi--A--3--1 /var/lib/lxc/01-ceph-osd-loi-A-3-1/rootfs/var/lib/ceph/osd/ceph-24 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-root--ceph--osd--loi--A--3--2 /var/lib/lxc/02-ceph-osd-loi-A-3-2 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/osd--loi--3--2-A25 /var/lib/lxc/02-ceph-osd-loi-A-3-2/rootfs/CEPH/A.25 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-journal--ceph--osd--loi--A--3--2 /var/lib/lxc/02-ceph-osd-loi-A-3-2/rootfs/var/lib/ceph/osd/ceph-25 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-root--ceph--osd--loi--A--3--3 /var/lib/lxc/03-ceph-osd-loi-A-3-3 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/osd--loi--3--3-A26 /var/lib/lxc/03-ceph-osd-loi-A-3-3/rootfs/CEPH/A.26 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-journal--ceph--osd--loi--A--3--3 /var/lib/lxc/03-ceph-osd-loi-A-3-3/rootfs/var/lib/ceph/osd/ceph-26 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-root--ceph--osd--loi--B--3--1 /var/lib/lxc/04-ceph-osd-loi-B-3-1 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/osd--loi--3--1-B24 /var/lib/lxc/04-ceph-osd-loi-B-3-1/rootfs/CEPH/B.24 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-journal--ceph--osd--loi--B--3--1 /var/lib/lxc/04-ceph-osd-loi-B-3-1/rootfs/var/lib/ceph/osd/ceph-24 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-root--ceph--osd--loi--B--3--2 /var/lib/lxc/05-ceph-osd-loi-B-3-2 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/osd--loi--3--2-B25 /var/lib/lxc/05-ceph-osd-loi-B-3-2/rootfs/CEPH/B.25 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-journal--ceph--osd--loi--B--3--2 /var/lib/lxc/05-ceph-osd-loi-B-3-2/rootfs/var/lib/ceph/osd/ceph-25 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-root--ceph--osd--loi--B--3--3 /var/lib/lxc/06-ceph-osd-loi-B-3-3 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/osd--loi--3--3-B26 /var/lib/lxc/06-ceph-osd-loi-B-3-3/rootfs/CEPH/B.26 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
/dev/mapper/LocalDisk-journal--ceph--osd--loi--B--3--3 /var/lib/lxc/06-ceph-osd-loi-B-3-3/rootfs/var/lib/ceph/osd/ceph-26 xfs rw,noatime,attr2,nobarrier,inode64,noquota 0 0
fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0