Ceph » Linux kernel client

[4]kdb> dumpall

[dumpall]kdb> pid R

KDB current process is kworker/4:1(pid=29967)

[dumpall]kdb> -dumpcommon

[dumpcommon]kdb> set BTAPROMPT 0

[dumpcommon]kdb> set LINES 10000

[dumpcommon]kdb> -summary

sysname Linux

release 3.10.0-rc6-ceph-00091-g2dd322b

version #1 SMP Wed Jun 19 16:06:38 PDT 2013

machine x86_64

nodename plana56

domainname (none)

ccversion CCVERSION

date 2013-06-22 22:32:35 tz_minuteswest 420

uptime 05:37

load avg 26.15 8.19 3.64

MemTotal: 2038247 kB

MemFree: 748512 kB

Buffers: 256 kB

[dumpcommon]kdb> -cpu

Currently on cpu 4

Available cpus: 0-3(I), 4, 5-7(I)

[dumpcommon]kdb> -ps

7 idle processes (state I) and

112 sleeping system daemon (state M) processes suppressed,

use 'ps A' to see all.

Task Addr Pid Parent [*] cpu State Thread Command

0xffff88020dd03f20 29967 2 1 4 R 0xffff88020dd043a8 *kworker/4:1

0xffff880225ca8000 1 0 0 0 S 0xffff880225ca8488 init

0xffff88020cc40000 492 1 0 3 S 0xffff88020cc40488 upstart-udev-br

0xffff88020cfd9f90 501 1 0 2 S 0xffff88020cfda418 udevd

0xffff88020cd69f90 538 1 0 2 S 0xffff88020cd6a418 dbus-daemon

0xffff88020b35bf20 949 1 0 5 S 0xffff88020b35c3a8 rpc.idmapd

0xffff880224e3bf20 1036 1 0 3 S 0xffff880224e3c3a8 upstart-socket-

0xffff88020dc5deb0 1040 1 0 2 S 0xffff88020dc5e338 rpcbind

0xffff880210080000 1104 1 0 0 S 0xffff880210080488 rpc.statd

0xffff88020cfdbf20 1242 1 0 1 S 0xffff88020cfdc3a8 getty

0xffff88020dc5bf20 1247 1 0 4 S 0xffff88020dc5c3a8 getty

0xffff88020c5ebf20 1254 1 0 4 S 0xffff88020c5ec3a8 getty

0xffff88020c5edeb0 1255 1 0 5 S 0xffff88020c5ee338 getty

0xffff88020cf83f20 1257 1 0 4 S 0xffff88020cf843a8 getty

0xffff88020c178000 1273 1 0 1 S 0xffff88020c178488 xinetd

0xffff88020d37deb0 1275 1 0 2 S 0xffff88020d37e338 atd

0xffff88020b0f3f20 1319 1 0 5 S 0xffff88020b0f43a8 irqbalance

0xffff88020c748000 1492 1 0 3 S 0xffff88020c748488 nrpe

0xffff88020dc63f20 1550 1 0 5 S 0xffff88020dc643a8 rpc.mountd

0xffff88020b358000 1578 1 0 3 S 0xffff88020b358488 pkcsslotd

0xffff880210083f20 1591 1 0 0 S 0xffff8802100843a8 rwhod

0xffff88020dc58000 1603 1591 0 3 S 0xffff88020dc58488 rwhod

0xffff88020c32bf20 1692 1 0 6 S 0xffff88020c32c3a8 getty

0xffff88020c329f90 1693 1 0 1 S 0xffff88020c32a418 getty

0xffff880210023f20 2232 1 0 0 S 0xffff8802100243a8 ntpd

0xffff88020dc60000 25450 2 0 1 D 0xffff88020dc60488 kworker/1:1

0xffff88019e0a5eb0 29928 2 0 6 D 0xffff88019e0a6338 kworker/6:1

0xffff88020dd03f20 29967 2 1 4 R 0xffff88020dd043a8 *kworker/4:1

0xffff88020c133f20 11216 2 0 3 D 0xffff88020c1343a8 kworker/u64:1

0xffff880222028000 12266 1 0 0 S 0xffff880222028488 sshd

0xffff88022202bf20 12282 12266 0 0 S 0xffff88022202c3a8 sshd

0xffff880224829f90 12316 1 0 2 S 0xffff88022482a418 rsyslogd

0xffff880210058000 12317 1 0 2 S 0xffff880210058488 rs:main Q:Reg

0xffff8802258cbf20 12318 1 0 0 S 0xffff8802258cc3a8 rsyslogd

0xffff880224b1deb0 12319 1 0 1 S 0xffff880224b1e338 rsyslogd

0xffff88020dd01f90 12553 1 0 7 S 0xffff88020dd02418 cron

0xffff88020d9d1f90 16744 1 0 3 S 0xffff88020d9d2418 sshd

0xffff880224a00000 17512 501 0 1 S 0xffff880224a00488 udevd

0xffff88020cc45eb0 17542 12282 0 6 S 0xffff88020cc46338 bash

0xffff880223030000 17544 17542 0 6 S 0xffff880223030488 ffsb.sh

0xffff880223855eb0 19210 17544 0 0 S 0xffff880223856338 ffsb

0xffff88020c179f90 19250 17544 0 3 S 0xffff88020c17a418 ffsb

0xffff880223850000 19251 17544 0 0 D 0xffff880223850488 ffsb

0xffff880223853f20 19252 17544 0 3 D 0xffff8802238543a8 ffsb

0xffff880223851f90 19253 17544 0 0 D 0xffff880223852418 ffsb

0xffff88020d0cdeb0 19254 17544 0 6 D 0xffff88020d0ce338 ffsb

0xffff88020d0c8000 19255 17544 0 0 D 0xffff88020d0c8488 ffsb

0xffff88019e0a0000 19256 17544 0 3 D 0xffff88019e0a0488 ffsb

0xffff88019e0a3f20 19257 17544 0 7 D 0xffff88019e0a43a8 ffsb

0xffff88019e0a1f90 19258 17544 0 3 D 0xffff88019e0a2418 ffsb

0xffff880224f08000 19259 17544 0 2 D 0xffff880224f08488 ffsb

0xffff880224828000 19260 17544 0 1 D 0xffff880224828488 ffsb

0xffff88022482bf20 19261 17544 0 0 D 0xffff88022482c3a8 ffsb

0xffff88022482deb0 19262 17544 0 7 D 0xffff88022482e338 ffsb

0xffff88020dd05eb0 19263 17544 0 1 D 0xffff88020dd06338 ffsb

0xffff88020dd00000 19264 17544 0 3 D 0xffff88020dd00488 ffsb

0xffff88020d08deb0 19265 17544 0 0 D 0xffff88020d08e338 ffsb

0xffff88020c5b8000 19266 17544 0 7 D 0xffff88020c5b8488 ffsb

0xffff88020c5bdeb0 19267 17544 0 7 D 0xffff88020c5be338 ffsb

0xffff880225e83f20 19268 17544 0 3 D 0xffff880225e843a8 ffsb

0xffff88020d2d5eb0 19269 17544 0 0 D 0xffff88020d2d6338 ffsb

0xffff88020d2d1f90 19270 17544 0 0 D 0xffff88020d2d2418 ffsb

0xffff88020d2d3f20 19271 17544 0 3 D 0xffff88020d2d43a8 ffsb

0xffff88020dc61f90 19272 17544 0 0 D 0xffff88020dc62418 ffsb

0xffff88020dc65eb0 19273 17544 0 3 D 0xffff88020dc66338 ffsb

0xffff88020cfddeb0 19274 17544 0 6 D 0xffff88020cfde338 ffsb

0xffff88020cfd8000 19275 17544 0 2 D 0xffff88020cfd8488 ffsb

0xffff880223795eb0 19276 17544 0 3 D 0xffff880223796338 ffsb

0xffff880223790000 19277 17544 0 4 D 0xffff880223790488 ffsb

0xffff88020dcd3f20 19278 17544 0 4 D 0xffff88020dcd43a8 ffsb

0xffff880224e38000 19279 17544 0 6 D 0xffff880224e38488 ffsb

0xffff880224e3deb0 19280 17544 0 5 D 0xffff880224e3e338 ffsb

0xffff880224e39f90 19281 17544 0 0 D 0xffff880224e3a418 ffsb

0xffff880210059f90 19282 17544 0 1 D 0xffff88021005a418 ffsb

0xffff8801e8920000 19247 2 0 0 D 0xffff8801e8920488 kworker/0:1

0xffff8801e8921f90 19283 2 0 4 D 0xffff8801e8922418 kworker/4:0

0xffff88020d871f90 19285 2 0 4 R 0xffff88020d872418 kworker/4:3

0xffff88020ce93f20 19287 2 0 0 D 0xffff88020ce943a8 kworker/0:3

[dumpcommon]kdb> -dmesg 600

<4>[17533.026513] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17533.305786] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17533.548387] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17533.560224] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17533.560224]

<4>[17533.592541] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17533.603875] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17533.603875]

<4>[17535.002318] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17535.622836] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17535.635858] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17535.635858]

<4>[17536.194110] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17536.220037] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17536.220037]

<4>[17537.204423] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17537.216409] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17537.216409]

<4>[17539.232540] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17539.244239] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17539.244239]

<4>[17539.636142] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17539.681878] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17539.695057] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17539.695057]

<4>[17539.726893] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17539.740351] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17539.740351]

<4>[17541.480993] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17542.496935] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17542.531752] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17543.031175] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17543.066613] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17543.379091] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17544.233901] libceph: osd1 10.214.131.6:6808 socket error on read

<4>[17544.466703] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17544.701796] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17545.178602] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17546.879851] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17548.047776] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17548.806604] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17549.788432] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17549.801209] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17551.530479] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17551.645252] libceph: osd4 10.214.131.24:6800 socket error on read

<4>[17552.039592] libceph: osd5 10.214.131.24:6809 socket error on read

<4>[17552.491139] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17552.997848] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17554.148475] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17554.439676] libceph: osd1 10.214.131.6:6808 socket error on read

<4>[17554.668289] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17555.022507] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17555.036487] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17555.036487]

<4>[17555.326965] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17555.363084] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17557.038761] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17557.052047] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17557.052047]

<4>[17557.903326] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17557.916751] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17557.916751]

<4>[17558.807014] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17558.849574] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17559.480226] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17560.939397] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17560.952991] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17560.952991]

<4>[17563.060683] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17564.468034] libceph: osd0 10.214.131.6:6800 socket error on read

<4>[17564.546618] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17564.561124] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17564.561124]

<4>[17568.660832] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17568.921904] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17570.987567] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17571.355165] libceph: osd0 10.214.131.6:6800 socket error on write

<4>[17571.566214] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17571.899464] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17573.094100] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17573.104350] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17573.104350]

<4>[17573.637771] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17574.071533] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17575.241888] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17575.290380] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17575.304391] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17575.304391]

<4>[17575.587316] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17576.039354] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17576.790371] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17577.076261] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17577.805040] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17578.865286] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17579.201121] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17581.212965] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17581.829552] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17584.490562] libceph: osd5 10.214.131.24:6809 socket error on read

<4>[17585.939918] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17586.184549] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17587.115863] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17587.119286] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17587.123722] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17587.123722]

<4>[17587.981582] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17588.631562] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17592.098284] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17592.211747] libceph: osd3 10.214.131.24:6804 socket error on read

<4>[17592.224591] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17592.224591]

<4>[17592.988563] libceph: osd2 10.214.131.6:6804 socket error on read

<4>[17593.133189] libceph: osd2 10.214.131.6:6804 socket error on write

<4>[17593.506824] libceph: osd1 10.214.131.6:6808 socket error on read

<4>[17594.009964] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17594.703762] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17597.127834] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17597.597392] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17597.612196] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17597.612196]

<4>[17598.893909] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17600.273864] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17600.476443] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17602.012767] libceph: osd4 10.214.131.24:6800 socket error on read

<4>[17602.375769] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17605.741750] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17605.845332] libceph: osd5 10.214.131.24:6809 socket error on write

<4>[17606.589856] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17606.603909] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17606.603909]

<4>[17606.790072] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17609.475310] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17609.486628] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17609.486628]

<4>[17611.430599] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17611.443762] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17611.443762]

<4>[17612.213691] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17613.169551] libceph: osd2 10.214.131.6:6804 socket error on read

<4>[17613.737234] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17614.795950] libceph: osd1 10.214.131.6:6808 socket error on read

<4>[17618.470117] libceph: osd2 10.214.131.6:6804 socket error on write

<4>[17618.666911] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17618.678700] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17618.678700]

<4>[17619.622901] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17620.051985] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17620.064819] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17620.064819]

<4>[17620.333916] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17624.771099] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17625.503820] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17626.496739] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17626.509040] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17626.509040]

<4>[17626.632422] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17626.644609] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17626.644609]

<4>[17627.754056] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17629.050099] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17630.089157] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17634.844652] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17635.166961] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17636.786645] libceph: osd3 10.214.131.24:6804 socket error on write

<4>[17636.798107] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17636.798107]

<4>[17637.107751] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17638.146838] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17639.004101] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17640.068454] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17640.110164] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17640.555220] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17640.608122] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17640.608122]

<4>[17641.170014] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17641.826747] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17643.169332] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17643.239154] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17643.455348] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17644.570704] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17644.584060] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17644.584060]

<4>[17645.530896] libceph: osd0 10.214.131.6:6800 socket error on read

<4>[17651.258837] libceph: osd5 10.214.131.24:6809 socket error on read

<4>[17653.158165] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17653.168263] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17653.168263]

<4>[17653.640934] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17653.652207] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17653.652207]

<4>[17653.768853] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17655.619574] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17656.541316] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17658.432820] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17658.445536] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17658.445536]

<4>[17659.125996] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17660.231395] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17660.282051] libceph: osd5 10.214.131.24:6809 socket error on write

<4>[17663.622767] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17663.678556] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17664.033800] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17664.803811] libceph: osd2 10.214.131.6:6804 socket error on read

<4>[17667.966439] libceph: osd0 10.214.131.6:6800 socket error on read

<4>[17669.013576] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17669.412517] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17669.712890] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17671.116700] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17671.914702] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17672.038049] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17672.379980] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17673.393492] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17675.245090] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17678.375568] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17678.587283] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17679.451751] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17680.491038] libceph: osd3 10.214.131.24:6804 socket error on write

<4>[17680.499479] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17680.499479]

<4>[17680.692682] libceph: osd3 10.214.131.24:6804 socket error on read

<4>[17680.705085] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17680.705085]

<4>[17681.026845] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17681.638215] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17683.084331] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17684.798383] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17686.907250] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17687.635351] libceph: osd2 10.214.131.6:6804 socket error on write

<4>[17689.307434] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17689.372548] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17689.925830] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17692.487908] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17692.754723] libceph: osd1 10.214.131.6:6808 socket error on write

<4>[17693.474343] libceph: osd5 10.214.131.24:6809 socket error on write

<4>[17693.692379] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17694.835062] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17695.606397] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17698.555244] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17698.677550] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17698.758742] libceph: osd0 10.214.131.6:6800 socket error on read

<4>[17699.014722] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17702.135852] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17702.621096] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17706.359738] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17706.373519] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17706.373519]

<4>[17708.281440] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17709.367172] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17709.972851] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17712.054650] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17712.066092] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17712.066092]

<4>[17712.590032] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17712.767382] libceph: osd0 10.214.131.6:6800 socket error on read

<4>[17713.685098] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17713.698129] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17713.698129]

<4>[17716.062376] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17719.577941] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17720.874451] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17722.646150] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17725.365118] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17726.363520] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17726.376119] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17726.376119]

<4>[17726.482694] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17726.745879] libceph: osd5 10.214.131.24:6809 socket error on read

<4>[17727.916552] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN)

<4>[17728.493797] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17728.517094] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17729.898608] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17730.083932] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17730.095588] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17730.095588]

<4>[17731.235819] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17732.159519] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17732.171497] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17732.171497]

<4>[17742.977181] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17742.988882] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17742.988882]

<4>[17743.090374] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17747.310262] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17751.382439] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17751.395387] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17751.395387]

<4>[17751.455678] libceph: osd2 10.214.131.6:6804 socket error on read

<4>[17752.159530] libceph: osd0 10.214.131.6:6800 socket error on read

<4>[17754.262501] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17754.274612] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17754.274612]

<4>[17754.425617] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN)

<4>[17755.559934] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<4>[17756.277738] libceph: osd1 10.214.131.6:6808 socket error on read

<4>[17758.774973] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN)

<4>[17758.789031] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17758.789031]

<4>[17760.970301] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17762.411042] libceph: osd3 10.214.131.24:6804 socket error on write

<4>[17762.424563] rbd: obj_request ffff8801d98cb7a0 already marked done

<4>[17762.424563]

<4>[17763.255145] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN)

<4>[17764.658962] libceph: osd5 10.214.131.24:6809 socket error on read

<4>[17765.070353] libceph: osd2 10.214.131.6:6804 socket error on read

<4>[17766.447631] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN)

<4>[17767.843423] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN)

<5>[17778.790982] Key type ceph unregistered

<6>[18270.726611] kgdb: Unregistered I/O driver kgdboc, debugger disabled.

<6>[18270.732993] kgdb: Registered I/O driver kgdboc.

<12>[18277.329116] init: cron main process (18413) killed by TERM signal

<6>[18283.899773] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro,user_xattr,user_xattr

<6>[18355.068892] device fsid ca09882a-4f69-4c7f-9b65-76b369fe2315 devid 1 transid 7 /dev/sdd

<6>[18355.080794] btrfs flagging fs with big metadata feature

<6>[18355.977555] device fsid ac2c23bb-10fe-4bad-9c33-a211eef9da8a devid 1 transid 7 /dev/sdb

<6>[18355.992357] btrfs flagging fs with big metadata feature

<6>[18356.889823] device fsid 7c4979d7-9726-47f6-9991-6ddd974fa2c9 devid 1 transid 7 /dev/sdc

<6>[18356.903542] btrfs flagging fs with big metadata feature

<6>[18739.278247] kgdb: Unregistered I/O driver kgdboc, debugger disabled.

<6>[18739.284631] kgdb: Registered I/O driver kgdboc.

<12>[18743.729481] init: cron main process (24867) killed by TERM signal

<6>[18750.270916] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro,user_xattr,user_xattr

<5>[18909.912979] Key type ceph registered

<6>[18909.916825] libceph: loaded (mon/osd proto 15/24)

<6>[18910.001391] ceph: loaded (mds proto 32)

<6>[18910.008436] libceph: client4143 fsid fdbd22ee-8e25-4fe2-a6e5-09e250792f49

<6>[18910.017962] libceph: mon2 10.214.132.14:6789 session established

<6>[18957.728828] kgdb: Unregistered I/O driver kgdboc, debugger disabled.

<6>[18957.735260] kgdb: Registered I/O driver kgdboc.

<12>[18963.730725] init: cron main process (32370) killed by TERM signal

<6>[18970.571722] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro,user_xattr,user_xattr

<6>[19061.897253] libceph: client4105 fsid a0bafb5c-6ebb-41ea-8fc9-4766f12d3837

<6>[19061.907074] libceph: mon0 10.214.132.16:6789 session established

<6>[19310.941929] kgdb: Unregistered I/O driver kgdboc, debugger disabled.

<6>[19310.948310] kgdb: Registered I/O driver kgdboc.

<12>[19316.676000] init: cron main process (6262) killed by TERM signal

<6>[19323.013033] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro,user_xattr,user_xattr

<6>[19423.187276] libceph: client4143 fsid 77de2efd-1a1a-4a18-b063-47db86244ea7

<6>[19423.197171] libceph: mon2 10.214.132.14:6789 session established

<6>[19444.799182] libceph: osd2 down

<6>[19444.802352] libceph: osd2 up

<6>[19448.846897] libceph: osd0 weight 0x0 (out)

<4>[19453.875514] libceph: osd2 10.214.132.16:6808 socket closed (con state OPEN)

<4>[19454.249085] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING)

<4>[19454.698506] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING)

<4>[19455.849256] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING)

<4>[19457.670307] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING)

<6>[19460.048351] libceph: osd0 weight 0x10000 (in)

<4>[19461.742708] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING)

<6>[19464.274855] libceph: osd2 down

<6>[19464.277937] libceph: osd2 up

<6>[19466.701862] libceph: osd5 weight 0x0 (out)

<6>[19473.388263] libceph: osd4 weight 0x0 (out)

<4>[19473.424303] libceph: osd1 10.214.132.16:6800 socket closed (con state OPEN)

<4>[19473.503478] libceph: osd3 10.214.132.14:6800 socket closed (con state OPEN)

<4>[19478.113622] libceph: osd3 10.214.132.14:6800 socket closed (con state OPEN)

<4>[19478.120818] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING)

<4>[19478.713039] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING)

<4>[19479.864775] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING)

<4>[19481.745104] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING)

<4>[19485.878080] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING)

<4>[19488.242249] libceph: osd5 10.214.132.14:6808 socket closed (con state OPEN)

<6>[19493.603194] libceph: osd4 weight 0x10000 (in)

<4>[19493.760153] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN)

<4>[19493.897309] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING)

<6>[19500.143974] libceph: osd3 down

<6>[19500.147100] libceph: osd5 down

<6>[19501.287213] libceph: osd5 up

<4>[19502.613979] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN)

<6>[19502.887613] libceph: osd3 up

<4>[19503.389534] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN)

<4>[19505.203671] libceph: osd1 10.214.132.16:6800 socket closed (con state OPEN)

<6>[19549.477787] libceph: osd5 weight 0x10000 (in)

<6>[19555.270712] libceph: osd0 weight 0x0 (out)

<4>[19555.624170] libceph: osd5 10.214.132.14:6817 socket error on write

<6>[19561.972152] libceph: osd0 weight 0x10000 (in)

<4>[19567.109775] libceph: osd2 10.214.132.16:6812 socket closed (con state OPEN)

<4>[19572.035081] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING)

<4>[19572.740668] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING)

<4>[19573.920275] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING)

<4>[19575.771970] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING)

<4>[19579.933897] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING)

<6>[19580.809766] libceph: osd2 down

<6>[19581.971431] libceph: osd2 up

<4>[19582.099009] libceph: osd0 10.214.132.16:6801 socket closed (con state OPEN)

<4>[19582.112255] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN)

<4>[19585.095083] libceph: osd1 10.214.132.16:6800 socket closed (con state OPEN)

<6>[19625.979741] libceph: osd0 weight 0x0 (out)

<4>[19626.059263] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN)

<6>[19637.726796] libceph: osd0 weight 0x10000 (in)

<6>[19654.210343] libceph: osd1 weight 0x0 (out)

<6>[19752.920435] libceph: osd1 weight 0x10000 (in)

<4>[19776.765662] libceph: osd0 10.214.132.16:6801 socket error on write

<4>[19776.796030] libceph: osd0 10.214.132.16:6801 socket closed (con state CONNECTING)

<4>[19778.015116] libceph: osd0 10.214.132.16:6801 socket closed (con state CONNECTING)

<3>[19799.355399] INFO: rcu_sched self-detected stall on CPU { 6} (t=2100 jiffies g=245350 c=245349 q=2640)

<4>[19799.364789] CPU: 6 PID: 19284 Comm: kworker/6:2 Tainted: G W 3.10.0-rc6-ceph-00091-g2dd322b #1

<4>[19799.374303] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<3>[19799.375424] INFO: rcu_sched detected stalls on CPUs/tasks: { 6} (detected by 0, t=2102 jiffies, g=245350, c=245349, q=2640)

<6>[19799.375425] Task dump for CPU 6:

<6>[19799.375429] kworker/6:2 R running task 0 19284 2 0x00000000

<6>[19799.375442] Workqueue: ceph-msgr con_work [libceph]

<4>[19799.375445] ffff880125ff7de8 ffffffff8105f3da ffffffff8105f36f ffff8802272d3a00

<4>[19799.375447] 0000000000000000 00000006272d2f98 ffff880125ff7fd8 ffff8802272d2f80

<4>[19799.375450] ffffffffa05666d0 0000000000000000 0000000000000000 ffffffffa055940e

<4>[19799.375451] Call Trace:

<4>[19799.375457] [<ffffffff8105f3da>] ? process_one_work+0x1da/0x540

<4>[19799.375459] [<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

<4>[19799.375462] [<ffffffff810605bc>] worker_thread+0x11c/0x370

<4>[19799.375464] [<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

<4>[19799.375468] [<ffffffff8106727a>] kthread+0xea/0xf0

<4>[19799.375471] [<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

<4>[19799.375476] [<ffffffff8163ff9c>] ret_from_fork+0x7c/0xb0

<4>[19799.375478] [<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

<4>[19799.480150] Workqueue: ceph-msgr con_work [libceph]

<4>[19799.485054] ffffffff81c4ca00 ffff8802272c3db8 ffffffff81630b82 ffff8802272c3e38

<4>[19799.492523] ffffffff810e285a 0000000000000006 ffff8802272cd4e0 ffff8802272c3de8

<4>[19799.499995] ffffffff810e644c 0000000000000086 0000000000000001 0000000000000006

<4>[19799.507469] Call Trace:

<4>[19799.509929] <IRQ> [<ffffffff81630b82>] dump_stack+0x19/0x1b

<4>[19799.515718] [<ffffffff810e285a>] rcu_check_callbacks+0x21a/0x710

<4>[19799.521831] [<ffffffff810e644c>] ? acct_account_cputime+0x1c/0x20

<4>[19799.528034] [<ffffffff81050f68>] update_process_times+0x48/0x80

<4>[19799.534062] [<ffffffff8109b616>] tick_sched_handle.isra.10+0x36/0x50

<4>[19799.540524] [<ffffffff8109b71c>] tick_sched_timer+0x4c/0x80

<4>[19799.546203] [<ffffffff8106a841>] __run_hrtimer+0x81/0x1e0

<4>[19799.551709] [<ffffffff8109b6d0>] ? tick_nohz_handler+0xa0/0xa0

<4>[19799.557647] [<ffffffff8106b147>] hrtimer_interrupt+0x107/0x260

<4>[19799.563588] [<ffffffff81641b69>] smp_apic_timer_interrupt+0x69/0x99

<4>[19799.569964] [<ffffffff81640caf>] apic_timer_interrupt+0x6f/0x80

<4>[19799.575987] <EOI> [<ffffffff8112edec>] ? shrink_inactive_list+0x18c/0x400

<4>[19799.582995] [<ffffffff81637590>] ? _raw_spin_unlock_irq+0x30/0x40

<4>[19799.589195] [<ffffffff81637595>] ? _raw_spin_unlock_irq+0x35/0x40

<4>[19799.595397] [<ffffffff8112edec>] shrink_inactive_list+0x18c/0x400

<4>[19799.601595] [<ffffffff8112f66d>] shrink_lruvec+0x2cd/0x4d0

<4>[19799.607187] [<ffffffff8119855b>] ? bdi_queue_work+0x8b/0xf0

<4>[19799.612869] [<ffffffff8112fc1c>] do_try_to_free_pages+0x11c/0x3a0

<4>[19799.619068] [<ffffffff81130066>] try_to_free_pages+0xd6/0x1b0

<4>[19799.624922] [<ffffffff811375b0>] ? next_zone+0x30/0x40

<4>[19799.630165] [<ffffffff81125406>] __alloc_pages_nodemask+0x596/0x8f0

<4>[19799.636541] [<ffffffff8115bb1a>] alloc_pages_current+0xba/0x170

<4>[19799.642569] [<ffffffff81516d3e>] sk_page_frag_refill+0x7e/0x130

<4>[19799.648593] [<ffffffff8156f5a5>] tcp_sendmsg+0x305/0xe50

<4>[19799.654010] [<ffffffff8159af99>] inet_sendmsg+0xb9/0xf0

<4>[19799.659339] [<ffffffff8159aee5>] ? inet_sendmsg+0x5/0xf0

<4>[19799.664760] [<ffffffff81510de2>] sock_sendmsg+0xc2/0xe0

<4>[19799.670090] [<ffffffff812ee35b>] ? chksum_update+0x1b/0x30

<4>[19799.675686] [<ffffffff812ea1e8>] ? crypto_shash_update+0x18/0x30

<4>[19799.681814] [<ffffffffa0000056>] ? crc32c+0x56/0x7c [libcrc32c]

<4>[19799.687842] [<ffffffff81510e40>] kernel_sendmsg+0x40/0x60

<4>[19799.693353] [<ffffffffa05424d8>] con_work+0xc28/0x2d50 [libceph]

<4>[19799.699468] [<ffffffff81080bb3>] ? idle_balance+0x133/0x180

<4>[19799.705145] [<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

<4>[19799.711257] [<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

<4>[19799.717371] [<ffffffff8105f3da>] process_one_work+0x1da/0x540

<4>[19799.723220] [<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

<4>[19799.729247] [<ffffffff810605bc>] worker_thread+0x11c/0x370

<4>[19799.734840] [<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

<4>[19799.741388] [<ffffffff8106727a>] kthread+0xea/0xf0

<4>[19799.746283] [<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

<4>[19799.752658] [<ffffffff8163ff9c>] ret_from_fork+0x7c/0xb0

<4>[19799.758076] [<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

<3>[19820.788280] INFO: rcu_sched self-detected stall on CPU

<3>[19820.788286] INFO: rcu_sched self-detected stall on CP

<4>[19820.788287]

<4>[19820.788288]

<4>[19820.788288] }

<4>[19820.788291] (t=2101 jiffies g=245351 c=245350 q=5233)

<4>[19820.788294] CPU: 6 PID: 19284 Comm: kworker/6:2 Tainted: G W 3.10.0-rc6-ceph-00091-g2dd322b #1

<4>[19820.788295] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[19820.788308] Workqueue: ceph-msgr con_work [libceph

<4>[19820.788308]

<4>[19820.788309] ffffffff81c4ca0

<4>[19820.788310] ffff8802272c3db

<4>[19820.788310] ffffffff81630b8

<4>[19820.788310] ffff8802272c3e3

<4>[19820.788311]

<4>[19820.788311] ffffffff810e285

<4>[19820.788311] 000000000000000

<4>[19820.788312] ffff8802272cd4e

<4>[19820.788312] ffff8802272c3de

<4>[19820.788312]

<4>[19820.788313] ffffffff810e644

<4>[19820.788313] 000000000000008

<4>[19820.788313] 000000000000000

<4>[19820.788314] 000000000000000

<4>[19820.788314]

<4>[19820.788315] Call Trace:

<4>[19820.788316] <IRQ>

<4>[19820.788317]

<4>[19820.788322] [<ffffffff81630b82>] dump_stack+0x19/0x1b

<4>[19820.788322]

<4>[19820.788326] [<ffffffff810e285a>] rcu_check_callbacks+0x21a/0x710

<4>[19820.788326]

<4>[19820.788328] [<ffffffff810e644c>] ? acct_account_cputime+0x1c/0x20

<4>[19820.788328]

<4>[19820.788332] [<ffffffff81050f68>] update_process_times+0x48/0x80

<4>[19820.788332]

<4>[19820.788335] [<ffffffff8109b616>] tick_sched_handle.isra.10+0x36/0x50

<4>[19820.788336]

<4>[19820.788337] [<ffffffff8109b71c>] tick_sched_timer+0x4c/0x80

<4>[19820.788337]

<4>[19820.788340] [<ffffffff8106a841>] __run_hrtimer+0x81/0x1e0

<4>[19820.788340]

<4>[19820.788342] [<ffffffff8109b6d0>] ? tick_nohz_handler+0xa0/0xa0

<4>[19820.788342]

<4>[19820.788344] [<ffffffff8106b147>] hrtimer_interrupt+0x107/0x260

<4>[19820.788344]

<4>[19820.788348] [<ffffffff81641b69>] smp_apic_timer_interrupt+0x69/0x99

<4>[19820.788348]

<4>[19820.788350] [<ffffffff81640caf>] apic_timer_interrupt+0x6f/0x80

<4>[19820.788350] <EOI>

<4>[19820.788351]

<4>[19820.788353] [<ffffffff8112edec>] ? shrink_inactive_list+0x18c/0x400

<4>[19820.788354]

<4>[19820.788356] [<ffffffff81637590>] ? _raw_spin_unlock_irq+0x30/0x40

<4>[19820.788356]

<4>[19820.788357] [<ffffffff81637595>] ? _raw_spin_unlock_irq+0x35/0x40

<4>[19820.788358]

<4>[19820.788359] [<ffffffff8112edec>] shrink_inactive_list+0x18c/0x400

<4>[19820.788359]

<4>[19820.788361] [<ffffffff8112f66d>] shrink_lruvec+0x2cd/0x4d0

<4>[19820.788361]

<4>[19820.788362] [<ffffffff8112d66c>] ? shrink_slab+0x1dc/0x300

<4>[19820.788363]

<4>[19820.788365] [<ffffffff81125c90>] ? page_alloc_cpu_notify+0x60/0x60

<4>[19820.788366]

<4>[19820.788367] [<ffffffff8112fc1c>] do_try_to_free_pages+0x11c/0x3a0

<4>[19820.788367]

<4>[19820.788368] [<ffffffff81125c90>] ? page_alloc_cpu_notify+0x60/0x60

<4>[19820.788369]

<4>[19820.788370] [<ffffffff81130066>] try_to_free_pages+0xd6/0x1b0

<4>[19820.788370]

<4>[19820.788372] [<ffffffff81125406>] __alloc_pages_nodemask+0x596/0x8f0

<4>[19820.788372]

<4>[19820.788375] [<ffffffff8115bb1a>] alloc_pages_current+0xba/0x170

<4>[19820.788376]

<4>[19820.788378] [<ffffffff81516d3e>] sk_page_frag_refill+0x7e/0x130

<4>[19820.788379]

<4>[19820.788381] [<ffffffff8156f5a5>] tcp_sendmsg+0x305/0xe50

<4>[19820.788381]

<4>[19820.788383] [<ffffffff8159af99>] inet_sendmsg+0xb9/0xf0

<4>[19820.788383]

<4>[19820.788384] [<ffffffff8159aee5>] ? inet_sendmsg+0x5/0xf0

<4>[19820.788384]

<4>[19820.788387] [<ffffffff81510de2>] sock_sendmsg+0xc2/0xe0

<4>[19820.788387]

<4>[19820.788389] [<ffffffff812ee35b>] ? chksum_update+0x1b/0x30

<4>[19820.788390]

<4>[19820.788391] [<ffffffff812ea1e8>] ? crypto_shash_update+0x18/0x30

<4>[19820.788402]

<4>[19820.788404] [<ffffffffa0000056>] ? crc32c+0x56/0x7c [libcrc32c]

<4>[19820.788404]

<4>[19820.788406] [<ffffffff81510e40>] kernel_sendmsg+0x40/0x60

<4>[19820.788406]

<4>[19820.788411] [<ffffffffa05424d8>] con_work+0xc28/0x2d50 [libceph]

<4>[19820.788411]

<4>[19820.788413] [<ffffffff81080bb3>] ? idle_balance+0x133/0x180

<4>[19820.788413]

<4>[19820.788415] [<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

<4>[19820.788416]

<4>[19820.788417] [<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

<4>[19820.788417]

<4>[19820.788420] [<ffffffff8105f3da>] process_one_work+0x1da/0x540

<4>[19820.788420]

<4>[19820.788421] [<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

<4>[19820.788422]

<4>[19820.788423] [<ffffffff810605bc>] worker_thread+0x11c/0x370

<4>[19820.788423]

<4>[19820.788424] [<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

<4>[19820.788425]

<4>[19820.788426] [<ffffffff8106727a>] kthread+0xea/0xf0

<4>[19820.788427]

<4>[19820.788429] [<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

<4>[19820.788429]

<4>[19820.788430] [<ffffffff8163ff9c>] ret_from_fork+0x7c/0xb0

<4>[19820.788431]

<4>[19820.788432] [<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

<4>[19821.112660]

<4>[19821.114348] 4} (t=2133 jiffies g=245351 c=245350 q=5264)

<4>[19821.118574] CPU: 4 PID: 19283 Comm: kworker/4:0 Tainted: G W 3.10.0-rc6-ceph-00091-g2dd322b #1

<4>[19821.128090] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[19821.135619] Workqueue: ceph-msgr con_work [libceph]

<4>[19821.140525] ffffffff81c4ca00 ffff880227283db8 ffffffff81630b82 ffff880227283e38

<4>[19821.147998] ffffffff810e285a 0000000000000004 ffff88022728d4e0 ffff880227283de8

<4>[19821.155476] ffffffff810e644c 0000000000000086 0000000000000001 0000000000000004

<4>[19821.162953] Call Trace:

<4>[19821.165413] <IRQ> [<ffffffff81630b82>] dump_stack+0x19/0x1b

<4>[19821.171207] [<ffffffff810e285a>] rcu_check_callbacks+0x21a/0x710

<4>[19821.177323] [<ffffffff810e644c>] ? acct_account_cputime+0x1c/0x20

<4>[19821.183526] [<ffffffff81050f68>] update_process_times+0x48/0x80

<4>[19821.189556] [<ffffffff8109b616>] tick_sched_handle.isra.10+0x36/0x50

<4>[19821.196017] [<ffffffff8109b71c>] tick_sched_timer+0x4c/0x80

<4>[19821.201698] [<ffffffff8106a841>] __run_hrtimer+0x81/0x1e0

<4>[19821.207204] [<ffffffff8109b6d0>] ? tick_nohz_handler+0xa0/0xa0

<4>[19821.213146] [<ffffffff8106b147>] hrtimer_interrupt+0x107/0x260

<4>[19821.219087] [<ffffffff81641b69>] smp_apic_timer_interrupt+0x69/0x99

<4>[19821.225463] [<ffffffff81640caf>] apic_timer_interrupt+0x6f/0x80

<4>[19821.231488] <EOI> [<ffffffff8112edec>] ? shrink_inactive_list+0x18c/0x400

<4>[19821.238496] [<ffffffff81637590>] ? _raw_spin_unlock_irq+0x30/0x40

<4>[19821.244773] [<ffffffff81637595>] ? _raw_spin_unlock_irq+0x35/0x40

<4>[19821.250975] [<ffffffff8112edec>] shrink_inactive_list+0x18c/0x400

<4>[19821.257177] [<ffffffff8112f66d>] shrink_lruvec+0x2cd/0x4d0

<4>[19821.262771] [<ffffffff8119855b>] ? bdi_queue_work+0x8b/0xf0

<4>[19821.268451] [<ffffffff8112fc1c>] do_try_to_free_pages+0x11c/0x3a0

<4>[19821.274652] [<ffffffff81130066>] try_to_free_pages+0xd6/0x1b0

<4>[19821.280508] [<ffffffff811375b0>] ? next_zone+0x30/0x40

<4>[19821.285756] [<ffffffff81125406>] __alloc_pages_nodemask+0x596/0x8f0

<4>[19821.292136] [<ffffffff8115bb1a>] alloc_pages_current+0xba/0x170

<4>[19821.298165] [<ffffffff81516d3e>] sk_page_frag_refill+0x7e/0x130

<4>[19821.304195] [<ffffffff8156f5a5>] tcp_sendmsg+0x305/0xe50

<4>[19821.309615] [<ffffffff8159af99>] inet_sendmsg+0xb9/0xf0

<4>[19821.314947] [<ffffffff8159aee5>] ? inet_sendmsg+0x5/0xf0

<4>[19821.320368] [<ffffffff81510de2>] sock_sendmsg+0xc2/0xe0

<4>[19821.325701] [<ffffffff812ee35b>] ? chksum_update+0x1b/0x30

<4>[19821.331295] [<ffffffff812ea1e8>] ? crypto_shash_update+0x18/0x30

<4>[19821.337423] [<ffffffffa0000056>] ? crc32c+0x56/0x7c [libcrc32c]

<4>[19821.343454] [<ffffffff81510e40>] kernel_sendmsg+0x40/0x60

<4>[19821.348970] [<ffffffffa05424d8>] con_work+0xc28/0x2d50 [libceph]

<4>[19821.355088] [<ffffffff810888d5>] ? cpuacct_charge+0x5/0xa0

<4>[19821.360683] [<ffffffff81080bb3>] ? idle_balance+0x133/0x180

<4>[19821.366365] [<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

<4>[19821.372480] [<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

<4>[19821.378596] [<ffffffff8105f3da>] process_one_work+0x1da/0x540

<4>[19821.384454] [<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

<4>[19821.390493] [<ffffffff81637b5c>] ? retint_restore_args+0xe/0xe

<4>[19821.396436] [<ffffffff810605bc>] worker_thread+0x11c/0x370

<4>[19821.402030] [<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

<4>[19821.408580] [<ffffffff8106727a>] kthread+0xea/0xf0

<4>[19821.413479] [<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

<4>[19821.419859] [<ffffffff8163ff9c>] ret_from_fork+0x7c/0xb0

<4>[19821.425282] [<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

<6>[19821.508678] libceph: osd0 weight 0x0 (out)

<4>[19821.510853] libceph: osd0 10.214.132.16:6801 socket closed (con state CONNECTING)

<6>[19822.738359] libceph: osd2 weight 0x0 (out)

<4>[19823.371107] libceph: osd2 10.214.132.16:6808 socket closed (con state OPEN)

<6>[19823.373330] libceph: osd0 down

<6>[19824.040224] libceph: osd0 up

<6>[19824.670804] libceph: osd0 weight 0x10000 (in)

<6>[19825.568053] libceph: osd5 weight 0x0 (out)

<6>[19826.613710] libceph: osd5 weight 0x10000 (in)

<6>[19827.491250] libceph: osd5 weight 0x0 (out)

<1>[19828.585548] BUG: unable to handle kernel NULL pointer dereference at (null)

<1>[19828.593437] IP: [<ffffffff813185cb>] rb_erase+0x1bb/0x370

<4>[19828.598865] PGD 0

<4>[19828.600899] Oops: 0002 [#1] SMP

[dumpcommon]kdb> -bt

Stack traceback for pid 29967

0xffff88020dd03f20 29967 2 1 4 R 0xffff88020dd043a8 *kworker/4:1

ffff88020b257b48 0000000000000018 0000000000000000 ffff88020b257b68

ffffffffa05487bc ffff8802204e4000 ffff880224ec7950 ffff88020b257b98

ffffffffa0548abf ffff8802204e4030 ffff880224ec7950 0000000000000000

Call Trace:

[<ffffffffa05487bc>] ? __remove_osd+0x3c/0xa0 [libceph]

[<ffffffffa0548abf>] ? __reset_osd+0x12f/0x170 [libceph]

[<ffffffffa054a6de>] ? osd_reset+0x7e/0x2b0 [libceph]

[<ffffffffa0541e21>] ? con_work+0x571/0x2d50 [libceph]

[<ffffffff81080bb3>] ? idle_balance+0x133/0x180

[<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

[<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

[<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

[<ffffffff8105f3da>] ? process_one_work+0x1da/0x540

[<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

[<ffffffff810605bc>] ? worker_thread+0x11c/0x370

[<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106727a>] ? kthread+0xea/0xf0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8163ff9c>] ? ret_from_fork+0x7c/0xb0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

[dumpall]kdb> -bta

7 idle processes (state I) and

112 sleeping system daemon (state M) processes suppressed,

use 'ps A' to see all.

Stack traceback for pid 29967

0xffff88020dd03f20 29967 2 1 4 R 0xffff88020dd043a8 *kworker/4:1

ffff88020b257b48 0000000000000018 0000000000000000 ffff88020b257b68

ffffffffa05487bc ffff8802204e4000 ffff880224ec7950 ffff88020b257b98

ffffffffa0548abf ffff8802204e4030 ffff880224ec7950 0000000000000000

Call Trace:

[<ffffffffa05487bc>] ? __remove_osd+0x3c/0xa0 [libceph]

[<ffffffffa0548abf>] ? __reset_osd+0x12f/0x170 [libceph]

[<ffffffffa054a6de>] ? osd_reset+0x7e/0x2b0 [libceph]

[<ffffffffa0541e21>] ? con_work+0x571/0x2d50 [libceph]

[<ffffffff81080bb3>] ? idle_balance+0x133/0x180

[<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

[<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

[<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

[<ffffffff8105f3da>] ? process_one_work+0x1da/0x540

[<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

[<ffffffff810605bc>] ? worker_thread+0x11c/0x370

[<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106727a>] ? kthread+0xea/0xf0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8163ff9c>] ? ret_from_fork+0x7c/0xb0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

Stack traceback for pid 1

0xffff880225ca8000 1 0 0 0 S 0xffff880225ca8488 init

ffff880225c8b8d8 0000000000000046 ffffffff8105eb15 ffffffff81c905c0

ffff880225ca8000 ffff880225c8bfd8 ffff880225c8bfd8 ffff880225c8bfd8

ffffffff81c10440 ffff880225ca8000 0000000000000000 0000000000000000

Call Trace:

[<ffffffff8105eb15>] ? flush_work+0x5/0x280

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff813122e6>] ? cpumask_next_and+0x36/0x50

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e1ce>] ? might_fault+0x8e/0xa0

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 492

0xffff88020cc40000 492 1 0 3 S 0xffff88020cc40488 upstart-udev-br

ffff88020c3218d8 0000000000000046 0000000000000002 0000000000000000

ffff88020cc40000 ffff88020c321fd8 ffff88020c321fd8 ffff88020c321fd8

ffff880225d2deb0 ffff88020cc40000 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff815c5885>] ? unix_write_space+0x5/0xc0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff811899e5>] ? touch_atime+0xd5/0x170

[<ffffffff811899e5>] ? touch_atime+0xd5/0x170

[<ffffffff81510aa8>] ? sock_recvmsg+0xd8/0xf0

[<ffffffff8117830f>] ? pipe_read+0x7f/0x560

[<ffffffff8117830f>] ? pipe_read+0x7f/0x560

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff8106793a>] ? remove_wait_queue+0x5a/0x70

[<ffffffff810450ec>] ? do_wait+0x12c/0x260

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 501

0xffff88020cfd9f90 501 1 0 2 S 0xffff88020cfda418 udevd

ffff88020cf7dde8 0000000000000046 0000000000000002 0000000000000000

ffff88020cfd9f90 ffff88020cf7dfd8 ffff88020cf7dfd8 ffff88020cf7dfd8

ffff880225d2bf20 ffff88020cfd9f90 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff811b60e8>] SyS_epoll_wait+0x418/0x4c0

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 538

0xffff88020cd69f90 538 1 0 2 S 0xffff88020cd6a418 dbus-daemon

ffff88020c20fa38 0000000000000046 0000000000000002 0000000000000000

ffff88020cd69f90 ffff88020c20ffd8 ffff88020c20ffd8 ffff88020c20ffd8

ffff880225d2bf20 ffff88020cd69f90 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183e8f>] do_sys_poll+0x40f/0x4d0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff8113e1ce>] ? might_fault+0x8e/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81511e56>] ? ___sys_recvmsg+0x166/0x2a0

[<ffffffff8163b5cc>] ? __do_page_fault+0x2ac/0x520

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81637590>] ? _raw_spin_unlock_irq+0x30/0x40

[<ffffffff81071bb5>] ? finish_task_switch+0x85/0x110

[<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

[<ffffffff8118402b>] SyS_poll+0x6b/0x100

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 949

0xffff88020b35bf20 949 1 0 5 S 0xffff88020b35c3a8 rpc.idmapd

ffff88020d8b9de8 0000000000000046 0000000000000002 0000000000000002

ffff88020b35bf20 ffff88020d8b9fd8 ffff88020d8b9fd8 ffff88020d8b9fd8

ffff880225d39f90 ffff88020b35bf20 ffff880200000000 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff8106f103>] ? lg_local_unlock+0x23/0x40

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff811b60e8>] SyS_epoll_wait+0x418/0x4c0

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1036

0xffff880224e3bf20 1036 1 0 3 S 0xffff880224e3c3a8 upstart-socket-

ffff880220f398d8 0000000000000046 0000000000000000 ffffffff810679d3

ffff880224e3bf20 ffff880220f39fd8 ffff880220f39fd8 ffff880220f39fd8

ffff880225d2deb0 ffff880224e3bf20 0000000000000000 0000000000000000

Call Trace:

[<ffffffff810679d3>] ? add_wait_queue+0x23/0x60

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff815c5885>] ? unix_write_space+0x5/0xc0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff811899e5>] ? touch_atime+0xd5/0x170

[<ffffffff81510aa8>] ? sock_recvmsg+0xd8/0xf0

[<ffffffff8117830f>] ? pipe_read+0x7f/0x560

[<ffffffff8117830f>] ? pipe_read+0x7f/0x560

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff8106793a>] ? remove_wait_queue+0x5a/0x70

[<ffffffff810450ec>] ? do_wait+0x12c/0x260

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1040

0xffff88020dc5deb0 1040 1 0 2 S 0xffff88020dc5e338 rpcbind

ffff880220effa38 0000000000000046 00001207b08cc9a4 0000000001c9c37f

ffff88020dc5deb0 ffff880220efffd8 ffff880220efffd8 ffff880220efffd8

ffff880225d2bf20 ffff88020dc5deb0 0000000000000000 ffff880220effef8

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff816346fc>] schedule_hrtimeout_range_clock+0x12c/0x170

[<ffffffff8106a340>] ? update_rmtp+0x70/0x70

[<ffffffff8106af64>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183e8f>] do_sys_poll+0x40f/0x4d0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff8118ebd4>] ? mntput+0x24/0x40

[<ffffffff81170cb9>] ? __fput+0x189/0x240

[<ffffffff8109330c>] ? ktime_get_ts+0x4c/0xe0

[<ffffffff81182d25>] ? poll_select_set_timeout+0x85/0xa0

[<ffffffff8118402b>] SyS_poll+0x6b/0x100

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1104

0xffff880210080000 1104 1 0 0 S 0xffff880210080488 rpc.statd

ffff88020c1b38d8 0000000000000046 0000000000000002 0000000000000000

ffff880210080000 ffff88020c1b3fd8 ffff88020c1b3fd8 ffff88020c1b3fd8

ffffffff81c10440 ffff880210080000 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff810709a7>] ? __wake_up_sync_key+0x37/0x80

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff8106b953>] ? up_read+0x23/0x40

[<ffffffff8163b5cc>] ? __do_page_fault+0x2ac/0x520

[<ffffffff8118ebd4>] ? mntput+0x24/0x40

[<ffffffff81170cb9>] ? __fput+0x189/0x240

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1242

0xffff88020cfdbf20 1242 1 0 1 S 0xffff88020cfdc3a8 getty

ffff88020db8bcc8 0000000000000046 0000000000000002 ffff880227212f80

ffff88020cfdbf20 ffff88020db8bfd8 ffff88020db8bfd8 ffff88020db8bfd8

ffff880225d29f90 ffff88020cfdbf20 ffff88020db8bcb8 ffff880221386000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81632f2d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba368>] ? n_tty_read+0x388/0x960

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba258>] n_tty_read+0x278/0x960

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b61b5>] tty_read+0x95/0x100

[<ffffffff8116f8d0>] vfs_read+0xb0/0x180

[<ffffffff8116fb55>] SyS_read+0x55/0xa0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1247

0xffff88020dc5bf20 1247 1 0 4 S 0xffff88020dc5c3a8 getty

ffff88020dd8fcc8 0000000000000046 0000000000000002 ffff880227292f80

ffff88020dc5bf20 ffff88020dd8ffd8 ffff88020dd8ffd8 ffff88020dd8ffd8

ffff880225d38000 ffff88020dc5bf20 ffff88020dd8fcb8 ffff8802100cb000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81632f2d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba368>] ? n_tty_read+0x388/0x960

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba258>] n_tty_read+0x278/0x960

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b61b5>] tty_read+0x95/0x100

[<ffffffff8116f8d0>] vfs_read+0xb0/0x180

[<ffffffff8116fb55>] SyS_read+0x55/0xa0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1254

0xffff88020c5ebf20 1254 1 0 4 S 0xffff88020c5ec3a8 getty

ffff880220f51cc8 0000000000000046 0000000000000002 ffff880227292f80

ffff88020c5ebf20 ffff880220f51fd8 ffff880220f51fd8 ffff880220f51fd8

ffff880225d38000 ffff88020c5ebf20 ffff880220f51cb8 ffff8802100ce800

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81632f2d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba368>] ? n_tty_read+0x388/0x960

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba258>] n_tty_read+0x278/0x960

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b61b5>] tty_read+0x95/0x100

[<ffffffff8116f8d0>] vfs_read+0xb0/0x180

[<ffffffff8116fb55>] SyS_read+0x55/0xa0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1255

0xffff88020c5edeb0 1255 1 0 5 S 0xffff88020c5ee338 getty

ffff880224dbdcc8 0000000000000046 0000000000000002 ffff8802272b2f80

ffff88020c5edeb0 ffff880224dbdfd8 ffff880224dbdfd8 ffff880224dbdfd8

ffff880225d39f90 ffff88020c5edeb0 ffff880224dbdcb8 ffff88020bc7c000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81632f2d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba368>] ? n_tty_read+0x388/0x960

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba258>] n_tty_read+0x278/0x960

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b61b5>] tty_read+0x95/0x100

[<ffffffff8116f8d0>] vfs_read+0xb0/0x180

[<ffffffff8116fb55>] SyS_read+0x55/0xa0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1257

0xffff88020cf83f20 1257 1 0 4 S 0xffff88020cf843a8 getty

ffff880220e8bcc8 0000000000000046 0000000000000002 ffff880227292f80

ffff88020cf83f20 ffff880220e8bfd8 ffff880220e8bfd8 ffff880220e8bfd8

ffff880225d38000 ffff88020cf83f20 ffff880220e8bcb8 ffff8802100cf000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81632f2d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba368>] ? n_tty_read+0x388/0x960

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba258>] n_tty_read+0x278/0x960

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b61b5>] tty_read+0x95/0x100

[<ffffffff8116f8d0>] vfs_read+0xb0/0x180

[<ffffffff8116fb55>] SyS_read+0x55/0xa0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1273

0xffff88020c178000 1273 1 0 1 S 0xffff88020c178488 xinetd

ffff8802249878d8 0000000000000046 0000000000000000 ffffffff810679d3

ffff88020c178000 ffff880224987fd8 ffff880224987fd8 ffff880224987fd8

ffff880225d29f90 ffff88020c178000 ffff880200000000 0000000000000000

Call Trace:

[<ffffffff810679d3>] ? add_wait_queue+0x23/0x60

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff81076fce>] ? try_to_wake_up+0x2e/0x2f0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff8151aecb>] ? skb_queue_tail+0x2b/0x60

[<ffffffff8111d135>] ? find_get_page+0x5/0xf0

[<ffffffff8111d135>] ? find_get_page+0x5/0xf0

[<ffffffff8113e32a>] ? __do_fault+0x14a/0x470

[<ffffffff8113e32a>] ? __do_fault+0x14a/0x470

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff8106b953>] ? up_read+0x23/0x40

[<ffffffff8163b5cc>] ? __do_page_fault+0x2ac/0x520

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1275

0xffff88020d37deb0 1275 1 0 2 S 0xffff88020d37e338 atd

ffff880224f27e58 0000000000000046 0000034630b8a000 000000000000c350

ffff88020d37deb0 ffff880224f27fd8 ffff880224f27fd8 ffff880224f27fd8

ffff880225d2bf20 ffff88020d37deb0 0000000000000002 ffff880224f27eb8

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163450c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6c8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a340>] ? update_rmtp+0x70/0x70

[<ffffffff8106af64>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b806>] SyS_nanosleep+0x76/0x90

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1319

0xffff88020b0f3f20 1319 1 0 5 S 0xffff88020b0f43a8 irqbalance

ffff88020cf13e58 0000000000000046 000000022268ef88 000000000000c350

ffff88020b0f3f20 ffff88020cf13fd8 ffff88020cf13fd8 ffff88020cf13fd8

ffff880225d39f90 ffff88020b0f3f20 00007f9933324000 ffff88020cf13eb8

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163450c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6c8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a340>] ? update_rmtp+0x70/0x70

[<ffffffff8106af64>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b806>] SyS_nanosleep+0x76/0x90

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1492

0xffff88020c748000 1492 1 0 3 S 0xffff88020c748488 nrpe

ffff88020ca6d8d8 0000000000000046 00001205f16a6d1a 000000000007a11f

ffff88020c748000 ffff88020ca6dfd8 ffff88020ca6dfd8 ffff88020ca6dfd8

ffff880225d2deb0 ffff88020c748000 0000000000000000 ffff88020ca6dd18

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff816346fc>] schedule_hrtimeout_range_clock+0x12c/0x170

[<ffffffff8106a340>] ? update_rmtp+0x70/0x70

[<ffffffff8106af64>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff810e4b71>] ? delayacct_end+0x71/0xa0

[<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810e4b89>] ? delayacct_end+0x89/0xa0

[<ffffffff811530f3>] ? swap_info_get+0x63/0xe0

[<ffffffff811b1534>] ? fsnotify_clear_marks_by_inode+0x34/0x130

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff8118ea7c>] ? mntput_no_expire+0x2c/0x160

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8109330c>] ? ktime_get_ts+0x4c/0xe0

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1550

0xffff88020dc63f20 1550 1 0 5 S 0xffff88020dc643a8 rpc.mountd

ffff88020c7478d8 0000000000000046 0000000000000002 0000000000000000

ffff88020dc63f20 ffff88020c747fd8 ffff88020c747fd8 ffff88020c747fd8

ffff880225d39f90 ffff88020dc63f20 ffff88022fff9e00 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff8107214c>] ? ttwu_stat+0x4c/0x140

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff8106b953>] ? up_read+0x23/0x40

[<ffffffff8163b5cc>] ? __do_page_fault+0x2ac/0x520

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1578

0xffff88020b358000 1578 1 0 3 S 0xffff88020b358488 pkcsslotd

ffff88020cac5e58 0000000000000046 00000002540be400 000000000000c350

ffff88020b358000 ffff88020cac5fd8 ffff88020cac5fd8 ffff88020cac5fd8

ffff880225d2deb0 ffff88020b358000 ffff88020cac5f38 ffff88020cac5eb8

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163450c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6c8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a340>] ? update_rmtp+0x70/0x70

[<ffffffff8106af64>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b806>] SyS_nanosleep+0x76/0x90

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1591

0xffff880210083f20 1591 1 0 0 S 0xffff8802100843a8 rwhod

ffff88020c3aba48 0000000000000046 ffff88020c3aba18 0000000000000246

ffff880210083f20 ffff88020c3abfd8 ffff88020c3abfd8 ffff88020c3abfd8

ffffffff81c10440 ffff880210083f20 0000000000000002 ffff880220e3b180

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81632f2d>] schedule_timeout+0x1bd/0x270

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81520edb>] __skb_recv_datagram+0x2fb/0x380

[<ffffffff81520fa0>] ? skb_recv_datagram+0x40/0x40

[<ffffffff8158ec77>] udp_recvmsg+0xa7/0x410

[<ffffffff8159aebd>] inet_recvmsg+0xcd/0xf0

[<ffffffff81510aa8>] sock_recvmsg+0xd8/0xf0

[<ffffffff81186912>] ? dput+0x32/0x240

[<ffffffff81513439>] SYSC_recvfrom+0xe9/0x170

[<ffffffff81170cb9>] ? __fput+0x189/0x240

[<ffffffff81170d7e>] ? ____fput+0xe/0x10

[<ffffffff8131e7b4>] ? lockdep_sys_exit_thunk+0x35/0x67

[<ffffffff8151446e>] SyS_recvfrom+0xe/0x10

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1603

0xffff88020dc58000 1603 1591 0 3 S 0xffff88020dc58488 rwhod

ffff880216493e58 0000000000000046 00000029e8d60800 000000000000c350

ffff88020dc58000 ffff880216493fd8 ffff880216493fd8 ffff880216493fd8

ffff880225d2deb0 ffff88020dc58000 0000000000000002 ffff880216493eb8

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163450c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6c8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a340>] ? update_rmtp+0x70/0x70

[<ffffffff8106af64>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b806>] SyS_nanosleep+0x76/0x90

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1692

0xffff88020c32bf20 1692 1 0 6 S 0xffff88020c32c3a8 getty

ffff88020d819cc8 0000000000000046 0000000000000002 ffff8802272d2f80

ffff88020c32bf20 ffff88020d819fd8 ffff88020d819fd8 ffff88020d819fd8

ffff880225d3bf20 ffff88020c32bf20 ffff88020d819cb8 ffff88020dc0e000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81632f2d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba368>] ? n_tty_read+0x388/0x960

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba258>] n_tty_read+0x278/0x960

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b61b5>] tty_read+0x95/0x100

[<ffffffff8116f8d0>] vfs_read+0xb0/0x180

[<ffffffff8116fb55>] SyS_read+0x55/0xa0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1693

0xffff88020c329f90 1693 1 0 1 S 0xffff88020c32a418 getty

ffff880224e35cc8 0000000000000046 ffff88020cb591c8 ffff8802272b2f80

ffff88020c329f90 ffff880224e35fd8 ffff880224e35fd8 ffff880224e35fd8

ffff880225d29f90 ffff88020c329f90 ffff880224e35cb8 ffff88020cb59800

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81632f2d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba368>] ? n_tty_read+0x388/0x960

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba258>] n_tty_read+0x278/0x960

[<ffffffff81077290>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b61b5>] tty_read+0x95/0x100

[<ffffffff8116f8d0>] vfs_read+0xb0/0x180

[<ffffffff8116fb55>] SyS_read+0x55/0xa0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 2232

0xffff880210023f20 2232 1 0 0 S 0xffff8802100243a8 ntpd

ffff8802100938d8 0000000000000046 0000000000000002 0000000000000000

ffff880210023f20 ffff880210093fd8 ffff880210093fd8 ffff880210093fd8

ffffffff81c10440 ffff880210023f20 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff8105593a>] ? signal_delivered+0x5a/0x70

[<ffffffff8100aed2>] ? fpu_finit+0x22/0x40

[<ffffffff8100b146>] ? init_fpu+0x56/0xc0

[<ffffffff8100c12e>] ? __restore_xstate_sig+0x22e/0x510

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 25450

0xffff88020dc60000 25450 2 0 1 D 0xffff88020dc60488 kworker/1:1

ffff880222017c00 0000000000000046 0000000000000001 0000000000000000

ffff88020dc60000 ffff880222017fd8 ffff880222017fd8 ffff880222017fd8

ffff880225d29f90 ffff88020dc60000 ffff880224ec7968 ffff88020dc60000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81636da5>] rwsem_down_read_failed+0xa5/0xf9

[<ffffffff8131e674>] call_rwsem_down_read_failed+0x14/0x30

[<ffffffff816347c7>] ? down_read+0x47/0x50

[<ffffffffa05497c7>] ? handle_timeout+0x57/0x1f0 [libceph]

[<ffffffffa05497c7>] handle_timeout+0x57/0x1f0 [libceph]

[<ffffffff8105f3da>] process_one_work+0x1da/0x540

[<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

[<ffffffff81050ad3>] ? mod_timer+0x113/0x230

[<ffffffff810605bc>] worker_thread+0x11c/0x370

[<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106727a>] kthread+0xea/0xf0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8163ff9c>] ret_from_fork+0x7c/0xb0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

Stack traceback for pid 29928

0xffff88019e0a5eb0 29928 2 0 6 D 0xffff88019e0a6338 kworker/6:1

ffff8801b5ef79e8 0000000000000046 000000000000000d ffff8801b5ef79f8

ffff88019e0a5eb0 ffff8801b5ef7fd8 ffff8801b5ef7fd8 ffff8801b5ef7fd8

ffff88020d870000 ffff88019e0a5eb0 ffff880224ec7968 ffff88019e0a5eb0

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81636c45>] rwsem_down_write_failed+0xf5/0x1b0

[<ffffffff8131e6a3>] call_rwsem_down_write_failed+0x13/0x20

[<ffffffff81634821>] ? down_write+0x51/0x58

[<ffffffffa054b9f5>] ? ceph_osdc_handle_map+0xe5/0x580 [libceph]

[<ffffffffa053e82b>] ? ceph_check_fsid+0x2b/0x80 [libceph]

[<ffffffffa054b9f5>] ceph_osdc_handle_map+0xe5/0x580 [libceph]

[<ffffffffa054bfc8>] dispatch+0x138/0x9b0 [libceph]

[<ffffffffa053fe38>] ? ceph_tcp_recvmsg+0x48/0x60 [libceph]

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffffa05431f8>] con_work+0x1948/0x2d50 [libceph]

[<ffffffff81080bb3>] ? idle_balance+0x133/0x180

[<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

[<ffffffff81071b78>] ? finish_task_switch+0x48/0x110

[<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

[<ffffffff8105f3da>] process_one_work+0x1da/0x540

[<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

[<ffffffff81050ad3>] ? mod_timer+0x113/0x230

[<ffffffff810605bc>] worker_thread+0x11c/0x370

[<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106727a>] kthread+0xea/0xf0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8163ff9c>] ret_from_fork+0x7c/0xb0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

Stack traceback for pid 11216

0xffff88020c133f20 11216 2 0 3 D 0xffff88020c1343a8 kworker/u64:1

ffff88022489f700 0000000000000046 000000004c8be1d4 ffff88022489f6b8

ffff88020c133f20 ffff88022489ffd8 ffff88022489ffd8 ffff88022489ffd8

ffff880225d2deb0 ffff88020c133f20 ffff880224ec7968 ffff88020c133f20

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81636da5>] rwsem_down_read_failed+0xa5/0xf9

[<ffffffffa053f271>] ? ceph_msg_data_create+0x41/0x60 [libceph]

[<ffffffff8131e674>] call_rwsem_down_read_failed+0x14/0x30

[<ffffffff816347c7>] ? down_read+0x47/0x50

[<ffffffffa0549abe>] ? ceph_osdc_start_request+0x3e/0x130 [libceph]

[<ffffffffa0549abe>] ceph_osdc_start_request+0x3e/0x130 [libceph]

[<ffffffffa069b4ce>] ceph_writepages_start+0x94e/0xfa0 [ceph]

[<ffffffff81128923>] do_writepages+0x23/0x40

[<ffffffff811980ce>] __writeback_single_inode+0x4e/0x2a0

[<ffffffff811995c0>] writeback_sb_inodes+0x280/0x3e0

[<ffffffff811997be>] __writeback_inodes_wb+0x9e/0xd0

[<ffffffff811999eb>] wb_writeback+0x1fb/0x310

[<ffffffff8119b2fb>] wb_do_writeback+0xbb/0x1e0

[<ffffffff81061f6f>] ? set_worker_desc+0x6f/0x80

[<ffffffff8119b49a>] bdi_writeback_workfn+0x7a/0x200

[<ffffffff8105f3da>] process_one_work+0x1da/0x540

[<ffffffff8105f36f>] ? process_one_work+0x16f/0x540

[<ffffffff810605bc>] worker_thread+0x11c/0x370

[<ffffffff810604a0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106727a>] kthread+0xea/0xf0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8163ff9c>] ret_from_fork+0x7c/0xb0

[<ffffffff81067190>] ? flush_kthread_worker+0x150/0x150

Stack traceback for pid 12266

0xffff880222028000 12266 1 0 0 S 0xffff880222028488 sshd

ffff88020dc13a38 0000000000000046 0000000000000002 0000000000000000

ffff880222028000 ffff88020dc13fd8 ffff88020dc13fd8 ffff88020dc13fd8

ffffffff81c10440 ffff880222028000 ffff88020dc13b08 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff8113ee8f>] ? __pte_alloc+0x3f/0xc0

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183e8f>] do_sys_poll+0x40f/0x4d0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff8113f1bf>] ? do_wp_page+0x19f/0x6f0

[<ffffffff8163763b>] ? _raw_spin_unlock+0x2b/0x40

[<ffffffff8113f2ba>] ? do_wp_page+0x29a/0x6f0

[<ffffffff81140f24>] ? handle_pte_fault+0x264/0x8a0

[<ffffffff8163b47c>] ? __do_page_fault+0x15c/0x520

[<ffffffff8163b47c>] ? __do_page_fault+0x15c/0x520

[<ffffffff810797c5>] ? sched_clock_cpu+0xc5/0x120

[<ffffffff8106b953>] ? up_read+0x23/0x40

[<ffffffff8163b5cc>] ? __do_page_fault+0x2ac/0x520

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81637590>] ? _raw_spin_unlock_irq+0x30/0x40

[<ffffffff810562ed>] ? do_sigaction+0x12d/0x1f0

[<ffffffff8118402b>] SyS_poll+0x6b/0x100

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 12282

0xffff88022202bf20 12282 12266 0 0 S 0xffff88022202c3a8 sshd

ffff88020c8bf8d8 0000000000000046 0000000000000000 ffffffff810679d3

ffff88022202bf20 ffff88020c8bffd8 ffff88020c8bffd8 ffff88020c8bffd8

ffffffff81c10440 ffff88022202bf20 0000000000000000 0000000000000000

Call Trace:

[<ffffffff810679d3>] ? add_wait_queue+0x23/0x60

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff810888d5>] ? cpuacct_charge+0x5/0xa0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff81517b92>] ? release_sock+0x172/0x1e0

[<ffffffff81570eb6>] ? tcp_recvmsg+0x396/0xc80

[<ffffffff8159adf5>] ? inet_recvmsg+0x5/0xf0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff81511783>] ? sock_aio_read+0x23/0x30

[<ffffffff8116e9fa>] ? do_sync_read+0x7a/0xb0

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 12316

0xffff880224829f90 12316 1 0 2 S 0xffff88022482a418 rsyslogd

ffff88018c59d8d8 0000000000000046 0000120b68a414f1 0000000001c9c37e

ffff880224829f90 ffff88018c59dfd8 ffff88018c59dfd8 ffff88018c59dfd8

ffff880225d2bf20 ffff880224829f90 ffff88018c59d8b8 ffff88018c59dd18

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff816346fc>] schedule_hrtimeout_range_clock+0x12c/0x170

[<ffffffff8106a340>] ? update_rmtp+0x70/0x70

[<ffffffff8106af64>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff811530f3>] ? swap_info_get+0x63/0xe0

[<ffffffff811530f3>] ? swap_info_get+0x63/0xe0

[<ffffffff811530f3>] ? swap_info_get+0x63/0xe0

[<ffffffff811530f3>] ? swap_info_get+0x63/0xe0

[<ffffffff811529fd>] ? delete_from_swap_cache+0x3d/0x70

[<ffffffff8163763b>] ? _raw_spin_unlock+0x2b/0x40

[<ffffffff811541a1>] ? swapcache_free+0x41/0x50

[<ffffffff81152a20>] ? delete_from_swap_cache+0x60/0x70

[<ffffffff8163763b>] ? _raw_spin_unlock+0x2b/0x40

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff8106b953>] ? up_read+0x23/0x40

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8109330c>] ? ktime_get_ts+0x4c/0xe0

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 12317

0xffff880210058000 12317 1 0 2 S 0xffff880210058488 rs:main Q:Reg

ffff88020cd87c28 0000000000000046 000000000000005f ffffffff826c2340

ffff880210058000 ffff88020cd87fd8 ffff88020cd87fd8 ffff88020cd87fd8

ffff880225d2bf20 ffff880210058000 0000000000000000 ffff88020cd87cd0

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff810a47f7>] futex_wait_queue_me+0xc7/0x100

[<ffffffff810a5c37>] futex_wait+0x1a7/0x2c0

[<ffffffff810a309d>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81212cdf>] ? ext4_file_write+0x9f/0x420

[<ffffffff8163763b>] ? _raw_spin_unlock+0x2b/0x40

[<ffffffff811b024a>] ? fsnotify+0x8a/0x2f0

[<ffffffff810a72c4>] do_futex+0x344/0xbb0

[<ffffffff811b03ca>] ? fsnotify+0x20a/0x2f0

[<ffffffff811b024a>] ? fsnotify+0x8a/0x2f0

[<ffffffff810a7c77>] SyS_futex+0x147/0x1a0

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 12318

0xffff8802258cbf20 12318 1 0 0 S 0xffff8802258cc3a8 rsyslogd

ffff88020c5fd8d8 0000000000000046 0000000000000000 ffff88020c77d800

ffff8802258cbf20 ffff88020c5fdfd8 ffff88020c5fdfd8 ffff88020c5fdfd8

ffffffff81c10440 ffff8802258cbf20 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163471d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810679fd>] ? add_wait_queue+0x4d/0x60

[<ffffffff81182aa5>] ? __pollwait+0x75/0xf0

[<ffffffff81634753>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182929>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183311>] do_select+0x5d1/0x6e0

[<ffffffff81182d45>] ? do_select+0x5/0x6e0

[<ffffffff81165a80>] ? kmem_cache_free+0x1b0/0x1c0

[<ffffffff81182b20>] ? __pollwait+0xf0/0xf0

[<ffffffff8163446e>] ? mutex_unlock+0xe/0x10

[<ffffffff815c8776>] ? unix_dgram_recvmsg+0x316/0x4e0

[<ffffffff8107afb3>] ? select_task_rq_fair+0x53/0x8a0

[<ffffffff8107afb3>] ? select_task_rq_fair+0x53/0x8a0

[<ffffffff8107214c>] ? ttwu_stat+0x4c/0x140

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff8113e185>] ? might_fault+0x45/0xa0

[<ffffffff81183634>] core_sys_select+0x214/0x340

[<ffffffff81183462>] ? core_sys_select+0x42/0x340

[<ffffffff81511e56>] ? ___sys_recvmsg+0x166/0x2a0

[<ffffffff810a79c5>] ? do_futex+0xa45/0xbb0

[<ffffffff8118c47e>] ? fget_light+0x9e/0x130

[<ffffffff81183821>] SyS_select+0xc1/0x100

[<ffffffff8131e73e>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 12319

0xffff880224b1deb0 12319 1 0 1 S 0xffff880224b1e338 rsyslogd

ffff8801e8977e28 0000000000000046 0000000000000286 ffffffff81c317c0

ffff880224b1deb0 ffff8801e8977fd8 ffff8801e8977fd8 ffff8801e8977fd8

ffff880225d29f90 ffff880224b1deb0 ffff8801e8977e38 0000000000000000

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff81042b31>] do_syslog+0x4d1/0x550

[<ffffffff81067a70>] ? __init_waitqueue_head+0x60/0x60

[<ffffffff811e0a66>] kmsg_read+0x56/0x70

[<ffffffff811d2bf3>] proc_reg_read+0x43/0x70

[<ffffffff8116f8d0>] vfs_read+0xb0/0x180

[<ffffffff8116fb55>] SyS_read+0x55/0xa0

[<ffffffff81640042>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 12553

0xffff88020dd01f90 12553 1 0 7 S 0xffff88020dd02418 cron

ffff8801ee239e58 0000000000000046 0000000df8475800 000000000000c350

ffff88020dd01f90 ffff8801ee239fd8 ffff8801ee239fd8 ffff8801ee239fd8

ffff880225d3deb0 ffff88020dd01f90 0000000000000002 ffff8801ee239eb8

Call Trace:

[<ffffffff81635f09>] schedule+0x29/0x70

[<ffffffff8163450c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6c8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a340>] ? update_rmtp+0x70/0x70