[4]kdb> dumpall [dumpall]kdb> pid R KDB current process is kworker/4:1(pid=29967) [dumpall]kdb> -dumpcommon [dumpcommon]kdb> set BTAPROMPT 0 [dumpcommon]kdb> set LINES 10000 [dumpcommon]kdb> -summary sysname Linux release 3.10.0-rc6-ceph-00091-g2dd322b version #1 SMP Wed Jun 19 16:06:38 PDT 2013 machine x86_64 nodename plana56 domainname (none) ccversion CCVERSION date 2013-06-22 22:32:35 tz_minuteswest 420 uptime 05:37 load avg 26.15 8.19 3.64 MemTotal: 2038247 kB MemFree: 748512 kB Buffers: 256 kB [dumpcommon]kdb> -cpu Currently on cpu 4 Available cpus: 0-3(I), 4, 5-7(I) [dumpcommon]kdb> -ps 7 idle processes (state I) and 112 sleeping system daemon (state M) processes suppressed, use 'ps A' to see all. Task Addr Pid Parent [*] cpu State Thread Command 0xffff88020dd03f20 29967 2 1 4 R 0xffff88020dd043a8 *kworker/4:1 0xffff880225ca8000 1 0 0 0 S 0xffff880225ca8488 init 0xffff88020cc40000 492 1 0 3 S 0xffff88020cc40488 upstart-udev-br 0xffff88020cfd9f90 501 1 0 2 S 0xffff88020cfda418 udevd 0xffff88020cd69f90 538 1 0 2 S 0xffff88020cd6a418 dbus-daemon 0xffff88020b35bf20 949 1 0 5 S 0xffff88020b35c3a8 rpc.idmapd 0xffff880224e3bf20 1036 1 0 3 S 0xffff880224e3c3a8 upstart-socket- 0xffff88020dc5deb0 1040 1 0 2 S 0xffff88020dc5e338 rpcbind 0xffff880210080000 1104 1 0 0 S 0xffff880210080488 rpc.statd 0xffff88020cfdbf20 1242 1 0 1 S 0xffff88020cfdc3a8 getty 0xffff88020dc5bf20 1247 1 0 4 S 0xffff88020dc5c3a8 getty 0xffff88020c5ebf20 1254 1 0 4 S 0xffff88020c5ec3a8 getty 0xffff88020c5edeb0 1255 1 0 5 S 0xffff88020c5ee338 getty 0xffff88020cf83f20 1257 1 0 4 S 0xffff88020cf843a8 getty 0xffff88020c178000 1273 1 0 1 S 0xffff88020c178488 xinetd 0xffff88020d37deb0 1275 1 0 2 S 0xffff88020d37e338 atd 0xffff88020b0f3f20 1319 1 0 5 S 0xffff88020b0f43a8 irqbalance 0xffff88020c748000 1492 1 0 3 S 0xffff88020c748488 nrpe 0xffff88020dc63f20 1550 1 0 5 S 0xffff88020dc643a8 rpc.mountd 0xffff88020b358000 1578 1 0 3 S 0xffff88020b358488 pkcsslotd 0xffff880210083f20 1591 1 0 0 S 0xffff8802100843a8 rwhod 0xffff88020dc58000 1603 1591 0 3 S 0xffff88020dc58488 rwhod 0xffff88020c32bf20 1692 1 0 6 S 0xffff88020c32c3a8 getty 0xffff88020c329f90 1693 1 0 1 S 0xffff88020c32a418 getty 0xffff880210023f20 2232 1 0 0 S 0xffff8802100243a8 ntpd 0xffff88020dc60000 25450 2 0 1 D 0xffff88020dc60488 kworker/1:1 0xffff88019e0a5eb0 29928 2 0 6 D 0xffff88019e0a6338 kworker/6:1 0xffff88020dd03f20 29967 2 1 4 R 0xffff88020dd043a8 *kworker/4:1 0xffff88020c133f20 11216 2 0 3 D 0xffff88020c1343a8 kworker/u64:1 0xffff880222028000 12266 1 0 0 S 0xffff880222028488 sshd 0xffff88022202bf20 12282 12266 0 0 S 0xffff88022202c3a8 sshd 0xffff880224829f90 12316 1 0 2 S 0xffff88022482a418 rsyslogd 0xffff880210058000 12317 1 0 2 S 0xffff880210058488 rs:main Q:Reg 0xffff8802258cbf20 12318 1 0 0 S 0xffff8802258cc3a8 rsyslogd 0xffff880224b1deb0 12319 1 0 1 S 0xffff880224b1e338 rsyslogd 0xffff88020dd01f90 12553 1 0 7 S 0xffff88020dd02418 cron 0xffff88020d9d1f90 16744 1 0 3 S 0xffff88020d9d2418 sshd 0xffff880224a00000 17512 501 0 1 S 0xffff880224a00488 udevd 0xffff88020cc45eb0 17542 12282 0 6 S 0xffff88020cc46338 bash 0xffff880223030000 17544 17542 0 6 S 0xffff880223030488 ffsb.sh 0xffff880223855eb0 19210 17544 0 0 S 0xffff880223856338 ffsb 0xffff88020c179f90 19250 17544 0 3 S 0xffff88020c17a418 ffsb 0xffff880223850000 19251 17544 0 0 D 0xffff880223850488 ffsb 0xffff880223853f20 19252 17544 0 3 D 0xffff8802238543a8 ffsb 0xffff880223851f90 19253 17544 0 0 D 0xffff880223852418 ffsb 0xffff88020d0cdeb0 19254 17544 0 6 D 0xffff88020d0ce338 ffsb 0xffff88020d0c8000 19255 17544 0 0 D 0xffff88020d0c8488 ffsb 0xffff88019e0a0000 19256 17544 0 3 D 0xffff88019e0a0488 ffsb 0xffff88019e0a3f20 19257 17544 0 7 D 0xffff88019e0a43a8 ffsb 0xffff88019e0a1f90 19258 17544 0 3 D 0xffff88019e0a2418 ffsb 0xffff880224f08000 19259 17544 0 2 D 0xffff880224f08488 ffsb 0xffff880224828000 19260 17544 0 1 D 0xffff880224828488 ffsb 0xffff88022482bf20 19261 17544 0 0 D 0xffff88022482c3a8 ffsb 0xffff88022482deb0 19262 17544 0 7 D 0xffff88022482e338 ffsb 0xffff88020dd05eb0 19263 17544 0 1 D 0xffff88020dd06338 ffsb 0xffff88020dd00000 19264 17544 0 3 D 0xffff88020dd00488 ffsb 0xffff88020d08deb0 19265 17544 0 0 D 0xffff88020d08e338 ffsb 0xffff88020c5b8000 19266 17544 0 7 D 0xffff88020c5b8488 ffsb 0xffff88020c5bdeb0 19267 17544 0 7 D 0xffff88020c5be338 ffsb 0xffff880225e83f20 19268 17544 0 3 D 0xffff880225e843a8 ffsb 0xffff88020d2d5eb0 19269 17544 0 0 D 0xffff88020d2d6338 ffsb 0xffff88020d2d1f90 19270 17544 0 0 D 0xffff88020d2d2418 ffsb 0xffff88020d2d3f20 19271 17544 0 3 D 0xffff88020d2d43a8 ffsb 0xffff88020dc61f90 19272 17544 0 0 D 0xffff88020dc62418 ffsb 0xffff88020dc65eb0 19273 17544 0 3 D 0xffff88020dc66338 ffsb 0xffff88020cfddeb0 19274 17544 0 6 D 0xffff88020cfde338 ffsb 0xffff88020cfd8000 19275 17544 0 2 D 0xffff88020cfd8488 ffsb 0xffff880223795eb0 19276 17544 0 3 D 0xffff880223796338 ffsb 0xffff880223790000 19277 17544 0 4 D 0xffff880223790488 ffsb 0xffff88020dcd3f20 19278 17544 0 4 D 0xffff88020dcd43a8 ffsb 0xffff880224e38000 19279 17544 0 6 D 0xffff880224e38488 ffsb 0xffff880224e3deb0 19280 17544 0 5 D 0xffff880224e3e338 ffsb 0xffff880224e39f90 19281 17544 0 0 D 0xffff880224e3a418 ffsb 0xffff880210059f90 19282 17544 0 1 D 0xffff88021005a418 ffsb 0xffff8801e8920000 19247 2 0 0 D 0xffff8801e8920488 kworker/0:1 0xffff8801e8921f90 19283 2 0 4 D 0xffff8801e8922418 kworker/4:0 0xffff88020d871f90 19285 2 0 4 R 0xffff88020d872418 kworker/4:3 0xffff88020ce93f20 19287 2 0 0 D 0xffff88020ce943a8 kworker/0:3 [dumpcommon]kdb> -dmesg 600 <4>[17533.026513] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17533.305786] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17533.548387] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17533.560224] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17533.560224] <4>[17533.592541] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17533.603875] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17533.603875] <4>[17535.002318] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17535.622836] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17535.635858] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17535.635858] <4>[17536.194110] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17536.220037] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17536.220037] <4>[17537.204423] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17537.216409] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17537.216409] <4>[17539.232540] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17539.244239] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17539.244239] <4>[17539.636142] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17539.681878] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17539.695057] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17539.695057] <4>[17539.726893] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17539.740351] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17539.740351] <4>[17541.480993] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17542.496935] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17542.531752] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17543.031175] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17543.066613] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17543.379091] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17544.233901] libceph: osd1 10.214.131.6:6808 socket error on read <4>[17544.466703] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17544.701796] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17545.178602] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17546.879851] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17548.047776] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17548.806604] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17549.788432] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17549.801209] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17551.530479] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17551.645252] libceph: osd4 10.214.131.24:6800 socket error on read <4>[17552.039592] libceph: osd5 10.214.131.24:6809 socket error on read <4>[17552.491139] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17552.997848] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17554.148475] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17554.439676] libceph: osd1 10.214.131.6:6808 socket error on read <4>[17554.668289] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17555.022507] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17555.036487] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17555.036487] <4>[17555.326965] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17555.363084] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17557.038761] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17557.052047] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17557.052047] <4>[17557.903326] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17557.916751] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17557.916751] <4>[17558.807014] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17558.849574] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17559.480226] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17560.939397] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17560.952991] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17560.952991] <4>[17563.060683] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17564.468034] libceph: osd0 10.214.131.6:6800 socket error on read <4>[17564.546618] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17564.561124] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17564.561124] <4>[17568.660832] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17568.921904] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17570.987567] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17571.355165] libceph: osd0 10.214.131.6:6800 socket error on write <4>[17571.566214] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17571.899464] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17573.094100] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17573.104350] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17573.104350] <4>[17573.637771] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17574.071533] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17575.241888] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17575.290380] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17575.304391] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17575.304391] <4>[17575.587316] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17576.039354] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17576.790371] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17577.076261] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17577.805040] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17578.865286] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17579.201121] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17581.212965] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17581.829552] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17584.490562] libceph: osd5 10.214.131.24:6809 socket error on read <4>[17585.939918] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17586.184549] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17587.115863] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17587.119286] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17587.123722] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17587.123722] <4>[17587.981582] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17588.631562] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17592.098284] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17592.211747] libceph: osd3 10.214.131.24:6804 socket error on read <4>[17592.224591] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17592.224591] <4>[17592.988563] libceph: osd2 10.214.131.6:6804 socket error on read <4>[17593.133189] libceph: osd2 10.214.131.6:6804 socket error on write <4>[17593.506824] libceph: osd1 10.214.131.6:6808 socket error on read <4>[17594.009964] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17594.703762] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17597.127834] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17597.597392] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17597.612196] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17597.612196] <4>[17598.893909] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17600.273864] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17600.476443] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17602.012767] libceph: osd4 10.214.131.24:6800 socket error on read <4>[17602.375769] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17605.741750] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17605.845332] libceph: osd5 10.214.131.24:6809 socket error on write <4>[17606.589856] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17606.603909] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17606.603909] <4>[17606.790072] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17609.475310] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17609.486628] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17609.486628] <4>[17611.430599] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17611.443762] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17611.443762] <4>[17612.213691] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17613.169551] libceph: osd2 10.214.131.6:6804 socket error on read <4>[17613.737234] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17614.795950] libceph: osd1 10.214.131.6:6808 socket error on read <4>[17618.470117] libceph: osd2 10.214.131.6:6804 socket error on write <4>[17618.666911] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17618.678700] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17618.678700] <4>[17619.622901] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17620.051985] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17620.064819] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17620.064819] <4>[17620.333916] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17624.771099] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17625.503820] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17626.496739] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17626.509040] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17626.509040] <4>[17626.632422] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17626.644609] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17626.644609] <4>[17627.754056] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17629.050099] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17630.089157] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17634.844652] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17635.166961] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17636.786645] libceph: osd3 10.214.131.24:6804 socket error on write <4>[17636.798107] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17636.798107] <4>[17637.107751] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17638.146838] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17639.004101] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17640.068454] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17640.110164] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17640.555220] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17640.608122] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17640.608122] <4>[17641.170014] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17641.826747] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17643.169332] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17643.239154] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17643.455348] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17644.570704] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17644.584060] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17644.584060] <4>[17645.530896] libceph: osd0 10.214.131.6:6800 socket error on read <4>[17651.258837] libceph: osd5 10.214.131.24:6809 socket error on read <4>[17653.158165] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17653.168263] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17653.168263] <4>[17653.640934] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17653.652207] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17653.652207] <4>[17653.768853] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17655.619574] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17656.541316] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17658.432820] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17658.445536] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17658.445536] <4>[17659.125996] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17660.231395] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17660.282051] libceph: osd5 10.214.131.24:6809 socket error on write <4>[17663.622767] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17663.678556] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17664.033800] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17664.803811] libceph: osd2 10.214.131.6:6804 socket error on read <4>[17667.966439] libceph: osd0 10.214.131.6:6800 socket error on read <4>[17669.013576] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17669.412517] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17669.712890] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17671.116700] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17671.914702] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17672.038049] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17672.379980] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17673.393492] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17675.245090] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17678.375568] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17678.587283] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17679.451751] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17680.491038] libceph: osd3 10.214.131.24:6804 socket error on write <4>[17680.499479] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17680.499479] <4>[17680.692682] libceph: osd3 10.214.131.24:6804 socket error on read <4>[17680.705085] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17680.705085] <4>[17681.026845] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17681.638215] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17683.084331] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17684.798383] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17686.907250] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17687.635351] libceph: osd2 10.214.131.6:6804 socket error on write <4>[17689.307434] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17689.372548] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17689.925830] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17692.487908] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17692.754723] libceph: osd1 10.214.131.6:6808 socket error on write <4>[17693.474343] libceph: osd5 10.214.131.24:6809 socket error on write <4>[17693.692379] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17694.835062] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17695.606397] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17698.555244] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17698.677550] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17698.758742] libceph: osd0 10.214.131.6:6800 socket error on read <4>[17699.014722] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17702.135852] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17702.621096] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17706.359738] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17706.373519] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17706.373519] <4>[17708.281440] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17709.367172] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17709.972851] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17712.054650] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17712.066092] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17712.066092] <4>[17712.590032] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17712.767382] libceph: osd0 10.214.131.6:6800 socket error on read <4>[17713.685098] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17713.698129] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17713.698129] <4>[17716.062376] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17719.577941] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17720.874451] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17722.646150] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17725.365118] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17726.363520] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17726.376119] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17726.376119] <4>[17726.482694] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17726.745879] libceph: osd5 10.214.131.24:6809 socket error on read <4>[17727.916552] libceph: osd4 10.214.131.24:6800 socket closed (con state OPEN) <4>[17728.493797] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17728.517094] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17729.898608] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17730.083932] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17730.095588] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17730.095588] <4>[17731.235819] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17732.159519] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17732.171497] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17732.171497] <4>[17742.977181] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17742.988882] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17742.988882] <4>[17743.090374] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17747.310262] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17751.382439] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17751.395387] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17751.395387] <4>[17751.455678] libceph: osd2 10.214.131.6:6804 socket error on read <4>[17752.159530] libceph: osd0 10.214.131.6:6800 socket error on read <4>[17754.262501] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17754.274612] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17754.274612] <4>[17754.425617] libceph: osd0 10.214.131.6:6800 socket closed (con state OPEN) <4>[17755.559934] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <4>[17756.277738] libceph: osd1 10.214.131.6:6808 socket error on read <4>[17758.774973] libceph: osd3 10.214.131.24:6804 socket closed (con state OPEN) <4>[17758.789031] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17758.789031] <4>[17760.970301] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17762.411042] libceph: osd3 10.214.131.24:6804 socket error on write <4>[17762.424563] rbd: obj_request ffff8801d98cb7a0 already marked done <4>[17762.424563] <4>[17763.255145] libceph: osd1 10.214.131.6:6808 socket closed (con state OPEN) <4>[17764.658962] libceph: osd5 10.214.131.24:6809 socket error on read <4>[17765.070353] libceph: osd2 10.214.131.6:6804 socket error on read <4>[17766.447631] libceph: osd5 10.214.131.24:6809 socket closed (con state OPEN) <4>[17767.843423] libceph: osd2 10.214.131.6:6804 socket closed (con state OPEN) <5>[17778.790982] Key type ceph unregistered <6>[18270.726611] kgdb: Unregistered I/O driver kgdboc, debugger disabled. <6>[18270.732993] kgdb: Registered I/O driver kgdboc. <12>[18277.329116] init: cron main process (18413) killed by TERM signal <6>[18283.899773] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro,user_xattr,user_xattr <6>[18355.068892] device fsid ca09882a-4f69-4c7f-9b65-76b369fe2315 devid 1 transid 7 /dev/sdd <6>[18355.080794] btrfs flagging fs with big metadata feature <6>[18355.977555] device fsid ac2c23bb-10fe-4bad-9c33-a211eef9da8a devid 1 transid 7 /dev/sdb <6>[18355.992357] btrfs flagging fs with big metadata feature <6>[18356.889823] device fsid 7c4979d7-9726-47f6-9991-6ddd974fa2c9 devid 1 transid 7 /dev/sdc <6>[18356.903542] btrfs flagging fs with big metadata feature <6>[18739.278247] kgdb: Unregistered I/O driver kgdboc, debugger disabled. <6>[18739.284631] kgdb: Registered I/O driver kgdboc. <12>[18743.729481] init: cron main process (24867) killed by TERM signal <6>[18750.270916] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro,user_xattr,user_xattr <5>[18909.912979] Key type ceph registered <6>[18909.916825] libceph: loaded (mon/osd proto 15/24) <6>[18910.001391] ceph: loaded (mds proto 32) <6>[18910.008436] libceph: client4143 fsid fdbd22ee-8e25-4fe2-a6e5-09e250792f49 <6>[18910.017962] libceph: mon2 10.214.132.14:6789 session established <6>[18957.728828] kgdb: Unregistered I/O driver kgdboc, debugger disabled. <6>[18957.735260] kgdb: Registered I/O driver kgdboc. <12>[18963.730725] init: cron main process (32370) killed by TERM signal <6>[18970.571722] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro,user_xattr,user_xattr <6>[19061.897253] libceph: client4105 fsid a0bafb5c-6ebb-41ea-8fc9-4766f12d3837 <6>[19061.907074] libceph: mon0 10.214.132.16:6789 session established <6>[19310.941929] kgdb: Unregistered I/O driver kgdboc, debugger disabled. <6>[19310.948310] kgdb: Registered I/O driver kgdboc. <12>[19316.676000] init: cron main process (6262) killed by TERM signal <6>[19323.013033] EXT4-fs (sda1): re-mounted. Opts: errors=remount-ro,user_xattr,user_xattr <6>[19423.187276] libceph: client4143 fsid 77de2efd-1a1a-4a18-b063-47db86244ea7 <6>[19423.197171] libceph: mon2 10.214.132.14:6789 session established <6>[19444.799182] libceph: osd2 down <6>[19444.802352] libceph: osd2 up <6>[19448.846897] libceph: osd0 weight 0x0 (out) <4>[19453.875514] libceph: osd2 10.214.132.16:6808 socket closed (con state OPEN) <4>[19454.249085] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING) <4>[19454.698506] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING) <4>[19455.849256] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING) <4>[19457.670307] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING) <6>[19460.048351] libceph: osd0 weight 0x10000 (in) <4>[19461.742708] libceph: osd2 10.214.132.16:6808 socket closed (con state CONNECTING) <6>[19464.274855] libceph: osd2 down <6>[19464.277937] libceph: osd2 up <6>[19466.701862] libceph: osd5 weight 0x0 (out) <6>[19473.388263] libceph: osd4 weight 0x0 (out) <4>[19473.424303] libceph: osd1 10.214.132.16:6800 socket closed (con state OPEN) <4>[19473.503478] libceph: osd3 10.214.132.14:6800 socket closed (con state OPEN) <4>[19478.113622] libceph: osd3 10.214.132.14:6800 socket closed (con state OPEN) <4>[19478.120818] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING) <4>[19478.713039] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING) <4>[19479.864775] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING) <4>[19481.745104] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING) <4>[19485.878080] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING) <4>[19488.242249] libceph: osd5 10.214.132.14:6808 socket closed (con state OPEN) <6>[19493.603194] libceph: osd4 weight 0x10000 (in) <4>[19493.760153] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN) <4>[19493.897309] libceph: osd3 10.214.132.14:6800 socket closed (con state CONNECTING) <6>[19500.143974] libceph: osd3 down <6>[19500.147100] libceph: osd5 down <6>[19501.287213] libceph: osd5 up <4>[19502.613979] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN) <6>[19502.887613] libceph: osd3 up <4>[19503.389534] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN) <4>[19505.203671] libceph: osd1 10.214.132.16:6800 socket closed (con state OPEN) <6>[19549.477787] libceph: osd5 weight 0x10000 (in) <6>[19555.270712] libceph: osd0 weight 0x0 (out) <4>[19555.624170] libceph: osd5 10.214.132.14:6817 socket error on write <6>[19561.972152] libceph: osd0 weight 0x10000 (in) <4>[19567.109775] libceph: osd2 10.214.132.16:6812 socket closed (con state OPEN) <4>[19572.035081] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING) <4>[19572.740668] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING) <4>[19573.920275] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING) <4>[19575.771970] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING) <4>[19579.933897] libceph: osd2 10.214.132.16:6812 socket closed (con state CONNECTING) <6>[19580.809766] libceph: osd2 down <6>[19581.971431] libceph: osd2 up <4>[19582.099009] libceph: osd0 10.214.132.16:6801 socket closed (con state OPEN) <4>[19582.112255] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN) <4>[19585.095083] libceph: osd1 10.214.132.16:6800 socket closed (con state OPEN) <6>[19625.979741] libceph: osd0 weight 0x0 (out) <4>[19626.059263] libceph: osd4 10.214.132.14:6804 socket closed (con state OPEN) <6>[19637.726796] libceph: osd0 weight 0x10000 (in) <6>[19654.210343] libceph: osd1 weight 0x0 (out) <6>[19752.920435] libceph: osd1 weight 0x10000 (in) <4>[19776.765662] libceph: osd0 10.214.132.16:6801 socket error on write <4>[19776.796030] libceph: osd0 10.214.132.16:6801 socket closed (con state CONNECTING) <4>[19778.015116] libceph: osd0 10.214.132.16:6801 socket closed (con state CONNECTING) <3>[19799.355399] INFO: rcu_sched self-detected stall on CPU { 6} (t=2100 jiffies g=245350 c=245349 q=2640) <4>[19799.364789] CPU: 6 PID: 19284 Comm: kworker/6:2 Tainted: G W 3.10.0-rc6-ceph-00091-g2dd322b #1 <4>[19799.374303] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <3>[19799.375424] INFO: rcu_sched detected stalls on CPUs/tasks: { 6} (detected by 0, t=2102 jiffies, g=245350, c=245349, q=2640) <6>[19799.375425] Task dump for CPU 6: <6>[19799.375429] kworker/6:2 R running task 0 19284 2 0x00000000 <6>[19799.375442] Workqueue: ceph-msgr con_work [libceph] <4>[19799.375445] ffff880125ff7de8 ffffffff8105f3da ffffffff8105f36f ffff8802272d3a00 <4>[19799.375447] 0000000000000000 00000006272d2f98 ffff880125ff7fd8 ffff8802272d2f80 <4>[19799.375450] ffffffffa05666d0 0000000000000000 0000000000000000 ffffffffa055940e <4>[19799.375451] Call Trace: <4>[19799.375457] [] ? process_one_work+0x1da/0x540 <4>[19799.375459] [] ? process_one_work+0x16f/0x540 <4>[19799.375462] [] worker_thread+0x11c/0x370 <4>[19799.375464] [] ? manage_workers.isra.20+0x2e0/0x2e0 <4>[19799.375468] [] kthread+0xea/0xf0 <4>[19799.375471] [] ? flush_kthread_worker+0x150/0x150 <4>[19799.375476] [] ret_from_fork+0x7c/0xb0 <4>[19799.375478] [] ? flush_kthread_worker+0x150/0x150 <4>[19799.480150] Workqueue: ceph-msgr con_work [libceph] <4>[19799.485054] ffffffff81c4ca00 ffff8802272c3db8 ffffffff81630b82 ffff8802272c3e38 <4>[19799.492523] ffffffff810e285a 0000000000000006 ffff8802272cd4e0 ffff8802272c3de8 <4>[19799.499995] ffffffff810e644c 0000000000000086 0000000000000001 0000000000000006 <4>[19799.507469] Call Trace: <4>[19799.509929] [] dump_stack+0x19/0x1b <4>[19799.515718] [] rcu_check_callbacks+0x21a/0x710 <4>[19799.521831] [] ? acct_account_cputime+0x1c/0x20 <4>[19799.528034] [] update_process_times+0x48/0x80 <4>[19799.534062] [] tick_sched_handle.isra.10+0x36/0x50 <4>[19799.540524] [] tick_sched_timer+0x4c/0x80 <4>[19799.546203] [] __run_hrtimer+0x81/0x1e0 <4>[19799.551709] [] ? tick_nohz_handler+0xa0/0xa0 <4>[19799.557647] [] hrtimer_interrupt+0x107/0x260 <4>[19799.563588] [] smp_apic_timer_interrupt+0x69/0x99 <4>[19799.569964] [] apic_timer_interrupt+0x6f/0x80 <4>[19799.575987] [] ? shrink_inactive_list+0x18c/0x400 <4>[19799.582995] [] ? _raw_spin_unlock_irq+0x30/0x40 <4>[19799.589195] [] ? _raw_spin_unlock_irq+0x35/0x40 <4>[19799.595397] [] shrink_inactive_list+0x18c/0x400 <4>[19799.601595] [] shrink_lruvec+0x2cd/0x4d0 <4>[19799.607187] [] ? bdi_queue_work+0x8b/0xf0 <4>[19799.612869] [] do_try_to_free_pages+0x11c/0x3a0 <4>[19799.619068] [] try_to_free_pages+0xd6/0x1b0 <4>[19799.624922] [] ? next_zone+0x30/0x40 <4>[19799.630165] [] __alloc_pages_nodemask+0x596/0x8f0 <4>[19799.636541] [] alloc_pages_current+0xba/0x170 <4>[19799.642569] [] sk_page_frag_refill+0x7e/0x130 <4>[19799.648593] [] tcp_sendmsg+0x305/0xe50 <4>[19799.654010] [] inet_sendmsg+0xb9/0xf0 <4>[19799.659339] [] ? inet_sendmsg+0x5/0xf0 <4>[19799.664760] [] sock_sendmsg+0xc2/0xe0 <4>[19799.670090] [] ? chksum_update+0x1b/0x30 <4>[19799.675686] [] ? crypto_shash_update+0x18/0x30 <4>[19799.681814] [] ? crc32c+0x56/0x7c [libcrc32c] <4>[19799.687842] [] kernel_sendmsg+0x40/0x60 <4>[19799.693353] [] con_work+0xc28/0x2d50 [libceph] <4>[19799.699468] [] ? idle_balance+0x133/0x180 <4>[19799.705145] [] ? finish_task_switch+0x48/0x110 <4>[19799.711257] [] ? finish_task_switch+0x48/0x110 <4>[19799.717371] [] process_one_work+0x1da/0x540 <4>[19799.723220] [] ? process_one_work+0x16f/0x540 <4>[19799.729247] [] worker_thread+0x11c/0x370 <4>[19799.734840] [] ? manage_workers.isra.20+0x2e0/0x2e0 <4>[19799.741388] [] kthread+0xea/0xf0 <4>[19799.746283] [] ? flush_kthread_worker+0x150/0x150 <4>[19799.752658] [] ret_from_fork+0x7c/0xb0 <4>[19799.758076] [] ? flush_kthread_worker+0x150/0x150 <3>[19820.788280] INFO: rcu_sched self-detected stall on CPU <3>[19820.788286] INFO: rcu_sched self-detected stall on CP <4>[19820.788287] <4>[19820.788288] <4>[19820.788288] } <4>[19820.788291] (t=2101 jiffies g=245351 c=245350 q=5233) <4>[19820.788294] CPU: 6 PID: 19284 Comm: kworker/6:2 Tainted: G W 3.10.0-rc6-ceph-00091-g2dd322b #1 <4>[19820.788295] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[19820.788308] Workqueue: ceph-msgr con_work [libceph <4>[19820.788308] <4>[19820.788309] ffffffff81c4ca0 <4>[19820.788310] ffff8802272c3db <4>[19820.788310] ffffffff81630b8 <4>[19820.788310] ffff8802272c3e3 <4>[19820.788311] <4>[19820.788311] ffffffff810e285 <4>[19820.788311] 000000000000000 <4>[19820.788312] ffff8802272cd4e <4>[19820.788312] ffff8802272c3de <4>[19820.788312] <4>[19820.788313] ffffffff810e644 <4>[19820.788313] 000000000000008 <4>[19820.788313] 000000000000000 <4>[19820.788314] 000000000000000 <4>[19820.788314] <4>[19820.788315] Call Trace: <4>[19820.788316] <4>[19820.788317] <4>[19820.788322] [] dump_stack+0x19/0x1b <4>[19820.788322] <4>[19820.788326] [] rcu_check_callbacks+0x21a/0x710 <4>[19820.788326] <4>[19820.788328] [] ? acct_account_cputime+0x1c/0x20 <4>[19820.788328] <4>[19820.788332] [] update_process_times+0x48/0x80 <4>[19820.788332] <4>[19820.788335] [] tick_sched_handle.isra.10+0x36/0x50 <4>[19820.788336] <4>[19820.788337] [] tick_sched_timer+0x4c/0x80 <4>[19820.788337] <4>[19820.788340] [] __run_hrtimer+0x81/0x1e0 <4>[19820.788340] <4>[19820.788342] [] ? tick_nohz_handler+0xa0/0xa0 <4>[19820.788342] <4>[19820.788344] [] hrtimer_interrupt+0x107/0x260 <4>[19820.788344] <4>[19820.788348] [] smp_apic_timer_interrupt+0x69/0x99 <4>[19820.788348] <4>[19820.788350] [] apic_timer_interrupt+0x6f/0x80 <4>[19820.788350] <4>[19820.788351] <4>[19820.788353] [] ? shrink_inactive_list+0x18c/0x400 <4>[19820.788354] <4>[19820.788356] [] ? _raw_spin_unlock_irq+0x30/0x40 <4>[19820.788356] <4>[19820.788357] [] ? _raw_spin_unlock_irq+0x35/0x40 <4>[19820.788358] <4>[19820.788359] [] shrink_inactive_list+0x18c/0x400 <4>[19820.788359] <4>[19820.788361] [] shrink_lruvec+0x2cd/0x4d0 <4>[19820.788361] <4>[19820.788362] [] ? shrink_slab+0x1dc/0x300 <4>[19820.788363] <4>[19820.788365] [] ? page_alloc_cpu_notify+0x60/0x60 <4>[19820.788366] <4>[19820.788367] [] do_try_to_free_pages+0x11c/0x3a0 <4>[19820.788367] <4>[19820.788368] [] ? page_alloc_cpu_notify+0x60/0x60 <4>[19820.788369] <4>[19820.788370] [] try_to_free_pages+0xd6/0x1b0 <4>[19820.788370] <4>[19820.788372] [] __alloc_pages_nodemask+0x596/0x8f0 <4>[19820.788372] <4>[19820.788375] [] alloc_pages_current+0xba/0x170 <4>[19820.788376] <4>[19820.788378] [] sk_page_frag_refill+0x7e/0x130 <4>[19820.788379] <4>[19820.788381] [] tcp_sendmsg+0x305/0xe50 <4>[19820.788381] <4>[19820.788383] [] inet_sendmsg+0xb9/0xf0 <4>[19820.788383] <4>[19820.788384] [] ? inet_sendmsg+0x5/0xf0 <4>[19820.788384] <4>[19820.788387] [] sock_sendmsg+0xc2/0xe0 <4>[19820.788387] <4>[19820.788389] [] ? chksum_update+0x1b/0x30 <4>[19820.788390] <4>[19820.788391] [] ? crypto_shash_update+0x18/0x30 <4>[19820.788402] <4>[19820.788404] [] ? crc32c+0x56/0x7c [libcrc32c] <4>[19820.788404] <4>[19820.788406] [] kernel_sendmsg+0x40/0x60 <4>[19820.788406] <4>[19820.788411] [] con_work+0xc28/0x2d50 [libceph] <4>[19820.788411] <4>[19820.788413] [] ? idle_balance+0x133/0x180 <4>[19820.788413] <4>[19820.788415] [] ? finish_task_switch+0x48/0x110 <4>[19820.788416] <4>[19820.788417] [] ? finish_task_switch+0x48/0x110 <4>[19820.788417] <4>[19820.788420] [] process_one_work+0x1da/0x540 <4>[19820.788420] <4>[19820.788421] [] ? process_one_work+0x16f/0x540 <4>[19820.788422] <4>[19820.788423] [] worker_thread+0x11c/0x370 <4>[19820.788423] <4>[19820.788424] [] ? manage_workers.isra.20+0x2e0/0x2e0 <4>[19820.788425] <4>[19820.788426] [] kthread+0xea/0xf0 <4>[19820.788427] <4>[19820.788429] [] ? flush_kthread_worker+0x150/0x150 <4>[19820.788429] <4>[19820.788430] [] ret_from_fork+0x7c/0xb0 <4>[19820.788431] <4>[19820.788432] [] ? flush_kthread_worker+0x150/0x150 <4>[19821.112660] <4>[19821.114348] 4} (t=2133 jiffies g=245351 c=245350 q=5264) <4>[19821.118574] CPU: 4 PID: 19283 Comm: kworker/4:0 Tainted: G W 3.10.0-rc6-ceph-00091-g2dd322b #1 <4>[19821.128090] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[19821.135619] Workqueue: ceph-msgr con_work [libceph] <4>[19821.140525] ffffffff81c4ca00 ffff880227283db8 ffffffff81630b82 ffff880227283e38 <4>[19821.147998] ffffffff810e285a 0000000000000004 ffff88022728d4e0 ffff880227283de8 <4>[19821.155476] ffffffff810e644c 0000000000000086 0000000000000001 0000000000000004 <4>[19821.162953] Call Trace: <4>[19821.165413] [] dump_stack+0x19/0x1b <4>[19821.171207] [] rcu_check_callbacks+0x21a/0x710 <4>[19821.177323] [] ? acct_account_cputime+0x1c/0x20 <4>[19821.183526] [] update_process_times+0x48/0x80 <4>[19821.189556] [] tick_sched_handle.isra.10+0x36/0x50 <4>[19821.196017] [] tick_sched_timer+0x4c/0x80 <4>[19821.201698] [] __run_hrtimer+0x81/0x1e0 <4>[19821.207204] [] ? tick_nohz_handler+0xa0/0xa0 <4>[19821.213146] [] hrtimer_interrupt+0x107/0x260 <4>[19821.219087] [] smp_apic_timer_interrupt+0x69/0x99 <4>[19821.225463] [] apic_timer_interrupt+0x6f/0x80 <4>[19821.231488] [] ? shrink_inactive_list+0x18c/0x400 <4>[19821.238496] [] ? _raw_spin_unlock_irq+0x30/0x40 <4>[19821.244773] [] ? _raw_spin_unlock_irq+0x35/0x40 <4>[19821.250975] [] shrink_inactive_list+0x18c/0x400 <4>[19821.257177] [] shrink_lruvec+0x2cd/0x4d0 <4>[19821.262771] [] ? bdi_queue_work+0x8b/0xf0 <4>[19821.268451] [] do_try_to_free_pages+0x11c/0x3a0 <4>[19821.274652] [] try_to_free_pages+0xd6/0x1b0 <4>[19821.280508] [] ? next_zone+0x30/0x40 <4>[19821.285756] [] __alloc_pages_nodemask+0x596/0x8f0 <4>[19821.292136] [] alloc_pages_current+0xba/0x170 <4>[19821.298165] [] sk_page_frag_refill+0x7e/0x130 <4>[19821.304195] [] tcp_sendmsg+0x305/0xe50 <4>[19821.309615] [] inet_sendmsg+0xb9/0xf0 <4>[19821.314947] [] ? inet_sendmsg+0x5/0xf0 <4>[19821.320368] [] sock_sendmsg+0xc2/0xe0 <4>[19821.325701] [] ? chksum_update+0x1b/0x30 <4>[19821.331295] [] ? crypto_shash_update+0x18/0x30 <4>[19821.337423] [] ? crc32c+0x56/0x7c [libcrc32c] <4>[19821.343454] [] kernel_sendmsg+0x40/0x60 <4>[19821.348970] [] con_work+0xc28/0x2d50 [libceph] <4>[19821.355088] [] ? cpuacct_charge+0x5/0xa0 <4>[19821.360683] [] ? idle_balance+0x133/0x180 <4>[19821.366365] [] ? finish_task_switch+0x48/0x110 <4>[19821.372480] [] ? finish_task_switch+0x48/0x110 <4>[19821.378596] [] process_one_work+0x1da/0x540 <4>[19821.384454] [] ? process_one_work+0x16f/0x540 <4>[19821.390493] [] ? retint_restore_args+0xe/0xe <4>[19821.396436] [] worker_thread+0x11c/0x370 <4>[19821.402030] [] ? manage_workers.isra.20+0x2e0/0x2e0 <4>[19821.408580] [] kthread+0xea/0xf0 <4>[19821.413479] [] ? flush_kthread_worker+0x150/0x150 <4>[19821.419859] [] ret_from_fork+0x7c/0xb0 <4>[19821.425282] [] ? flush_kthread_worker+0x150/0x150 <6>[19821.508678] libceph: osd0 weight 0x0 (out) <4>[19821.510853] libceph: osd0 10.214.132.16:6801 socket closed (con state CONNECTING) <6>[19822.738359] libceph: osd2 weight 0x0 (out) <4>[19823.371107] libceph: osd2 10.214.132.16:6808 socket closed (con state OPEN) <6>[19823.373330] libceph: osd0 down <6>[19824.040224] libceph: osd0 up <6>[19824.670804] libceph: osd0 weight 0x10000 (in) <6>[19825.568053] libceph: osd5 weight 0x0 (out) <6>[19826.613710] libceph: osd5 weight 0x10000 (in) <6>[19827.491250] libceph: osd5 weight 0x0 (out) <1>[19828.585548] BUG: unable to handle kernel NULL pointer dereference at (null) <1>[19828.593437] IP: [] rb_erase+0x1bb/0x370 <4>[19828.598865] PGD 0 <4>[19828.600899] Oops: 0002 [#1] SMP [dumpcommon]kdb> -bt Stack traceback for pid 29967 0xffff88020dd03f20 29967 2 1 4 R 0xffff88020dd043a8 *kworker/4:1 ffff88020b257b48 0000000000000018 0000000000000000 ffff88020b257b68 ffffffffa05487bc ffff8802204e4000 ffff880224ec7950 ffff88020b257b98 ffffffffa0548abf ffff8802204e4030 ffff880224ec7950 0000000000000000 Call Trace: [] ? __remove_osd+0x3c/0xa0 [libceph] [] ? __reset_osd+0x12f/0x170 [libceph] [] ? osd_reset+0x7e/0x2b0 [libceph] [] ? con_work+0x571/0x2d50 [libceph] [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] ? process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] ? worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] ? kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ? ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 [dumpall]kdb> -bta 7 idle processes (state I) and 112 sleeping system daemon (state M) processes suppressed, use 'ps A' to see all. Stack traceback for pid 29967 0xffff88020dd03f20 29967 2 1 4 R 0xffff88020dd043a8 *kworker/4:1 ffff88020b257b48 0000000000000018 0000000000000000 ffff88020b257b68 ffffffffa05487bc ffff8802204e4000 ffff880224ec7950 ffff88020b257b98 ffffffffa0548abf ffff8802204e4030 ffff880224ec7950 0000000000000000 Call Trace: [] ? __remove_osd+0x3c/0xa0 [libceph] [] ? __reset_osd+0x12f/0x170 [libceph] [] ? osd_reset+0x7e/0x2b0 [libceph] [] ? con_work+0x571/0x2d50 [libceph] [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] ? process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] ? worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] ? kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ? ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 1 0xffff880225ca8000 1 0 0 0 S 0xffff880225ca8488 init ffff880225c8b8d8 0000000000000046 ffffffff8105eb15 ffffffff81c905c0 ffff880225ca8000 ffff880225c8bfd8 ffff880225c8bfd8 ffff880225c8bfd8 ffffffff81c10440 ffff880225ca8000 0000000000000000 0000000000000000 Call Trace: [] ? flush_work+0x5/0x280 [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? cpumask_next_and+0x36/0x50 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x8e/0xa0 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 492 0xffff88020cc40000 492 1 0 3 S 0xffff88020cc40488 upstart-udev-br ffff88020c3218d8 0000000000000046 0000000000000002 0000000000000000 ffff88020cc40000 ffff88020c321fd8 ffff88020c321fd8 ffff88020c321fd8 ffff880225d2deb0 ffff88020cc40000 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? unix_write_space+0x5/0xc0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? touch_atime+0xd5/0x170 [] ? touch_atime+0xd5/0x170 [] ? sock_recvmsg+0xd8/0xf0 [] ? pipe_read+0x7f/0x560 [] ? pipe_read+0x7f/0x560 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? trace_hardirqs_on+0xd/0x10 [] ? remove_wait_queue+0x5a/0x70 [] ? do_wait+0x12c/0x260 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 501 0xffff88020cfd9f90 501 1 0 2 S 0xffff88020cfda418 udevd ffff88020cf7dde8 0000000000000046 0000000000000002 0000000000000000 ffff88020cfd9f90 ffff88020cf7dfd8 ffff88020cf7dfd8 ffff88020cf7dfd8 ffff880225d2bf20 ffff88020cfd9f90 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] schedule_hrtimeout_range+0x13/0x20 [] SyS_epoll_wait+0x418/0x4c0 [] ? try_to_wake_up+0x2f0/0x2f0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 538 0xffff88020cd69f90 538 1 0 2 S 0xffff88020cd6a418 dbus-daemon ffff88020c20fa38 0000000000000046 0000000000000002 0000000000000000 ffff88020cd69f90 ffff88020c20ffd8 ffff88020c20ffd8 ffff88020c20ffd8 ffff880225d2bf20 ffff88020cd69f90 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_sys_poll+0x40f/0x4d0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? might_fault+0x8e/0xa0 [] ? might_fault+0x45/0xa0 [] ? ___sys_recvmsg+0x166/0x2a0 [] ? __do_page_fault+0x2ac/0x520 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? finish_task_switch+0x85/0x110 [] ? finish_task_switch+0x48/0x110 [] SyS_poll+0x6b/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 949 0xffff88020b35bf20 949 1 0 5 S 0xffff88020b35c3a8 rpc.idmapd ffff88020d8b9de8 0000000000000046 0000000000000002 0000000000000002 ffff88020b35bf20 ffff88020d8b9fd8 ffff88020d8b9fd8 ffff88020d8b9fd8 ffff880225d39f90 ffff88020b35bf20 ffff880200000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? lg_local_unlock+0x23/0x40 [] schedule_hrtimeout_range+0x13/0x20 [] SyS_epoll_wait+0x418/0x4c0 [] ? try_to_wake_up+0x2f0/0x2f0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1036 0xffff880224e3bf20 1036 1 0 3 S 0xffff880224e3c3a8 upstart-socket- ffff880220f398d8 0000000000000046 0000000000000000 ffffffff810679d3 ffff880224e3bf20 ffff880220f39fd8 ffff880220f39fd8 ffff880220f39fd8 ffff880225d2deb0 ffff880224e3bf20 0000000000000000 0000000000000000 Call Trace: [] ? add_wait_queue+0x23/0x60 [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? unix_write_space+0x5/0xc0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? touch_atime+0xd5/0x170 [] ? sock_recvmsg+0xd8/0xf0 [] ? pipe_read+0x7f/0x560 [] ? pipe_read+0x7f/0x560 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? trace_hardirqs_on+0xd/0x10 [] ? remove_wait_queue+0x5a/0x70 [] ? do_wait+0x12c/0x260 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1040 0xffff88020dc5deb0 1040 1 0 2 S 0xffff88020dc5e338 rpcbind ffff880220effa38 0000000000000046 00001207b08cc9a4 0000000001c9c37f ffff88020dc5deb0 ffff880220efffd8 ffff880220efffd8 ffff880220efffd8 ffff880225d2bf20 ffff88020dc5deb0 0000000000000000 ffff880220effef8 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x12c/0x170 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_sys_poll+0x40f/0x4d0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? mntput+0x24/0x40 [] ? __fput+0x189/0x240 [] ? ktime_get_ts+0x4c/0xe0 [] ? poll_select_set_timeout+0x85/0xa0 [] SyS_poll+0x6b/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1104 0xffff880210080000 1104 1 0 0 S 0xffff880210080488 rpc.statd ffff88020c1b38d8 0000000000000046 0000000000000002 0000000000000000 ffff880210080000 ffff88020c1b3fd8 ffff88020c1b3fd8 ffff88020c1b3fd8 ffffffff81c10440 ffff880210080000 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __wake_up_sync_key+0x37/0x80 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? up_read+0x23/0x40 [] ? __do_page_fault+0x2ac/0x520 [] ? mntput+0x24/0x40 [] ? __fput+0x189/0x240 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1242 0xffff88020cfdbf20 1242 1 0 1 S 0xffff88020cfdc3a8 getty ffff88020db8bcc8 0000000000000046 0000000000000002 ffff880227212f80 ffff88020cfdbf20 ffff88020db8bfd8 ffff88020db8bfd8 ffff88020db8bfd8 ffff880225d29f90 ffff88020cfdbf20 ffff88020db8bcb8 ffff880221386000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1247 0xffff88020dc5bf20 1247 1 0 4 S 0xffff88020dc5c3a8 getty ffff88020dd8fcc8 0000000000000046 0000000000000002 ffff880227292f80 ffff88020dc5bf20 ffff88020dd8ffd8 ffff88020dd8ffd8 ffff88020dd8ffd8 ffff880225d38000 ffff88020dc5bf20 ffff88020dd8fcb8 ffff8802100cb000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1254 0xffff88020c5ebf20 1254 1 0 4 S 0xffff88020c5ec3a8 getty ffff880220f51cc8 0000000000000046 0000000000000002 ffff880227292f80 ffff88020c5ebf20 ffff880220f51fd8 ffff880220f51fd8 ffff880220f51fd8 ffff880225d38000 ffff88020c5ebf20 ffff880220f51cb8 ffff8802100ce800 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1255 0xffff88020c5edeb0 1255 1 0 5 S 0xffff88020c5ee338 getty ffff880224dbdcc8 0000000000000046 0000000000000002 ffff8802272b2f80 ffff88020c5edeb0 ffff880224dbdfd8 ffff880224dbdfd8 ffff880224dbdfd8 ffff880225d39f90 ffff88020c5edeb0 ffff880224dbdcb8 ffff88020bc7c000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1257 0xffff88020cf83f20 1257 1 0 4 S 0xffff88020cf843a8 getty ffff880220e8bcc8 0000000000000046 0000000000000002 ffff880227292f80 ffff88020cf83f20 ffff880220e8bfd8 ffff880220e8bfd8 ffff880220e8bfd8 ffff880225d38000 ffff88020cf83f20 ffff880220e8bcb8 ffff8802100cf000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1273 0xffff88020c178000 1273 1 0 1 S 0xffff88020c178488 xinetd ffff8802249878d8 0000000000000046 0000000000000000 ffffffff810679d3 ffff88020c178000 ffff880224987fd8 ffff880224987fd8 ffff880224987fd8 ffff880225d29f90 ffff88020c178000 ffff880200000000 0000000000000000 Call Trace: [] ? add_wait_queue+0x23/0x60 [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? try_to_wake_up+0x2e/0x2f0 [] ? __pollwait+0xf0/0xf0 [] ? skb_queue_tail+0x2b/0x60 [] ? find_get_page+0x5/0xf0 [] ? find_get_page+0x5/0xf0 [] ? __do_fault+0x14a/0x470 [] ? __do_fault+0x14a/0x470 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? up_read+0x23/0x40 [] ? __do_page_fault+0x2ac/0x520 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1275 0xffff88020d37deb0 1275 1 0 2 S 0xffff88020d37e338 atd ffff880224f27e58 0000000000000046 0000034630b8a000 000000000000c350 ffff88020d37deb0 ffff880224f27fd8 ffff880224f27fd8 ffff880224f27fd8 ffff880225d2bf20 ffff88020d37deb0 0000000000000002 ffff880224f27eb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1319 0xffff88020b0f3f20 1319 1 0 5 S 0xffff88020b0f43a8 irqbalance ffff88020cf13e58 0000000000000046 000000022268ef88 000000000000c350 ffff88020b0f3f20 ffff88020cf13fd8 ffff88020cf13fd8 ffff88020cf13fd8 ffff880225d39f90 ffff88020b0f3f20 00007f9933324000 ffff88020cf13eb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1492 0xffff88020c748000 1492 1 0 3 S 0xffff88020c748488 nrpe ffff88020ca6d8d8 0000000000000046 00001205f16a6d1a 000000000007a11f ffff88020c748000 ffff88020ca6dfd8 ffff88020ca6dfd8 ffff88020ca6dfd8 ffff880225d2deb0 ffff88020c748000 0000000000000000 ffff88020ca6dd18 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x12c/0x170 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __pollwait+0xf0/0xf0 [] ? delayacct_end+0x71/0xa0 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? delayacct_end+0x89/0xa0 [] ? swap_info_get+0x63/0xe0 [] ? fsnotify_clear_marks_by_inode+0x34/0x130 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? mntput_no_expire+0x2c/0x160 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? ktime_get_ts+0x4c/0xe0 [] SyS_select+0xc1/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1550 0xffff88020dc63f20 1550 1 0 5 S 0xffff88020dc643a8 rpc.mountd ffff88020c7478d8 0000000000000046 0000000000000002 0000000000000000 ffff88020dc63f20 ffff88020c747fd8 ffff88020c747fd8 ffff88020c747fd8 ffff880225d39f90 ffff88020dc63f20 ffff88022fff9e00 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? ttwu_stat+0x4c/0x140 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? up_read+0x23/0x40 [] ? __do_page_fault+0x2ac/0x520 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1578 0xffff88020b358000 1578 1 0 3 S 0xffff88020b358488 pkcsslotd ffff88020cac5e58 0000000000000046 00000002540be400 000000000000c350 ffff88020b358000 ffff88020cac5fd8 ffff88020cac5fd8 ffff88020cac5fd8 ffff880225d2deb0 ffff88020b358000 ffff88020cac5f38 ffff88020cac5eb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1591 0xffff880210083f20 1591 1 0 0 S 0xffff8802100843a8 rwhod ffff88020c3aba48 0000000000000046 ffff88020c3aba18 0000000000000246 ffff880210083f20 ffff88020c3abfd8 ffff88020c3abfd8 ffff88020c3abfd8 ffffffff81c10440 ffff880210083f20 0000000000000002 ffff880220e3b180 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? trace_hardirqs_on+0xd/0x10 [] __skb_recv_datagram+0x2fb/0x380 [] ? skb_recv_datagram+0x40/0x40 [] udp_recvmsg+0xa7/0x410 [] inet_recvmsg+0xcd/0xf0 [] sock_recvmsg+0xd8/0xf0 [] ? dput+0x32/0x240 [] SYSC_recvfrom+0xe9/0x170 [] ? __fput+0x189/0x240 [] ? ____fput+0xe/0x10 [] ? lockdep_sys_exit_thunk+0x35/0x67 [] SyS_recvfrom+0xe/0x10 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1603 0xffff88020dc58000 1603 1591 0 3 S 0xffff88020dc58488 rwhod ffff880216493e58 0000000000000046 00000029e8d60800 000000000000c350 ffff88020dc58000 ffff880216493fd8 ffff880216493fd8 ffff880216493fd8 ffff880225d2deb0 ffff88020dc58000 0000000000000002 ffff880216493eb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1692 0xffff88020c32bf20 1692 1 0 6 S 0xffff88020c32c3a8 getty ffff88020d819cc8 0000000000000046 0000000000000002 ffff8802272d2f80 ffff88020c32bf20 ffff88020d819fd8 ffff88020d819fd8 ffff88020d819fd8 ffff880225d3bf20 ffff88020c32bf20 ffff88020d819cb8 ffff88020dc0e000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1693 0xffff88020c329f90 1693 1 0 1 S 0xffff88020c32a418 getty ffff880224e35cc8 0000000000000046 ffff88020cb591c8 ffff8802272b2f80 ffff88020c329f90 ffff880224e35fd8 ffff880224e35fd8 ffff880224e35fd8 ffff880225d29f90 ffff88020c329f90 ffff880224e35cb8 ffff88020cb59800 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 2232 0xffff880210023f20 2232 1 0 0 S 0xffff8802100243a8 ntpd ffff8802100938d8 0000000000000046 0000000000000002 0000000000000000 ffff880210023f20 ffff880210093fd8 ffff880210093fd8 ffff880210093fd8 ffffffff81c10440 ffff880210023f20 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? signal_delivered+0x5a/0x70 [] ? fpu_finit+0x22/0x40 [] ? init_fpu+0x56/0xc0 [] ? __restore_xstate_sig+0x22e/0x510 [] ? might_fault+0x45/0xa0 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 25450 0xffff88020dc60000 25450 2 0 1 D 0xffff88020dc60488 kworker/1:1 ffff880222017c00 0000000000000046 0000000000000001 0000000000000000 ffff88020dc60000 ffff880222017fd8 ffff880222017fd8 ffff880222017fd8 ffff880225d29f90 ffff88020dc60000 ffff880224ec7968 ffff88020dc60000 Call Trace: [] schedule+0x29/0x70 [] rwsem_down_read_failed+0xa5/0xf9 [] call_rwsem_down_read_failed+0x14/0x30 [] ? down_read+0x47/0x50 [] ? handle_timeout+0x57/0x1f0 [libceph] [] handle_timeout+0x57/0x1f0 [libceph] [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] ? mod_timer+0x113/0x230 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 29928 0xffff88019e0a5eb0 29928 2 0 6 D 0xffff88019e0a6338 kworker/6:1 ffff8801b5ef79e8 0000000000000046 000000000000000d ffff8801b5ef79f8 ffff88019e0a5eb0 ffff8801b5ef7fd8 ffff8801b5ef7fd8 ffff8801b5ef7fd8 ffff88020d870000 ffff88019e0a5eb0 ffff880224ec7968 ffff88019e0a5eb0 Call Trace: [] schedule+0x29/0x70 [] rwsem_down_write_failed+0xf5/0x1b0 [] call_rwsem_down_write_failed+0x13/0x20 [] ? down_write+0x51/0x58 [] ? ceph_osdc_handle_map+0xe5/0x580 [libceph] [] ? ceph_check_fsid+0x2b/0x80 [libceph] [] ceph_osdc_handle_map+0xe5/0x580 [libceph] [] dispatch+0x138/0x9b0 [libceph] [] ? ceph_tcp_recvmsg+0x48/0x60 [libceph] [] ? trace_hardirqs_on+0xd/0x10 [] con_work+0x1948/0x2d50 [libceph] [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] ? mod_timer+0x113/0x230 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 11216 0xffff88020c133f20 11216 2 0 3 D 0xffff88020c1343a8 kworker/u64:1 ffff88022489f700 0000000000000046 000000004c8be1d4 ffff88022489f6b8 ffff88020c133f20 ffff88022489ffd8 ffff88022489ffd8 ffff88022489ffd8 ffff880225d2deb0 ffff88020c133f20 ffff880224ec7968 ffff88020c133f20 Call Trace: [] schedule+0x29/0x70 [] rwsem_down_read_failed+0xa5/0xf9 [] ? ceph_msg_data_create+0x41/0x60 [libceph] [] call_rwsem_down_read_failed+0x14/0x30 [] ? down_read+0x47/0x50 [] ? ceph_osdc_start_request+0x3e/0x130 [libceph] [] ceph_osdc_start_request+0x3e/0x130 [libceph] [] ceph_writepages_start+0x94e/0xfa0 [ceph] [] do_writepages+0x23/0x40 [] __writeback_single_inode+0x4e/0x2a0 [] writeback_sb_inodes+0x280/0x3e0 [] __writeback_inodes_wb+0x9e/0xd0 [] wb_writeback+0x1fb/0x310 [] wb_do_writeback+0xbb/0x1e0 [] ? set_worker_desc+0x6f/0x80 [] bdi_writeback_workfn+0x7a/0x200 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 12266 0xffff880222028000 12266 1 0 0 S 0xffff880222028488 sshd ffff88020dc13a38 0000000000000046 0000000000000002 0000000000000000 ffff880222028000 ffff88020dc13fd8 ffff88020dc13fd8 ffff88020dc13fd8 ffffffff81c10440 ffff880222028000 ffff88020dc13b08 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] ? __pte_alloc+0x3f/0xc0 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] ? might_fault+0x45/0xa0 [] do_sys_poll+0x40f/0x4d0 [] ? __pollwait+0xf0/0xf0 [] ? do_wp_page+0x19f/0x6f0 [] ? _raw_spin_unlock+0x2b/0x40 [] ? do_wp_page+0x29a/0x6f0 [] ? handle_pte_fault+0x264/0x8a0 [] ? __do_page_fault+0x15c/0x520 [] ? __do_page_fault+0x15c/0x520 [] ? sched_clock_cpu+0xc5/0x120 [] ? up_read+0x23/0x40 [] ? __do_page_fault+0x2ac/0x520 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? do_sigaction+0x12d/0x1f0 [] SyS_poll+0x6b/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 12282 0xffff88022202bf20 12282 12266 0 0 S 0xffff88022202c3a8 sshd ffff88020c8bf8d8 0000000000000046 0000000000000000 ffffffff810679d3 ffff88022202bf20 ffff88020c8bffd8 ffff88020c8bffd8 ffff88020c8bffd8 ffffffff81c10440 ffff88022202bf20 0000000000000000 0000000000000000 Call Trace: [] ? add_wait_queue+0x23/0x60 [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? cpuacct_charge+0x5/0xa0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? release_sock+0x172/0x1e0 [] ? tcp_recvmsg+0x396/0xc80 [] ? inet_recvmsg+0x5/0xf0 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? sock_aio_read+0x23/0x30 [] ? do_sync_read+0x7a/0xb0 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 12316 0xffff880224829f90 12316 1 0 2 S 0xffff88022482a418 rsyslogd ffff88018c59d8d8 0000000000000046 0000120b68a414f1 0000000001c9c37e ffff880224829f90 ffff88018c59dfd8 ffff88018c59dfd8 ffff88018c59dfd8 ffff880225d2bf20 ffff880224829f90 ffff88018c59d8b8 ffff88018c59dd18 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x12c/0x170 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? swap_info_get+0x63/0xe0 [] ? swap_info_get+0x63/0xe0 [] ? swap_info_get+0x63/0xe0 [] ? swap_info_get+0x63/0xe0 [] ? delete_from_swap_cache+0x3d/0x70 [] ? _raw_spin_unlock+0x2b/0x40 [] ? swapcache_free+0x41/0x50 [] ? delete_from_swap_cache+0x60/0x70 [] ? _raw_spin_unlock+0x2b/0x40 [] ? core_sys_select+0x42/0x340 [] ? core_sys_select+0x42/0x340 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? up_read+0x23/0x40 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? ktime_get_ts+0x4c/0xe0 [] SyS_select+0xc1/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 12317 0xffff880210058000 12317 1 0 2 S 0xffff880210058488 rs:main Q:Reg ffff88020cd87c28 0000000000000046 000000000000005f ffffffff826c2340 ffff880210058000 ffff88020cd87fd8 ffff88020cd87fd8 ffff88020cd87fd8 ffff880225d2bf20 ffff880210058000 0000000000000000 ffff88020cd87cd0 Call Trace: [] schedule+0x29/0x70 [] futex_wait_queue_me+0xc7/0x100 [] futex_wait+0x1a7/0x2c0 [] ? trace_hardirqs_on+0xd/0x10 [] ? ext4_file_write+0x9f/0x420 [] ? _raw_spin_unlock+0x2b/0x40 [] ? fsnotify+0x8a/0x2f0 [] do_futex+0x344/0xbb0 [] ? fsnotify+0x20a/0x2f0 [] ? fsnotify+0x8a/0x2f0 [] SyS_futex+0x147/0x1a0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 12318 0xffff8802258cbf20 12318 1 0 0 S 0xffff8802258cc3a8 rsyslogd ffff88020c5fd8d8 0000000000000046 0000000000000000 ffff88020c77d800 ffff8802258cbf20 ffff88020c5fdfd8 ffff88020c5fdfd8 ffff88020c5fdfd8 ffffffff81c10440 ffff8802258cbf20 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? add_wait_queue+0x4d/0x60 [] ? __pollwait+0x75/0xf0 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? kmem_cache_free+0x1b0/0x1c0 [] ? __pollwait+0xf0/0xf0 [] ? mutex_unlock+0xe/0x10 [] ? unix_dgram_recvmsg+0x316/0x4e0 [] ? select_task_rq_fair+0x53/0x8a0 [] ? select_task_rq_fair+0x53/0x8a0 [] ? ttwu_stat+0x4c/0x140 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? ___sys_recvmsg+0x166/0x2a0 [] ? do_futex+0xa45/0xbb0 [] ? fget_light+0x9e/0x130 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 12319 0xffff880224b1deb0 12319 1 0 1 S 0xffff880224b1e338 rsyslogd ffff8801e8977e28 0000000000000046 0000000000000286 ffffffff81c317c0 ffff880224b1deb0 ffff8801e8977fd8 ffff8801e8977fd8 ffff8801e8977fd8 ffff880225d29f90 ffff880224b1deb0 ffff8801e8977e38 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] do_syslog+0x4d1/0x550 [] ? __init_waitqueue_head+0x60/0x60 [] kmsg_read+0x56/0x70 [] proc_reg_read+0x43/0x70 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 12553 0xffff88020dd01f90 12553 1 0 7 S 0xffff88020dd02418 cron ffff8801ee239e58 0000000000000046 0000000df8475800 000000000000c350 ffff88020dd01f90 ffff8801ee239fd8 ffff8801ee239fd8 ffff8801ee239fd8 ffff880225d3deb0 ffff88020dd01f90 0000000000000002 ffff8801ee239eb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 16744 0xffff88020d9d1f90 16744 1 0 3 S 0xffff88020d9d2418 sshd ffff8802220c38d8 0000000000000046 0000000000000002 0000000000000000 ffff88020d9d1f90 ffff8802220c3fd8 ffff8802220c3fd8 ffff8802220c3fd8 ffff880225d2deb0 ffff88020d9d1f90 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? fsnotify_clear_marks_by_inode+0x34/0x130 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __d_lookup+0x5/0x1c0 [] ? __d_lookup+0x5/0x1c0 [] ? mix_pool_bytes.constprop.16+0x4d/0xd0 [] ? mix_pool_bytes.constprop.16+0x4d/0xd0 [] ? __d_lookup+0xd8/0x1c0 [] ? __d_lookup+0x5/0x1c0 [] ? fsnotify_clear_marks_by_inode+0x34/0x130 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? mntput_no_expire+0x2c/0x160 [] ? lg_local_unlock+0x23/0x40 [] ? mntput_no_expire+0x49/0x160 [] ? mntput+0x24/0x40 [] ? __fput+0x189/0x240 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 17512 0xffff880224a00000 17512 501 0 1 S 0xffff880224a00488 udevd ffff88022483fde8 0000000000000046 0000000000000000 0000000000000000 ffff880224a00000 ffff88022483ffd8 ffff88022483ffd8 ffff88022483ffd8 ffff880225d29f90 ffff880224a00000 ffff880225700800 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] schedule_hrtimeout_range+0x13/0x20 [] SyS_epoll_wait+0x418/0x4c0 [] ? try_to_wake_up+0x2f0/0x2f0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 17542 0xffff88020cc45eb0 17542 12282 0 6 S 0xffff88020cc46338 bash ffff88020cb85e58 0000000000000046 ffff88020cb85e68 ffffffff8104447d ffff88020cc45eb0 ffff88020cb85fd8 ffff88020cb85fd8 ffff88020cb85fd8 ffff880225d3bf20 ffff88020cc45eb0 ffff88020cb85ef0 0000000000000000 Call Trace: [] ? wait_consider_task+0x9d/0xbe0 [] schedule+0x29/0x70 [] do_wait+0x1e9/0x260 [] SyS_wait4+0xa3/0x100 [] ? might_fault+0x45/0xa0 [] ? task_stopped_code+0x50/0x50 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 17544 0xffff880223030000 17544 17542 0 6 S 0xffff880223030488 ffsb.sh ffff88020c6fbe58 0000000000000046 ffff88020c6fbe68 ffffffff8104447d ffff880223030000 ffff88020c6fbfd8 ffff88020c6fbfd8 ffff88020c6fbfd8 ffff880225d3bf20 ffff880223030000 ffff88020c6fbef0 0000000000000000 Call Trace: [] ? wait_consider_task+0x9d/0xbe0 [] schedule+0x29/0x70 [] do_wait+0x1e9/0x260 [] SyS_wait4+0xa3/0x100 [] ? might_fault+0x45/0xa0 [] ? task_stopped_code+0x50/0x50 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19210 0xffff880223855eb0 19210 17544 0 0 S 0xffff880223856338 ffsb ffff88020cb0dc28 0000000000000046 ffff88020cb0dbf8 ffffffff826c0840 ffff880223855eb0 ffff88020cb0dfd8 ffff88020cb0dfd8 ffff88020cb0dfd8 ffffffff81c10440 ffff880223855eb0 0000000000000000 ffff88020cb0dcd0 Call Trace: [] schedule+0x29/0x70 [] futex_wait_queue_me+0xc7/0x100 [] futex_wait+0x1a7/0x2c0 [] ? pipe_write+0x31b/0x5d0 [] ? fsnotify+0x8a/0x2f0 [] do_futex+0x344/0xbb0 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? finish_task_switch+0x85/0x110 [] ? finish_task_switch+0x48/0x110 [] SyS_futex+0x147/0x1a0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19250 0xffff88020c179f90 19250 17544 0 3 S 0xffff88020c17a418 ffsb ffff88020cb3d8d8 0000000000000046 0000120610402434 00000000000f423f ffff88020c179f90 ffff88020cb3dfd8 ffff88020cb3dfd8 ffff88020cb3dfd8 ffff880225d2deb0 ffff88020c179f90 0000000000000002 ffff88020cb3dd18 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x12c/0x170 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? delayacct_end+0x71/0xa0 [] ? delayacct_end+0x71/0xa0 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? __wake_up+0x32/0x70 [] ? __wake_up+0x32/0x70 [] ? swap_info_get+0x63/0xe0 [] ? trace_hardirqs_on+0xd/0x10 [] ? core_sys_select+0x42/0x340 [] ? core_sys_select+0x42/0x340 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? trace_hardirqs_off_thunk+0x3a/0x3c [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? up_read+0x23/0x40 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? ktime_get_ts+0x4c/0xe0 [] SyS_select+0xc1/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19251 0xffff880223850000 19251 17544 0 0 D 0xffff880223850488 ffsb ffff88020c1259d8 0000000000000046 0000000000000000 0000000000000046 ffff880223850000 ffff88020c125fd8 ffff88020c125fd8 ffff88020c125fd8 ffff8801e8920000 ffff880223850000 ffff88020c1259d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] ? cpuacct_charge+0x5/0xa0 [] ? cpuacct_charge+0x5/0xa0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19252 0xffff880223853f20 19252 17544 0 3 D 0xffff8802238543a8 ffsb ffff8802220dd9d8 0000000000000046 0000000000000000 0000000000000046 ffff880223853f20 ffff8802220ddfd8 ffff8802220ddfd8 ffff8802220ddfd8 ffff880225d2deb0 ffff880223853f20 ffff8802220dd9d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? might_fault+0x45/0xa0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19253 0xffff880223851f90 19253 17544 0 0 D 0xffff880223852418 ffsb ffff88020c91d9d8 0000000000000046 0000000000000000 0000000000000046 ffff880223851f90 ffff88020c91dfd8 ffff88020c91dfd8 ffff88020c91dfd8 ffffffff81c10440 ffff880223851f90 ffff88020c91d9d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? select_task_rq_fair+0x53/0x8a0 [] ? finish_task_switch+0x48/0x110 [] ? ttwu_stat+0x4c/0x140 [] ? try_to_wake_up+0x2e/0x2f0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19254 0xffff88020d0cdeb0 19254 17544 0 6 D 0xffff88020d0ce338 ffsb ffff88020b2cb9d8 0000000000000046 0000000000000000 0000000000000046 ffff88020d0cdeb0 ffff88020b2cbfd8 ffff88020b2cbfd8 ffff88020b2cbfd8 ffff880225d3bf20 ffff88020d0cdeb0 ffff88020b2cb9d8 ffff8802272d42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? do_raw_spin_lock+0x76/0x140 [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19255 0xffff88020d0c8000 19255 17544 0 0 D 0xffff88020d0c8488 ffsb ffff88020bd059d8 0000000000000046 0000000000000000 0000000000000046 ffff88020d0c8000 ffff88020bd05fd8 ffff88020bd05fd8 ffff88020bd05fd8 ffffffff81c10440 ffff88020d0c8000 ffff88020bd059d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? do_raw_spin_lock+0x76/0x140 [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19256 0xffff88019e0a0000 19256 17544 0 3 D 0xffff88019e0a0488 ffsb ffff880224daf9d8 0000000000000046 0000000000000000 0000000000000046 ffff88019e0a0000 ffff880224daffd8 ffff880224daffd8 ffff880224daffd8 ffff880223791f90 ffff88019e0a0000 ffff880224daf9d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19257 0xffff88019e0a3f20 19257 17544 0 7 D 0xffff88019e0a43a8 ffsb ffff88020bdd19d8 0000000000000046 0000000000000000 0000000000000046 ffff88019e0a3f20 ffff88020bdd1fd8 ffff88020bdd1fd8 ffff88020bdd1fd8 ffff880225d3deb0 ffff88019e0a3f20 ffff88020bdd19d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19258 0xffff88019e0a1f90 19258 17544 0 3 D 0xffff88019e0a2418 ffsb ffff88020b3079d8 0000000000000046 0000000000000000 0000000000000046 ffff88019e0a1f90 ffff88020b307fd8 ffff88020b307fd8 ffff88020b307fd8 ffff880223791f90 ffff88019e0a1f90 ffff88020b3079d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19259 0xffff880224f08000 19259 17544 0 2 D 0xffff880224f08488 ffsb ffff88020bc719d8 0000000000000046 0000000000000000 0000000000000046 ffff880224f08000 ffff88020bc71fd8 ffff88020bc71fd8 ffff88020bc71fd8 ffff880225d2bf20 ffff880224f08000 ffff88020bc719d8 ffff8802272542d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? select_task_rq_fair+0x53/0x8a0 [] ? finish_task_switch+0x48/0x110 [] ? ttwu_stat+0x4c/0x140 [] ? try_to_wake_up+0x2e/0x2f0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19260 0xffff880224828000 19260 17544 0 1 D 0xffff880224828488 ffsb ffff88020c2419d8 0000000000000046 0000000000000000 0000000000000046 ffff880224828000 ffff88020c241fd8 ffff88020c241fd8 ffff88020c241fd8 ffff88020dc60000 ffff880224828000 ffff88020c2419d8 ffff8802272342d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19261 0xffff88022482bf20 19261 17544 0 0 D 0xffff88022482c3a8 ffsb ffff88020bc779d8 0000000000000046 0000000000000000 0000000000000046 ffff88022482bf20 ffff88020bc77fd8 ffff88020bc77fd8 ffff88020bc77fd8 ffffffff81c10440 ffff88022482bf20 ffff88020bc779d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? do_wp_page+0x333/0x6f0 [] ? handle_pte_fault+0x264/0x8a0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19262 0xffff88022482deb0 19262 17544 0 7 D 0xffff88022482e338 ffsb ffff88020c2dbe38 0000000000000046 ffff88020c2dbe58 0000000000000246 ffff88022482deb0 ffff88020c2dbfd8 ffff88020c2dbfd8 ffff88020c2dbfd8 ffff880225d3deb0 ffff88022482deb0 ffff88020c2dbee8 ffff8800bfd40538 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_llseek+0x45/0x170 [ceph] [] ? fget_light+0x3c/0x130 [] ceph_llseek+0x45/0x170 [ceph] [] vfs_llseek+0x33/0x40 [] SyS_lseek+0x7e/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19263 0xffff88020dd05eb0 19263 17544 0 1 D 0xffff88020dd06338 ffsb ffff88020c9299d8 0000000000000046 0000000000000000 0000000000000046 ffff88020dd05eb0 ffff88020c929fd8 ffff88020c929fd8 ffff88020c929fd8 ffff88020dc60000 ffff88020dd05eb0 ffff88020c9299d8 ffff8802272342d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19264 0xffff88020dd00000 19264 17544 0 3 D 0xffff88020dd00488 ffsb ffff88020c5919d8 0000000000000046 0000000000000000 0000000000000046 ffff88020dd00000 ffff88020c591fd8 ffff88020c591fd8 ffff88020c591fd8 ffff880223791f90 ffff88020dd00000 ffff88020c5919d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19265 0xffff88020d08deb0 19265 17544 0 0 D 0xffff88020d08e338 ffsb ffff88020ce799d8 0000000000000046 0000000000000000 0000000000000046 ffff88020d08deb0 ffff88020ce79fd8 ffff88020ce79fd8 ffff88020ce79fd8 ffffffff81c10440 ffff88020d08deb0 ffff88020ce799d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? do_wp_page+0x333/0x6f0 [] ? handle_pte_fault+0x264/0x8a0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19266 0xffff88020c5b8000 19266 17544 0 7 D 0xffff88020c5b8488 ffsb ffff88020c4899d8 0000000000000046 0000000000000000 0000000000000046 ffff88020c5b8000 ffff88020c489fd8 ffff88020c489fd8 ffff88020c489fd8 ffff88019e0a3f20 ffff88020c5b8000 ffff88020c4899d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? do_raw_spin_lock+0x76/0x140 [] ? do_raw_spin_lock+0x76/0x140 [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19267 0xffff88020c5bdeb0 19267 17544 0 7 D 0xffff88020c5be338 ffsb ffff88020c8039d8 0000000000000046 0000000000000000 0000000000000046 ffff88020c5bdeb0 ffff88020c803fd8 ffff88020c803fd8 ffff88020c803fd8 ffff880225d3deb0 ffff88020c5bdeb0 ffff88020c8039d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __atomic_notifier_call_chain+0x86/0xa0 [] ? __atomic_notifier_call_chain+0x5/0xa0 [] ? ttwu_stat+0x4c/0x140 [] ? try_to_wake_up+0x2e/0x2f0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19268 0xffff880225e83f20 19268 17544 0 3 D 0xffff880225e843a8 ffsb ffff88020b3e59d8 0000000000000046 0000000000000000 0000000000000046 ffff880225e83f20 ffff88020b3e5fd8 ffff88020b3e5fd8 ffff88020b3e5fd8 ffff880225d2deb0 ffff880225e83f20 ffff88020b3e59d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19269 0xffff88020d2d5eb0 19269 17544 0 0 D 0xffff88020d2d6338 ffsb ffff88020d0df9d8 0000000000000046 0000000000000000 0000000000000046 ffff88020d2d5eb0 ffff88020d0dffd8 ffff88020d0dffd8 ffff88020d0dffd8 ffff8801e8920000 ffff88020d2d5eb0 ffff88020d0df9d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19270 0xffff88020d2d1f90 19270 17544 0 0 D 0xffff88020d2d2418 ffsb ffff88020c9d19d8 0000000000000046 0000000000000000 0000000000000046 ffff88020d2d1f90 ffff88020c9d1fd8 ffff88020c9d1fd8 ffff88020c9d1fd8 ffffffff81c10440 ffff88020d2d1f90 ffff88020c9d19d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19271 0xffff88020d2d3f20 19271 17544 0 3 D 0xffff88020d2d43a8 ffsb ffff88020bc8b9d8 0000000000000046 0000000000000000 0000000000000046 ffff88020d2d3f20 ffff88020bc8bfd8 ffff88020bc8bfd8 ffff88020bc8bfd8 ffff880225d2deb0 ffff88020d2d3f20 ffff88020bc8b9d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19272 0xffff88020dc61f90 19272 17544 0 0 D 0xffff88020dc62418 ffsb ffff88020b3959d8 0000000000000046 0000000000000000 0000000000000046 ffff88020dc61f90 ffff88020b395fd8 ffff88020b395fd8 ffff88020b395fd8 ffffffff81c10440 ffff88020dc61f90 ffff88020b3959d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19273 0xffff88020dc65eb0 19273 17544 0 3 D 0xffff88020dc66338 ffsb ffff88022216b9d8 0000000000000046 0000000000000000 0000000000000046 ffff88020dc65eb0 ffff88022216bfd8 ffff88022216bfd8 ffff88022216bfd8 ffff880223791f90 ffff88020dc65eb0 ffff88022216b9d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19274 0xffff88020cfddeb0 19274 17544 0 6 D 0xffff88020cfde338 ffsb ffff88020bd619d8 0000000000000046 0000000000000000 0000000000000046 ffff88020cfddeb0 ffff88020bd61fd8 ffff88020bd61fd8 ffff88020bd61fd8 ffff880225d3bf20 ffff88020cfddeb0 ffff88020bd619d8 ffff8802272d42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19275 0xffff88020cfd8000 19275 17544 0 2 D 0xffff88020cfd8488 ffsb ffff88020c5019d8 0000000000000046 0000000000000000 0000000000000046 ffff88020cfd8000 ffff88020c501fd8 ffff88020c501fd8 ffff88020c501fd8 ffff88020b063f20 ffff88020cfd8000 ffff88020c5019d8 ffff8802272542d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19276 0xffff880223795eb0 19276 17544 0 3 D 0xffff880223796338 ffsb ffff88020c461e38 0000000000000046 ffff88020c461e58 0000000000000246 ffff880223795eb0 ffff88020c461fd8 ffff88020c461fd8 ffff88020c461fd8 ffff880223791f90 ffff880223795eb0 ffff88020c461ee8 ffff8800bfd40d30 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_llseek+0x45/0x170 [ceph] [] ? fget_light+0x3c/0x130 [] ceph_llseek+0x45/0x170 [ceph] [] vfs_llseek+0x33/0x40 [] SyS_lseek+0x7e/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19277 0xffff880223790000 19277 17544 0 4 D 0xffff880223790488 ffsb ffff88020b29fbd8 0000000000000046 ffff88020b29fbf8 0000000000000246 ffff880223790000 ffff88020b29ffd8 ffff88020b29ffd8 ffff88020b29ffd8 ffff880225d38000 ffff880223790000 ffff880225c80740 ffff8801e5b1fcc0 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_aio_write+0xab/0xa90 [ceph] [] ceph_aio_write+0xab/0xa90 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? finish_task_switch+0x48/0x110 [] ? vfs_write+0x1bb/0x1e0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19278 0xffff88020dcd3f20 19278 17544 0 4 D 0xffff88020dcd43a8 ffsb ffff88020c25b9d8 0000000000000046 0000000000000000 0000000000000046 ffff88020dcd3f20 ffff88020c25bfd8 ffff88020c25bfd8 ffff88020c25bfd8 ffff8801e8921f90 ffff88020dcd3f20 ffff88020c25b9d8 ffff8802272942d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19279 0xffff880224e38000 19279 17544 0 6 D 0xffff880224e38488 ffsb ffff88022208dbd8 0000000000000046 ffff88022208dbf8 0000000000000246 ffff880224e38000 ffff88022208dfd8 ffff88022208dfd8 ffff88022208dfd8 ffff880225d3bf20 ffff880224e38000 0000000000000000 ffff8800bfd41d20 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_aio_write+0xab/0xa90 [ceph] [] ceph_aio_write+0xab/0xa90 [ceph] [] ? select_task_rq_fair+0x53/0x8a0 [] ? finish_task_switch+0x48/0x110 [] ? ttwu_stat+0x4c/0x140 [] ? try_to_wake_up+0x2e/0x2f0 [] ? vfs_write+0x1bb/0x1e0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19280 0xffff880224e3deb0 19280 17544 0 5 D 0xffff880224e3e338 ffsb ffff880225dcd9d8 0000000000000046 0000000000000000 0000000000000046 ffff880224e3deb0 ffff880225dcdfd8 ffff880225dcdfd8 ffff880225dcdfd8 ffff880225d39f90 ffff880224e3deb0 ffff880225dcd9d8 ffff8802272b42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? select_task_rq_fair+0x53/0x8a0 [] ? finish_task_switch+0x48/0x110 [] ? ttwu_stat+0x4c/0x140 [] ? try_to_wake_up+0x2e/0x2f0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19281 0xffff880224e39f90 19281 17544 0 0 D 0xffff880224e3a418 ffsb ffff88020c36dbd8 0000000000000046 ffff88020c36dbf8 0000000000000246 ffff880224e39f90 ffff88020c36dfd8 ffff88020c36dfd8 ffff88020c36dfd8 ffff8801e8920000 ffff880224e39f90 0000000100000001 ffff8800bfd41d20 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_aio_write+0xab/0xa90 [ceph] [] ceph_aio_write+0xab/0xa90 [ceph] [] ? select_task_rq_fair+0x53/0x8a0 [] ? finish_task_switch+0x48/0x110 [] ? ttwu_stat+0x4c/0x140 [] ? try_to_wake_up+0x2e/0x2f0 [] ? vfs_write+0x1bb/0x1e0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19282 0xffff880210059f90 19282 17544 0 1 D 0xffff88021005a418 ffsb ffff8802220159d8 0000000000000046 0000000000000000 0000000000000046 ffff880210059f90 ffff880222015fd8 ffff880222015fd8 ffff880222015fd8 ffff880225d29f90 ffff880210059f90 ffff8802220159d8 ffff8802272342d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8f0/0xa90 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? handle_pte_fault+0x475/0x8a0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 19247 0xffff8801e8920000 19247 2 0 0 D 0xffff8801e8920488 kworker/0:1 ffff88020b255ab0 0000000000000046 ffff88015d00e450 0000000000000000 ffff8801e8920000 ffff88020b255fd8 ffff88020b255fd8 ffff88020b255fd8 ffff88020b35deb0 ffff8801e8920000 ffff880224ec7968 ffff8801e8920000 Call Trace: [] schedule+0x29/0x70 [] rwsem_down_read_failed+0xa5/0xf9 [] call_rwsem_down_read_failed+0x14/0x30 [] ? down_read+0x47/0x50 [] ? osd_reset+0x48/0x2b0 [libceph] [] osd_reset+0x48/0x2b0 [libceph] [] con_work+0x571/0x2d50 [libceph] [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 19283 0xffff8801e8921f90 19283 2 0 4 D 0xffff8801e8922418 kworker/4:0 ffff8801307cd9e8 0000000000000046 000000000000000d ffff8801307cd9f8 ffff8801e8921f90 ffff8801307cdfd8 ffff8801307cdfd8 ffff8801307cdfd8 ffff88020dd03f20 ffff8801e8921f90 ffff880224ec7968 ffff8801e8921f90 Call Trace: [] schedule+0x29/0x70 [] rwsem_down_write_failed+0xf5/0x1b0 [] call_rwsem_down_write_failed+0x13/0x20 [] ? down_write+0x51/0x58 [] ? ceph_osdc_handle_map+0xe5/0x580 [libceph] [] ? ceph_check_fsid+0x2b/0x80 [libceph] [] ceph_osdc_handle_map+0xe5/0x580 [libceph] [] dispatch+0x138/0x9b0 [libceph] [] ? ceph_tcp_recvmsg+0x48/0x60 [libceph] [] ? trace_hardirqs_on+0xd/0x10 [] con_work+0x1948/0x2d50 [libceph] [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] ? rcu_irq_exit+0x68/0xb0 [] ? retint_restore_args+0xe/0xe [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] ? retint_restore_args+0xe/0xe [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 19285 0xffff88020d871f90 19285 2 0 4 R 0xffff88020d872418 kworker/4:3 ffff88010f7f3b58 0000000000000046 ffff88010f7f3b58 ffff88010f7f3b18 ffff88020d871f90 ffff88010f7f3fd8 ffff88010f7f3fd8 ffff88010f7f3fd8 ffff8801e8921f90 ffff88020d871f90 ffffffffffffffff ffff88010f7f2000 Call Trace: [] __cond_resched+0x2a/0x40 [] _cond_resched+0x32/0x40 [] mutex_lock_nested+0x2a/0x360 [] ? trace_hardirqs_on+0xd/0x10 [] con_work+0x1956/0x2d50 [libceph] [] ? cpuacct_charge+0x5/0xa0 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 19287 0xffff88020ce93f20 19287 2 0 0 D 0xffff88020ce943a8 kworker/0:3 ffff88016d0239e8 0000000000000046 000000000000000d ffff88016d0239f8 ffff88020ce93f20 ffff88016d023fd8 ffff88016d023fd8 ffff88016d023fd8 ffff8801e8920000 ffff88020ce93f20 ffff880224ec7968 ffff88020ce93f20 Call Trace: [] schedule+0x29/0x70 [] rwsem_down_write_failed+0xf5/0x1b0 [] call_rwsem_down_write_failed+0x13/0x20 [] ? down_write+0x51/0x58 [] ? ceph_osdc_handle_map+0xe5/0x580 [libceph] [] ? ceph_check_fsid+0x2b/0x80 [libceph] [] ceph_osdc_handle_map+0xe5/0x580 [libceph] [] dispatch+0x138/0x9b0 [libceph] [] ? ceph_tcp_recvmsg+0x48/0x60 [libceph] [] ? trace_hardirqs_on+0xd/0x10 [] con_work+0x1948/0x2d50 [libceph] [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150