Ceph » Linux kernel client

[4]kdb> dumpall

[dumpall]kdb> pid R

KDB current process is kworker/4:1(pid=17381)

[dumpall]kdb> -dumpcommon

[dumpcommon]kdb> set BTAPROMPT 0

[dumpcommon]kdb> set LINES 10000

[dumpcommon]kdb> -summary

sysname Linux

release 3.10.0-rc7-ceph-00049-g9e41c3a

version #1 SMP Thu Jun 27 06:41:24 PDT 2013

machine x86_64

nodename plana72

domainname (none)

ccversion CCVERSION

date 2013-06-28 12:37:39 tz_minuteswest 420

uptime 09:33

load avg 29.95 11.20 5.32

MemTotal: 2038245 kB

MemFree: 722832 kB

Buffers: 200 kB

[dumpcommon]kdb> -cpu

Currently on cpu 4

Available cpus: 0-3(I), 4, 5-7(I)

[dumpcommon]kdb> -ps

7 idle processes (state I) and

111 sleeping system daemon (state M) processes suppressed,

use 'ps A' to see all.

Task Addr Pid Parent [*] cpu State Thread Command

0xffff880216983f20 17381 2 1 4 R 0xffff8802169843a8 *kworker/4:1

0xffff880225ca8000 1 0 0 0 S 0xffff880225ca8488 init

0xffff88020c711f90 494 1 0 3 S 0xffff88020c712418 upstart-udev-br

0xffff88020cf23f20 498 1 0 1 S 0xffff88020cf243a8 udevd

0xffff88020cf25eb0 538 1 0 0 S 0xffff88020cf26338 dbus-daemon

0xffff880215678000 923 1 0 5 S 0xffff880215678488 rpc.idmapd

0xffff8802156cdeb0 983 1 0 1 S 0xffff8802156ce338 upstart-socket-

0xffff880219f5deb0 990 1 0 3 S 0xffff880219f5e338 rpcbind

0xffff880215618000 1049 1 0 0 S 0xffff880215618488 rpc.statd

0xffff88020cfebf20 1207 1 0 0 S 0xffff88020cfec3a8 getty

0xffff88020c713f20 1222 1 0 6 S 0xffff88020c7143a8 getty

0xffff88020c588000 1237 1 0 6 S 0xffff88020c588488 getty

0xffff8802244c5eb0 1238 1 0 7 S 0xffff8802244c6338 getty

0xffff880224d4deb0 1240 1 0 6 S 0xffff880224d4e338 getty

0xffff8802156a1f90 1246 1 0 0 S 0xffff8802156a2418 atd

0xffff880224d4bf20 1247 1 0 0 S 0xffff880224d4c3a8 xinetd

0xffff880215665eb0 1255 1 0 2 S 0xffff880215666338 nrpe

0xffff88020c67bf20 1333 1 0 1 S 0xffff88020c67c3a8 irqbalance

0xffff88020bbb3f20 1349 1 0 6 S 0xffff88020bbb43a8 rpc.mountd

0xffff8802156cbf20 1400 1 0 2 S 0xffff8802156cc3a8 pkcsslotd

0xffff880221e61f90 1409 1 0 5 S 0xffff880221e62418 rwhod

0xffff88020ce51f90 1411 1409 0 1 S 0xffff88020ce52418 rwhod

0xffff88020c715eb0 1528 1 0 5 S 0xffff88020c716338 getty

0xffff88020c710000 1529 1 0 0 S 0xffff88020c710488 getty

0xffff880219f59f90 1874 1 0 6 S 0xffff880219f5a418 ntpd

0xffff880216983f20 17381 2 1 4 R 0xffff8802169843a8 *kworker/4:1

0xffff8801377f3f20 27902 2 0 5 D 0xffff8801377f43a8 kworker/u64:1

0xffff880100079f90 28210 2 0 0 D 0xffff88010007a418 kworker/0:2

0xffff88021e420000 28529 2 0 0 D 0xffff88021e420488 kworker/0:1

0xffff880222099f90 29322 1 0 0 S 0xffff88022209a418 sshd

0xffff88020c859f90 29338 29322 0 2 S 0xffff88020c85a418 sshd

0xffff88010007deb0 29372 1 0 7 S 0xffff88010007e338 rsyslogd

0xffff880224665eb0 29373 1 0 1 S 0xffff880224666338 rs:main Q:Reg

0xffff8801000f3f20 29374 1 0 1 S 0xffff8801000f43a8 rsyslogd

0xffff8801000f0000 29375 1 0 1 S 0xffff8801000f0488 rsyslogd

0xffff880225dc3f20 29606 1 0 1 S 0xffff880225dc43a8 cron

0xffff88020d2a9f90 31164 1 0 1 S 0xffff88020d2aa418 sshd

0xffff88020d0ebf20 31726 498 0 6 S 0xffff88020d0ec3a8 udevd

0xffff88020d0edeb0 31727 498 0 7 S 0xffff88020d0ee338 udevd

0xffff88020c51deb0 31763 29338 0 7 S 0xffff88020c51e338 bash

0xffff88020db51f90 31765 31763 0 7 S 0xffff88020db52418 ffsb.sh

0xffff8801377f5eb0 984 31765 0 0 S 0xffff8801377f6338 ffsb

0xffff8802227ddeb0 1018 31765 0 3 S 0xffff8802227de338 ffsb

0xffff88020cf20000 1019 31765 0 6 D 0xffff88020cf20488 ffsb

0xffff88020cf21f90 1020 31765 0 3 D 0xffff88020cf22418 ffsb

0xffff880224295eb0 1021 31765 0 5 D 0xffff880224296338 ffsb

0xffff880224293f20 1022 31765 0 6 D 0xffff8802242943a8 ffsb

0xffff880224290000 1023 31765 0 3 D 0xffff880224290488 ffsb

0xffff880224291f90 1024 31765 0 2 D 0xffff880224292418 ffsb

0xffff880224f48000 1025 31765 0 7 D 0xffff880224f48488 ffsb

0xffff880100078000 1026 31765 0 3 D 0xffff880100078488 ffsb

0xffff88010007bf20 1027 31765 0 3 D 0xffff88010007c3a8 ffsb

0xffff8802156a3f20 1028 31765 0 0 D 0xffff8802156a43a8 ffsb

0xffff8802156a5eb0 1029 31765 0 0 D 0xffff8802156a6338 ffsb

0xffff8802156a0000 1030 31765 0 2 D 0xffff8802156a0488 ffsb

0xffff88020bbb1f90 1031 31765 0 3 D 0xffff88020bbb2418 ffsb

0xffff880225de3f20 1032 31765 0 7 D 0xffff880225de43a8 ffsb

0xffff88020d0adeb0 1033 31765 0 1 D 0xffff88020d0ae338 ffsb

0xffff880219f0bf20 1034 31765 0 2 D 0xffff880219f0c3a8 ffsb

0xffff880219f0deb0 1035 31765 0 2 D 0xffff880219f0e338 ffsb

0xffff880219f08000 1036 31765 0 7 D 0xffff880219f08488 ffsb

0xffff880215663f20 1037 31765 0 3 D 0xffff8802156643a8 ffsb

0xffff880225d19f90 1038 31765 0 0 D 0xffff880225d1a418 ffsb

0xffff880216981f90 1039 31765 0 5 D 0xffff880216982418 ffsb

0xffff880216985eb0 1040 31765 0 0 D 0xffff880216986338 ffsb

0xffff880216980000 1041 31765 0 2 D 0xffff880216980488 ffsb

0xffff880224971f90 1042 31765 0 4 D 0xffff880224972418 ffsb

0xffff880219d81f90 1043 31765 0 7 D 0xffff880219d82418 ffsb

0xffff880219d85eb0 1044 31765 0 6 D 0xffff880219d86338 ffsb

0xffff880219d83f20 1045 31765 0 7 D 0xffff880219d843a8 ffsb

0xffff880225e4bf20 1046 31765 0 7 D 0xffff880225e4c3a8 ffsb

0xffff8801f4915eb0 1047 31765 0 3 D 0xffff8801f4916338 ffsb

0xffff8801f4910000 1048 31765 0 1 D 0xffff8801f4910488 ffsb

0xffff8801f4911f90 1050 31765 0 5 D 0xffff8801f4912418 ffsb

0xffff8801d8503f20 1051 31765 0 7 D 0xffff8801d85043a8 ffsb

0xffff88015ecd9f90 988 2 0 4 R 0xffff88015ecda418 kworker/4:2

0xffff88011e7bdeb0 1052 2 0 0 D 0xffff88011e7be338 kworker/0:3

0xffff88015ecdbf20 1053 2 0 0 D 0xffff88015ecdc3a8 kworker/0:4

0xffff88020db53f20 1054 2 0 0 D 0xffff88020db543a8 kworker/0:5

0xffff88011e7bbf20 1083 2 0 4 R 0xffff88011e7bc3a8 kworker/4:3

[dumpcommon]kdb> -dmesg 600

<4>[34289.402817] ffffffff81c89c78 ffffffff81c89b00 ffff880225d45e58 ffffffff814ed5df

<4>[34289.410282] 0000000000000000 000000000260d2e4 0000000000000000 000000000260d2e4

<4>[34289.417748] Call Trace:

<4>[34289.420209] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34289.426144] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34289.431992] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34289.437318] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34289.443166] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<6>[34294.755885] libceph: osd4 weight 0x10000 (in)

<3>[34318.746693] INFO: rcu_sched self-detected stall on CPU { 4} (t=2100 jiffies g=656695 c=656694 q=2625)

<6>[34318.756134] sending NMI to all CPUs:

<4>[34318.759729] NMI backtrace for cpu 0

<4>[34318.763236] CPU: 0 PID: 1054 Comm: kworker/0:5 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34318.772661] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34318.780185] Workqueue: ceph-msgr con_work [libceph]

<4>[34318.785090] task: ffff88020db53f20 ti: ffff88019caa8000 task.ti: ffff88019caa8000

<4>[34318.792604] RIP: 0010:[<ffffffff8112ecae>] [<ffffffff8112ecae>] putback_inactive_pages+0xbe/0x280

<3>[34318.796729] INFO: rcu_sched detected stalls on CPUs/tasks: { 4} (detected by 6, t=2105 jiffies, g=656695, c=656694, q=2625)

<4>[34318.812774] RSP: 0018:ffff88019caa9498 EFLAGS: 00000002

<4>[34318.818104] RAX: 0000000000000000 RBX: ffffea000606c1e0 RCX: 020000000000282d

<4>[34318.825257] RDX: 0000000000000002 RSI: 0000000000000003 RDI: ffffea000606c1c0

<4>[34318.832410] RBP: ffff88019caa9508 R08: dead000000200200 R09: 0000000000000001

<4>[34318.839564] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88019caa9570

<4>[34318.846716] R13: ffff88022fffa2b8 R14: ffffea000606c1c0 R15: ffff88022fff9e00

<4>[34318.853869] FS: 0000000000000000(0000) GS:ffff880227200000(0000) knlGS:0000000000000000

<4>[34318.861991] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34318.867754] CR2: 00000000026369a8 CR3: 0000000001c0b000 CR4: 00000000000007f0

<4>[34318.874908] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34318.882061] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34318.889214] Stack:

<4>[34318.891238] 0000000000000002 ffff880200000000 0000000000000003 ffff88022fffa280

<4>[34318.898709] ffff88022fffa2f8 ffff88019caa94c8 ffff88019caa94c8 ffff88019caa94c8

<4>[34318.906175] ffff88019caa9508 ffff88022fffa2b8 ffff88019caa9750 ffff88022fff9e00

<4>[34318.913644] Call Trace:

<4>[34318.916107] [<ffffffff8112f0e3>] shrink_inactive_list+0x273/0x400

<4>[34318.922306] [<ffffffff8112f87d>] shrink_lruvec+0x2cd/0x4d0

<4>[34318.927897] [<ffffffff8119879b>] ? bdi_queue_work+0x8b/0xf0

<4>[34318.933573] [<ffffffff8112fe2c>] do_try_to_free_pages+0x11c/0x3a0

<4>[34318.939770] [<ffffffff81130276>] try_to_free_pages+0xd6/0x1b0

<4>[34318.945621] [<ffffffff811377c0>] ? next_zone+0x30/0x40

<4>[34318.950864] [<ffffffff81125616>] __alloc_pages_nodemask+0x596/0x8f0

<4>[34318.957237] [<ffffffff8115bd2a>] alloc_pages_current+0xba/0x170

<4>[34318.963263] [<ffffffff81516f6e>] sk_page_frag_refill+0x7e/0x130

<4>[34318.969290] [<ffffffff8156f7d5>] tcp_sendmsg+0x305/0xe50

<4>[34318.974707] [<ffffffff8159b1c9>] inet_sendmsg+0xb9/0xf0

<4>[34318.980036] [<ffffffff8159b115>] ? inet_sendmsg+0x5/0xf0

<4>[34318.985454] [<ffffffff81511012>] sock_sendmsg+0xc2/0xe0

<4>[34318.990785] [<ffffffff812ee5cb>] ? chksum_update+0x1b/0x30

<4>[34318.996376] [<ffffffff812ea458>] ? crypto_shash_update+0x18/0x30

<4>[34319.002498] [<ffffffffa0000056>] ? crc32c+0x56/0x7c [libcrc32c]

<4>[34319.008523] [<ffffffff81511070>] kernel_sendmsg+0x40/0x60

<4>[34319.014030] [<ffffffffa07a34d8>] con_work+0xc28/0x2d50 [libceph]

<4>[34319.020142] [<ffffffff810889a5>] ? cpuacct_charge+0x5/0xa0

<4>[34319.025732] [<ffffffff81080c83>] ? idle_balance+0x133/0x180

<4>[34319.031408] [<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

<4>[34319.037519] [<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

<4>[34319.043632] [<ffffffff8105f3fa>] process_one_work+0x1da/0x540

<4>[34319.049481] [<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

<4>[34319.055506] [<ffffffff810605dc>] worker_thread+0x11c/0x370

<4>[34319.061095] [<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

<4>[34319.067643] [<ffffffff8106729a>] kthread+0xea/0xf0

<4>[34319.072538] [<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

<4>[34319.078911] [<ffffffff8164029c>] ret_from_fork+0x7c/0xb0

<4>[34319.084327] [<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

<4>[34319.110298] NMI backtrace for cpu 1

<4>[34319.113805] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34319.122798] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34319.130314] task: ffff880225d29f90 ti: ffff880225d30000 task.ti: ffff880225d30000

<4>[34319.137829] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34319.145796] RSP: 0018:ffff880225d31dc8 EFLAGS: 00000046

<4>[34319.151123] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34319.158276] RDX: 0000000000000000 RSI: ffff880225d31fd8 RDI: 0000000000000001

<4>[34319.165430] RBP: ffff880225d31df8 R08: ffff88022723a840 R09: 0000000000001068

<4>[34319.172583] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34319.179736] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34319.186889] FS: 0000000000000000(0000) GS:ffff880227220000(0000) knlGS:0000000000000000

<4>[34319.195013] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34319.200775] CR2: 00007f80e99e1000 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34319.207927] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34319.215081] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34319.222232] Stack:

<4>[34319.224258] ffff880225d31df8 0000000181093504 ffff88022723a840 00001f31a1e6fda0

<4>[34319.231730] ffffffff81c89c78 ffffffff81c89b00 ffff880225d31e58 ffffffff814ed5df

<4>[34319.239202] 0000000000000000 0000000001c7daac 0000000000000000 0000000001c7daac

<4>[34319.246674] Call Trace:

<4>[34319.249141] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34319.255080] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34319.260934] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34319.266263] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34319.272116] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34319.297531] NMI backtrace for cpu 2

<4>[34319.301041] CPU: 2 PID: 0 Comm: swapper/2 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34319.310031] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34319.317546] task: ffff880225d2bf20 ti: ffff880225d32000 task.ti: ffff880225d32000

<4>[34319.325062] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34319.333028] RSP: 0018:ffff880225d33dc8 EFLAGS: 00000046

<4>[34319.338355] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001

<4>[34319.345508] RDX: 0000000000000000 RSI: ffff880225d33fd8 RDI: 0000000000000002

<4>[34319.352661] RBP: ffff880225d33df8 R08: ffff88022725a840 R09: 00000000000027d8

<4>[34319.359814] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000003

<4>[34319.366968] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003

<4>[34319.374123] FS: 0000000000000000(0000) GS:ffff880227240000(0000) knlGS:0000000000000000

<4>[34319.382246] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34319.388007] CR2: 00007f77d73e3044 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34319.395162] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34319.402316] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34319.409467] Stack:

<4>[34319.411492] ffff880225d33df8 0000000281093504 ffff88022725a840 00001f31a17487f2

<4>[34319.418963] ffffffff81c89c20 ffffffff81c89b00 ffff880225d33e58 ffffffff814ed5df

<4>[34319.427669] 0000000000000000 0000000001a1b8aa 0000000000000000 0000000001a1b8aa

<4>[34319.435142] Call Trace:

<4>[34319.437609] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34319.443547] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34319.449400] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34319.454732] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34319.460587] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34319.485996] NMI backtrace for cpu 3

<4>[34319.489502] CPU: 3 PID: 0 Comm: swapper/3 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34319.498494] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34319.506008] task: ffff880225d2deb0 ti: ffff880225d34000 task.ti: ffff880225d34000

<4>[34319.513524] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34319.521487] RSP: 0018:ffff880225d35dc8 EFLAGS: 00000046

<4>[34319.526815] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34319.533968] RDX: 0000000000000000 RSI: ffff880225d35fd8 RDI: 0000000000000003

<4>[34319.541120] RBP: ffff880225d35df8 R08: ffff88022727a840 R09: 0000000000002fa8

<4>[34319.548273] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34319.555424] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34319.562576] FS: 0000000000000000(0000) GS:ffff880227260000(0000) knlGS:0000000000000000

<4>[34319.570707] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34319.576469] CR2: 00007f82a0a5c1d4 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34319.583621] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34319.590775] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34319.597930] Stack:

<4>[34319.599955] ffff880225d35df8 0000000381093504 ffff88022727a840 00001f31a1e6a34f

<4>[34319.607425] ffffffff81c89c78 ffffffff81c89b00 ffff880225d35e58 ffffffff814ed5df

<4>[34319.614895] 0000000000000000 000000000260d70f 0000000000000000 000000000260d70f

<4>[34319.622368] Call Trace:

<4>[34319.624835] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34319.630771] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34319.636626] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34319.641957] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34319.647809] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34319.673236] NMI backtrace for cpu 4

<4>[34319.676745] CPU: 4 PID: 988 Comm: kworker/4:2 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34319.686088] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34319.693618] Workqueue: ceph-msgr con_work [libceph]

<4>[34319.698524] task: ffff88015ecd9f90 ti: ffff880224ca2000 task.ti: ffff880224ca2000

<4>[34319.706040] RIP: 0010:[<ffffffff8131d59a>] [<ffffffff8131d59a>] delay_tsc+0x2a/0x60

<4>[34319.713833] RSP: 0018:ffff880227283d18 EFLAGS: 00000083

<4>[34319.719162] RAX: 0000000031bdd8b3 RBX: 00000000000003e9 RCX: 0000000000000004

<4>[34319.726317] RDX: 0000000000000018 RSI: 0000000031bdd89b RDI: 000000000003a724

<4>[34319.733472] RBP: ffff880227283d18 R08: 0000000000000004 R09: 0000000000000000

<4>[34319.740627] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000001000

<4>[34319.747782] R13: ffffffff81ce1bf0 R14: 0000000000000400 R15: 0000000000000092

<4>[34319.754938] FS: 0000000000000000(0000) GS:ffff880227280000(0000) knlGS:0000000000000000

<4>[34319.763063] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34319.768827] CR2: 00007f92b3cc1120 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34319.775982] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34319.783138] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34319.790292] Stack:

<4>[34319.792317] ffff880227283d28 ffffffff8131d509 ffff880227283d48 ffffffff81025eaa

<4>[34319.799793] 0000000000000002 000000000000b022 ffff880227283d98 ffffffff81026c9d

<4>[34319.807271] 0000000000000023 0000000000000005 ffff880227283db8 0000000000002710

<4>[34319.814746] Call Trace:

<4>[34319.817208] <IRQ>

<4>[34319.819133]

<4>[34319.820652] [<ffffffff8131d509>] __const_udelay+0x29/0x30

<4>[34319.824855] [<ffffffff81025eaa>] native_safe_apic_wait_icr_idle+0x2a/0x60

<4>[34319.831751] [<ffffffff81026c9d>] default_send_IPI_mask_sequence_phys+0xed/0x100

<4>[34319.839186] [<ffffffff8102a0e7>] physflat_send_IPI_all+0x17/0x20

<4>[34319.845301] [<ffffffff81026e1a>] arch_trigger_all_cpu_backtrace+0x5a/0x90

<4>[34319.852198] [<ffffffff810e296a>] rcu_check_callbacks+0x21a/0x660

<4>[34319.858312] [<ffffffff810e64ac>] ? acct_account_cputime+0x1c/0x20

<4>[34319.864516] [<ffffffff81050f88>] update_process_times+0x48/0x80

<4>[34319.870545] [<ffffffff8109b726>] tick_sched_handle.isra.10+0x36/0x50

<4>[34319.877006] [<ffffffff8109b82c>] tick_sched_timer+0x4c/0x80

<4>[34319.882685] [<ffffffff8106a861>] __run_hrtimer+0x81/0x1e0

<4>[34319.888193] [<ffffffff8109b7e0>] ? tick_nohz_handler+0xa0/0xa0

<4>[34319.894134] [<ffffffff8106b167>] hrtimer_interrupt+0x107/0x260

<4>[34319.900078] [<ffffffff81641e69>] smp_apic_timer_interrupt+0x69/0x99

<4>[34319.906453] [<ffffffff81640faf>] apic_timer_interrupt+0x6f/0x80

<4>[34319.912477] <EOI>

<4>[34319.914402]

<4>[34319.915921] [<ffffffff8112effc>] ? shrink_inactive_list+0x18c/0x400

<4>[34319.920993] [<ffffffff81637790>] ? _raw_spin_unlock_irq+0x30/0x40

<4>[34319.927195] [<ffffffff81637795>] ? _raw_spin_unlock_irq+0x35/0x40

<4>[34319.933397] [<ffffffff8112effc>] shrink_inactive_list+0x18c/0x400

<4>[34319.939601] [<ffffffff8112684a>] ? global_dirtyable_memory+0x1a/0x60

<4>[34319.946065] [<ffffffff8112f87d>] shrink_lruvec+0x2cd/0x4d0

<4>[34319.951659] [<ffffffff8119879b>] ? bdi_queue_work+0x8b/0xf0

<4>[34319.957338] [<ffffffff8112fe2c>] do_try_to_free_pages+0x11c/0x3a0

<4>[34319.963539] [<ffffffff81130276>] try_to_free_pages+0xd6/0x1b0

<4>[34319.969393] [<ffffffff81125616>] __alloc_pages_nodemask+0x596/0x8f0

<4>[34319.975772] [<ffffffff8115bd2a>] alloc_pages_current+0xba/0x170

<4>[34319.981800] [<ffffffff81516f6e>] sk_page_frag_refill+0x7e/0x130

<4>[34319.987829] [<ffffffff8156f7d5>] tcp_sendmsg+0x305/0xe50

<4>[34319.993249] [<ffffffff8159b1c9>] inet_sendmsg+0xb9/0xf0

<4>[34319.998582] [<ffffffff8159b115>] ? inet_sendmsg+0x5/0xf0

<4>[34320.004003] [<ffffffff81511012>] sock_sendmsg+0xc2/0xe0

<4>[34320.009337] [<ffffffff812ea458>] ? crypto_shash_update+0x18/0x30

<4>[34320.015463] [<ffffffffa0000056>] ? crc32c+0x56/0x7c [libcrc32c]

<4>[34320.021493] [<ffffffff81511070>] kernel_sendmsg+0x40/0x60

<4>[34320.027007] [<ffffffffa07a34d8>] con_work+0xc28/0x2d50 [libceph]

<4>[34320.033124] [<ffffffff810889a5>] ? cpuacct_charge+0x5/0xa0

<4>[34320.038717] [<ffffffff81080c83>] ? idle_balance+0x133/0x180

<4>[34320.044396] [<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

<4>[34320.050510] [<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

<4>[34320.056624] [<ffffffff8105f3fa>] process_one_work+0x1da/0x540

<4>[34320.062477] [<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

<4>[34320.068529] [<ffffffff810605dc>] worker_thread+0x11c/0x370

<4>[34320.074123] [<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

<4>[34320.080672] [<ffffffff8106729a>] kthread+0xea/0xf0

<4>[34320.085571] [<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

<4>[34320.091947] [<ffffffff8164029c>] ret_from_fork+0x7c/0xb0

<4>[34320.097367] [<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

<4>[34320.123418] NMI backtrace for cpu 5

<4>[34320.126926] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34320.135918] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34320.143433] task: ffff880225d39f90 ti: ffff880225d40000 task.ti: ffff880225d40000

<4>[34320.150947] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34320.158912] RSP: 0018:ffff880225d41dc8 EFLAGS: 00000046

<4>[34320.164239] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34320.171392] RDX: 0000000000000000 RSI: ffff880225d41fd8 RDI: 0000000000000005

<4>[34320.178544] RBP: ffff880225d41df8 R08: ffff8802272ba840 R09: 0000000000001838

<4>[34320.185696] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34320.192849] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34320.200003] FS: 0000000000000000(0000) GS:ffff8802272a0000(0000) knlGS:0000000000000000

<4>[34320.208124] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34320.213888] CR2: 00007f3ebc574000 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34320.221040] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34320.228194] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34320.235346] Stack:

<4>[34320.237372] ffff880225d41df8 0000000581093504 ffff8802272ba840 00001f31f2f61bde

<4>[34320.244842] ffffffff81c89c78 ffffffff81c89b00 ffff880225d41e58 ffffffff814ed5df

<4>[34320.252315] 0000000000000000 00000000026153c0 0000000000000000 00000000026153c0

<4>[34320.259788] Call Trace:

<4>[34320.262255] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34320.268192] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34320.274045] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34320.279375] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34320.285228] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34320.310644] NMI backtrace for cpu 6

<4>[34320.314152] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34320.323142] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34320.330659] task: ffff880225d3bf20 ti: ffff880225d42000 task.ti: ffff880225d42000

<4>[34320.338173] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34320.346135] RSP: 0018:ffff880225d43dc8 EFLAGS: 00000046

<4>[34320.351466] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34320.359776] RDX: 0000000000000000 RSI: ffff880225d43fd8 RDI: 0000000000000006

<4>[34320.366928] RBP: ffff880225d43df8 R08: ffff8802272da840 R09: 0000000000001068

<4>[34320.374083] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34320.381235] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34320.388387] FS: 0000000000000000(0000) GS:ffff8802272c0000(0000) knlGS:0000000000000000

<4>[34320.396512] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34320.402275] CR2: 0000000000a23d60 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34320.409427] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34320.416582] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34320.423735] Stack:

<4>[34320.425759] ffff880225d43df8 0000000681093504 ffff8802272da840 00001f31f2f62d35

<4>[34320.433229] ffffffff81c89c78 ffffffff81c89b00 ffff880225d43e58 ffffffff814ed5df

<4>[34320.440704] 0000000000000000 00000000026145a0 0000000000000000 00000000026145a0

<4>[34320.448177] Call Trace:

<4>[34320.450643] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34320.456579] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34320.462434] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34320.467765] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34320.473617] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34320.499049] NMI backtrace for cpu 7

<4>[34320.502557] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34320.511550] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34320.519063] task: ffff880225d3deb0 ti: ffff880225d44000 task.ti: ffff880225d44000

<4>[34320.526577] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34320.534540] RSP: 0018:ffff880225d45dc8 EFLAGS: 00000046

<4>[34320.539868] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34320.547020] RDX: 0000000000000000 RSI: ffff880225d45fd8 RDI: 0000000000000007

<4>[34320.554173] RBP: ffff880225d45df8 R08: ffff8802272fa840 R09: 0000000000003778

<4>[34320.561325] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34320.568516] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34320.575669] FS: 0000000000000000(0000) GS:ffff8802272e0000(0000) knlGS:0000000000000000

<4>[34320.583792] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34320.589554] CR2: 00007fff90652ff0 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34320.596707] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34320.603860] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34320.611011] Stack:

<4>[34320.613036] ffff880225d45df8 0000000781093504 ffff8802272fa840 00001f31f2f679a2

<4>[34320.620506] ffffffff81c89c78 ffffffff81c89b00 ffff880225d45e58 ffffffff814ed5df

<4>[34320.627976] 0000000000000000 000000000260ec8f 0000000000000000 000000000260ec8f

<4>[34320.635444] Call Trace:

<4>[34320.637911] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34320.643849] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34320.649702] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34320.655031] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34320.660884] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<3>[34341.710486] INFO: rcu_sched self-detected stall on CPU { 4} (t=2101 jiffies g=656696 c=656695 q=3333)

<6>[34341.719916] sending NMI to all CPUs:

<3>[34341.720494] INFO: rcu_sched detected stalls on CPUs/tasks: { 4} (detected by 0, t=2102 jiffies, g=656696, c=656695, q=3333)

<4>[34341.734677] NMI backtrace for cpu 0

<4>[34341.738184] CPU: 0 PID: 1054 Comm: kworker/0:5 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34341.747610] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34341.755134] Workqueue: ceph-msgr con_work [libceph]

<4>[34341.760038] task: ffff88020db53f20 ti: ffff88019caa8000 task.ti: ffff88019caa8000

<4>[34341.767552] RIP: 0010:[<ffffffff8112ef54>] [<ffffffff8112ef54>] shrink_inactive_list+0xe4/0x400

<4>[34341.776384] RSP: 0018:ffff88019caa9518 EFLAGS: 00000202

<4>[34341.781713] RAX: 0000000000000001 RBX: ffff88022fffa2b8 RCX: 0000000000000000

<4>[34341.788866] RDX: ffff88022fffa280 RSI: 0000000000000000 RDI: ffff88022fffa280

<4>[34341.796019] RBP: ffff88019caa95c8 R08: 00000000000036e1 R09: 0000000000000000

<4>[34341.803173] R10: 0000000000000001 R11: 0000000000000001 R12: ffff88019caa9750

<4>[34341.810326] R13: ffff88022fff9e00 R14: ffff88019caa9570 R15: 0000000000000020

<4>[34341.817480] FS: 0000000000000000(0000) GS:ffff880227200000(0000) knlGS:0000000000000000

<4>[34341.825602] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34341.831363] CR2: 00000000026369a8 CR3: 0000000001c0b000 CR4: 00000000000007f0

<4>[34341.838516] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34341.845670] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34341.852822] Stack:

<4>[34341.854846] ffff880100000000 0000000000000000 0000000000000801 0000000000000040

<4>[34341.862313] ffff880200000000 0000001625f56000 00000000ffffffe0 0000000000000001

<4>[34341.869784] ffff88022fffa280 0000000000000002 ffff88019caa9598 ffff88019caa9570

<4>[34341.877251] Call Trace:

<4>[34341.879713] [<ffffffff8112f87d>] shrink_lruvec+0x2cd/0x4d0

<4>[34341.885304] [<ffffffff8119879b>] ? bdi_queue_work+0x8b/0xf0

<4>[34341.890981] [<ffffffff8112fe2c>] do_try_to_free_pages+0x11c/0x3a0

<4>[34341.897179] [<ffffffff81130276>] try_to_free_pages+0xd6/0x1b0

<4>[34341.903033] [<ffffffff81125616>] __alloc_pages_nodemask+0x596/0x8f0

<4>[34341.909405] [<ffffffff8115bd2a>] alloc_pages_current+0xba/0x170

<4>[34341.915432] [<ffffffff81516f6e>] sk_page_frag_refill+0x7e/0x130

<4>[34341.921459] [<ffffffff8156f7d5>] tcp_sendmsg+0x305/0xe50

<4>[34341.926876] [<ffffffff8159b1c9>] inet_sendmsg+0xb9/0xf0

<4>[34341.932206] [<ffffffff8159b115>] ? inet_sendmsg+0x5/0xf0

<4>[34341.937624] [<ffffffff81511012>] sock_sendmsg+0xc2/0xe0

<4>[34341.942955] [<ffffffff812ee5cb>] ? chksum_update+0x1b/0x30

<4>[34341.948546] [<ffffffff812ea458>] ? crypto_shash_update+0x18/0x30

<4>[34341.954668] [<ffffffffa0000056>] ? crc32c+0x56/0x7c [libcrc32c]

<4>[34341.960692] [<ffffffff81511070>] kernel_sendmsg+0x40/0x60

<4>[34341.966199] [<ffffffffa07a34d8>] con_work+0xc28/0x2d50 [libceph]

<4>[34341.972311] [<ffffffff810889a5>] ? cpuacct_charge+0x5/0xa0

<4>[34341.977901] [<ffffffff81080c83>] ? idle_balance+0x133/0x180

<4>[34341.983578] [<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

<4>[34341.989690] [<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

<4>[34341.995802] [<ffffffff8105f3fa>] process_one_work+0x1da/0x540

<4>[34342.001651] [<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

<4>[34342.007676] [<ffffffff810605dc>] worker_thread+0x11c/0x370

<4>[34342.013265] [<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

<4>[34342.019811] [<ffffffff8106729a>] kthread+0xea/0xf0

<4>[34342.024706] [<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

<4>[34342.031079] [<ffffffff8164029c>] ret_from_fork+0x7c/0xb0

<4>[34342.036496] [<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

<4>[34342.062455] NMI backtrace for cpu 1

<4>[34342.065963] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34342.074955] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34342.082472] task: ffff880225d29f90 ti: ffff880225d30000 task.ti: ffff880225d30000

<4>[34342.089986] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34342.097951] RSP: 0018:ffff880225d31dc8 EFLAGS: 00000046

<4>[34342.103278] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34342.110433] RDX: 0000000000000000 RSI: ffff880225d31fd8 RDI: 0000000000000001

<4>[34342.117585] RBP: ffff880225d31df8 R08: ffff88022723a840 R09: 0000000000001068

<4>[34342.124739] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34342.131892] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34342.139046] FS: 0000000000000000(0000) GS:ffff880227220000(0000) knlGS:0000000000000000

<4>[34342.147169] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34342.152932] CR2: 00007f80e99e1000 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34342.160085] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34342.167238] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34342.174391] Stack:

<4>[34342.176416] ffff880225d31df8 0000000181093504 ffff88022723a840 00001f36fa6c95e2

<4>[34342.183889] ffffffff81c89c78 ffffffff81c89b00 ffff880225d31e58 ffffffff814ed5df

<4>[34342.191360] 0000000000000000 000000000260be45 0000000000000000 000000000260be45

<4>[34342.198832] Call Trace:

<4>[34342.201299] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34342.207237] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34342.213091] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34342.218422] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34342.224275] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34342.249681] NMI backtrace for cpu 2

<4>[34342.253188] CPU: 2 PID: 0 Comm: swapper/2 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34342.262181] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34342.269697] task: ffff880225d2bf20 ti: ffff880225d32000 task.ti: ffff880225d32000

<4>[34342.277213] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34342.285180] RSP: 0018:ffff880225d33dc8 EFLAGS: 00000046

<4>[34342.290508] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34342.297662] RDX: 0000000000000000 RSI: ffff880225d33fd8 RDI: 0000000000000002

<4>[34342.304813] RBP: ffff880225d33df8 R08: ffff88022725a840 R09: 0000000000002fa8

<4>[34342.311967] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34342.320270] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34342.327422] FS: 0000000000000000(0000) GS:ffff880227240000(0000) knlGS:0000000000000000

<4>[34342.335545] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34342.341307] CR2: 00007f77d73e3044 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34342.348458] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34342.355612] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34342.362764] Stack:

<4>[34342.364790] ffff880225d33df8 0000000281093504 ffff88022725a840 00001f36fa6c3c3e

<4>[34342.372260] ffffffff81c89c78 ffffffff81c89b00 ffff880225d33e58 ffffffff814ed5df

<4>[34342.379730] 0000000000000000 0000000001c9194a 0000000000000000 0000000001c9194a

<4>[34342.387202] Call Trace:

<4>[34342.389670] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34342.395607] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34342.401462] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34342.406792] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34342.412643] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34342.438056] NMI backtrace for cpu 3

<4>[34342.441563] CPU: 3 PID: 0 Comm: swapper/3 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34342.450553] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34342.458069] task: ffff880225d2deb0 ti: ffff880225d34000 task.ti: ffff880225d34000

<4>[34342.465581] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34342.473546] RSP: 0018:ffff880225d35dc8 EFLAGS: 00000046

<4>[34342.478874] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001

<4>[34342.486026] RDX: 0000000000000000 RSI: ffff880225d35fd8 RDI: 0000000000000003

<4>[34342.493179] RBP: ffff880225d35df8 R08: ffff88022727a840 R09: 0000000000002fa8

<4>[34342.500331] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000003

<4>[34342.507485] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003

<4>[34342.514637] FS: 0000000000000000(0000) GS:ffff880227260000(0000) knlGS:0000000000000000

<4>[34342.522759] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34342.528520] CR2: 00007f82a0a5c1d4 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34342.535674] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34342.542828] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34342.549980] Stack:

<4>[34342.552005] ffff880225d35df8 0000000381093504 ffff88022727a840 00001f36fa6cdaf4

<4>[34342.559475] ffffffff81c89c20 ffffffff81c89b00 ffff880225d35e58 ffffffff814ed5df

<4>[34342.566943] 0000000000000000 00000000026060db 0000000000000000 00000000026060db

<4>[34342.574412] Call Trace:

<4>[34342.576876] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34342.582856] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34342.588708] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34342.594039] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34342.599890] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34342.625325] NMI backtrace for cpu 4

<4>[34342.628834] CPU: 4 PID: 988 Comm: kworker/4:2 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34342.638175] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34342.645703] Workqueue: ceph-msgr con_work [libceph]

<4>[34342.650610] task: ffff88015ecd9f90 ti: ffff880224ca2000 task.ti: ffff880224ca2000

<4>[34342.658127] RIP: 0010:[<ffffffff8131d59a>] [<ffffffff8131d59a>] delay_tsc+0x2a/0x60

<4>[34342.665922] RSP: 0018:ffff880227283d18 EFLAGS: 00000083

<4>[34342.671252] RAX: 00000000fe188365 RBX: 00000000000003e9 RCX: 0000000000000004

<4>[34342.678408] RDX: 0000000000000015 RSI: 00000000fe188350 RDI: 000000000003a724

<4>[34342.685564] RBP: ffff880227283d18 R08: 0000000000000004 R09: 0000000000000000

<4>[34342.692719] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000001000

<4>[34342.699873] R13: ffffffff81ce1bf0 R14: 0000000000000400 R15: 0000000000000092

<4>[34342.707030] FS: 0000000000000000(0000) GS:ffff880227280000(0000) knlGS:0000000000000000

<4>[34342.715154] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34342.720917] CR2: 00007f92b3cc1120 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34342.728072] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34342.735226] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34342.742382] Stack:

<4>[34342.744410] ffff880227283d28 ffffffff8131d509 ffff880227283d48 ffffffff81025eaa

<4>[34342.751886] 0000000000000002 000000000000b022 ffff880227283d98 ffffffff81026c9d

<4>[34342.759363] 0000000000000023 0000000000000005 ffff880227283db8 0000000000002710

<4>[34342.766837] Call Trace:

<4>[34342.769298] <IRQ>

<4>[34342.771226]

<4>[34342.772746] [<ffffffff8131d509>] __const_udelay+0x29/0x30

<4>[34342.776949] [<ffffffff81025eaa>] native_safe_apic_wait_icr_idle+0x2a/0x60

<4>[34342.783847] [<ffffffff81026c9d>] default_send_IPI_mask_sequence_phys+0xed/0x100

<4>[34342.791284] [<ffffffff8102a0e7>] physflat_send_IPI_all+0x17/0x20

<4>[34342.797399] [<ffffffff81026e1a>] arch_trigger_all_cpu_backtrace+0x5a/0x90

<4>[34342.804297] [<ffffffff810e296a>] rcu_check_callbacks+0x21a/0x660

<4>[34342.810410] [<ffffffff810e64ac>] ? acct_account_cputime+0x1c/0x20

<4>[34342.816614] [<ffffffff81050f88>] update_process_times+0x48/0x80

<4>[34342.822643] [<ffffffff8109b726>] tick_sched_handle.isra.10+0x36/0x50

<4>[34342.829105] [<ffffffff8109b82c>] tick_sched_timer+0x4c/0x80

<4>[34342.834786] [<ffffffff8106a861>] __run_hrtimer+0x81/0x1e0

<4>[34342.840293] [<ffffffff8109b7e0>] ? tick_nohz_handler+0xa0/0xa0

<4>[34342.846233] [<ffffffff8106b167>] hrtimer_interrupt+0x107/0x260

<4>[34342.852175] [<ffffffff81641e69>] smp_apic_timer_interrupt+0x69/0x99

<4>[34342.858552] [<ffffffff81640faf>] apic_timer_interrupt+0x6f/0x80

<4>[34342.864575] <EOI>

<4>[34342.866502]

<4>[34342.868021] [<ffffffff8112f0fd>] ? shrink_inactive_list+0x28d/0x400

<4>[34342.873094] [<ffffffff81637790>] ? _raw_spin_unlock_irq+0x30/0x40

<4>[34342.879298] [<ffffffff81637795>] ? _raw_spin_unlock_irq+0x35/0x40

<4>[34342.885500] [<ffffffff8112f0fd>] shrink_inactive_list+0x28d/0x400

<4>[34342.891701] [<ffffffff8112f87d>] shrink_lruvec+0x2cd/0x4d0

<4>[34342.897294] [<ffffffff8119879b>] ? bdi_queue_work+0x8b/0xf0

<4>[34342.902974] [<ffffffff8112fe2c>] do_try_to_free_pages+0x11c/0x3a0

<4>[34342.909175] [<ffffffff81130276>] try_to_free_pages+0xd6/0x1b0

<4>[34342.915029] [<ffffffff811377c0>] ? next_zone+0x30/0x40

<4>[34342.920276] [<ffffffff81125616>] __alloc_pages_nodemask+0x596/0x8f0

<4>[34342.926653] [<ffffffff8115bd2a>] alloc_pages_current+0xba/0x170

<4>[34342.932683] [<ffffffff81516f6e>] sk_page_frag_refill+0x7e/0x130

<4>[34342.938713] [<ffffffff8156f7d5>] tcp_sendmsg+0x305/0xe50

<4>[34342.944133] [<ffffffff8159b1c9>] inet_sendmsg+0xb9/0xf0

<4>[34342.949465] [<ffffffff8159b115>] ? inet_sendmsg+0x5/0xf0

<4>[34342.954885] [<ffffffff81511012>] sock_sendmsg+0xc2/0xe0

<4>[34342.960219] [<ffffffff812ea458>] ? crypto_shash_update+0x18/0x30

<4>[34342.966345] [<ffffffffa0000056>] ? crc32c+0x56/0x7c [libcrc32c]

<4>[34342.972375] [<ffffffff81511070>] kernel_sendmsg+0x40/0x60

<4>[34342.977889] [<ffffffffa07a34d8>] con_work+0xc28/0x2d50 [libceph]

<4>[34342.984005] [<ffffffff810889a5>] ? cpuacct_charge+0x5/0xa0

<4>[34342.989597] [<ffffffff81080c83>] ? idle_balance+0x133/0x180

<4>[34342.995278] [<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

<4>[34343.001392] [<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

<4>[34343.007507] [<ffffffff8105f3fa>] process_one_work+0x1da/0x540

<4>[34343.013360] [<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

<4>[34343.019387] [<ffffffff810605dc>] worker_thread+0x11c/0x370

<4>[34343.024980] [<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

<4>[34343.031530] [<ffffffff8106729a>] kthread+0xea/0xf0

<4>[34343.036429] [<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

<4>[34343.042805] [<ffffffff8164029c>] ret_from_fork+0x7c/0xb0

<4>[34343.048224] [<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

<4>[34343.074310] NMI backtrace for cpu 5

<4>[34343.077817] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34343.086808] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34343.094323] task: ffff880225d39f90 ti: ffff880225d40000 task.ti: ffff880225d40000

<4>[34343.101836] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34343.109799] RSP: 0018:ffff880225d41dc8 EFLAGS: 00000046

<4>[34343.115127] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34343.122279] RDX: 0000000000000000 RSI: ffff880225d41fd8 RDI: 0000000000000005

<4>[34343.129431] RBP: ffff880225d41df8 R08: ffff8802272ba840 R09: 0000000000001838

<4>[34343.136584] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34343.143737] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34343.150890] FS: 0000000000000000(0000) GS:ffff8802272a0000(0000) knlGS:0000000000000000

<4>[34343.159015] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34343.164777] CR2: 00007f3ebc574000 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34343.171930] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34343.179082] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34343.186236] Stack:

<4>[34343.188261] ffff880225d41df8 0000000581093504 ffff8802272ba840 00001f374a4a710c

<4>[34343.195732] ffffffff81c89c78 ffffffff81c89b00 ffff880225d41e58 ffffffff814ed5df

<4>[34343.203206] 0000000000000000 0000000002614f50 0000000000000000 0000000002614f50

<4>[34343.210678] Call Trace:

<4>[34343.213144] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34343.219083] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34343.224935] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34343.230266] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34343.236118] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34343.262665] NMI backtrace for cpu 6

<4>[34343.266173] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34343.275164] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34343.282681] task: ffff880225d3bf20 ti: ffff880225d42000 task.ti: ffff880225d42000

<4>[34343.290196] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34343.298162] RSP: 0018:ffff880225d43dc8 EFLAGS: 00000046

<4>[34343.303490] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34343.310643] RDX: 0000000000000000 RSI: ffff880225d43fd8 RDI: 0000000000000006

<4>[34343.317795] RBP: ffff880225d43df8 R08: ffff8802272da840 R09: 0000000000001068

<4>[34343.324948] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34343.332101] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34343.339254] FS: 0000000000000000(0000) GS:ffff8802272c0000(0000) knlGS:0000000000000000

<4>[34343.347378] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34343.353141] CR2: 0000000000a23d60 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34343.360294] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34343.367447] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34343.374599] Stack:

<4>[34343.376626] ffff880225d43df8 0000000681093504 ffff8802272da840 00001f374919fd92

<4>[34343.384098] ffffffff81c89c78 ffffffff81c89b00 ffff880225d43e58 ffffffff814ed5df

<4>[34343.391569] 0000000000000000 000000000260deeb 0000000000000000 000000000260deeb

<4>[34343.399040] Call Trace:

<4>[34343.401506] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34343.407444] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34343.413298] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34343.418628] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34343.424481] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<4>[34343.449904] NMI backtrace for cpu 7

<4>[34343.453412] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1

<4>[34343.462404] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011

<4>[34343.469918] task: ffff880225d3deb0 ti: ffff880225d44000 task.ti: ffff880225d44000

<4>[34343.477433] RIP: 0010:[<ffffffff81373c77>] [<ffffffff81373c77>] intel_idle+0xa7/0x100

<4>[34343.485398] RSP: 0018:ffff880225d45dc8 EFLAGS: 00000046

<4>[34343.490726] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001

<4>[34343.497877] RDX: 0000000000000000 RSI: ffff880225d45fd8 RDI: 0000000000000007

<4>[34343.505030] RBP: ffff880225d45df8 R08: ffff8802272fa840 R09: 0000000000003778

<4>[34343.512183] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004

<4>[34343.519336] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004

<4>[34343.526488] FS: 0000000000000000(0000) GS:ffff8802272e0000(0000) knlGS:0000000000000000

<4>[34343.534612] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b

<4>[34343.540374] CR2: 00007fff90652ff0 CR3: 0000000001c0b000 CR4: 00000000000007e0

<4>[34343.547527] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

<4>[34343.554679] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400

<4>[34343.561831] Stack:

<4>[34343.563857] ffff880225d45df8 0000000781093504 ffff8802272fa840 00001f37491a0150

<4>[34343.571326] ffffffff81c89c78 ffffffff81c89b00 ffff880225d45e58 ffffffff814ed5df

<4>[34343.578797] 0000000000000000 000000000260d7fc 0000000000000000 000000000260d7fc

<4>[34343.586267] Call Trace:

<4>[34343.588732] [<ffffffff814ed5df>] cpuidle_enter_state+0x4f/0xe0

<4>[34343.594671] [<ffffffff814ed73e>] cpuidle_idle_call+0xce/0x240

<4>[34343.600524] [<ffffffff8100ac1e>] arch_cpu_idle+0xe/0x30

<4>[34343.605853] [<ffffffff81091dee>] cpu_startup_entry+0xce/0x2b0

<4>[34343.611706] [<ffffffff81624dc7>] start_secondary+0x26b/0x26d

<6>[34343.649514] libceph: osd0 down

<6>[34343.652702] libceph: osd3 weight 0x0 (out)

<4>[34343.683502] libceph: osd4 10.214.132.25:6813 socket error on write

<4>[34343.689775] libceph: osd0 10.214.132.26:6808 socket closed (con state OPEN)

<1>[34344.361875] BUG: unable to handle kernel NULL pointer dereference at (null)

<1>[34344.369766] IP: [<ffffffff81318823>] rb_erase+0x1a3/0x370

<4>[34344.375194] PGD 0

<4>[34344.377229] Oops: 0000 [#1] SMP

[dumpcommon]kdb> -bt

Stack traceback for pid 17381

0xffff880216983f20 17381 2 1 4 R 0xffff8802169843a8 *kworker/4:1

ffff8800c791bb48 0000000000000018 0000000000000000 ffff8800c791bb68

ffffffffa07a97bc ffff8801fa10c000 ffff880114b3f950 ffff8800c791bb98

ffffffffa07a9abf ffff8801fa10c030 ffff880114b3f950 0000000000000000

Call Trace:

[<ffffffffa07a97bc>] ? __remove_osd+0x3c/0xa0 [libceph]

[<ffffffffa07a9abf>] ? __reset_osd+0x12f/0x170 [libceph]

[<ffffffffa07ab6de>] ? osd_reset+0x7e/0x2b0 [libceph]

[<ffffffffa07a2e21>] ? con_work+0x571/0x2d50 [libceph]

[<ffffffff810889a5>] ? cpuacct_charge+0x5/0xa0

[<ffffffff81080c83>] ? idle_balance+0x133/0x180

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff8105f3fa>] ? process_one_work+0x1da/0x540

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff810605dc>] ? worker_thread+0x11c/0x370

[<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106729a>] ? kthread+0xea/0xf0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8164029c>] ? ret_from_fork+0x7c/0xb0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

[dumpall]kdb> -bta

7 idle processes (state I) and

111 sleeping system daemon (state M) processes suppressed,

use 'ps A' to see all.

Stack traceback for pid 17381

0xffff880216983f20 17381 2 1 4 R 0xffff8802169843a8 *kworker/4:1

ffff8800c791bb48 0000000000000018 0000000000000000 ffff8800c791bb68

ffffffffa07a97bc ffff8801fa10c000 ffff880114b3f950 ffff8800c791bb98

ffffffffa07a9abf ffff8801fa10c030 ffff880114b3f950 0000000000000000

Call Trace:

[<ffffffffa07a97bc>] ? __remove_osd+0x3c/0xa0 [libceph]

[<ffffffffa07a9abf>] ? __reset_osd+0x12f/0x170 [libceph]

[<ffffffffa07ab6de>] ? osd_reset+0x7e/0x2b0 [libceph]

[<ffffffffa07a2e21>] ? con_work+0x571/0x2d50 [libceph]

[<ffffffff810889a5>] ? cpuacct_charge+0x5/0xa0

[<ffffffff81080c83>] ? idle_balance+0x133/0x180

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff8105f3fa>] ? process_one_work+0x1da/0x540

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff810605dc>] ? worker_thread+0x11c/0x370

[<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106729a>] ? kthread+0xea/0xf0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8164029c>] ? ret_from_fork+0x7c/0xb0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

Stack traceback for pid 1

0xffff880225ca8000 1 0 0 0 S 0xffff880225ca8488 init

ffff880225c8b8d8 0000000000000046 ffffffff8105eb35 ffffffff81c90600

ffff880225ca8000 ffff880225c8bfd8 ffff880225c8bfd8 ffff880225c8bfd8

ffffffff81c10440 ffff880225ca8000 0000000000000000 0000000000000000

Call Trace:

[<ffffffff8105eb35>] ? flush_work+0x5/0x280

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e3de>] ? might_fault+0x8e/0xa0

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 494

0xffff88020c711f90 494 1 0 3 S 0xffff88020c712418 upstart-udev-br

ffff88020cb458d8 0000000000000046 0000000000000002 0000000000000000

ffff88020c711f90 ffff88020cb45fd8 ffff88020cb45fd8 ffff88020cb45fd8

ffff880225d2deb0 ffff88020c711f90 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff815c5ab5>] ? unix_write_space+0x5/0xc0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81510cd8>] ? sock_recvmsg+0xd8/0xf0

[<ffffffff8117854f>] ? pipe_read+0x7f/0x560

[<ffffffff8117854f>] ? pipe_read+0x7f/0x560

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff8106795a>] ? remove_wait_queue+0x5a/0x70

[<ffffffff8104510c>] ? do_wait+0x12c/0x260

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 498

0xffff88020cf23f20 498 1 0 1 S 0xffff88020cf243a8 udevd

ffff88020d347de8 0000000000000046 0000000000000002 0000000000000000

ffff88020cf23f20 ffff88020d347fd8 ffff88020d347fd8 ffff88020d347fd8

ffff880225d29f90 ffff88020cf23f20 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff811b6358>] SyS_epoll_wait+0x418/0x4c0

[<ffffffff8116fa0e>] ? vfs_write+0x18e/0x1e0

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 538

0xffff88020cf25eb0 538 1 0 0 S 0xffff88020cf26338 dbus-daemon

ffff88020c5e3a38 0000000000000046 0000000000000002 0000000000000000

ffff88020cf25eb0 ffff88020c5e3fd8 ffff88020c5e3fd8 ffff88020c5e3fd8

ffffffff81c10440 ffff88020cf25eb0 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff811840cf>] do_sys_poll+0x40f/0x4d0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff8113e3de>] ? might_fault+0x8e/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81512086>] ? ___sys_recvmsg+0x166/0x2a0

[<ffffffff8163b85c>] ? __do_page_fault+0x2ac/0x520

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81637790>] ? _raw_spin_unlock_irq+0x30/0x40

[<ffffffff81071c55>] ? finish_task_switch+0x85/0x110

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff8118426b>] SyS_poll+0x6b/0x100

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 923

0xffff880215678000 923 1 0 5 S 0xffff880215678488 rpc.idmapd

ffff88020b0a3de8 0000000000000046 0000000000000002 0000000000000002

ffff880215678000 ffff88020b0a3fd8 ffff88020b0a3fd8 ffff88020b0a3fd8

ffff880225d39f90 ffff880215678000 ffff880200000000 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff8106f1a3>] ? lg_local_unlock+0x23/0x40

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff811b6358>] SyS_epoll_wait+0x418/0x4c0

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 983

0xffff8802156cdeb0 983 1 0 1 S 0xffff8802156ce338 upstart-socket-

ffff8802245878d8 0000000000000046 0000000000000000 ffffffff810679f3

ffff8802156cdeb0 ffff880224587fd8 ffff880224587fd8 ffff880224587fd8

ffff880225d29f90 ffff8802156cdeb0 0000000000000000 0000000000000000

Call Trace:

[<ffffffff810679f3>] ? add_wait_queue+0x23/0x60

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff815c5ab5>] ? unix_write_space+0x5/0xc0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81189c25>] ? touch_atime+0xd5/0x170

[<ffffffff81510cd8>] ? sock_recvmsg+0xd8/0xf0

[<ffffffff8117854f>] ? pipe_read+0x7f/0x560

[<ffffffff8117854f>] ? pipe_read+0x7f/0x560

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff8106795a>] ? remove_wait_queue+0x5a/0x70

[<ffffffff8104510c>] ? do_wait+0x12c/0x260

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 990

0xffff880219f5deb0 990 1 0 3 S 0xffff880219f5e338 rpcbind

ffff880215407a38 0000000000000046 00001f37ef85cfbf 0000000001c9c37f

ffff880219f5deb0 ffff880215407fd8 ffff880215407fd8 ffff880215407fd8

ffff880225d2deb0 ffff880219f5deb0 ffff880215407a18 ffff880215407ef8

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163490c>] schedule_hrtimeout_range_clock+0x12c/0x170

[<ffffffff8106a360>] ? update_rmtp+0x70/0x70

[<ffffffff8106af84>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff811840cf>] do_sys_poll+0x40f/0x4d0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff8118ee14>] ? mntput+0x24/0x40

[<ffffffff81170ef9>] ? __fput+0x189/0x240

[<ffffffff8109341c>] ? ktime_get_ts+0x4c/0xe0

[<ffffffff81182f65>] ? poll_select_set_timeout+0x85/0xa0

[<ffffffff8118426b>] SyS_poll+0x6b/0x100

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1049

0xffff880215618000 1049 1 0 0 S 0xffff880215618488 rpc.statd

ffff880224dd78d8 0000000000000046 0000000000000002 0000000000000000

ffff880215618000 ffff880224dd7fd8 ffff880224dd7fd8 ffff880224dd7fd8

ffffffff81c10440 ffff880215618000 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff81070a47>] ? __wake_up_sync_key+0x37/0x80

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff8106b973>] ? up_read+0x23/0x40

[<ffffffff8163b85c>] ? __do_page_fault+0x2ac/0x520

[<ffffffff8118ee14>] ? mntput+0x24/0x40

[<ffffffff81170ef9>] ? __fput+0x189/0x240

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1207

0xffff88020cfebf20 1207 1 0 0 S 0xffff88020cfec3a8 getty

ffff880224d35cc8 0000000000000046 ffff880224d35cb8 ffff880227212f80

ffff88020cfebf20 ffff880224d35fd8 ffff880224d35fd8 ffff880224d35fd8

ffffffff81c10440 ffff88020cfebf20 ffff880224d35cb8 ffff88020db6f000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163313d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba658>] ? n_tty_read+0x388/0x960

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba548>] n_tty_read+0x278/0x960

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b64a5>] tty_read+0x95/0x100

[<ffffffff8116fb10>] vfs_read+0xb0/0x180

[<ffffffff8116fd95>] SyS_read+0x55/0xa0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1222

0xffff88020c713f20 1222 1 0 6 S 0xffff88020c7143a8 getty

ffff880224f89cc8 0000000000000046 0000000000000002 ffff8802272d2f80

ffff88020c713f20 ffff880224f89fd8 ffff880224f89fd8 ffff880224f89fd8

ffff880225d3bf20 ffff88020c713f20 ffff880224f89cb8 ffff880219f54000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163313d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba658>] ? n_tty_read+0x388/0x960

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba548>] n_tty_read+0x278/0x960

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b64a5>] tty_read+0x95/0x100

[<ffffffff8116fb10>] vfs_read+0xb0/0x180

[<ffffffff8116fd95>] SyS_read+0x55/0xa0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1237

0xffff88020c588000 1237 1 0 6 S 0xffff88020c588488 getty

ffff8802155f5cc8 0000000000000046 0000000000000002 ffff8802272d2f80

ffff88020c588000 ffff8802155f5fd8 ffff8802155f5fd8 ffff8802155f5fd8

ffff880225d3bf20 ffff88020c588000 ffff8802155f5cb8 ffff880219f55000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163313d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba658>] ? n_tty_read+0x388/0x960

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba548>] n_tty_read+0x278/0x960

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b64a5>] tty_read+0x95/0x100

[<ffffffff8116fb10>] vfs_read+0xb0/0x180

[<ffffffff8116fd95>] SyS_read+0x55/0xa0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1238

0xffff8802244c5eb0 1238 1 0 7 S 0xffff8802244c6338 getty

ffff88020b927cc8 0000000000000046 0000000000000002 ffff8802272f2f80

ffff8802244c5eb0 ffff88020b927fd8 ffff88020b927fd8 ffff88020b927fd8

ffff880225d3deb0 ffff8802244c5eb0 ffff88020b927cb8 ffff88020b30c800

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163313d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba658>] ? n_tty_read+0x388/0x960

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba548>] n_tty_read+0x278/0x960

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b64a5>] tty_read+0x95/0x100

[<ffffffff8116fb10>] vfs_read+0xb0/0x180

[<ffffffff8116fd95>] SyS_read+0x55/0xa0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1240

0xffff880224d4deb0 1240 1 0 6 S 0xffff880224d4e338 getty

ffff880224fd3cc8 0000000000000046 0000000000000002 ffff8802272d2f80

ffff880224d4deb0 ffff880224fd3fd8 ffff880224fd3fd8 ffff880224fd3fd8

ffff880225d3bf20 ffff880224d4deb0 ffff880224fd3cb8 ffff880219f56000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163313d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba658>] ? n_tty_read+0x388/0x960

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba548>] n_tty_read+0x278/0x960

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b64a5>] tty_read+0x95/0x100

[<ffffffff8116fb10>] vfs_read+0xb0/0x180

[<ffffffff8116fd95>] SyS_read+0x55/0xa0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1246

0xffff8802156a1f90 1246 1 0 0 S 0xffff8802156a2418 atd

ffff88020b09be58 0000000000000046 0000034630b8a000 000000000000c350

ffff8802156a1f90 ffff88020b09bfd8 ffff88020b09bfd8 ffff88020b09bfd8

ffffffff81c10440 ffff8802156a1f90 0000000000000002 ffff88020b09beb8

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163471c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6e8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a360>] ? update_rmtp+0x70/0x70

[<ffffffff8106af84>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b826>] SyS_nanosleep+0x76/0x90

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1247

0xffff880224d4bf20 1247 1 0 0 S 0xffff880224d4c3a8 xinetd

ffff880224ec98d8 0000000000000046 0000000000000000 ffffffff810679f3

ffff880224d4bf20 ffff880224ec9fd8 ffff880224ec9fd8 ffff880224ec9fd8

ffffffff81c10440 ffff880224d4bf20 ffff880200000000 0000000000000000

Call Trace:

[<ffffffff810679f3>] ? add_wait_queue+0x23/0x60

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff8107709e>] ? try_to_wake_up+0x2e/0x2f0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff8151b0fb>] ? skb_queue_tail+0x2b/0x60

[<ffffffff8111d345>] ? find_get_page+0x5/0xf0

[<ffffffff8111d345>] ? find_get_page+0x5/0xf0

[<ffffffff8113e53a>] ? __do_fault+0x14a/0x470

[<ffffffff8113e53a>] ? __do_fault+0x14a/0x470

[<ffffffff8163783b>] ? _raw_spin_unlock+0x2b/0x40

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff8106b973>] ? up_read+0x23/0x40

[<ffffffff8163b85c>] ? __do_page_fault+0x2ac/0x520

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1255

0xffff880215665eb0 1255 1 0 2 S 0xffff880215666338 nrpe

ffff88020ce4d8d8 0000000000000046 00001f37a70a6981 000000000007a11e

ffff880215665eb0 ffff88020ce4dfd8 ffff88020ce4dfd8 ffff88020ce4dfd8

ffff880225d2bf20 ffff880215665eb0 0000000000000000 ffff88020ce4dd18

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163490c>] schedule_hrtimeout_range_clock+0x12c/0x170

[<ffffffff8106a360>] ? update_rmtp+0x70/0x70

[<ffffffff8106af84>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff8162e61c>] ? get_partial_node.isra.63+0x54/0x1da

[<ffffffff8131ae95>] ? number.isra.1+0x355/0x390

[<ffffffff8163783b>] ? _raw_spin_unlock+0x2b/0x40

[<ffffffff8162e77d>] ? get_partial_node.isra.63+0x1b5/0x1da

[<ffffffff81153303>] ? swap_info_get+0x63/0xe0

[<ffffffff81153303>] ? swap_info_get+0x63/0xe0

[<ffffffff811411f3>] ? handle_pte_fault+0x323/0x8a0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff8106b973>] ? up_read+0x23/0x40

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8109341c>] ? ktime_get_ts+0x4c/0xe0

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1333

0xffff88020c67bf20 1333 1 0 1 S 0xffff88020c67c3a8 irqbalance

ffff880224477e58 0000000000000046 000000024f967cc8 000000000000c350

ffff88020c67bf20 ffff880224477fd8 ffff880224477fd8 ffff880224477fd8

ffff880225d29f90 ffff88020c67bf20 00007f80e99e2000 ffff880224477eb8

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163471c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6e8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a360>] ? update_rmtp+0x70/0x70

[<ffffffff8106af84>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b826>] SyS_nanosleep+0x76/0x90

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1349

0xffff88020bbb3f20 1349 1 0 6 S 0xffff88020bbb43a8 rpc.mountd

ffff88020db058d8 0000000000000046 0000000000000002 0000000000000000

ffff88020bbb3f20 ffff88020db05fd8 ffff88020db05fd8 ffff88020db05fd8

ffff880225d3bf20 ffff88020bbb3f20 ffff88022fff9e00 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff810721ec>] ? ttwu_stat+0x4c/0x140

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff8106b973>] ? up_read+0x23/0x40

[<ffffffff8163b85c>] ? __do_page_fault+0x2ac/0x520

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1400

0xffff8802156cbf20 1400 1 0 2 S 0xffff8802156cc3a8 pkcsslotd

ffff8802148e7e58 0000000000000046 00000002540be400 000000000000c350

ffff8802156cbf20 ffff8802148e7fd8 ffff8802148e7fd8 ffff8802148e7fd8

ffff880225d2bf20 ffff8802156cbf20 0000000000000000 ffff8802148e7eb8

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163471c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6e8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a360>] ? update_rmtp+0x70/0x70

[<ffffffff8106af84>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b826>] SyS_nanosleep+0x76/0x90

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1409

0xffff880221e61f90 1409 1 0 5 S 0xffff880221e62418 rwhod

ffff880219fffa48 0000000000000046 ffff880219fffa18 0000000000000246

ffff880221e61f90 ffff880219ffffd8 ffff880219ffffd8 ffff880219ffffd8

ffff880225d39f90 ffff880221e61f90 0000000000000002 ffff880222640000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163313d>] schedule_timeout+0x1bd/0x270

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff8152110b>] __skb_recv_datagram+0x2fb/0x380

[<ffffffff815211d0>] ? skb_recv_datagram+0x40/0x40

[<ffffffff8158eea7>] udp_recvmsg+0xa7/0x410

[<ffffffff8159b0ed>] inet_recvmsg+0xcd/0xf0

[<ffffffff81510cd8>] sock_recvmsg+0xd8/0xf0

[<ffffffff81186d00>] ? dput+0x1e0/0x240

[<ffffffff81513669>] SYSC_recvfrom+0xe9/0x170

[<ffffffff81170ef9>] ? __fput+0x189/0x240

[<ffffffff81170fbe>] ? ____fput+0xe/0x10

[<ffffffff8131ea24>] ? lockdep_sys_exit_thunk+0x35/0x67

[<ffffffff8151469e>] SyS_recvfrom+0xe/0x10

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1411

0xffff88020ce51f90 1411 1409 0 1 S 0xffff88020ce52418 rwhod

ffff88022464fe58 0000000000000046 00000029e8d60800 000000000000c350

ffff88020ce51f90 ffff88022464ffd8 ffff88022464ffd8 ffff88022464ffd8

ffff880225d29f90 ffff88020ce51f90 0000000000000002 ffff88022464feb8

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163471c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6e8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a360>] ? update_rmtp+0x70/0x70

[<ffffffff8106af84>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b826>] SyS_nanosleep+0x76/0x90

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1528

0xffff88020c715eb0 1528 1 0 5 S 0xffff88020c716338 getty

ffff88022445fcc8 0000000000000046 0000000000000002 ffff8802272b2f80

ffff88020c715eb0 ffff88022445ffd8 ffff88022445ffd8 ffff88022445ffd8

ffff880225d39f90 ffff88020c715eb0 ffff88022445fcb8 ffff88020cba1000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163313d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba658>] ? n_tty_read+0x388/0x960

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba548>] n_tty_read+0x278/0x960

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b64a5>] tty_read+0x95/0x100

[<ffffffff8116fb10>] vfs_read+0xb0/0x180

[<ffffffff8116fd95>] SyS_read+0x55/0xa0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1529

0xffff88020c710000 1529 1 0 0 S 0xffff88020c710488 getty

ffff880221ce1cc8 0000000000000046 ffff88020c8a09c8 ffff8802272b2f80

ffff88020c710000 ffff880221ce1fd8 ffff880221ce1fd8 ffff880221ce1fd8

ffffffff81c10440 ffff88020c710000 ffff880221ce1cb8 ffff88020c8a7800

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163313d>] schedule_timeout+0x1bd/0x270

[<ffffffff813ba658>] ? n_tty_read+0x388/0x960

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff813ba548>] n_tty_read+0x278/0x960

[<ffffffff81077360>] ? try_to_wake_up+0x2f0/0x2f0

[<ffffffff813b64a5>] tty_read+0x95/0x100

[<ffffffff8116fb10>] vfs_read+0xb0/0x180

[<ffffffff8116fd95>] SyS_read+0x55/0xa0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 1874

0xffff880219f59f90 1874 1 0 6 S 0xffff880219f5a418 ntpd

ffff88020c53b8d8 0000000000000046 0000000000000002 0000000000000000

ffff880219f59f90 ffff88020c53bfd8 ffff88020c53bfd8 ffff88020c53bfd8

ffff880225d3bf20 ffff880219f59f90 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff8105595a>] ? signal_delivered+0x5a/0x70

[<ffffffff8100ae82>] ? fpu_finit+0x22/0x40

[<ffffffff8100b0f6>] ? init_fpu+0x56/0xc0

[<ffffffff8100c0de>] ? __restore_xstate_sig+0x22e/0x510

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 27902

0xffff8801377f3f20 27902 2 0 5 D 0xffff8801377f43a8 kworker/u64:1

ffff8801a71ed748 0000000000000046 ffff8801a71ed768 0000000000000246

ffff8801377f3f20 ffff8801a71edfd8 ffff8801a71edfd8 ffff8801a71edfd8

ffff880225d39f90 ffff8801377f3f20 ffffffff81636fa5 ffff880114b3fa28

Call Trace:

[<ffffffff81636fa5>] ? rwsem_down_read_failed+0xa5/0xf9

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163643e>] schedule_preempt_disabled+0xe/0x10

[<ffffffff816337bc>] mutex_lock_nested+0x15c/0x360

[<ffffffffa07aaac8>] ? ceph_osdc_start_request+0x48/0x130 [libceph]

[<ffffffffa07aaac8>] ceph_osdc_start_request+0x48/0x130 [libceph]

[<ffffffffa07e248e>] ceph_writepages_start+0x94e/0xfa0 [ceph]

[<ffffffff81128b33>] do_writepages+0x23/0x40

[<ffffffff8119830e>] __writeback_single_inode+0x4e/0x2a0

[<ffffffff81199800>] writeback_sb_inodes+0x280/0x3e0

[<ffffffff811999fe>] __writeback_inodes_wb+0x9e/0xd0

[<ffffffff81199c2b>] wb_writeback+0x1fb/0x310

[<ffffffff8119b53b>] wb_do_writeback+0xbb/0x1e0

[<ffffffff81061f8f>] ? set_worker_desc+0x6f/0x80

[<ffffffff8119b6da>] bdi_writeback_workfn+0x7a/0x200

[<ffffffff8105f3fa>] process_one_work+0x1da/0x540

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff810605dc>] worker_thread+0x11c/0x370

[<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106729a>] kthread+0xea/0xf0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8164029c>] ret_from_fork+0x7c/0xb0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

Stack traceback for pid 28210

0xffff880100079f90 28210 2 0 0 D 0xffff88010007a418 kworker/0:2

ffff880211a8fab8 0000000000000046 ffff880211a8fad8 0000000000000246

ffff880100079f90 ffff880211a8ffd8 ffff880211a8ffd8 ffff880211a8ffd8

ffff88015ecdbf20 ffff880100079f90 0000000000000000 ffff880114b3fa28

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163643e>] schedule_preempt_disabled+0xe/0x10

[<ffffffff816337bc>] mutex_lock_nested+0x15c/0x360

[<ffffffffa07ad081>] ? dispatch+0x1f1/0x9b0 [libceph]

[<ffffffffa07ad081>] dispatch+0x1f1/0x9b0 [libceph]

[<ffffffffa07a0e38>] ? ceph_tcp_recvmsg+0x48/0x60 [libceph]

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffffa07a41f8>] con_work+0x1948/0x2d50 [libceph]

[<ffffffff810889a5>] ? cpuacct_charge+0x5/0xa0

[<ffffffff81080c83>] ? idle_balance+0x133/0x180

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff8105f3fa>] process_one_work+0x1da/0x540

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff810605dc>] worker_thread+0x11c/0x370

[<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106729a>] kthread+0xea/0xf0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8164029c>] ret_from_fork+0x7c/0xb0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

Stack traceback for pid 28529

0xffff88021e420000 28529 2 0 0 D 0xffff88021e420488 kworker/0:1

ffff880219067a88 0000000000000046 ffff880219067aa8 0000000000000246

ffff88021e420000 ffff880219067fd8 ffff880219067fd8 ffff880219067fd8

ffff8801000f5eb0 ffff88021e420000 ffff880114b3f288 ffff880114b3fa28

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163643e>] schedule_preempt_disabled+0xe/0x10

[<ffffffff816337bc>] mutex_lock_nested+0x15c/0x360

[<ffffffffa07acbc9>] ? ceph_osdc_handle_map+0x2b9/0x580 [libceph]

[<ffffffffa07acbc9>] ceph_osdc_handle_map+0x2b9/0x580 [libceph]

[<ffffffffa07a8050>] dispatch+0x120/0x7c0 [libceph]

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffffa07a41f8>] con_work+0x1948/0x2d50 [libceph]

[<ffffffff810889a5>] ? cpuacct_charge+0x5/0xa0

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff8105f3fa>] process_one_work+0x1da/0x540

[<ffffffff8105f38f>] ? process_one_work+0x16f/0x540

[<ffffffff810605dc>] worker_thread+0x11c/0x370

[<ffffffff810604c0>] ? manage_workers.isra.20+0x2e0/0x2e0

[<ffffffff8106729a>] kthread+0xea/0xf0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

[<ffffffff8164029c>] ret_from_fork+0x7c/0xb0

[<ffffffff810671b0>] ? flush_kthread_worker+0x150/0x150

Stack traceback for pid 29322

0xffff880222099f90 29322 1 0 0 S 0xffff88022209a418 sshd

ffff88020c455a38 0000000000000046 0000000000000002 0000000000000000

ffff880222099f90 ffff88020c455fd8 ffff88020c455fd8 ffff88020c455fd8

ffffffff81c10440 ffff880222099f90 ffff88020c455b08 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff8113f09f>] ? __pte_alloc+0x3f/0xc0

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff811840cf>] do_sys_poll+0x40f/0x4d0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff8113f3cf>] ? do_wp_page+0x19f/0x6f0

[<ffffffff8163783b>] ? _raw_spin_unlock+0x2b/0x40

[<ffffffff8113f4ca>] ? do_wp_page+0x29a/0x6f0

[<ffffffff81141134>] ? handle_pte_fault+0x264/0x8a0

[<ffffffff8163b70c>] ? __do_page_fault+0x15c/0x520

[<ffffffff8163b70c>] ? __do_page_fault+0x15c/0x520

[<ffffffff81079895>] ? sched_clock_cpu+0xc5/0x120

[<ffffffff8106b973>] ? up_read+0x23/0x40

[<ffffffff8163b85c>] ? __do_page_fault+0x2ac/0x520

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81637790>] ? _raw_spin_unlock_irq+0x30/0x40

[<ffffffff8105630d>] ? do_sigaction+0x12d/0x1f0

[<ffffffff8118426b>] SyS_poll+0x6b/0x100

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 29338

0xffff88020c859f90 29338 29322 0 2 S 0xffff88020c85a418 sshd

ffff880224a2d8d8 0000000000000046 0000000000000000 ffffffff810679f3

ffff88020c859f90 ffff880224a2dfd8 ffff880224a2dfd8 ffff880224a2dfd8

ffff880225d2bf20 ffff88020c859f90 0000000000000000 0000000000000000

Call Trace:

[<ffffffff810679f3>] ? add_wait_queue+0x23/0x60

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81517dc2>] ? release_sock+0x172/0x1e0

[<ffffffff815710e6>] ? tcp_recvmsg+0x396/0xc80

[<ffffffff8159b025>] ? inet_recvmsg+0x5/0xf0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff815119b3>] ? sock_aio_read+0x23/0x30

[<ffffffff8116ec0a>] ? do_sync_read+0x7a/0xb0

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 29372

0xffff88010007deb0 29372 1 0 7 S 0xffff88010007e338 rsyslogd

ffff88020cbc18d8 0000000000000046 00001f3d3b0fba0a 0000000001c9c37e

ffff88010007deb0 ffff88020cbc1fd8 ffff88020cbc1fd8 ffff88020cbc1fd8

ffff880225d3deb0 ffff88010007deb0 0000000000000002 ffff88020cbc1d18

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163490c>] schedule_hrtimeout_range_clock+0x12c/0x170

[<ffffffff8106a360>] ? update_rmtp+0x70/0x70

[<ffffffff8106af84>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff81080c83>] ? idle_balance+0x133/0x180

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff810e4bd1>] ? delayacct_end+0x71/0xa0

[<ffffffff810e4bd1>] ? delayacct_end+0x71/0xa0

[<ffffffff81071c18>] ? finish_task_switch+0x48/0x110

[<ffffffff81153303>] ? swap_info_get+0x63/0xe0

[<ffffffff81070ae2>] ? __wake_up+0x32/0x70

[<ffffffff81070ae2>] ? __wake_up+0x32/0x70

[<ffffffff81153303>] ? swap_info_get+0x63/0xe0

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8106b973>] ? up_read+0x23/0x40

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8109341c>] ? ktime_get_ts+0x4c/0xe0

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 29373

0xffff880224665eb0 29373 1 0 1 S 0xffff880224666338 rs:main Q:Reg

ffff8801a5b11c28 0000000000000046 0000000000000080 ffffffff826c2020

ffff880224665eb0 ffff8801a5b11fd8 ffff8801a5b11fd8 ffff8801a5b11fd8

ffff880225d29f90 ffff880224665eb0 0000000000000000 ffff8801a5b11cd0

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff810a4907>] futex_wait_queue_me+0xc7/0x100

[<ffffffff810a5d47>] futex_wait+0x1a7/0x2c0

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81212f4f>] ? ext4_file_write+0x9f/0x420

[<ffffffff8163783b>] ? _raw_spin_unlock+0x2b/0x40

[<ffffffff811b04ba>] ? fsnotify+0x8a/0x2f0

[<ffffffff810a73d4>] do_futex+0x344/0xbb0

[<ffffffff811b063a>] ? fsnotify+0x20a/0x2f0

[<ffffffff811b04ba>] ? fsnotify+0x8a/0x2f0

[<ffffffff810a7d87>] SyS_futex+0x147/0x1a0

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 29374

0xffff8801000f3f20 29374 1 0 1 S 0xffff8801000f43a8 rsyslogd

ffff88020b0018d8 0000000000000046 ffff880200000000 ffff88013a83f900

ffff8801000f3f20 ffff88020b001fd8 ffff88020b001fd8 ffff88020b001fd8

ffff880225d29f90 ffff8801000f3f20 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81182ce5>] ? __pollwait+0x75/0xf0

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff8163467e>] ? mutex_unlock+0xe/0x10

[<ffffffff815c89a6>] ? unix_dgram_recvmsg+0x316/0x4e0

[<ffffffff8107b083>] ? select_task_rq_fair+0x53/0x8a0

[<ffffffff8107b083>] ? select_task_rq_fair+0x53/0x8a0

[<ffffffff810721ec>] ? ttwu_stat+0x4c/0x140

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff81512086>] ? ___sys_recvmsg+0x166/0x2a0

[<ffffffff810a7ad5>] ? do_futex+0xa45/0xbb0

[<ffffffff8118c6be>] ? fget_light+0x9e/0x130

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 29375

0xffff8801000f0000 29375 1 0 1 S 0xffff8801000f0488 rsyslogd

ffff88020c949e28 0000000000000046 0000000000000286 ffffffff81c31800

ffff8801000f0000 ffff88020c949fd8 ffff88020c949fd8 ffff88020c949fd8

ffff880225d29f90 ffff8801000f0000 ffff88020c949e38 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff81042b51>] do_syslog+0x4d1/0x550

[<ffffffff81067a90>] ? __init_waitqueue_head+0x60/0x60

[<ffffffff811e0cd6>] kmsg_read+0x56/0x70

[<ffffffff811d2e63>] proc_reg_read+0x43/0x70

[<ffffffff8116fb10>] vfs_read+0xb0/0x180

[<ffffffff8116fd95>] SyS_read+0x55/0xa0

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 29606

0xffff880225dc3f20 29606 1 0 1 S 0xffff880225dc43a8 cron

ffff88020cc97e58 0000000000000046 0000000df8475800 000000000000c350

ffff880225dc3f20 ffff88020cc97fd8 ffff88020cc97fd8 ffff88020cc97fd8

ffff880225d29f90 ffff880225dc3f20 0000000000000002 ffff88020cc97eb8

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163471c>] do_nanosleep+0x9c/0xd0

[<ffffffff8106b6e8>] hrtimer_nanosleep+0xb8/0x180

[<ffffffff8106a360>] ? update_rmtp+0x70/0x70

[<ffffffff8106af84>] ? hrtimer_start_range_ns+0x14/0x20

[<ffffffff8106b826>] SyS_nanosleep+0x76/0x90

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b

Stack traceback for pid 31164

0xffff88020d2a9f90 31164 1 0 1 S 0xffff88020d2aa418 sshd

ffff88020b9938d8 0000000000000046 0000000000000002 0000000000000000

ffff88020d2a9f90 ffff88020b993fd8 ffff88020b993fd8 ffff88020b993fd8

ffff880225d29f90 ffff88020d2a9f90 0000000000000000 0000000000000000

Call Trace:

[<ffffffff81636109>] schedule+0x29/0x70

[<ffffffff8163492d>] schedule_hrtimeout_range_clock+0x14d/0x170

[<ffffffff810a31ad>] ? trace_hardirqs_on+0xd/0x10

[<ffffffff81067a1d>] ? add_wait_queue+0x4d/0x60

[<ffffffff81634963>] schedule_hrtimeout_range+0x13/0x20

[<ffffffff81182b69>] poll_schedule_timeout+0x49/0x70

[<ffffffff81183551>] do_select+0x5d1/0x6e0

[<ffffffff81182f85>] ? do_select+0x5/0x6e0

[<ffffffff811b17a4>] ? fsnotify_clear_marks_by_inode+0x34/0x130

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81182d60>] ? __pollwait+0xf0/0xf0

[<ffffffff81187ca5>] ? __d_lookup+0x5/0x1c0

[<ffffffff81187ca5>] ? __d_lookup+0x5/0x1c0

[<ffffffff813e0fed>] ? mix_pool_bytes.constprop.16+0x4d/0xd0

[<ffffffff813e0fed>] ? mix_pool_bytes.constprop.16+0x4d/0xd0

[<ffffffff81187d78>] ? __d_lookup+0xd8/0x1c0

[<ffffffff81187ca5>] ? __d_lookup+0x5/0x1c0

[<ffffffff811b17a4>] ? fsnotify_clear_marks_by_inode+0x34/0x130

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff8113e395>] ? might_fault+0x45/0xa0

[<ffffffff81183874>] core_sys_select+0x214/0x340

[<ffffffff811836a2>] ? core_sys_select+0x42/0x340

[<ffffffff8118ecbc>] ? mntput_no_expire+0x2c/0x160

[<ffffffff8106f1a3>] ? lg_local_unlock+0x23/0x40

[<ffffffff8118ecd9>] ? mntput_no_expire+0x49/0x160

[<ffffffff8118ee14>] ? mntput+0x24/0x40

[<ffffffff81170ef9>] ? __fput+0x189/0x240

[<ffffffff81183a61>] SyS_select+0xc1/0x100

[<ffffffff8131e9ae>] ? trace_hardirqs_on_thunk+0x3a/0x3f

[<ffffffff81640342>] system_call_fastpath+0x16/0x1b