[4]kdb> dumpall [dumpall]kdb> pid R KDB current process is kworker/4:1(pid=17381) [dumpall]kdb> -dumpcommon [dumpcommon]kdb> set BTAPROMPT 0 [dumpcommon]kdb> set LINES 10000 [dumpcommon]kdb> -summary sysname Linux release 3.10.0-rc7-ceph-00049-g9e41c3a version #1 SMP Thu Jun 27 06:41:24 PDT 2013 machine x86_64 nodename plana72 domainname (none) ccversion CCVERSION date 2013-06-28 12:37:39 tz_minuteswest 420 uptime 09:33 load avg 29.95 11.20 5.32 MemTotal: 2038245 kB MemFree: 722832 kB Buffers: 200 kB [dumpcommon]kdb> -cpu Currently on cpu 4 Available cpus: 0-3(I), 4, 5-7(I) [dumpcommon]kdb> -ps 7 idle processes (state I) and 111 sleeping system daemon (state M) processes suppressed, use 'ps A' to see all. Task Addr Pid Parent [*] cpu State Thread Command 0xffff880216983f20 17381 2 1 4 R 0xffff8802169843a8 *kworker/4:1 0xffff880225ca8000 1 0 0 0 S 0xffff880225ca8488 init 0xffff88020c711f90 494 1 0 3 S 0xffff88020c712418 upstart-udev-br 0xffff88020cf23f20 498 1 0 1 S 0xffff88020cf243a8 udevd 0xffff88020cf25eb0 538 1 0 0 S 0xffff88020cf26338 dbus-daemon 0xffff880215678000 923 1 0 5 S 0xffff880215678488 rpc.idmapd 0xffff8802156cdeb0 983 1 0 1 S 0xffff8802156ce338 upstart-socket- 0xffff880219f5deb0 990 1 0 3 S 0xffff880219f5e338 rpcbind 0xffff880215618000 1049 1 0 0 S 0xffff880215618488 rpc.statd 0xffff88020cfebf20 1207 1 0 0 S 0xffff88020cfec3a8 getty 0xffff88020c713f20 1222 1 0 6 S 0xffff88020c7143a8 getty 0xffff88020c588000 1237 1 0 6 S 0xffff88020c588488 getty 0xffff8802244c5eb0 1238 1 0 7 S 0xffff8802244c6338 getty 0xffff880224d4deb0 1240 1 0 6 S 0xffff880224d4e338 getty 0xffff8802156a1f90 1246 1 0 0 S 0xffff8802156a2418 atd 0xffff880224d4bf20 1247 1 0 0 S 0xffff880224d4c3a8 xinetd 0xffff880215665eb0 1255 1 0 2 S 0xffff880215666338 nrpe 0xffff88020c67bf20 1333 1 0 1 S 0xffff88020c67c3a8 irqbalance 0xffff88020bbb3f20 1349 1 0 6 S 0xffff88020bbb43a8 rpc.mountd 0xffff8802156cbf20 1400 1 0 2 S 0xffff8802156cc3a8 pkcsslotd 0xffff880221e61f90 1409 1 0 5 S 0xffff880221e62418 rwhod 0xffff88020ce51f90 1411 1409 0 1 S 0xffff88020ce52418 rwhod 0xffff88020c715eb0 1528 1 0 5 S 0xffff88020c716338 getty 0xffff88020c710000 1529 1 0 0 S 0xffff88020c710488 getty 0xffff880219f59f90 1874 1 0 6 S 0xffff880219f5a418 ntpd 0xffff880216983f20 17381 2 1 4 R 0xffff8802169843a8 *kworker/4:1 0xffff8801377f3f20 27902 2 0 5 D 0xffff8801377f43a8 kworker/u64:1 0xffff880100079f90 28210 2 0 0 D 0xffff88010007a418 kworker/0:2 0xffff88021e420000 28529 2 0 0 D 0xffff88021e420488 kworker/0:1 0xffff880222099f90 29322 1 0 0 S 0xffff88022209a418 sshd 0xffff88020c859f90 29338 29322 0 2 S 0xffff88020c85a418 sshd 0xffff88010007deb0 29372 1 0 7 S 0xffff88010007e338 rsyslogd 0xffff880224665eb0 29373 1 0 1 S 0xffff880224666338 rs:main Q:Reg 0xffff8801000f3f20 29374 1 0 1 S 0xffff8801000f43a8 rsyslogd 0xffff8801000f0000 29375 1 0 1 S 0xffff8801000f0488 rsyslogd 0xffff880225dc3f20 29606 1 0 1 S 0xffff880225dc43a8 cron 0xffff88020d2a9f90 31164 1 0 1 S 0xffff88020d2aa418 sshd 0xffff88020d0ebf20 31726 498 0 6 S 0xffff88020d0ec3a8 udevd 0xffff88020d0edeb0 31727 498 0 7 S 0xffff88020d0ee338 udevd 0xffff88020c51deb0 31763 29338 0 7 S 0xffff88020c51e338 bash 0xffff88020db51f90 31765 31763 0 7 S 0xffff88020db52418 ffsb.sh 0xffff8801377f5eb0 984 31765 0 0 S 0xffff8801377f6338 ffsb 0xffff8802227ddeb0 1018 31765 0 3 S 0xffff8802227de338 ffsb 0xffff88020cf20000 1019 31765 0 6 D 0xffff88020cf20488 ffsb 0xffff88020cf21f90 1020 31765 0 3 D 0xffff88020cf22418 ffsb 0xffff880224295eb0 1021 31765 0 5 D 0xffff880224296338 ffsb 0xffff880224293f20 1022 31765 0 6 D 0xffff8802242943a8 ffsb 0xffff880224290000 1023 31765 0 3 D 0xffff880224290488 ffsb 0xffff880224291f90 1024 31765 0 2 D 0xffff880224292418 ffsb 0xffff880224f48000 1025 31765 0 7 D 0xffff880224f48488 ffsb 0xffff880100078000 1026 31765 0 3 D 0xffff880100078488 ffsb 0xffff88010007bf20 1027 31765 0 3 D 0xffff88010007c3a8 ffsb 0xffff8802156a3f20 1028 31765 0 0 D 0xffff8802156a43a8 ffsb 0xffff8802156a5eb0 1029 31765 0 0 D 0xffff8802156a6338 ffsb 0xffff8802156a0000 1030 31765 0 2 D 0xffff8802156a0488 ffsb 0xffff88020bbb1f90 1031 31765 0 3 D 0xffff88020bbb2418 ffsb 0xffff880225de3f20 1032 31765 0 7 D 0xffff880225de43a8 ffsb 0xffff88020d0adeb0 1033 31765 0 1 D 0xffff88020d0ae338 ffsb 0xffff880219f0bf20 1034 31765 0 2 D 0xffff880219f0c3a8 ffsb 0xffff880219f0deb0 1035 31765 0 2 D 0xffff880219f0e338 ffsb 0xffff880219f08000 1036 31765 0 7 D 0xffff880219f08488 ffsb 0xffff880215663f20 1037 31765 0 3 D 0xffff8802156643a8 ffsb 0xffff880225d19f90 1038 31765 0 0 D 0xffff880225d1a418 ffsb 0xffff880216981f90 1039 31765 0 5 D 0xffff880216982418 ffsb 0xffff880216985eb0 1040 31765 0 0 D 0xffff880216986338 ffsb 0xffff880216980000 1041 31765 0 2 D 0xffff880216980488 ffsb 0xffff880224971f90 1042 31765 0 4 D 0xffff880224972418 ffsb 0xffff880219d81f90 1043 31765 0 7 D 0xffff880219d82418 ffsb 0xffff880219d85eb0 1044 31765 0 6 D 0xffff880219d86338 ffsb 0xffff880219d83f20 1045 31765 0 7 D 0xffff880219d843a8 ffsb 0xffff880225e4bf20 1046 31765 0 7 D 0xffff880225e4c3a8 ffsb 0xffff8801f4915eb0 1047 31765 0 3 D 0xffff8801f4916338 ffsb 0xffff8801f4910000 1048 31765 0 1 D 0xffff8801f4910488 ffsb 0xffff8801f4911f90 1050 31765 0 5 D 0xffff8801f4912418 ffsb 0xffff8801d8503f20 1051 31765 0 7 D 0xffff8801d85043a8 ffsb 0xffff88015ecd9f90 988 2 0 4 R 0xffff88015ecda418 kworker/4:2 0xffff88011e7bdeb0 1052 2 0 0 D 0xffff88011e7be338 kworker/0:3 0xffff88015ecdbf20 1053 2 0 0 D 0xffff88015ecdc3a8 kworker/0:4 0xffff88020db53f20 1054 2 0 0 D 0xffff88020db543a8 kworker/0:5 0xffff88011e7bbf20 1083 2 0 4 R 0xffff88011e7bc3a8 kworker/4:3 [dumpcommon]kdb> -dmesg 600 <4>[34289.402817] ffffffff81c89c78 ffffffff81c89b00 ffff880225d45e58 ffffffff814ed5df <4>[34289.410282] 0000000000000000 000000000260d2e4 0000000000000000 000000000260d2e4 <4>[34289.417748] Call Trace: <4>[34289.420209] [] cpuidle_enter_state+0x4f/0xe0 <4>[34289.426144] [] cpuidle_idle_call+0xce/0x240 <4>[34289.431992] [] arch_cpu_idle+0xe/0x30 <4>[34289.437318] [] cpu_startup_entry+0xce/0x2b0 <4>[34289.443166] [] start_secondary+0x26b/0x26d <6>[34294.755885] libceph: osd4 weight 0x10000 (in) <3>[34318.746693] INFO: rcu_sched self-detected stall on CPU { 4} (t=2100 jiffies g=656695 c=656694 q=2625) <6>[34318.756134] sending NMI to all CPUs: <4>[34318.759729] NMI backtrace for cpu 0 <4>[34318.763236] CPU: 0 PID: 1054 Comm: kworker/0:5 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34318.772661] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34318.780185] Workqueue: ceph-msgr con_work [libceph] <4>[34318.785090] task: ffff88020db53f20 ti: ffff88019caa8000 task.ti: ffff88019caa8000 <4>[34318.792604] RIP: 0010:[] [] putback_inactive_pages+0xbe/0x280 <3>[34318.796729] INFO: rcu_sched detected stalls on CPUs/tasks: { 4} (detected by 6, t=2105 jiffies, g=656695, c=656694, q=2625) <4>[34318.812774] RSP: 0018:ffff88019caa9498 EFLAGS: 00000002 <4>[34318.818104] RAX: 0000000000000000 RBX: ffffea000606c1e0 RCX: 020000000000282d <4>[34318.825257] RDX: 0000000000000002 RSI: 0000000000000003 RDI: ffffea000606c1c0 <4>[34318.832410] RBP: ffff88019caa9508 R08: dead000000200200 R09: 0000000000000001 <4>[34318.839564] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88019caa9570 <4>[34318.846716] R13: ffff88022fffa2b8 R14: ffffea000606c1c0 R15: ffff88022fff9e00 <4>[34318.853869] FS: 0000000000000000(0000) GS:ffff880227200000(0000) knlGS:0000000000000000 <4>[34318.861991] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34318.867754] CR2: 00000000026369a8 CR3: 0000000001c0b000 CR4: 00000000000007f0 <4>[34318.874908] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34318.882061] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34318.889214] Stack: <4>[34318.891238] 0000000000000002 ffff880200000000 0000000000000003 ffff88022fffa280 <4>[34318.898709] ffff88022fffa2f8 ffff88019caa94c8 ffff88019caa94c8 ffff88019caa94c8 <4>[34318.906175] ffff88019caa9508 ffff88022fffa2b8 ffff88019caa9750 ffff88022fff9e00 <4>[34318.913644] Call Trace: <4>[34318.916107] [] shrink_inactive_list+0x273/0x400 <4>[34318.922306] [] shrink_lruvec+0x2cd/0x4d0 <4>[34318.927897] [] ? bdi_queue_work+0x8b/0xf0 <4>[34318.933573] [] do_try_to_free_pages+0x11c/0x3a0 <4>[34318.939770] [] try_to_free_pages+0xd6/0x1b0 <4>[34318.945621] [] ? next_zone+0x30/0x40 <4>[34318.950864] [] __alloc_pages_nodemask+0x596/0x8f0 <4>[34318.957237] [] alloc_pages_current+0xba/0x170 <4>[34318.963263] [] sk_page_frag_refill+0x7e/0x130 <4>[34318.969290] [] tcp_sendmsg+0x305/0xe50 <4>[34318.974707] [] inet_sendmsg+0xb9/0xf0 <4>[34318.980036] [] ? inet_sendmsg+0x5/0xf0 <4>[34318.985454] [] sock_sendmsg+0xc2/0xe0 <4>[34318.990785] [] ? chksum_update+0x1b/0x30 <4>[34318.996376] [] ? crypto_shash_update+0x18/0x30 <4>[34319.002498] [] ? crc32c+0x56/0x7c [libcrc32c] <4>[34319.008523] [] kernel_sendmsg+0x40/0x60 <4>[34319.014030] [] con_work+0xc28/0x2d50 [libceph] <4>[34319.020142] [] ? cpuacct_charge+0x5/0xa0 <4>[34319.025732] [] ? idle_balance+0x133/0x180 <4>[34319.031408] [] ? finish_task_switch+0x48/0x110 <4>[34319.037519] [] ? finish_task_switch+0x48/0x110 <4>[34319.043632] [] process_one_work+0x1da/0x540 <4>[34319.049481] [] ? process_one_work+0x16f/0x540 <4>[34319.055506] [] worker_thread+0x11c/0x370 <4>[34319.061095] [] ? manage_workers.isra.20+0x2e0/0x2e0 <4>[34319.067643] [] kthread+0xea/0xf0 <4>[34319.072538] [] ? flush_kthread_worker+0x150/0x150 <4>[34319.078911] [] ret_from_fork+0x7c/0xb0 <4>[34319.084327] [] ? flush_kthread_worker+0x150/0x150 <4>[34319.110298] NMI backtrace for cpu 1 <4>[34319.113805] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34319.122798] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34319.130314] task: ffff880225d29f90 ti: ffff880225d30000 task.ti: ffff880225d30000 <4>[34319.137829] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34319.145796] RSP: 0018:ffff880225d31dc8 EFLAGS: 00000046 <4>[34319.151123] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34319.158276] RDX: 0000000000000000 RSI: ffff880225d31fd8 RDI: 0000000000000001 <4>[34319.165430] RBP: ffff880225d31df8 R08: ffff88022723a840 R09: 0000000000001068 <4>[34319.172583] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34319.179736] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34319.186889] FS: 0000000000000000(0000) GS:ffff880227220000(0000) knlGS:0000000000000000 <4>[34319.195013] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34319.200775] CR2: 00007f80e99e1000 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34319.207927] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34319.215081] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34319.222232] Stack: <4>[34319.224258] ffff880225d31df8 0000000181093504 ffff88022723a840 00001f31a1e6fda0 <4>[34319.231730] ffffffff81c89c78 ffffffff81c89b00 ffff880225d31e58 ffffffff814ed5df <4>[34319.239202] 0000000000000000 0000000001c7daac 0000000000000000 0000000001c7daac <4>[34319.246674] Call Trace: <4>[34319.249141] [] cpuidle_enter_state+0x4f/0xe0 <4>[34319.255080] [] cpuidle_idle_call+0xce/0x240 <4>[34319.260934] [] arch_cpu_idle+0xe/0x30 <4>[34319.266263] [] cpu_startup_entry+0xce/0x2b0 <4>[34319.272116] [] start_secondary+0x26b/0x26d <4>[34319.297531] NMI backtrace for cpu 2 <4>[34319.301041] CPU: 2 PID: 0 Comm: swapper/2 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34319.310031] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34319.317546] task: ffff880225d2bf20 ti: ffff880225d32000 task.ti: ffff880225d32000 <4>[34319.325062] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34319.333028] RSP: 0018:ffff880225d33dc8 EFLAGS: 00000046 <4>[34319.338355] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 <4>[34319.345508] RDX: 0000000000000000 RSI: ffff880225d33fd8 RDI: 0000000000000002 <4>[34319.352661] RBP: ffff880225d33df8 R08: ffff88022725a840 R09: 00000000000027d8 <4>[34319.359814] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000003 <4>[34319.366968] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 <4>[34319.374123] FS: 0000000000000000(0000) GS:ffff880227240000(0000) knlGS:0000000000000000 <4>[34319.382246] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34319.388007] CR2: 00007f77d73e3044 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34319.395162] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34319.402316] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34319.409467] Stack: <4>[34319.411492] ffff880225d33df8 0000000281093504 ffff88022725a840 00001f31a17487f2 <4>[34319.418963] ffffffff81c89c20 ffffffff81c89b00 ffff880225d33e58 ffffffff814ed5df <4>[34319.427669] 0000000000000000 0000000001a1b8aa 0000000000000000 0000000001a1b8aa <4>[34319.435142] Call Trace: <4>[34319.437609] [] cpuidle_enter_state+0x4f/0xe0 <4>[34319.443547] [] cpuidle_idle_call+0xce/0x240 <4>[34319.449400] [] arch_cpu_idle+0xe/0x30 <4>[34319.454732] [] cpu_startup_entry+0xce/0x2b0 <4>[34319.460587] [] start_secondary+0x26b/0x26d <4>[34319.485996] NMI backtrace for cpu 3 <4>[34319.489502] CPU: 3 PID: 0 Comm: swapper/3 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34319.498494] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34319.506008] task: ffff880225d2deb0 ti: ffff880225d34000 task.ti: ffff880225d34000 <4>[34319.513524] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34319.521487] RSP: 0018:ffff880225d35dc8 EFLAGS: 00000046 <4>[34319.526815] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34319.533968] RDX: 0000000000000000 RSI: ffff880225d35fd8 RDI: 0000000000000003 <4>[34319.541120] RBP: ffff880225d35df8 R08: ffff88022727a840 R09: 0000000000002fa8 <4>[34319.548273] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34319.555424] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34319.562576] FS: 0000000000000000(0000) GS:ffff880227260000(0000) knlGS:0000000000000000 <4>[34319.570707] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34319.576469] CR2: 00007f82a0a5c1d4 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34319.583621] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34319.590775] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34319.597930] Stack: <4>[34319.599955] ffff880225d35df8 0000000381093504 ffff88022727a840 00001f31a1e6a34f <4>[34319.607425] ffffffff81c89c78 ffffffff81c89b00 ffff880225d35e58 ffffffff814ed5df <4>[34319.614895] 0000000000000000 000000000260d70f 0000000000000000 000000000260d70f <4>[34319.622368] Call Trace: <4>[34319.624835] [] cpuidle_enter_state+0x4f/0xe0 <4>[34319.630771] [] cpuidle_idle_call+0xce/0x240 <4>[34319.636626] [] arch_cpu_idle+0xe/0x30 <4>[34319.641957] [] cpu_startup_entry+0xce/0x2b0 <4>[34319.647809] [] start_secondary+0x26b/0x26d <4>[34319.673236] NMI backtrace for cpu 4 <4>[34319.676745] CPU: 4 PID: 988 Comm: kworker/4:2 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34319.686088] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34319.693618] Workqueue: ceph-msgr con_work [libceph] <4>[34319.698524] task: ffff88015ecd9f90 ti: ffff880224ca2000 task.ti: ffff880224ca2000 <4>[34319.706040] RIP: 0010:[] [] delay_tsc+0x2a/0x60 <4>[34319.713833] RSP: 0018:ffff880227283d18 EFLAGS: 00000083 <4>[34319.719162] RAX: 0000000031bdd8b3 RBX: 00000000000003e9 RCX: 0000000000000004 <4>[34319.726317] RDX: 0000000000000018 RSI: 0000000031bdd89b RDI: 000000000003a724 <4>[34319.733472] RBP: ffff880227283d18 R08: 0000000000000004 R09: 0000000000000000 <4>[34319.740627] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000001000 <4>[34319.747782] R13: ffffffff81ce1bf0 R14: 0000000000000400 R15: 0000000000000092 <4>[34319.754938] FS: 0000000000000000(0000) GS:ffff880227280000(0000) knlGS:0000000000000000 <4>[34319.763063] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34319.768827] CR2: 00007f92b3cc1120 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34319.775982] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34319.783138] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34319.790292] Stack: <4>[34319.792317] ffff880227283d28 ffffffff8131d509 ffff880227283d48 ffffffff81025eaa <4>[34319.799793] 0000000000000002 000000000000b022 ffff880227283d98 ffffffff81026c9d <4>[34319.807271] 0000000000000023 0000000000000005 ffff880227283db8 0000000000002710 <4>[34319.814746] Call Trace: <4>[34319.817208] <4>[34319.819133] <4>[34319.820652] [] __const_udelay+0x29/0x30 <4>[34319.824855] [] native_safe_apic_wait_icr_idle+0x2a/0x60 <4>[34319.831751] [] default_send_IPI_mask_sequence_phys+0xed/0x100 <4>[34319.839186] [] physflat_send_IPI_all+0x17/0x20 <4>[34319.845301] [] arch_trigger_all_cpu_backtrace+0x5a/0x90 <4>[34319.852198] [] rcu_check_callbacks+0x21a/0x660 <4>[34319.858312] [] ? acct_account_cputime+0x1c/0x20 <4>[34319.864516] [] update_process_times+0x48/0x80 <4>[34319.870545] [] tick_sched_handle.isra.10+0x36/0x50 <4>[34319.877006] [] tick_sched_timer+0x4c/0x80 <4>[34319.882685] [] __run_hrtimer+0x81/0x1e0 <4>[34319.888193] [] ? tick_nohz_handler+0xa0/0xa0 <4>[34319.894134] [] hrtimer_interrupt+0x107/0x260 <4>[34319.900078] [] smp_apic_timer_interrupt+0x69/0x99 <4>[34319.906453] [] apic_timer_interrupt+0x6f/0x80 <4>[34319.912477] <4>[34319.914402] <4>[34319.915921] [] ? shrink_inactive_list+0x18c/0x400 <4>[34319.920993] [] ? _raw_spin_unlock_irq+0x30/0x40 <4>[34319.927195] [] ? _raw_spin_unlock_irq+0x35/0x40 <4>[34319.933397] [] shrink_inactive_list+0x18c/0x400 <4>[34319.939601] [] ? global_dirtyable_memory+0x1a/0x60 <4>[34319.946065] [] shrink_lruvec+0x2cd/0x4d0 <4>[34319.951659] [] ? bdi_queue_work+0x8b/0xf0 <4>[34319.957338] [] do_try_to_free_pages+0x11c/0x3a0 <4>[34319.963539] [] try_to_free_pages+0xd6/0x1b0 <4>[34319.969393] [] __alloc_pages_nodemask+0x596/0x8f0 <4>[34319.975772] [] alloc_pages_current+0xba/0x170 <4>[34319.981800] [] sk_page_frag_refill+0x7e/0x130 <4>[34319.987829] [] tcp_sendmsg+0x305/0xe50 <4>[34319.993249] [] inet_sendmsg+0xb9/0xf0 <4>[34319.998582] [] ? inet_sendmsg+0x5/0xf0 <4>[34320.004003] [] sock_sendmsg+0xc2/0xe0 <4>[34320.009337] [] ? crypto_shash_update+0x18/0x30 <4>[34320.015463] [] ? crc32c+0x56/0x7c [libcrc32c] <4>[34320.021493] [] kernel_sendmsg+0x40/0x60 <4>[34320.027007] [] con_work+0xc28/0x2d50 [libceph] <4>[34320.033124] [] ? cpuacct_charge+0x5/0xa0 <4>[34320.038717] [] ? idle_balance+0x133/0x180 <4>[34320.044396] [] ? finish_task_switch+0x48/0x110 <4>[34320.050510] [] ? finish_task_switch+0x48/0x110 <4>[34320.056624] [] process_one_work+0x1da/0x540 <4>[34320.062477] [] ? process_one_work+0x16f/0x540 <4>[34320.068529] [] worker_thread+0x11c/0x370 <4>[34320.074123] [] ? manage_workers.isra.20+0x2e0/0x2e0 <4>[34320.080672] [] kthread+0xea/0xf0 <4>[34320.085571] [] ? flush_kthread_worker+0x150/0x150 <4>[34320.091947] [] ret_from_fork+0x7c/0xb0 <4>[34320.097367] [] ? flush_kthread_worker+0x150/0x150 <4>[34320.123418] NMI backtrace for cpu 5 <4>[34320.126926] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34320.135918] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34320.143433] task: ffff880225d39f90 ti: ffff880225d40000 task.ti: ffff880225d40000 <4>[34320.150947] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34320.158912] RSP: 0018:ffff880225d41dc8 EFLAGS: 00000046 <4>[34320.164239] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34320.171392] RDX: 0000000000000000 RSI: ffff880225d41fd8 RDI: 0000000000000005 <4>[34320.178544] RBP: ffff880225d41df8 R08: ffff8802272ba840 R09: 0000000000001838 <4>[34320.185696] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34320.192849] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34320.200003] FS: 0000000000000000(0000) GS:ffff8802272a0000(0000) knlGS:0000000000000000 <4>[34320.208124] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34320.213888] CR2: 00007f3ebc574000 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34320.221040] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34320.228194] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34320.235346] Stack: <4>[34320.237372] ffff880225d41df8 0000000581093504 ffff8802272ba840 00001f31f2f61bde <4>[34320.244842] ffffffff81c89c78 ffffffff81c89b00 ffff880225d41e58 ffffffff814ed5df <4>[34320.252315] 0000000000000000 00000000026153c0 0000000000000000 00000000026153c0 <4>[34320.259788] Call Trace: <4>[34320.262255] [] cpuidle_enter_state+0x4f/0xe0 <4>[34320.268192] [] cpuidle_idle_call+0xce/0x240 <4>[34320.274045] [] arch_cpu_idle+0xe/0x30 <4>[34320.279375] [] cpu_startup_entry+0xce/0x2b0 <4>[34320.285228] [] start_secondary+0x26b/0x26d <4>[34320.310644] NMI backtrace for cpu 6 <4>[34320.314152] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34320.323142] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34320.330659] task: ffff880225d3bf20 ti: ffff880225d42000 task.ti: ffff880225d42000 <4>[34320.338173] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34320.346135] RSP: 0018:ffff880225d43dc8 EFLAGS: 00000046 <4>[34320.351466] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34320.359776] RDX: 0000000000000000 RSI: ffff880225d43fd8 RDI: 0000000000000006 <4>[34320.366928] RBP: ffff880225d43df8 R08: ffff8802272da840 R09: 0000000000001068 <4>[34320.374083] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34320.381235] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34320.388387] FS: 0000000000000000(0000) GS:ffff8802272c0000(0000) knlGS:0000000000000000 <4>[34320.396512] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34320.402275] CR2: 0000000000a23d60 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34320.409427] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34320.416582] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34320.423735] Stack: <4>[34320.425759] ffff880225d43df8 0000000681093504 ffff8802272da840 00001f31f2f62d35 <4>[34320.433229] ffffffff81c89c78 ffffffff81c89b00 ffff880225d43e58 ffffffff814ed5df <4>[34320.440704] 0000000000000000 00000000026145a0 0000000000000000 00000000026145a0 <4>[34320.448177] Call Trace: <4>[34320.450643] [] cpuidle_enter_state+0x4f/0xe0 <4>[34320.456579] [] cpuidle_idle_call+0xce/0x240 <4>[34320.462434] [] arch_cpu_idle+0xe/0x30 <4>[34320.467765] [] cpu_startup_entry+0xce/0x2b0 <4>[34320.473617] [] start_secondary+0x26b/0x26d <4>[34320.499049] NMI backtrace for cpu 7 <4>[34320.502557] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34320.511550] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34320.519063] task: ffff880225d3deb0 ti: ffff880225d44000 task.ti: ffff880225d44000 <4>[34320.526577] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34320.534540] RSP: 0018:ffff880225d45dc8 EFLAGS: 00000046 <4>[34320.539868] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34320.547020] RDX: 0000000000000000 RSI: ffff880225d45fd8 RDI: 0000000000000007 <4>[34320.554173] RBP: ffff880225d45df8 R08: ffff8802272fa840 R09: 0000000000003778 <4>[34320.561325] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34320.568516] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34320.575669] FS: 0000000000000000(0000) GS:ffff8802272e0000(0000) knlGS:0000000000000000 <4>[34320.583792] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34320.589554] CR2: 00007fff90652ff0 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34320.596707] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34320.603860] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34320.611011] Stack: <4>[34320.613036] ffff880225d45df8 0000000781093504 ffff8802272fa840 00001f31f2f679a2 <4>[34320.620506] ffffffff81c89c78 ffffffff81c89b00 ffff880225d45e58 ffffffff814ed5df <4>[34320.627976] 0000000000000000 000000000260ec8f 0000000000000000 000000000260ec8f <4>[34320.635444] Call Trace: <4>[34320.637911] [] cpuidle_enter_state+0x4f/0xe0 <4>[34320.643849] [] cpuidle_idle_call+0xce/0x240 <4>[34320.649702] [] arch_cpu_idle+0xe/0x30 <4>[34320.655031] [] cpu_startup_entry+0xce/0x2b0 <4>[34320.660884] [] start_secondary+0x26b/0x26d <3>[34341.710486] INFO: rcu_sched self-detected stall on CPU { 4} (t=2101 jiffies g=656696 c=656695 q=3333) <6>[34341.719916] sending NMI to all CPUs: <3>[34341.720494] INFO: rcu_sched detected stalls on CPUs/tasks: { 4} (detected by 0, t=2102 jiffies, g=656696, c=656695, q=3333) <4>[34341.734677] NMI backtrace for cpu 0 <4>[34341.738184] CPU: 0 PID: 1054 Comm: kworker/0:5 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34341.747610] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34341.755134] Workqueue: ceph-msgr con_work [libceph] <4>[34341.760038] task: ffff88020db53f20 ti: ffff88019caa8000 task.ti: ffff88019caa8000 <4>[34341.767552] RIP: 0010:[] [] shrink_inactive_list+0xe4/0x400 <4>[34341.776384] RSP: 0018:ffff88019caa9518 EFLAGS: 00000202 <4>[34341.781713] RAX: 0000000000000001 RBX: ffff88022fffa2b8 RCX: 0000000000000000 <4>[34341.788866] RDX: ffff88022fffa280 RSI: 0000000000000000 RDI: ffff88022fffa280 <4>[34341.796019] RBP: ffff88019caa95c8 R08: 00000000000036e1 R09: 0000000000000000 <4>[34341.803173] R10: 0000000000000001 R11: 0000000000000001 R12: ffff88019caa9750 <4>[34341.810326] R13: ffff88022fff9e00 R14: ffff88019caa9570 R15: 0000000000000020 <4>[34341.817480] FS: 0000000000000000(0000) GS:ffff880227200000(0000) knlGS:0000000000000000 <4>[34341.825602] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34341.831363] CR2: 00000000026369a8 CR3: 0000000001c0b000 CR4: 00000000000007f0 <4>[34341.838516] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34341.845670] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34341.852822] Stack: <4>[34341.854846] ffff880100000000 0000000000000000 0000000000000801 0000000000000040 <4>[34341.862313] ffff880200000000 0000001625f56000 00000000ffffffe0 0000000000000001 <4>[34341.869784] ffff88022fffa280 0000000000000002 ffff88019caa9598 ffff88019caa9570 <4>[34341.877251] Call Trace: <4>[34341.879713] [] shrink_lruvec+0x2cd/0x4d0 <4>[34341.885304] [] ? bdi_queue_work+0x8b/0xf0 <4>[34341.890981] [] do_try_to_free_pages+0x11c/0x3a0 <4>[34341.897179] [] try_to_free_pages+0xd6/0x1b0 <4>[34341.903033] [] __alloc_pages_nodemask+0x596/0x8f0 <4>[34341.909405] [] alloc_pages_current+0xba/0x170 <4>[34341.915432] [] sk_page_frag_refill+0x7e/0x130 <4>[34341.921459] [] tcp_sendmsg+0x305/0xe50 <4>[34341.926876] [] inet_sendmsg+0xb9/0xf0 <4>[34341.932206] [] ? inet_sendmsg+0x5/0xf0 <4>[34341.937624] [] sock_sendmsg+0xc2/0xe0 <4>[34341.942955] [] ? chksum_update+0x1b/0x30 <4>[34341.948546] [] ? crypto_shash_update+0x18/0x30 <4>[34341.954668] [] ? crc32c+0x56/0x7c [libcrc32c] <4>[34341.960692] [] kernel_sendmsg+0x40/0x60 <4>[34341.966199] [] con_work+0xc28/0x2d50 [libceph] <4>[34341.972311] [] ? cpuacct_charge+0x5/0xa0 <4>[34341.977901] [] ? idle_balance+0x133/0x180 <4>[34341.983578] [] ? finish_task_switch+0x48/0x110 <4>[34341.989690] [] ? finish_task_switch+0x48/0x110 <4>[34341.995802] [] process_one_work+0x1da/0x540 <4>[34342.001651] [] ? process_one_work+0x16f/0x540 <4>[34342.007676] [] worker_thread+0x11c/0x370 <4>[34342.013265] [] ? manage_workers.isra.20+0x2e0/0x2e0 <4>[34342.019811] [] kthread+0xea/0xf0 <4>[34342.024706] [] ? flush_kthread_worker+0x150/0x150 <4>[34342.031079] [] ret_from_fork+0x7c/0xb0 <4>[34342.036496] [] ? flush_kthread_worker+0x150/0x150 <4>[34342.062455] NMI backtrace for cpu 1 <4>[34342.065963] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34342.074955] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34342.082472] task: ffff880225d29f90 ti: ffff880225d30000 task.ti: ffff880225d30000 <4>[34342.089986] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34342.097951] RSP: 0018:ffff880225d31dc8 EFLAGS: 00000046 <4>[34342.103278] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34342.110433] RDX: 0000000000000000 RSI: ffff880225d31fd8 RDI: 0000000000000001 <4>[34342.117585] RBP: ffff880225d31df8 R08: ffff88022723a840 R09: 0000000000001068 <4>[34342.124739] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34342.131892] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34342.139046] FS: 0000000000000000(0000) GS:ffff880227220000(0000) knlGS:0000000000000000 <4>[34342.147169] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34342.152932] CR2: 00007f80e99e1000 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34342.160085] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34342.167238] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34342.174391] Stack: <4>[34342.176416] ffff880225d31df8 0000000181093504 ffff88022723a840 00001f36fa6c95e2 <4>[34342.183889] ffffffff81c89c78 ffffffff81c89b00 ffff880225d31e58 ffffffff814ed5df <4>[34342.191360] 0000000000000000 000000000260be45 0000000000000000 000000000260be45 <4>[34342.198832] Call Trace: <4>[34342.201299] [] cpuidle_enter_state+0x4f/0xe0 <4>[34342.207237] [] cpuidle_idle_call+0xce/0x240 <4>[34342.213091] [] arch_cpu_idle+0xe/0x30 <4>[34342.218422] [] cpu_startup_entry+0xce/0x2b0 <4>[34342.224275] [] start_secondary+0x26b/0x26d <4>[34342.249681] NMI backtrace for cpu 2 <4>[34342.253188] CPU: 2 PID: 0 Comm: swapper/2 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34342.262181] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34342.269697] task: ffff880225d2bf20 ti: ffff880225d32000 task.ti: ffff880225d32000 <4>[34342.277213] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34342.285180] RSP: 0018:ffff880225d33dc8 EFLAGS: 00000046 <4>[34342.290508] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34342.297662] RDX: 0000000000000000 RSI: ffff880225d33fd8 RDI: 0000000000000002 <4>[34342.304813] RBP: ffff880225d33df8 R08: ffff88022725a840 R09: 0000000000002fa8 <4>[34342.311967] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34342.320270] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34342.327422] FS: 0000000000000000(0000) GS:ffff880227240000(0000) knlGS:0000000000000000 <4>[34342.335545] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34342.341307] CR2: 00007f77d73e3044 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34342.348458] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34342.355612] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34342.362764] Stack: <4>[34342.364790] ffff880225d33df8 0000000281093504 ffff88022725a840 00001f36fa6c3c3e <4>[34342.372260] ffffffff81c89c78 ffffffff81c89b00 ffff880225d33e58 ffffffff814ed5df <4>[34342.379730] 0000000000000000 0000000001c9194a 0000000000000000 0000000001c9194a <4>[34342.387202] Call Trace: <4>[34342.389670] [] cpuidle_enter_state+0x4f/0xe0 <4>[34342.395607] [] cpuidle_idle_call+0xce/0x240 <4>[34342.401462] [] arch_cpu_idle+0xe/0x30 <4>[34342.406792] [] cpu_startup_entry+0xce/0x2b0 <4>[34342.412643] [] start_secondary+0x26b/0x26d <4>[34342.438056] NMI backtrace for cpu 3 <4>[34342.441563] CPU: 3 PID: 0 Comm: swapper/3 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34342.450553] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34342.458069] task: ffff880225d2deb0 ti: ffff880225d34000 task.ti: ffff880225d34000 <4>[34342.465581] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34342.473546] RSP: 0018:ffff880225d35dc8 EFLAGS: 00000046 <4>[34342.478874] RAX: 0000000000000010 RBX: 0000000000000004 RCX: 0000000000000001 <4>[34342.486026] RDX: 0000000000000000 RSI: ffff880225d35fd8 RDI: 0000000000000003 <4>[34342.493179] RBP: ffff880225d35df8 R08: ffff88022727a840 R09: 0000000000002fa8 <4>[34342.500331] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000003 <4>[34342.507485] R13: 0000000000000010 R14: 0000000000000002 R15: 0000000000000003 <4>[34342.514637] FS: 0000000000000000(0000) GS:ffff880227260000(0000) knlGS:0000000000000000 <4>[34342.522759] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34342.528520] CR2: 00007f82a0a5c1d4 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34342.535674] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34342.542828] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34342.549980] Stack: <4>[34342.552005] ffff880225d35df8 0000000381093504 ffff88022727a840 00001f36fa6cdaf4 <4>[34342.559475] ffffffff81c89c20 ffffffff81c89b00 ffff880225d35e58 ffffffff814ed5df <4>[34342.566943] 0000000000000000 00000000026060db 0000000000000000 00000000026060db <4>[34342.574412] Call Trace: <4>[34342.576876] [] cpuidle_enter_state+0x4f/0xe0 <4>[34342.582856] [] cpuidle_idle_call+0xce/0x240 <4>[34342.588708] [] arch_cpu_idle+0xe/0x30 <4>[34342.594039] [] cpu_startup_entry+0xce/0x2b0 <4>[34342.599890] [] start_secondary+0x26b/0x26d <4>[34342.625325] NMI backtrace for cpu 4 <4>[34342.628834] CPU: 4 PID: 988 Comm: kworker/4:2 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34342.638175] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34342.645703] Workqueue: ceph-msgr con_work [libceph] <4>[34342.650610] task: ffff88015ecd9f90 ti: ffff880224ca2000 task.ti: ffff880224ca2000 <4>[34342.658127] RIP: 0010:[] [] delay_tsc+0x2a/0x60 <4>[34342.665922] RSP: 0018:ffff880227283d18 EFLAGS: 00000083 <4>[34342.671252] RAX: 00000000fe188365 RBX: 00000000000003e9 RCX: 0000000000000004 <4>[34342.678408] RDX: 0000000000000015 RSI: 00000000fe188350 RDI: 000000000003a724 <4>[34342.685564] RBP: ffff880227283d18 R08: 0000000000000004 R09: 0000000000000000 <4>[34342.692719] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000001000 <4>[34342.699873] R13: ffffffff81ce1bf0 R14: 0000000000000400 R15: 0000000000000092 <4>[34342.707030] FS: 0000000000000000(0000) GS:ffff880227280000(0000) knlGS:0000000000000000 <4>[34342.715154] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34342.720917] CR2: 00007f92b3cc1120 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34342.728072] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34342.735226] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34342.742382] Stack: <4>[34342.744410] ffff880227283d28 ffffffff8131d509 ffff880227283d48 ffffffff81025eaa <4>[34342.751886] 0000000000000002 000000000000b022 ffff880227283d98 ffffffff81026c9d <4>[34342.759363] 0000000000000023 0000000000000005 ffff880227283db8 0000000000002710 <4>[34342.766837] Call Trace: <4>[34342.769298] <4>[34342.771226] <4>[34342.772746] [] __const_udelay+0x29/0x30 <4>[34342.776949] [] native_safe_apic_wait_icr_idle+0x2a/0x60 <4>[34342.783847] [] default_send_IPI_mask_sequence_phys+0xed/0x100 <4>[34342.791284] [] physflat_send_IPI_all+0x17/0x20 <4>[34342.797399] [] arch_trigger_all_cpu_backtrace+0x5a/0x90 <4>[34342.804297] [] rcu_check_callbacks+0x21a/0x660 <4>[34342.810410] [] ? acct_account_cputime+0x1c/0x20 <4>[34342.816614] [] update_process_times+0x48/0x80 <4>[34342.822643] [] tick_sched_handle.isra.10+0x36/0x50 <4>[34342.829105] [] tick_sched_timer+0x4c/0x80 <4>[34342.834786] [] __run_hrtimer+0x81/0x1e0 <4>[34342.840293] [] ? tick_nohz_handler+0xa0/0xa0 <4>[34342.846233] [] hrtimer_interrupt+0x107/0x260 <4>[34342.852175] [] smp_apic_timer_interrupt+0x69/0x99 <4>[34342.858552] [] apic_timer_interrupt+0x6f/0x80 <4>[34342.864575] <4>[34342.866502] <4>[34342.868021] [] ? shrink_inactive_list+0x28d/0x400 <4>[34342.873094] [] ? _raw_spin_unlock_irq+0x30/0x40 <4>[34342.879298] [] ? _raw_spin_unlock_irq+0x35/0x40 <4>[34342.885500] [] shrink_inactive_list+0x28d/0x400 <4>[34342.891701] [] shrink_lruvec+0x2cd/0x4d0 <4>[34342.897294] [] ? bdi_queue_work+0x8b/0xf0 <4>[34342.902974] [] do_try_to_free_pages+0x11c/0x3a0 <4>[34342.909175] [] try_to_free_pages+0xd6/0x1b0 <4>[34342.915029] [] ? next_zone+0x30/0x40 <4>[34342.920276] [] __alloc_pages_nodemask+0x596/0x8f0 <4>[34342.926653] [] alloc_pages_current+0xba/0x170 <4>[34342.932683] [] sk_page_frag_refill+0x7e/0x130 <4>[34342.938713] [] tcp_sendmsg+0x305/0xe50 <4>[34342.944133] [] inet_sendmsg+0xb9/0xf0 <4>[34342.949465] [] ? inet_sendmsg+0x5/0xf0 <4>[34342.954885] [] sock_sendmsg+0xc2/0xe0 <4>[34342.960219] [] ? crypto_shash_update+0x18/0x30 <4>[34342.966345] [] ? crc32c+0x56/0x7c [libcrc32c] <4>[34342.972375] [] kernel_sendmsg+0x40/0x60 <4>[34342.977889] [] con_work+0xc28/0x2d50 [libceph] <4>[34342.984005] [] ? cpuacct_charge+0x5/0xa0 <4>[34342.989597] [] ? idle_balance+0x133/0x180 <4>[34342.995278] [] ? finish_task_switch+0x48/0x110 <4>[34343.001392] [] ? finish_task_switch+0x48/0x110 <4>[34343.007507] [] process_one_work+0x1da/0x540 <4>[34343.013360] [] ? process_one_work+0x16f/0x540 <4>[34343.019387] [] worker_thread+0x11c/0x370 <4>[34343.024980] [] ? manage_workers.isra.20+0x2e0/0x2e0 <4>[34343.031530] [] kthread+0xea/0xf0 <4>[34343.036429] [] ? flush_kthread_worker+0x150/0x150 <4>[34343.042805] [] ret_from_fork+0x7c/0xb0 <4>[34343.048224] [] ? flush_kthread_worker+0x150/0x150 <4>[34343.074310] NMI backtrace for cpu 5 <4>[34343.077817] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34343.086808] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34343.094323] task: ffff880225d39f90 ti: ffff880225d40000 task.ti: ffff880225d40000 <4>[34343.101836] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34343.109799] RSP: 0018:ffff880225d41dc8 EFLAGS: 00000046 <4>[34343.115127] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34343.122279] RDX: 0000000000000000 RSI: ffff880225d41fd8 RDI: 0000000000000005 <4>[34343.129431] RBP: ffff880225d41df8 R08: ffff8802272ba840 R09: 0000000000001838 <4>[34343.136584] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34343.143737] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34343.150890] FS: 0000000000000000(0000) GS:ffff8802272a0000(0000) knlGS:0000000000000000 <4>[34343.159015] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34343.164777] CR2: 00007f3ebc574000 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34343.171930] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34343.179082] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34343.186236] Stack: <4>[34343.188261] ffff880225d41df8 0000000581093504 ffff8802272ba840 00001f374a4a710c <4>[34343.195732] ffffffff81c89c78 ffffffff81c89b00 ffff880225d41e58 ffffffff814ed5df <4>[34343.203206] 0000000000000000 0000000002614f50 0000000000000000 0000000002614f50 <4>[34343.210678] Call Trace: <4>[34343.213144] [] cpuidle_enter_state+0x4f/0xe0 <4>[34343.219083] [] cpuidle_idle_call+0xce/0x240 <4>[34343.224935] [] arch_cpu_idle+0xe/0x30 <4>[34343.230266] [] cpu_startup_entry+0xce/0x2b0 <4>[34343.236118] [] start_secondary+0x26b/0x26d <4>[34343.262665] NMI backtrace for cpu 6 <4>[34343.266173] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34343.275164] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34343.282681] task: ffff880225d3bf20 ti: ffff880225d42000 task.ti: ffff880225d42000 <4>[34343.290196] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34343.298162] RSP: 0018:ffff880225d43dc8 EFLAGS: 00000046 <4>[34343.303490] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34343.310643] RDX: 0000000000000000 RSI: ffff880225d43fd8 RDI: 0000000000000006 <4>[34343.317795] RBP: ffff880225d43df8 R08: ffff8802272da840 R09: 0000000000001068 <4>[34343.324948] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34343.332101] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34343.339254] FS: 0000000000000000(0000) GS:ffff8802272c0000(0000) knlGS:0000000000000000 <4>[34343.347378] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34343.353141] CR2: 0000000000a23d60 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34343.360294] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34343.367447] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34343.374599] Stack: <4>[34343.376626] ffff880225d43df8 0000000681093504 ffff8802272da840 00001f374919fd92 <4>[34343.384098] ffffffff81c89c78 ffffffff81c89b00 ffff880225d43e58 ffffffff814ed5df <4>[34343.391569] 0000000000000000 000000000260deeb 0000000000000000 000000000260deeb <4>[34343.399040] Call Trace: <4>[34343.401506] [] cpuidle_enter_state+0x4f/0xe0 <4>[34343.407444] [] cpuidle_idle_call+0xce/0x240 <4>[34343.413298] [] arch_cpu_idle+0xe/0x30 <4>[34343.418628] [] cpu_startup_entry+0xce/0x2b0 <4>[34343.424481] [] start_secondary+0x26b/0x26d <4>[34343.449904] NMI backtrace for cpu 7 <4>[34343.453412] CPU: 7 PID: 0 Comm: swapper/7 Tainted: G W 3.10.0-rc7-ceph-00049-g9e41c3a #1 <4>[34343.462404] Hardware name: Dell Inc. PowerEdge R410/01V648, BIOS 1.6.3 02/07/2011 <4>[34343.469918] task: ffff880225d3deb0 ti: ffff880225d44000 task.ti: ffff880225d44000 <4>[34343.477433] RIP: 0010:[] [] intel_idle+0xa7/0x100 <4>[34343.485398] RSP: 0018:ffff880225d45dc8 EFLAGS: 00000046 <4>[34343.490726] RAX: 0000000000000020 RBX: 0000000000000008 RCX: 0000000000000001 <4>[34343.497877] RDX: 0000000000000000 RSI: ffff880225d45fd8 RDI: 0000000000000007 <4>[34343.505030] RBP: ffff880225d45df8 R08: ffff8802272fa840 R09: 0000000000003778 <4>[34343.512183] R10: 0000000000000001 R11: 0000000000000000 R12: 0000000000000004 <4>[34343.519336] R13: 0000000000000020 R14: 0000000000000003 R15: 0000000000000004 <4>[34343.526488] FS: 0000000000000000(0000) GS:ffff8802272e0000(0000) knlGS:0000000000000000 <4>[34343.534612] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b <4>[34343.540374] CR2: 00007fff90652ff0 CR3: 0000000001c0b000 CR4: 00000000000007e0 <4>[34343.547527] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>[34343.554679] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>[34343.561831] Stack: <4>[34343.563857] ffff880225d45df8 0000000781093504 ffff8802272fa840 00001f37491a0150 <4>[34343.571326] ffffffff81c89c78 ffffffff81c89b00 ffff880225d45e58 ffffffff814ed5df <4>[34343.578797] 0000000000000000 000000000260d7fc 0000000000000000 000000000260d7fc <4>[34343.586267] Call Trace: <4>[34343.588732] [] cpuidle_enter_state+0x4f/0xe0 <4>[34343.594671] [] cpuidle_idle_call+0xce/0x240 <4>[34343.600524] [] arch_cpu_idle+0xe/0x30 <4>[34343.605853] [] cpu_startup_entry+0xce/0x2b0 <4>[34343.611706] [] start_secondary+0x26b/0x26d <6>[34343.649514] libceph: osd0 down <6>[34343.652702] libceph: osd3 weight 0x0 (out) <4>[34343.683502] libceph: osd4 10.214.132.25:6813 socket error on write <4>[34343.689775] libceph: osd0 10.214.132.26:6808 socket closed (con state OPEN) <1>[34344.361875] BUG: unable to handle kernel NULL pointer dereference at (null) <1>[34344.369766] IP: [] rb_erase+0x1a3/0x370 <4>[34344.375194] PGD 0 <4>[34344.377229] Oops: 0000 [#1] SMP [dumpcommon]kdb> -bt Stack traceback for pid 17381 0xffff880216983f20 17381 2 1 4 R 0xffff8802169843a8 *kworker/4:1 ffff8800c791bb48 0000000000000018 0000000000000000 ffff8800c791bb68 ffffffffa07a97bc ffff8801fa10c000 ffff880114b3f950 ffff8800c791bb98 ffffffffa07a9abf ffff8801fa10c030 ffff880114b3f950 0000000000000000 Call Trace: [] ? __remove_osd+0x3c/0xa0 [libceph] [] ? __reset_osd+0x12f/0x170 [libceph] [] ? osd_reset+0x7e/0x2b0 [libceph] [] ? con_work+0x571/0x2d50 [libceph] [] ? cpuacct_charge+0x5/0xa0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] ? process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] ? worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] ? kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ? ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 [dumpall]kdb> -bta 7 idle processes (state I) and 111 sleeping system daemon (state M) processes suppressed, use 'ps A' to see all. Stack traceback for pid 17381 0xffff880216983f20 17381 2 1 4 R 0xffff8802169843a8 *kworker/4:1 ffff8800c791bb48 0000000000000018 0000000000000000 ffff8800c791bb68 ffffffffa07a97bc ffff8801fa10c000 ffff880114b3f950 ffff8800c791bb98 ffffffffa07a9abf ffff8801fa10c030 ffff880114b3f950 0000000000000000 Call Trace: [] ? __remove_osd+0x3c/0xa0 [libceph] [] ? __reset_osd+0x12f/0x170 [libceph] [] ? osd_reset+0x7e/0x2b0 [libceph] [] ? con_work+0x571/0x2d50 [libceph] [] ? cpuacct_charge+0x5/0xa0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] ? process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] ? worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] ? kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ? ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 1 0xffff880225ca8000 1 0 0 0 S 0xffff880225ca8488 init ffff880225c8b8d8 0000000000000046 ffffffff8105eb35 ffffffff81c90600 ffff880225ca8000 ffff880225c8bfd8 ffff880225c8bfd8 ffff880225c8bfd8 ffffffff81c10440 ffff880225ca8000 0000000000000000 0000000000000000 Call Trace: [] ? flush_work+0x5/0x280 [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x8e/0xa0 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 494 0xffff88020c711f90 494 1 0 3 S 0xffff88020c712418 upstart-udev-br ffff88020cb458d8 0000000000000046 0000000000000002 0000000000000000 ffff88020c711f90 ffff88020cb45fd8 ffff88020cb45fd8 ffff88020cb45fd8 ffff880225d2deb0 ffff88020c711f90 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? unix_write_space+0x5/0xc0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? sock_recvmsg+0xd8/0xf0 [] ? pipe_read+0x7f/0x560 [] ? pipe_read+0x7f/0x560 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? trace_hardirqs_on+0xd/0x10 [] ? remove_wait_queue+0x5a/0x70 [] ? do_wait+0x12c/0x260 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 498 0xffff88020cf23f20 498 1 0 1 S 0xffff88020cf243a8 udevd ffff88020d347de8 0000000000000046 0000000000000002 0000000000000000 ffff88020cf23f20 ffff88020d347fd8 ffff88020d347fd8 ffff88020d347fd8 ffff880225d29f90 ffff88020cf23f20 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] schedule_hrtimeout_range+0x13/0x20 [] SyS_epoll_wait+0x418/0x4c0 [] ? vfs_write+0x18e/0x1e0 [] ? try_to_wake_up+0x2f0/0x2f0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 538 0xffff88020cf25eb0 538 1 0 0 S 0xffff88020cf26338 dbus-daemon ffff88020c5e3a38 0000000000000046 0000000000000002 0000000000000000 ffff88020cf25eb0 ffff88020c5e3fd8 ffff88020c5e3fd8 ffff88020c5e3fd8 ffffffff81c10440 ffff88020cf25eb0 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_sys_poll+0x40f/0x4d0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? might_fault+0x8e/0xa0 [] ? might_fault+0x45/0xa0 [] ? ___sys_recvmsg+0x166/0x2a0 [] ? __do_page_fault+0x2ac/0x520 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? finish_task_switch+0x85/0x110 [] ? finish_task_switch+0x48/0x110 [] SyS_poll+0x6b/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 923 0xffff880215678000 923 1 0 5 S 0xffff880215678488 rpc.idmapd ffff88020b0a3de8 0000000000000046 0000000000000002 0000000000000002 ffff880215678000 ffff88020b0a3fd8 ffff88020b0a3fd8 ffff88020b0a3fd8 ffff880225d39f90 ffff880215678000 ffff880200000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? lg_local_unlock+0x23/0x40 [] schedule_hrtimeout_range+0x13/0x20 [] SyS_epoll_wait+0x418/0x4c0 [] ? try_to_wake_up+0x2f0/0x2f0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 983 0xffff8802156cdeb0 983 1 0 1 S 0xffff8802156ce338 upstart-socket- ffff8802245878d8 0000000000000046 0000000000000000 ffffffff810679f3 ffff8802156cdeb0 ffff880224587fd8 ffff880224587fd8 ffff880224587fd8 ffff880225d29f90 ffff8802156cdeb0 0000000000000000 0000000000000000 Call Trace: [] ? add_wait_queue+0x23/0x60 [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? unix_write_space+0x5/0xc0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? touch_atime+0xd5/0x170 [] ? sock_recvmsg+0xd8/0xf0 [] ? pipe_read+0x7f/0x560 [] ? pipe_read+0x7f/0x560 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? trace_hardirqs_on+0xd/0x10 [] ? remove_wait_queue+0x5a/0x70 [] ? do_wait+0x12c/0x260 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 990 0xffff880219f5deb0 990 1 0 3 S 0xffff880219f5e338 rpcbind ffff880215407a38 0000000000000046 00001f37ef85cfbf 0000000001c9c37f ffff880219f5deb0 ffff880215407fd8 ffff880215407fd8 ffff880215407fd8 ffff880225d2deb0 ffff880219f5deb0 ffff880215407a18 ffff880215407ef8 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x12c/0x170 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_sys_poll+0x40f/0x4d0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? mntput+0x24/0x40 [] ? __fput+0x189/0x240 [] ? ktime_get_ts+0x4c/0xe0 [] ? poll_select_set_timeout+0x85/0xa0 [] SyS_poll+0x6b/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1049 0xffff880215618000 1049 1 0 0 S 0xffff880215618488 rpc.statd ffff880224dd78d8 0000000000000046 0000000000000002 0000000000000000 ffff880215618000 ffff880224dd7fd8 ffff880224dd7fd8 ffff880224dd7fd8 ffffffff81c10440 ffff880215618000 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __wake_up_sync_key+0x37/0x80 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? up_read+0x23/0x40 [] ? __do_page_fault+0x2ac/0x520 [] ? mntput+0x24/0x40 [] ? __fput+0x189/0x240 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1207 0xffff88020cfebf20 1207 1 0 0 S 0xffff88020cfec3a8 getty ffff880224d35cc8 0000000000000046 ffff880224d35cb8 ffff880227212f80 ffff88020cfebf20 ffff880224d35fd8 ffff880224d35fd8 ffff880224d35fd8 ffffffff81c10440 ffff88020cfebf20 ffff880224d35cb8 ffff88020db6f000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1222 0xffff88020c713f20 1222 1 0 6 S 0xffff88020c7143a8 getty ffff880224f89cc8 0000000000000046 0000000000000002 ffff8802272d2f80 ffff88020c713f20 ffff880224f89fd8 ffff880224f89fd8 ffff880224f89fd8 ffff880225d3bf20 ffff88020c713f20 ffff880224f89cb8 ffff880219f54000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1237 0xffff88020c588000 1237 1 0 6 S 0xffff88020c588488 getty ffff8802155f5cc8 0000000000000046 0000000000000002 ffff8802272d2f80 ffff88020c588000 ffff8802155f5fd8 ffff8802155f5fd8 ffff8802155f5fd8 ffff880225d3bf20 ffff88020c588000 ffff8802155f5cb8 ffff880219f55000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1238 0xffff8802244c5eb0 1238 1 0 7 S 0xffff8802244c6338 getty ffff88020b927cc8 0000000000000046 0000000000000002 ffff8802272f2f80 ffff8802244c5eb0 ffff88020b927fd8 ffff88020b927fd8 ffff88020b927fd8 ffff880225d3deb0 ffff8802244c5eb0 ffff88020b927cb8 ffff88020b30c800 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1240 0xffff880224d4deb0 1240 1 0 6 S 0xffff880224d4e338 getty ffff880224fd3cc8 0000000000000046 0000000000000002 ffff8802272d2f80 ffff880224d4deb0 ffff880224fd3fd8 ffff880224fd3fd8 ffff880224fd3fd8 ffff880225d3bf20 ffff880224d4deb0 ffff880224fd3cb8 ffff880219f56000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1246 0xffff8802156a1f90 1246 1 0 0 S 0xffff8802156a2418 atd ffff88020b09be58 0000000000000046 0000034630b8a000 000000000000c350 ffff8802156a1f90 ffff88020b09bfd8 ffff88020b09bfd8 ffff88020b09bfd8 ffffffff81c10440 ffff8802156a1f90 0000000000000002 ffff88020b09beb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1247 0xffff880224d4bf20 1247 1 0 0 S 0xffff880224d4c3a8 xinetd ffff880224ec98d8 0000000000000046 0000000000000000 ffffffff810679f3 ffff880224d4bf20 ffff880224ec9fd8 ffff880224ec9fd8 ffff880224ec9fd8 ffffffff81c10440 ffff880224d4bf20 ffff880200000000 0000000000000000 Call Trace: [] ? add_wait_queue+0x23/0x60 [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? try_to_wake_up+0x2e/0x2f0 [] ? __pollwait+0xf0/0xf0 [] ? skb_queue_tail+0x2b/0x60 [] ? find_get_page+0x5/0xf0 [] ? find_get_page+0x5/0xf0 [] ? __do_fault+0x14a/0x470 [] ? __do_fault+0x14a/0x470 [] ? _raw_spin_unlock+0x2b/0x40 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? up_read+0x23/0x40 [] ? __do_page_fault+0x2ac/0x520 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1255 0xffff880215665eb0 1255 1 0 2 S 0xffff880215666338 nrpe ffff88020ce4d8d8 0000000000000046 00001f37a70a6981 000000000007a11e ffff880215665eb0 ffff88020ce4dfd8 ffff88020ce4dfd8 ffff88020ce4dfd8 ffff880225d2bf20 ffff880215665eb0 0000000000000000 ffff88020ce4dd18 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x12c/0x170 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __pollwait+0xf0/0xf0 [] ? get_partial_node.isra.63+0x54/0x1da [] ? number.isra.1+0x355/0x390 [] ? _raw_spin_unlock+0x2b/0x40 [] ? get_partial_node.isra.63+0x1b5/0x1da [] ? swap_info_get+0x63/0xe0 [] ? swap_info_get+0x63/0xe0 [] ? handle_pte_fault+0x323/0x8a0 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? up_read+0x23/0x40 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? ktime_get_ts+0x4c/0xe0 [] SyS_select+0xc1/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1333 0xffff88020c67bf20 1333 1 0 1 S 0xffff88020c67c3a8 irqbalance ffff880224477e58 0000000000000046 000000024f967cc8 000000000000c350 ffff88020c67bf20 ffff880224477fd8 ffff880224477fd8 ffff880224477fd8 ffff880225d29f90 ffff88020c67bf20 00007f80e99e2000 ffff880224477eb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1349 0xffff88020bbb3f20 1349 1 0 6 S 0xffff88020bbb43a8 rpc.mountd ffff88020db058d8 0000000000000046 0000000000000002 0000000000000000 ffff88020bbb3f20 ffff88020db05fd8 ffff88020db05fd8 ffff88020db05fd8 ffff880225d3bf20 ffff88020bbb3f20 ffff88022fff9e00 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? ttwu_stat+0x4c/0x140 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? up_read+0x23/0x40 [] ? __do_page_fault+0x2ac/0x520 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1400 0xffff8802156cbf20 1400 1 0 2 S 0xffff8802156cc3a8 pkcsslotd ffff8802148e7e58 0000000000000046 00000002540be400 000000000000c350 ffff8802156cbf20 ffff8802148e7fd8 ffff8802148e7fd8 ffff8802148e7fd8 ffff880225d2bf20 ffff8802156cbf20 0000000000000000 ffff8802148e7eb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1409 0xffff880221e61f90 1409 1 0 5 S 0xffff880221e62418 rwhod ffff880219fffa48 0000000000000046 ffff880219fffa18 0000000000000246 ffff880221e61f90 ffff880219ffffd8 ffff880219ffffd8 ffff880219ffffd8 ffff880225d39f90 ffff880221e61f90 0000000000000002 ffff880222640000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? trace_hardirqs_on+0xd/0x10 [] __skb_recv_datagram+0x2fb/0x380 [] ? skb_recv_datagram+0x40/0x40 [] udp_recvmsg+0xa7/0x410 [] inet_recvmsg+0xcd/0xf0 [] sock_recvmsg+0xd8/0xf0 [] ? dput+0x1e0/0x240 [] SYSC_recvfrom+0xe9/0x170 [] ? __fput+0x189/0x240 [] ? ____fput+0xe/0x10 [] ? lockdep_sys_exit_thunk+0x35/0x67 [] SyS_recvfrom+0xe/0x10 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1411 0xffff88020ce51f90 1411 1409 0 1 S 0xffff88020ce52418 rwhod ffff88022464fe58 0000000000000046 00000029e8d60800 000000000000c350 ffff88020ce51f90 ffff88022464ffd8 ffff88022464ffd8 ffff88022464ffd8 ffff880225d29f90 ffff88020ce51f90 0000000000000002 ffff88022464feb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1528 0xffff88020c715eb0 1528 1 0 5 S 0xffff88020c716338 getty ffff88022445fcc8 0000000000000046 0000000000000002 ffff8802272b2f80 ffff88020c715eb0 ffff88022445ffd8 ffff88022445ffd8 ffff88022445ffd8 ffff880225d39f90 ffff88020c715eb0 ffff88022445fcb8 ffff88020cba1000 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1529 0xffff88020c710000 1529 1 0 0 S 0xffff88020c710488 getty ffff880221ce1cc8 0000000000000046 ffff88020c8a09c8 ffff8802272b2f80 ffff88020c710000 ffff880221ce1fd8 ffff880221ce1fd8 ffff880221ce1fd8 ffffffff81c10440 ffff88020c710000 ffff880221ce1cb8 ffff88020c8a7800 Call Trace: [] schedule+0x29/0x70 [] schedule_timeout+0x1bd/0x270 [] ? n_tty_read+0x388/0x960 [] ? trace_hardirqs_on+0xd/0x10 [] n_tty_read+0x278/0x960 [] ? try_to_wake_up+0x2f0/0x2f0 [] tty_read+0x95/0x100 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1874 0xffff880219f59f90 1874 1 0 6 S 0xffff880219f5a418 ntpd ffff88020c53b8d8 0000000000000046 0000000000000002 0000000000000000 ffff880219f59f90 ffff88020c53bfd8 ffff88020c53bfd8 ffff88020c53bfd8 ffff880225d3bf20 ffff880219f59f90 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? signal_delivered+0x5a/0x70 [] ? fpu_finit+0x22/0x40 [] ? init_fpu+0x56/0xc0 [] ? __restore_xstate_sig+0x22e/0x510 [] ? might_fault+0x45/0xa0 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 27902 0xffff8801377f3f20 27902 2 0 5 D 0xffff8801377f43a8 kworker/u64:1 ffff8801a71ed748 0000000000000046 ffff8801a71ed768 0000000000000246 ffff8801377f3f20 ffff8801a71edfd8 ffff8801a71edfd8 ffff8801a71edfd8 ffff880225d39f90 ffff8801377f3f20 ffffffff81636fa5 ffff880114b3fa28 Call Trace: [] ? rwsem_down_read_failed+0xa5/0xf9 [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_osdc_start_request+0x48/0x130 [libceph] [] ceph_osdc_start_request+0x48/0x130 [libceph] [] ceph_writepages_start+0x94e/0xfa0 [ceph] [] do_writepages+0x23/0x40 [] __writeback_single_inode+0x4e/0x2a0 [] writeback_sb_inodes+0x280/0x3e0 [] __writeback_inodes_wb+0x9e/0xd0 [] wb_writeback+0x1fb/0x310 [] wb_do_writeback+0xbb/0x1e0 [] ? set_worker_desc+0x6f/0x80 [] bdi_writeback_workfn+0x7a/0x200 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 28210 0xffff880100079f90 28210 2 0 0 D 0xffff88010007a418 kworker/0:2 ffff880211a8fab8 0000000000000046 ffff880211a8fad8 0000000000000246 ffff880100079f90 ffff880211a8ffd8 ffff880211a8ffd8 ffff880211a8ffd8 ffff88015ecdbf20 ffff880100079f90 0000000000000000 ffff880114b3fa28 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? dispatch+0x1f1/0x9b0 [libceph] [] dispatch+0x1f1/0x9b0 [libceph] [] ? ceph_tcp_recvmsg+0x48/0x60 [libceph] [] ? trace_hardirqs_on+0xd/0x10 [] con_work+0x1948/0x2d50 [libceph] [] ? cpuacct_charge+0x5/0xa0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 28529 0xffff88021e420000 28529 2 0 0 D 0xffff88021e420488 kworker/0:1 ffff880219067a88 0000000000000046 ffff880219067aa8 0000000000000246 ffff88021e420000 ffff880219067fd8 ffff880219067fd8 ffff880219067fd8 ffff8801000f5eb0 ffff88021e420000 ffff880114b3f288 ffff880114b3fa28 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_osdc_handle_map+0x2b9/0x580 [libceph] [] ceph_osdc_handle_map+0x2b9/0x580 [libceph] [] dispatch+0x120/0x7c0 [libceph] [] ? trace_hardirqs_on+0xd/0x10 [] con_work+0x1948/0x2d50 [libceph] [] ? cpuacct_charge+0x5/0xa0 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 29322 0xffff880222099f90 29322 1 0 0 S 0xffff88022209a418 sshd ffff88020c455a38 0000000000000046 0000000000000002 0000000000000000 ffff880222099f90 ffff88020c455fd8 ffff88020c455fd8 ffff88020c455fd8 ffffffff81c10440 ffff880222099f90 ffff88020c455b08 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] ? __pte_alloc+0x3f/0xc0 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] ? might_fault+0x45/0xa0 [] do_sys_poll+0x40f/0x4d0 [] ? __pollwait+0xf0/0xf0 [] ? do_wp_page+0x19f/0x6f0 [] ? _raw_spin_unlock+0x2b/0x40 [] ? do_wp_page+0x29a/0x6f0 [] ? handle_pte_fault+0x264/0x8a0 [] ? __do_page_fault+0x15c/0x520 [] ? __do_page_fault+0x15c/0x520 [] ? sched_clock_cpu+0xc5/0x120 [] ? up_read+0x23/0x40 [] ? __do_page_fault+0x2ac/0x520 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? do_sigaction+0x12d/0x1f0 [] SyS_poll+0x6b/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 29338 0xffff88020c859f90 29338 29322 0 2 S 0xffff88020c85a418 sshd ffff880224a2d8d8 0000000000000046 0000000000000000 ffffffff810679f3 ffff88020c859f90 ffff880224a2dfd8 ffff880224a2dfd8 ffff880224a2dfd8 ffff880225d2bf20 ffff88020c859f90 0000000000000000 0000000000000000 Call Trace: [] ? add_wait_queue+0x23/0x60 [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? release_sock+0x172/0x1e0 [] ? tcp_recvmsg+0x396/0xc80 [] ? inet_recvmsg+0x5/0xf0 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? sock_aio_read+0x23/0x30 [] ? do_sync_read+0x7a/0xb0 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 29372 0xffff88010007deb0 29372 1 0 7 S 0xffff88010007e338 rsyslogd ffff88020cbc18d8 0000000000000046 00001f3d3b0fba0a 0000000001c9c37e ffff88010007deb0 ffff88020cbc1fd8 ffff88020cbc1fd8 ffff88020cbc1fd8 ffff880225d3deb0 ffff88010007deb0 0000000000000002 ffff88020cbc1d18 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x12c/0x170 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? delayacct_end+0x71/0xa0 [] ? delayacct_end+0x71/0xa0 [] ? finish_task_switch+0x48/0x110 [] ? swap_info_get+0x63/0xe0 [] ? __wake_up+0x32/0x70 [] ? __wake_up+0x32/0x70 [] ? swap_info_get+0x63/0xe0 [] ? trace_hardirqs_on+0xd/0x10 [] ? core_sys_select+0x42/0x340 [] ? core_sys_select+0x42/0x340 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? might_fault+0x45/0xa0 [] ? up_read+0x23/0x40 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? ktime_get_ts+0x4c/0xe0 [] SyS_select+0xc1/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 29373 0xffff880224665eb0 29373 1 0 1 S 0xffff880224666338 rs:main Q:Reg ffff8801a5b11c28 0000000000000046 0000000000000080 ffffffff826c2020 ffff880224665eb0 ffff8801a5b11fd8 ffff8801a5b11fd8 ffff8801a5b11fd8 ffff880225d29f90 ffff880224665eb0 0000000000000000 ffff8801a5b11cd0 Call Trace: [] schedule+0x29/0x70 [] futex_wait_queue_me+0xc7/0x100 [] futex_wait+0x1a7/0x2c0 [] ? trace_hardirqs_on+0xd/0x10 [] ? ext4_file_write+0x9f/0x420 [] ? _raw_spin_unlock+0x2b/0x40 [] ? fsnotify+0x8a/0x2f0 [] do_futex+0x344/0xbb0 [] ? fsnotify+0x20a/0x2f0 [] ? fsnotify+0x8a/0x2f0 [] SyS_futex+0x147/0x1a0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 29374 0xffff8801000f3f20 29374 1 0 1 S 0xffff8801000f43a8 rsyslogd ffff88020b0018d8 0000000000000046 ffff880200000000 ffff88013a83f900 ffff8801000f3f20 ffff88020b001fd8 ffff88020b001fd8 ffff88020b001fd8 ffff880225d29f90 ffff8801000f3f20 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? add_wait_queue+0x4d/0x60 [] ? __pollwait+0x75/0xf0 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? __pollwait+0xf0/0xf0 [] ? mutex_unlock+0xe/0x10 [] ? unix_dgram_recvmsg+0x316/0x4e0 [] ? select_task_rq_fair+0x53/0x8a0 [] ? select_task_rq_fair+0x53/0x8a0 [] ? ttwu_stat+0x4c/0x140 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? ___sys_recvmsg+0x166/0x2a0 [] ? do_futex+0xa45/0xbb0 [] ? fget_light+0x9e/0x130 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 29375 0xffff8801000f0000 29375 1 0 1 S 0xffff8801000f0488 rsyslogd ffff88020c949e28 0000000000000046 0000000000000286 ffffffff81c31800 ffff8801000f0000 ffff88020c949fd8 ffff88020c949fd8 ffff88020c949fd8 ffff880225d29f90 ffff8801000f0000 ffff88020c949e38 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] do_syslog+0x4d1/0x550 [] ? __init_waitqueue_head+0x60/0x60 [] kmsg_read+0x56/0x70 [] proc_reg_read+0x43/0x70 [] vfs_read+0xb0/0x180 [] SyS_read+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 29606 0xffff880225dc3f20 29606 1 0 1 S 0xffff880225dc43a8 cron ffff88020cc97e58 0000000000000046 0000000df8475800 000000000000c350 ffff880225dc3f20 ffff88020cc97fd8 ffff88020cc97fd8 ffff88020cc97fd8 ffff880225d29f90 ffff880225dc3f20 0000000000000002 ffff88020cc97eb8 Call Trace: [] schedule+0x29/0x70 [] do_nanosleep+0x9c/0xd0 [] hrtimer_nanosleep+0xb8/0x180 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] SyS_nanosleep+0x76/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 31164 0xffff88020d2a9f90 31164 1 0 1 S 0xffff88020d2aa418 sshd ffff88020b9938d8 0000000000000046 0000000000000002 0000000000000000 ffff88020d2a9f90 ffff88020b993fd8 ffff88020b993fd8 ffff88020b993fd8 ffff880225d29f90 ffff88020d2a9f90 0000000000000000 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] ? trace_hardirqs_on+0xd/0x10 [] ? add_wait_queue+0x4d/0x60 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? fsnotify_clear_marks_by_inode+0x34/0x130 [] ? __pollwait+0xf0/0xf0 [] ? __pollwait+0xf0/0xf0 [] ? __d_lookup+0x5/0x1c0 [] ? __d_lookup+0x5/0x1c0 [] ? mix_pool_bytes.constprop.16+0x4d/0xd0 [] ? mix_pool_bytes.constprop.16+0x4d/0xd0 [] ? __d_lookup+0xd8/0x1c0 [] ? __d_lookup+0x5/0x1c0 [] ? fsnotify_clear_marks_by_inode+0x34/0x130 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? mntput_no_expire+0x2c/0x160 [] ? lg_local_unlock+0x23/0x40 [] ? mntput_no_expire+0x49/0x160 [] ? mntput+0x24/0x40 [] ? __fput+0x189/0x240 [] SyS_select+0xc1/0x100 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 31726 0xffff88020d0ebf20 31726 498 0 6 S 0xffff88020d0ec3a8 udevd ffff88020bb17de8 0000000000000046 0000000000000002 0000000000000000 ffff88020d0ebf20 ffff88020bb17fd8 ffff88020bb17fd8 ffff88020bb17fd8 ffff880225d3bf20 ffff88020d0ebf20 ffff88020bb17dc8 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] schedule_hrtimeout_range+0x13/0x20 [] SyS_epoll_wait+0x418/0x4c0 [] ? try_to_wake_up+0x2f0/0x2f0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 31727 0xffff88020d0edeb0 31727 498 0 7 S 0xffff88020d0ee338 udevd ffff88020bd5fde8 0000000000000046 0000000000000000 0000000000000000 ffff88020d0edeb0 ffff88020bd5ffd8 ffff88020bd5ffd8 ffff88020bd5ffd8 ffff880225d3deb0 ffff88020d0edeb0 ffff8802256e8800 0000000000000000 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x14d/0x170 [] schedule_hrtimeout_range+0x13/0x20 [] SyS_epoll_wait+0x418/0x4c0 [] ? try_to_wake_up+0x2f0/0x2f0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 31763 0xffff88020c51deb0 31763 29338 0 7 S 0xffff88020c51e338 bash ffff8801a7485e58 0000000000000046 ffff8801a7485e68 ffffffff8104449d ffff88020c51deb0 ffff8801a7485fd8 ffff8801a7485fd8 ffff8801a7485fd8 ffff880225d3deb0 ffff88020c51deb0 ffff8801a7485ef0 0000000000000000 Call Trace: [] ? wait_consider_task+0x9d/0xbe0 [] schedule+0x29/0x70 [] do_wait+0x1e9/0x260 [] SyS_wait4+0xa3/0x100 [] ? might_fault+0x45/0xa0 [] ? task_stopped_code+0x50/0x50 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 31765 0xffff88020db51f90 31765 31763 0 7 S 0xffff88020db52418 ffsb.sh ffff880224d21e58 0000000000000046 ffff880224d21e68 ffffffff8104449d ffff88020db51f90 ffff880224d21fd8 ffff880224d21fd8 ffff880224d21fd8 ffff880225d3deb0 ffff88020db51f90 ffff880224d21ef0 0000000000000000 Call Trace: [] ? wait_consider_task+0x9d/0xbe0 [] schedule+0x29/0x70 [] do_wait+0x1e9/0x260 [] SyS_wait4+0xa3/0x100 [] ? might_fault+0x45/0xa0 [] ? task_stopped_code+0x50/0x50 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 984 0xffff8801377f5eb0 984 31765 0 0 S 0xffff8801377f6338 ffsb ffff88020b3f3c28 0000000000000046 ffff88020b3f3bf8 ffffffff826c1e70 ffff8801377f5eb0 ffff88020b3f3fd8 ffff88020b3f3fd8 ffff88020b3f3fd8 ffff880224291f90 ffff8801377f5eb0 0000000000000000 ffff88020b3f3cd0 Call Trace: [] schedule+0x29/0x70 [] futex_wait_queue_me+0xc7/0x100 [] futex_wait+0x1a7/0x2c0 [] ? current_fs_time+0x16/0x60 [] ? pipe_write+0x31b/0x5d0 [] ? fsnotify+0x8a/0x2f0 [] do_futex+0x344/0xbb0 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? finish_task_switch+0x85/0x110 [] ? finish_task_switch+0x48/0x110 [] SyS_futex+0x147/0x1a0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1018 0xffff8802227ddeb0 1018 31765 0 3 S 0xffff8802227de338 ffsb ffff8802247b38d8 0000000000000046 00001f37ac11c10a 00000000000f423e ffff8802227ddeb0 ffff8802247b3fd8 ffff8802247b3fd8 ffff8802247b3fd8 ffff880225d2deb0 ffff8802227ddeb0 0000000000000002 ffff8802247b3d18 Call Trace: [] schedule+0x29/0x70 [] schedule_hrtimeout_range_clock+0x12c/0x170 [] ? update_rmtp+0x70/0x70 [] ? hrtimer_start_range_ns+0x14/0x20 [] schedule_hrtimeout_range+0x13/0x20 [] poll_schedule_timeout+0x49/0x70 [] do_select+0x5d1/0x6e0 [] ? do_select+0x5/0x6e0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? delayacct_end+0x71/0xa0 [] ? delayacct_end+0x71/0xa0 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? delayacct_end+0x89/0xa0 [] ? swap_info_get+0x63/0xe0 [] ? swap_info_get+0x63/0xe0 [] ? handle_pte_fault+0x323/0x8a0 [] ? core_sys_select+0x42/0x340 [] ? core_sys_select+0x42/0x340 [] core_sys_select+0x214/0x340 [] ? core_sys_select+0x42/0x340 [] ? trace_hardirqs_off_thunk+0x3a/0x3c [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? up_read+0x23/0x40 [] ? might_fault+0x45/0xa0 [] ? might_fault+0x45/0xa0 [] ? ktime_get_ts+0x4c/0xe0 [] SyS_select+0xc1/0x100 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1019 0xffff88020cf20000 1019 31765 0 6 D 0xffff88020cf20488 ffsb ffff880221e759d8 0000000000000046 0000000000000000 0000000000000046 ffff88020cf20000 ffff880221e75fd8 ffff880221e75fd8 ffff880221e75fd8 ffff880225d3bf20 ffff88020cf20000 ffff880221e759d8 ffff8802272d42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1020 0xffff88020cf21f90 1020 31765 0 3 D 0xffff88020cf22418 ffsb ffff88021541f9d8 0000000000000046 0000000000000000 0000000000000046 ffff88020cf21f90 ffff88021541ffd8 ffff88021541ffd8 ffff88021541ffd8 ffff880225d2deb0 ffff88020cf21f90 ffff88021541f9d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? do_raw_spin_lock+0x76/0x150 [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1021 0xffff880224295eb0 1021 31765 0 5 D 0xffff880224296338 ffsb ffff88020ce1b9d8 0000000000000046 0000000000000000 0000000000000046 ffff880224295eb0 ffff88020ce1bfd8 ffff88020ce1bfd8 ffff88020ce1bfd8 ffff880219d80000 ffff880224295eb0 ffff88020ce1b9d8 ffff8802272b42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? _raw_spin_unlock_irq+0x30/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1022 0xffff880224293f20 1022 31765 0 6 D 0xffff8802242943a8 ffsb ffff88020b0bb9d8 0000000000000046 0000000000000000 0000000000000046 ffff880224293f20 ffff88020b0bbfd8 ffff88020b0bbfd8 ffff88020b0bbfd8 ffff8801d8501f90 ffff880224293f20 ffff88020b0bb9d8 ffff8802272d42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1023 0xffff880224290000 1023 31765 0 3 D 0xffff880224290488 ffsb ffff88020ca579d8 0000000000000046 0000000000000000 0000000000000046 ffff880224290000 ffff88020ca57fd8 ffff88020ca57fd8 ffff88020ca57fd8 ffff8801377f3f20 ffff880224290000 ffff88020ca579d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? cpuacct_charge+0x5/0xa0 [] ? cpuacct_charge+0x5/0xa0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1024 0xffff880224291f90 1024 31765 0 2 D 0xffff880224292418 ffsb ffff88020c5ef9d8 0000000000000046 0000000000000000 0000000000000046 ffff880224291f90 ffff88020c5effd8 ffff88020c5effd8 ffff88020c5effd8 ffff880225d2bf20 ffff880224291f90 ffff88020c5ef9d8 ffff8802272542d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? handle_pte_fault+0x475/0x8a0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1025 0xffff880224f48000 1025 31765 0 7 D 0xffff880224f48488 ffsb ffff88020bf9d9d8 0000000000000046 0000000000000000 0000000000000046 ffff880224f48000 ffff88020bf9dfd8 ffff88020bf9dfd8 ffff88020bf9dfd8 ffff880225d3deb0 ffff880224f48000 ffff88020bf9d9d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1026 0xffff880100078000 1026 31765 0 3 D 0xffff880100078488 ffsb ffff88020c5e59d8 0000000000000046 0000000000000000 0000000000000046 ffff880100078000 ffff88020c5e5fd8 ffff88020c5e5fd8 ffff88020c5e5fd8 ffff880072095eb0 ffff880100078000 ffff88020c5e59d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1027 0xffff88010007bf20 1027 31765 0 3 D 0xffff88010007c3a8 ffsb ffff88020cb399d8 0000000000000046 0000000000000000 0000000000000046 ffff88010007bf20 ffff88020cb39fd8 ffff88020cb39fd8 ffff88020cb39fd8 ffff880225d2deb0 ffff88010007bf20 ffff88020cb399d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1028 0xffff8802156a3f20 1028 31765 0 0 D 0xffff8802156a43a8 ffsb ffff88020cad79d8 0000000000000046 0000000000000000 0000000000000046 ffff8802156a3f20 ffff88020cad7fd8 ffff88020cad7fd8 ffff88020cad7fd8 ffffffff81c10440 ffff8802156a3f20 ffff88020cad79d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1029 0xffff8802156a5eb0 1029 31765 0 0 D 0xffff8802156a6338 ffsb ffff88020bd059d8 0000000000000046 0000000000000000 0000000000000046 ffff8802156a5eb0 ffff88020bd05fd8 ffff88020bd05fd8 ffff88020bd05fd8 ffffffff81c10440 ffff8802156a5eb0 ffff88020bd059d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? finish_task_switch+0x48/0x110 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1030 0xffff8802156a0000 1030 31765 0 2 D 0xffff8802156a0488 ffsb ffff88020d3cf9d8 0000000000000046 0000000000000000 0000000000000046 ffff8802156a0000 ffff88020d3cffd8 ffff88020d3cffd8 ffff88020d3cffd8 ffff88013aa4deb0 ffff8802156a0000 ffff88020d3cf9d8 ffff8802272542d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? do_raw_spin_lock+0x76/0x150 [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1031 0xffff88020bbb1f90 1031 31765 0 3 D 0xffff88020bbb2418 ffsb ffff8802149659d8 0000000000000046 0000000000000000 0000000000000046 ffff88020bbb1f90 ffff880214965fd8 ffff880214965fd8 ffff880214965fd8 ffff880225d2deb0 ffff88020bbb1f90 ffff8802149659d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] ? rcu_irq_exit+0x68/0xb0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1032 0xffff880225de3f20 1032 31765 0 7 D 0xffff880225de43a8 ffsb ffff88020c8639d8 0000000000000046 0000000000000000 0000000000000046 ffff880225de3f20 ffff88020c863fd8 ffff88020c863fd8 ffff88020c863fd8 ffff880224f48000 ffff880225de3f20 ffff88020c8639d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1033 0xffff88020d0adeb0 1033 31765 0 1 D 0xffff88020d0ae338 ffsb ffff88020b1fd9d8 0000000000000046 0000000000000000 0000000000000046 ffff88020d0adeb0 ffff88020b1fdfd8 ffff88020b1fdfd8 ffff88020b1fdfd8 ffff880225d29f90 ffff88020d0adeb0 ffff88020b1fd9d8 ffff8802272342d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? select_task_rq_fair+0x53/0x8a0 [] ? finish_task_switch+0x48/0x110 [] ? ttwu_stat+0x4c/0x140 [] ? try_to_wake_up+0x2e/0x2f0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1034 0xffff880219f0bf20 1034 31765 0 2 D 0xffff880219f0c3a8 ffsb ffff880219c739d8 0000000000000046 0000000000000000 0000000000000046 ffff880219f0bf20 ffff880219c73fd8 ffff880219c73fd8 ffff880219c73fd8 ffff880225d2bf20 ffff880219f0bf20 ffff880219c739d8 ffff8802272542d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1035 0xffff880219f0deb0 1035 31765 0 2 D 0xffff880219f0e338 ffsb ffff8802155e99d8 0000000000000046 0000000000000000 0000000000000046 ffff880219f0deb0 ffff8802155e9fd8 ffff8802155e9fd8 ffff8802155e9fd8 ffff880225d2bf20 ffff880219f0deb0 ffff8802155e99d8 ffff8802272542d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1036 0xffff880219f08000 1036 31765 0 7 D 0xffff880219f08488 ffsb ffff88020cc519d8 0000000000000046 0000000000000000 0000000000000046 ffff880219f08000 ffff88020cc51fd8 ffff88020cc51fd8 ffff88020cc51fd8 ffff88020c9a0000 ffff880219f08000 ffff88020cc519d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? finish_task_switch+0x48/0x110 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1037 0xffff880215663f20 1037 31765 0 3 D 0xffff8802156643a8 ffsb ffff88020cf839d8 0000000000000046 0000000000000000 0000000000000046 ffff880215663f20 ffff88020cf83fd8 ffff88020cf83fd8 ffff88020cf83fd8 ffff880072095eb0 ffff880215663f20 ffff88020cf839d8 ffff8802272742d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? finish_task_switch+0x48/0x110 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1038 0xffff880225d19f90 1038 31765 0 0 D 0xffff880225d1a418 ffsb ffff8802149dd9d8 0000000000000046 0000000000000000 0000000000000046 ffff880225d19f90 ffff8802149ddfd8 ffff8802149ddfd8 ffff8802149ddfd8 ffffffff81c10440 ffff880225d19f90 ffff8802149dd9d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1039 0xffff880216981f90 1039 31765 0 5 D 0xffff880216982418 ffsb ffff88020b89b9d8 0000000000000046 0000000000000000 0000000000000046 ffff880216981f90 ffff88020b89bfd8 ffff88020b89bfd8 ffff88020b89bfd8 ffff880225d39f90 ffff880216981f90 ffff88020b89b9d8 ffff8802272b42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? do_raw_spin_lock+0x76/0x150 [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1040 0xffff880216985eb0 1040 31765 0 0 D 0xffff880216986338 ffsb ffff88020c4699d8 0000000000000046 0000000000000000 0000000000000046 ffff880216985eb0 ffff88020c469fd8 ffff88020c469fd8 ffff88020c469fd8 ffff88011e7bdeb0 ffff880216985eb0 ffff88020c4699d8 ffff8802272142d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1041 0xffff880216980000 1041 31765 0 2 D 0xffff880216980488 ffsb ffff88020cb019d8 0000000000000046 0000000000000000 0000000000000046 ffff880216980000 ffff88020cb01fd8 ffff88020cb01fd8 ffff88020cb01fd8 ffff880225d2bf20 ffff880216980000 ffff88020cb019d8 ffff8802272542d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1042 0xffff880224971f90 1042 31765 0 4 D 0xffff880224972418 ffsb ffff88020cc919d8 0000000000000046 0000000000000000 0000000000000046 ffff880224971f90 ffff88020cc91fd8 ffff88020cc91fd8 ffff88020cc91fd8 ffff880216983f20 ffff880224971f90 ffff88020cc919d8 ffff8802272942d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? cpuacct_charge+0x5/0xa0 [] ? cpuacct_charge+0x5/0xa0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1043 0xffff880219d81f90 1043 31765 0 7 D 0xffff880219d82418 ffsb ffff88020cc339d8 0000000000000046 0000000000000000 0000000000000046 ffff880219d81f90 ffff88020cc33fd8 ffff88020cc33fd8 ffff88020cc33fd8 ffff880225d3deb0 ffff880219d81f90 ffff88020cc339d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? cpuacct_charge+0x5/0xa0 [] ? cpuacct_charge+0x5/0xa0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1044 0xffff880219d85eb0 1044 31765 0 6 D 0xffff880219d86338 ffsb ffff88020c4afbd8 0000000000000046 ffff88020c4afbf8 0000000000000246 ffff880219d85eb0 ffff88020c4affd8 ffff88020c4affd8 ffff88020c4affd8 ffff880225d3bf20 ffff880219d85eb0 0000000000000000 ffff88021fbd8538 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_aio_write+0x9c/0xa50 [ceph] [] ceph_aio_write+0x9c/0xa50 [ceph] [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? trace_hardirqs_on+0xd/0x10 [] ? finish_task_switch+0x48/0x110 [] ? vfs_write+0x1bb/0x1e0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1045 0xffff880219d83f20 1045 31765 0 7 D 0xffff880219d843a8 ffsb ffff8802149e39d8 0000000000000046 0000000000000000 0000000000000046 ffff880219d83f20 ffff8802149e3fd8 ffff8802149e3fd8 ffff8802149e3fd8 ffff88020c9a0000 ffff880219d83f20 ffff8802149e39d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1046 0xffff880225e4bf20 1046 31765 0 7 D 0xffff880225e4c3a8 ffsb ffff880225e8d9d8 0000000000000046 0000000000000000 0000000000000046 ffff880225e4bf20 ffff880225e8dfd8 ffff880225e8dfd8 ffff880225e8dfd8 ffff880225d3deb0 ffff880225e4bf20 ffff880225e8d9d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? select_task_rq_fair+0x53/0x8a0 [] ? finish_task_switch+0x48/0x110 [] ? ttwu_stat+0x4c/0x140 [] ? try_to_wake_up+0x2e/0x2f0 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1047 0xffff8801f4915eb0 1047 31765 0 3 D 0xffff8801f4916338 ffsb ffff88020b23fe38 0000000000000046 ffff88020b23fe58 0000000000000246 ffff8801f4915eb0 ffff88020b23ffd8 ffff88020b23ffd8 ffff88020b23ffd8 ffff880225d2deb0 ffff8801f4915eb0 ffff88020b23fee8 ffff88021a882d10 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? ceph_llseek+0x45/0x170 [ceph] [] ? fget_light+0x3c/0x130 [] ceph_llseek+0x45/0x170 [ceph] [] vfs_llseek+0x33/0x40 [] SyS_lseek+0x7e/0x90 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1048 0xffff8801f4910000 1048 31765 0 1 D 0xffff8801f4910488 ffsb ffff88020becb9d8 0000000000000046 0000000000000000 0000000000000046 ffff8801f4910000 ffff88020becbfd8 ffff88020becbfd8 ffff88020becbfd8 ffff880225d29f90 ffff8801f4910000 ffff88020becb9d8 ffff8802272342d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1050 0xffff8801f4911f90 1050 31765 0 5 D 0xffff8801f4912418 ffsb ffff88020c8319d8 0000000000000046 0000000000000000 0000000000000046 ffff8801f4911f90 ffff88020c831fd8 ffff88020c831fd8 ffff88020c831fd8 ffff880225d39f90 ffff8801f4911f90 ffff88020c8319d8 ffff8802272b42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 1051 0xffff8801d8503f20 1051 31765 0 7 D 0xffff8801d85043a8 ffsb ffff88020cf459d8 0000000000000046 0000000000000000 0000000000000046 ffff8801d8503f20 ffff88020cf45fd8 ffff88020cf45fd8 ffff88020cf45fd8 ffff880225d3deb0 ffff8801d8503f20 ffff88020cf459d8 ffff8802272f42d8 Call Trace: [] ? __lock_page+0x70/0x70 [] schedule+0x29/0x70 [] io_schedule+0x8f/0xd0 [] sleep_on_page+0xe/0x20 [] __wait_on_bit_lock+0x5a/0xc0 [] __lock_page+0x67/0x70 [] ? autoremove_wake_function+0x40/0x40 [] find_lock_page+0x67/0x80 [] grab_cache_page_write_begin+0x6f/0xe0 [] ceph_write_begin+0x46/0xc0 [ceph] [] generic_file_buffered_write+0x11a/0x290 [] ceph_aio_write+0x8b0/0xa50 [ceph] [] ? __touch_cap+0x2f/0x100 [ceph] [] ? _raw_spin_unlock+0x2b/0x40 [] ? vfs_write+0x1bb/0x1e0 [] do_sync_write+0x7a/0xb0 [] vfs_write+0xce/0x1e0 [] ? fget_light+0x3c/0x130 [] SyS_write+0x55/0xa0 [] system_call_fastpath+0x16/0x1b Stack traceback for pid 988 0xffff88015ecd9f90 988 2 0 4 R 0xffff88015ecda418 kworker/4:2 ffff880224ca3ae8 0000000000000046 ffff88015ecd9f90 ffff880114b3f968 ffff88015ecd9f90 ffff880224ca3fd8 ffff880224ca3fd8 ffff880224ca3fd8 ffff880216983f20 ffff88015ecd9f90 ffffffff81636109 ffff880224ca2000 Call Trace: [] ? schedule+0x29/0x70 [] __cond_resched+0x2a/0x40 [] _cond_resched+0x32/0x40 [] mutex_lock_nested+0x2a/0x360 [] ? call_rwsem_down_read_failed+0x14/0x30 [] osd_reset+0x5e/0x2b0 [libceph] [] con_work+0x571/0x2d50 [libceph] [] ? cpuacct_charge+0x5/0xa0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 1052 0xffff88011e7bdeb0 1052 2 0 0 D 0xffff88011e7be338 kworker/0:3 ffff88013a49bb08 0000000000000046 ffff88013a49bb28 0000000000000246 ffff88011e7bdeb0 ffff88013a49bfd8 ffff88013a49bfd8 ffff88013a49bfd8 ffff8801000f5eb0 ffff88011e7bdeb0 0000000000000000 ffff880114b3fa28 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? alloc_msg+0xd0/0x2d0 [libceph] [] alloc_msg+0xd0/0x2d0 [libceph] [] con_work+0x1dff/0x2d50 [libceph] [] ? trace_hardirqs_off+0xd/0x10 [] ? _raw_spin_unlock_irqrestore+0x57/0x70 [] ? mod_timer+0x113/0x230 [] ? process_one_work+0x248/0x540 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 1053 0xffff88015ecdbf20 1053 2 0 0 D 0xffff88015ecdc3a8 kworker/0:4 ffff880186c3db08 0000000000000046 ffff880186c3db28 0000000000000246 ffff88015ecdbf20 ffff880186c3dfd8 ffff880186c3dfd8 ffff880186c3dfd8 ffff88021e420000 ffff88015ecdbf20 0000000000000000 ffff880114b3fa28 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? alloc_msg+0xd0/0x2d0 [libceph] [] alloc_msg+0xd0/0x2d0 [libceph] [] con_work+0x1dff/0x2d50 [libceph] [] ? cpuacct_charge+0x5/0xa0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 1054 0xffff88020db53f20 1054 2 0 0 D 0xffff88020db543a8 kworker/0:5 ffff88019caa9ab8 0000000000000046 ffff88019caa9ad8 0000000000000246 ffff88020db53f20 ffff88019caa9fd8 ffff88019caa9fd8 ffff88019caa9fd8 ffff88021e420000 ffff88020db53f20 0000000000000000 ffff880114b3fa28 Call Trace: [] schedule+0x29/0x70 [] schedule_preempt_disabled+0xe/0x10 [] mutex_lock_nested+0x15c/0x360 [] ? dispatch+0x1f1/0x9b0 [libceph] [] dispatch+0x1f1/0x9b0 [libceph] [] ? ceph_tcp_recvmsg+0x48/0x60 [libceph] [] ? trace_hardirqs_on+0xd/0x10 [] con_work+0x1948/0x2d50 [libceph] [] ? cpuacct_charge+0x5/0xa0 [] ? idle_balance+0x133/0x180 [] ? finish_task_switch+0x48/0x110 [] ? process_one_work+0x248/0x540 [] ? process_one_work+0x16f/0x540 [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150 Stack traceback for pid 1083 0xffff88011e7bbf20 1083 2 0 4 R 0xffff88011e7bc3a8 kworker/4:3 ffff880203bebc20 0000000000000046 ffff880225e44018 ffff880225c80720 ffff88011e7bbf20 ffff880203bebfd8 ffff880203bebfd8 ffff880203bebfd8 ffff88013aa48000 ffff88011e7bbf20 ffff880114b3f968 ffff88011e7bbf20 Call Trace: [] schedule+0x29/0x70 [] rwsem_down_read_failed+0xa5/0xf9 [] call_rwsem_down_read_failed+0x14/0x30 [] ? down_read+0x47/0x50 [] ? handle_osds_timeout+0x5c/0x170 [libceph] [] handle_osds_timeout+0x5c/0x170 [libceph] [] process_one_work+0x1da/0x540 [] ? process_one_work+0x16f/0x540 [] worker_thread+0x11c/0x370 [] ? manage_workers.isra.20+0x2e0/0x2e0 [] kthread+0xea/0xf0 [] ? flush_kthread_worker+0x150/0x150 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x150/0x150