[root@cartan1144 crash]# cd 127.0.0.1-2012-07-04-14:37:01/ cartan1334_127.0.0.1-2012-07-06-09:12:00/ [root@cartan1144 crash]# mv 127.0.0.1-2012-07-04-14:37:01/ cartan1206_127.0.0.1-2012-07-04-14:37:01/ [root@cartan1144 crash]# cd cartan1206_127.0.0.1-2012-07-04-14:37:01/ [root@cartan1144 cartan1206_127.0.0.1-2012-07-04-14:37:01]# crash /boot/System.map-2.6.32-131.17.1.bl6.Bull.27.0.x86_64 /usr/lib/debug/lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/vmlinux ./vmcore crash 5.1.1-2.bl6 Copyright (C) 2002-2011 Red Hat, Inc. Copyright (C) 2004, 2005, 2006 IBM Corporation Copyright (C) 1999-2006 Hewlett-Packard Co Copyright (C) 2005, 2006 Fujitsu Limited Copyright (C) 2006, 2007 VA Linux Systems Japan K.K. Copyright (C) 2005 NEC Corporation Copyright (C) 1999, 2002, 2007 Silicon Graphics, Inc. Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc. This program is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Enter "help copying" to see the conditions. This program has absolutely no warranty. Enter "help warranty" for details. GNU gdb (GDB) 7.0 Copyright (C) 2009 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-unknown-linux-gnu"... SYSTEM MAP: /boot/System.map-2.6.32-131.17.1.bl6.Bull.27.0.x86_64 DEBUG KERNEL: /usr/lib/debug/lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/vmlinux (2.6.32-131.17.1.bl6.Bull.27.0.x86_64) DUMPFILE: ./vmcore [PARTIAL DUMP] CPUS: 32 DATE: Wed Jul 4 14:36:44 2012 UPTIME: 1 days, 00:14:07 LOAD AVERAGE: 16.04, 17.66, 19.22 TASKS: 942 NODENAME: cartan1206 RELEASE: 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 VERSION: #1 SMP Mon Nov 7 15:21:24 CET 2011 MACHINE: x86_64 (2266 Mhz) MEMORY: 64 GB PANIC: "Kernel panic - not syncing: LBUG" PID: 16826 COMMAND: "%%U1234_malh" TASK: ffff880c782f27d0 [THREAD_INFO: ffff880c6fbb8000] CPU: 10 STATE: TASK_RUNNING (PANIC) crash> bt PID: 16826 TASK: ffff880c782f27d0 CPU: 10 COMMAND: "%%U1234_malh" #0 [ffff880c6fbbbad0] machine_kexec at ffffffff81027a4b #1 [ffff880c6fbbbb30] crash_kexec at ffffffff810a2db2 #2 [ffff880c6fbbbc00] panic at ffffffff8147d80a #3 [ffff880c6fbbbc80] lbug_with_loc at ffffffffa03c0ecb [libcfs] #4 [ffff880c6fbbbca0] cl_lock_hold_release at ffffffffa04dd5e2 [obdclass] #5 [ffff880c6fbbbcc0] cl_lock_release at ffffffffa04dedf2 [obdclass] #6 [ffff880c6fbbbcf0] cl_lock_link_fini at ffffffffa04e6338 [obdclass] #7 [ffff880c6fbbbd30] cl_io_unlock at ffffffffa04e6565 [obdclass] #8 [ffff880c6fbbbd70] cl_io_loop at ffffffffa04e7245 [obdclass] #9 [ffff880c6fbbbdb0] ll_file_io_generic at ffffffffa09f5ef7 [lustre] #10 [ffff880c6fbbbe20] ll_file_aio_read at ffffffffa09f61e9 [lustre] #11 [ffff880c6fbbbe80] ll_file_read at ffffffffa09f66a9 [lustre] #12 [ffff880c6fbbbef0] vfs_read at ffffffff8115e355 #13 [ffff880c6fbbbf30] sys_read at ffffffff8115e491 #14 [ffff880c6fbbbf80] system_call_fastpath at ffffffff81003172 RIP: 00000033e7cd89cd RSP: 00007fff93278bc8 RFLAGS: 00000202 RAX: 0000000000000000 RBX: ffffffff81003172 RCX: 00007fff93278b88 RDX: 0000000000100000 RSI: 00007ffe39238000 RDI: 0000000000000069 RBP: 0000000241500000 R8: 0000000241500000 R9: 0000000016837380 R10: 00007ffe4c5c5720 R11: 0000000000000293 R12: 0000000000100000 R13: 0000000241400000 R14: 00000000168372a0 R15: ffff880c6fbbbf78 ORIG_RAX: 0000000000000000 CS: 0033 SS: 002b crash> log | tail -100 LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lock@ffff88059fbbe778[1 3 0 1 1 00000000] W(2):[810032, 818223]@[0x101140000:0xd1f6:0x0] { LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lovsub@ffff88066f1d0de0: [0 ffff8804c6fe7470 W(2):[2431024, 2455599]@[0x2103bd57a:0xe:0x0]] [0 ffff8804c6fe72d0 R(1):[2431024, 2455599]@[0x2103bd57a:0xe:0x0]] LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) osc@ffff8805351917b8: ffff88060494d240 40100001 0x1fe0953368e5bb15 3 ffff880c73c49508 size: 3321161855 mtime: 1341405025 atime: 1341405025 ctime: 1341405025 blocks: 6363088 LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) } lock@ffff88059fbbe778 LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) queued. LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lock@ffff88059fbbe158[2 1 0 1 1 00000000] R(1):[809984, 810495]@[0x101140000:0xd1f6:0x0] { LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lovsub@ffff88066f1d08e0: [0 ffff8804c6fe70c8 R(1):[2430976, 2431487]@[0x2103bd57a:0xe:0x0]] LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) osc@ffff880535191a98: (null) 00000000 0x0 0 (null) size: 0 mtime: 0 atime: 0 ctime: 0 blocks: 0 LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) } lock@ffff88059fbbe158 LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) queuing. Pid: 16822, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] osc_lock_enqueue+0x775/0xa30 [osc] [] ? cfs_waitq_broadcast+0x17/0x20 [libcfs] [] ? cl_lock_state_signal+0x81/0x160 [obdclass] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] lov_lock_enqueue+0x20b/0x7c0 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] cl_enqueue_locked+0x6d/0x200 [obdclass] [] cl_lock_request+0x7e/0x1c0 [obdclass] [] cl_io_lock+0x3d5/0x570 [obdclass] [] cl_io_loop+0xba/0x190 [obdclass] [] ll_file_io_generic+0x3a7/0x560 [lustre] [] ll_file_aio_read+0x139/0x2c0 [lustre] [] ll_file_read+0x169/0x2a0 [lustre] [] vfs_read+0xb5/0x1a0 [] sys_read+0x51/0x90 [] system_call_fastpath+0x16/0x1b Lustre: DEBUG MARKER: Wed Jul 4 14:35:02 2012 LustreError: 11-0: an error occurred while communicating with QQ.P.BBO.BZ@o2ib2. The obd_ping operation failed with -107 Lustre: cdep4-OST02c4-osc-ffff88107812dc00: Connection to cdep4-OST02c4 (at QQ.P.BBO.BZ@o2ib2) was lost; in progress operations using this service will wait for recovery to complete LustreError: 167-0: This client was evicted by cdep4-OST02c4; in progress operations using this service will fail. LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) lock@ffff880c26e897b8[2 3 0 1 1 00000000] R(1):[787968, 18446744073709551615]@[0x102c40000:0x5307:0x0] { LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) lovsub@ffff880c5dfaafa0: [0 ffff880a2b418a20 R(1):[2364928, 18446744073709551615]@[0x2103bd57a:0xd:0x0]] LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) osc@ffff880a3834bee8: ffff8809d9539d80 40120002 0x1fe0953368e64d9e 3 ffff880c6f55c2f8 size: 3226075136 mtime: 1341405146 atime: 1341405146 ctime: 1341405146 blocks: 6190744 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) } lock@ffff880c26e897b8 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) dlmlock returned -5 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) lock@ffff880a166a9cb8[2 3 0 1 1 00000000] R(1):[787456, 18446744073709551615]@[0x102c40000:0x5307:0x0] { LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) lovsub@ffff880c5dfaa160: [0 ffff880a16d32268 R(1):[2364416, 18446744073709551615]@[0x2103bd57a:0xd:0x0]] LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) osc@ffff880a5fd3e088: ffff880a3e265000 40120002 0x1fe0953368e64da5 3 ffff881078136d20 size: 3226075136 mtime: 1341405146 atime: 1341405146 ctime: 1341405146 blocks: 6190744 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) } lock@ffff880a166a9cb8 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) dlmlock returned -5 LustreError: 22282:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace cdep4-OST02c4-osc-ffff88107812dc00 resource refcount nonzero (2) after lock cleanup; forcing cleanup. LustreError: 22282:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff880b2c57be40 (21255/0/0/0) (rc: 2) Lustre: cdep4-OST02c4-osc-ffff88107812dc00: Connection restored to cdep4-OST02c4 (at QQ.P.BBO.BZ@o2ib2) LustreError: 16825:0:(cl_lock.c:1412:cl_unuse_try()) result = -5, this is unlikely! LustreError: 16825:0:(cl_lock.c:1426:cl_unuse_locked()) lock@ffff880c26e89a58[1 3 0 1 0 00000000] R(1):[2364928, 18446744073709551615]@[0x2103bd57a:0xd:0x0] { LustreError: 16826:0:(cl_lock.c:1412:cl_unuse_try()) result = -5, this is unlikely! LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) lock@ffff880a0fb01a58[1 3 0 1 0 00000000] R(1):[2364416, 18446744073709551615]@[0x2103bd57a:0xd:0x0] { LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) vvp@ffff8809913bce20: LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) lov@ffff880a16d32268: 1 LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) 0 0: --- LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) } lock@ffff880a0fb01a58 LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) unuse return -5 LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) lock@ffff880a0fb01a58[1 3 0 0 0 00000000] R(1):[2364416, 18446744073709551615]@[0x2103bd57a:0xd:0x0] { LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) vvp@ffff8809913bce20: LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) lov@ffff880a16d32268: 1 LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) 0 0: --- LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) } lock@ffff880a0fb01a58 LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) failed at lock->cll_state != CLS_HELD. LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) LBUG Pid: 16826, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] lbug_with_loc+0x47/0xb0 [libcfs] [] cl_lock_hold_release+0x2a2/0x2b0 [obdclass] [] cl_lock_release+0x82/0x180 [obdclass] [] cl_lock_link_fini+0x68/0x160 [obdclass] [] cl_io_unlock+0x135/0x2e0 [obdclass] [] cl_io_loop+0xe5/0x190 [obdclass] [] ll_file_io_generic+0x3a7/0x560 [lustre] [] ll_file_aio_read+0x139/0x2c0 [lustre] [] ll_file_read+0x169/0x2a0 [lustre] [] vfs_read+0xb5/0x1a0 [] sys_read+0x51/0x90 [] system_call_fastpath+0x16/0x1b Kernel panic - not syncing: LBUG Pid: 16826, comm: %%U1234_malh Not tainted 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 #1 Call Trace: [] ? panic+0x78/0x143 [] ? lbug_with_loc+0x9b/0xb0 [libcfs] [] ? cl_lock_hold_release+0x2a2/0x2b0 [obdclass] [] ? cl_lock_release+0x82/0x180 [obdclass] [] ? cl_lock_link_fini+0x68/0x160 [obdclass] [] ? cl_io_unlock+0x135/0x2e0 [obdclass] [] ? cl_io_loop+0xe5/0x190 [obdclass] [] ? ll_file_io_generic+0x3a7/0x560 [lustre] [] ? ll_file_aio_read+0x139/0x2c0 [lustre] [] ? ll_file_read+0x169/0x2a0 [lustre] [] ? vfs_read+0xb5/0x1a0 [] ? sys_read+0x51/0x90 [] ? system_call_fastpath+0x16/0x1b crash> log Initializing cgroup subsys cpuset Initializing cgroup subsys cpu Linux version 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 (hpcdelivery@atlas.frec.bull.fr) (gcc version 4.4.5 20110214 (Bull 4.4.5-6) (GCC) ) #1 SMP Mon Nov 7 15:21:24 CET 2011 Command line: ro root=UUID=c897bbc4-3f40-4e1c-a0ce-1fa409531a01 rd_NO_LUKS rd_NO_LVM rd_NO_MD rd_NO_DM LANG=en_US.UTF-8 SYSFONT=latarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=us console=tty0 console=ttyS1,115200 crashkernel=192M rdloaddriver=ahci rdloaddriver=aacraid rdloaddriver=megaraid_sas cgroup_disable=memory selinux=0 nmi_watchdog=0 transparent_hugepage=never pciehp_disable tick_synchro=1 nointremap intel_iommu=off audit=0 KERNEL supported cpus: Intel GenuineIntel AMD AuthenticAMD Centaur CentaurHauls BIOS-provided physical RAM map: BIOS-e820: 0000000000000000 - 000000000009d000 (usable) BIOS-e820: 000000000009d000 - 00000000000a0000 (reserved) BIOS-e820: 00000000000e0000 - 0000000000100000 (reserved) BIOS-e820: 0000000000100000 - 0000000079b3b000 (usable) BIOS-e820: 0000000079b3b000 - 0000000079b88000 (ACPI data) BIOS-e820: 0000000079b88000 - 000000007ed0a000 (usable) BIOS-e820: 000000007ed0a000 - 000000007ed0e000 (ACPI data) BIOS-e820: 000000007ed0e000 - 000000007ed13000 (reserved) BIOS-e820: 000000007ed13000 - 000000007ed14000 (ACPI data) BIOS-e820: 000000007ed14000 - 000000007ed1f000 (ACPI NVS) BIOS-e820: 000000007ed1f000 - 000000007ed2f000 (ACPI data) BIOS-e820: 000000007ed2f000 - 000000007ed30000 (ACPI NVS) BIOS-e820: 000000007ed30000 - 000000007ed31000 (ACPI data) BIOS-e820: 000000007ed31000 - 000000007ed32000 (ACPI NVS) BIOS-e820: 000000007ed32000 - 000000007ed52000 (reserved) BIOS-e820: 000000007ed52000 - 000000007ed53000 (ACPI data) BIOS-e820: 000000007ed53000 - 000000007ed59000 (usable) BIOS-e820: 000000007ed59000 - 000000007f000000 (ACPI NVS) BIOS-e820: 000000007f000000 - 0000000090000000 (reserved) BIOS-e820: 00000000fc000000 - 00000000fd000000 (reserved) BIOS-e820: 00000000fed1c000 - 00000000fed20000 (reserved) BIOS-e820: 00000000ff800000 - 0000000100000000 (reserved) BIOS-e820: 0000000100000000 - 000000107fe8f000 (usable) BIOS-e820: 000000107fe8f000 - 000000107fedf000 (reserved) BIOS-e820: 000000107fedf000 - 000000107ffdf000 (ACPI NVS) BIOS-e820: 000000107ffdf000 - 000000107ffff000 (ACPI data) BIOS-e820: 000000107ffff000 - 0000001080000000 (usable) DMI 2.4 present. SMBIOS version 2.4 @ 0xFE120 e820 update range: 0000000000000000 - 0000000000001000 (usable) ==> (reserved) e820 remove range: 00000000000a0000 - 0000000000100000 (usable) last_pfn = 0x1080000 max_arch_pfn = 0x400000000 MTRR default type: write-back MTRR fixed ranges enabled: 00000-9FFFF write-back A0000-FFFFF uncachable MTRR variable ranges enabled: 0 base 00080000000 mask FFF80000000 uncachable 1 base FC000000000 mask FFF80000000 uncachable 2 disabled 3 disabled 4 disabled 5 disabled 6 disabled 7 disabled x86 PAT enabled: cpu 0, old 0x7040600070406, new 0x7010600070106 last_pfn = 0x7ed59 max_arch_pfn = 0x400000000 initial memory mapped : 0 - 20000000 init_memory_mapping: 0000000000000000-000000007ed59000 0000000000 - 007ec00000 page 2M 007ec00000 - 007ed59000 page 4k kernel direct mapping tables up to 7ed59000 @ 8000-c000 init_memory_mapping: 0000000100000000-0000001080000000 0100000000 - 1080000000 page 2M kernel direct mapping tables up to 1080000000 @ a000-4d000 RAMDISK: 37859000 - 37fefb26 ACPI: RSDP 00000000000fe020 00024 (v02 INTEL ) ACPI: XSDT 000000007ed52120 0008C (v01 INTEL TIANO 00000000 01000013) ACPI: FACP 000000007ed30000 000F4 (v04 INTEL TIANO 00000000 MSFT 01000013) ACPI: DSDT 000000007ed25000 09E8C (v02 INTEL TIANO 00000003 MSFT 01000013) ACPI: FACS 000000007ed2f000 00040 ACPI: APIC 000000007ed24000 004C4 (v02 INTEL TIANO 00000000 MSFT 01000013) ACPI: MCFG 000000007ed23000 0003C (v01 INTEL TIANO 00000001 MSFT 01000013) ACPI: HPET 000000007ed22000 00038 (v01 INTEL TIANO 00000001 MSFT 01000013) ACPI: SLIT 000000007ed21000 00050 (v01 INTEL TIANO 00000001 MSFT 01000013) ACPI: SRAT 000000007ed20000 00A30 (v02 INTEL TIANO 00000001 MSFT 01000013) ACPI: SSDT 0000000079b3b000 4C0B4 (v02 INTEL SSDT PM 00004000 INTL 20091214) ACPI: SPMI 000000007ed1f000 00040 (v05 INTEL INSYDE 00000001 ? 00000000) ACPI: SPCR 000000007ed13000 00050 (v01 INSYDE INSYDE 00000001 ? 00000001) ACPI: HEST 000000007ed0d000 000A8 (v01 INSYDE THURLEY 00000001 ? 00000001) ACPI: ERST 000000007ed0c000 00230 (v01 INSYDE THURLEY 00000001 ? 00000001) ACPI: BERT 000000007ed0b000 00030 (v01 INSYDE THURLEY 00000001 ? 00000001) ACPI: EINJ 000000007ed0a000 00130 (v01 INSYDE THURLEY 00000001 ? 00000001) ACPI: Local APIC address 0xfee00000 SRAT: PXM 0 -> APIC 0 -> Node 0 SRAT: PXM 1 -> APIC 32 -> Node 1 SRAT: PXM 2 -> APIC 64 -> Node 2 SRAT: PXM 3 -> APIC 96 -> Node 3 SRAT: PXM 0 -> APIC 16 -> Node 0 SRAT: PXM 1 -> APIC 48 -> Node 1 SRAT: PXM 2 -> APIC 80 -> Node 2 SRAT: PXM 3 -> APIC 112 -> Node 3 SRAT: PXM 0 -> APIC 4 -> Node 0 SRAT: PXM 1 -> APIC 36 -> Node 1 SRAT: PXM 2 -> APIC 68 -> Node 2 SRAT: PXM 3 -> APIC 100 -> Node 3 SRAT: PXM 0 -> APIC 20 -> Node 0 SRAT: PXM 1 -> APIC 52 -> Node 1 SRAT: PXM 2 -> APIC 84 -> Node 2 SRAT: PXM 3 -> APIC 116 -> Node 3 SRAT: PXM 0 -> APIC 2 -> Node 0 SRAT: PXM 1 -> APIC 34 -> Node 1 SRAT: PXM 2 -> APIC 66 -> Node 2 SRAT: PXM 3 -> APIC 98 -> Node 3 SRAT: PXM 0 -> APIC 18 -> Node 0 SRAT: PXM 1 -> APIC 50 -> Node 1 SRAT: PXM 2 -> APIC 82 -> Node 2 SRAT: PXM 3 -> APIC 114 -> Node 3 SRAT: PXM 0 -> APIC 6 -> Node 0 SRAT: PXM 1 -> APIC 38 -> Node 1 SRAT: PXM 2 -> APIC 70 -> Node 2 SRAT: PXM 3 -> APIC 102 -> Node 3 SRAT: PXM 0 -> APIC 22 -> Node 0 SRAT: PXM 1 -> APIC 54 -> Node 1 SRAT: PXM 2 -> APIC 86 -> Node 2 SRAT: PXM 3 -> APIC 118 -> Node 3 SRAT: Node 0 PXM 0 0-80000000 SRAT: Node 0 PXM 0 100000000-480000000 SRAT: Node 1 PXM 1 480000000-880000000 SRAT: Node 2 PXM 2 880000000-c80000000 SRAT: Node 3 PXM 3 c80000000-1080000000 NUMA: Using 31 for the hash shift. Bootmem setup node 0 0000000000000000-0000000480000000 NODE_DATA [0000000000048000 - 000000000004dfff] bootmap [0000000000100000 - 000000000018ffff] pages 90 (9 early reservations) ==> bootmem [0000000000 - 0480000000] #0 [0000000000 - 0000001000] BIOS data page ==> [0000000000 - 0000001000] #1 [0000006000 - 0000008000] TRAMPOLINE ==> [0000006000 - 0000008000] #2 [0001000000 - 0001cfaf94] TEXT DATA BSS ==> [0001000000 - 0001cfaf94] #3 [0037859000 - 0037fefb26] RAMDISK ==> [0037859000 - 0037fefb26] #4 [0000097c00 - 0000100000] BIOS reserved ==> [0000097c00 - 0000100000] #5 [0001cfb000 - 0001cfb0c4] BRK ==> [0001cfb000 - 0001cfb0c4] #6 [0000008000 - 000000a000] PGTABLE ==> [0000008000 - 000000a000] #7 [000000a000 - 0000048000] PGTABLE ==> [000000a000 - 0000048000] #8 [0000001000 - 0000001050] ACPI SLIT ==> [0000001000 - 0000001050] Bootmem setup node 1 0000000480000000-0000000880000000 NODE_DATA [0000000480000040 - 000000048000603f] bootmap [0000000480007000 - 0000000480086fff] pages 80 (9 early reservations) ==> bootmem [0480000000 - 0880000000] #0 [0000000000 - 0000001000] BIOS data page #1 [0000006000 - 0000008000] TRAMPOLINE #2 [0001000000 - 0001cfaf94] TEXT DATA BSS #3 [0037859000 - 0037fefb26] RAMDISK #4 [0000097c00 - 0000100000] BIOS reserved #5 [0001cfb000 - 0001cfb0c4] BRK #6 [0000008000 - 000000a000] PGTABLE #7 [000000a000 - 0000048000] PGTABLE #8 [0000001000 - 0000001050] ACPI SLIT Bootmem setup node 2 0000000880000000-0000000c80000000 NODE_DATA [0000000880000080 - 000000088000607f] bootmap [0000000880007000 - 0000000880086fff] pages 80 (9 early reservations) ==> bootmem [0880000000 - 0c80000000] #0 [0000000000 - 0000001000] BIOS data page #1 [0000006000 - 0000008000] TRAMPOLINE #2 [0001000000 - 0001cfaf94] TEXT DATA BSS #3 [0037859000 - 0037fefb26] RAMDISK #4 [0000097c00 - 0000100000] BIOS reserved #5 [0001cfb000 - 0001cfb0c4] BRK #6 [0000008000 - 000000a000] PGTABLE #7 [000000a000 - 0000048000] PGTABLE #8 [0000001000 - 0000001050] ACPI SLIT Bootmem setup node 3 0000000c80000000-0000001080000000 NODE_DATA [0000000c800000c0 - 0000000c800060bf] bootmap [0000000c80007000 - 0000000c80086fff] pages 80 (9 early reservations) ==> bootmem [0c80000000 - 1080000000] #0 [0000000000 - 0000001000] BIOS data page #1 [0000006000 - 0000008000] TRAMPOLINE #2 [0001000000 - 0001cfaf94] TEXT DATA BSS #3 [0037859000 - 0037fefb26] RAMDISK #4 [0000097c00 - 0000100000] BIOS reserved #5 [0001cfb000 - 0001cfb0c4] BRK #6 [0000008000 - 000000a000] PGTABLE #7 [000000a000 - 0000048000] PGTABLE #8 [0000001000 - 0000001050] ACPI SLIT found SMP MP-table at [ffff8800000fe270] fe270 Reserving 192MB of memory at 32MB for crashkernel (System RAM: 67584MB) [ffffea0000000000-ffffea000fbfffff] PMD -> [ffff880028600000-ffff8800365fffff] on node 0 [ffffea000fc00000-ffffea001dbfffff] PMD -> [ffff880480200000-ffff88048e1fffff] on node 1 [ffffea001dc00000-ffffea002bbfffff] PMD -> [ffff880880200000-ffff88088e1fffff] on node 2 [ffffea002bc00000-ffffea0039bfffff] PMD -> [ffff880c80200000-ffff880c8e1fffff] on node 3 Zone PFN ranges: DMA 0x00000001 -> 0x00001000 DMA32 0x00001000 -> 0x00100000 Normal 0x00100000 -> 0x01080000 Movable zone start PFN for each node early_node_map[9] active PFN ranges 0: 0x00000001 -> 0x0000009d 0: 0x00000100 -> 0x00079b3b 0: 0x00079b88 -> 0x0007ed0a 0: 0x0007ed53 -> 0x0007ed59 0: 0x00100000 -> 0x00480000 1: 0x00480000 -> 0x00880000 2: 0x00880000 -> 0x00c80000 3: 0x00c80000 -> 0x0107fe8f 3: 0x0107ffff -> 0x01080000 On node 0 totalpages: 4189279 DMA zone: 56 pages used for memmap DMA zone: 171 pages reserved DMA zone: 3769 pages, LIFO batch:0 DMA32 zone: 14280 pages used for memmap DMA32 zone: 500987 pages, LIFO batch:31 Normal zone: 50176 pages used for memmap Normal zone: 3619840 pages, LIFO batch:31 On node 1 totalpages: 4194304 Normal zone: 57344 pages used for memmap Normal zone: 4136960 pages, LIFO batch:31 On node 2 totalpages: 4194304 Normal zone: 57344 pages used for memmap Normal zone: 4136960 pages, LIFO batch:31 On node 3 totalpages: 4193936 Normal zone: 57344 pages used for memmap Normal zone: 4136592 pages, LIFO batch:31 ACPI: PM-Timer IO Port: 0x408 ACPI: Local APIC address 0xfee00000 ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled) ACPI: LAPIC (acpi_id[0x01] lapic_id[0x20] enabled) ACPI: LAPIC (acpi_id[0x02] lapic_id[0x40] enabled) ACPI: LAPIC (acpi_id[0x03] lapic_id[0x60] enabled) ACPI: LAPIC (acpi_id[0x04] lapic_id[0x10] enabled) ACPI: LAPIC (acpi_id[0x05] lapic_id[0x30] enabled) ACPI: LAPIC (acpi_id[0x06] lapic_id[0x50] enabled) ACPI: LAPIC (acpi_id[0x07] lapic_id[0x70] enabled) ACPI: LAPIC (acpi_id[0x08] lapic_id[0x04] enabled) ACPI: LAPIC (acpi_id[0x09] lapic_id[0x24] enabled) ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x44] enabled) ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x64] enabled) ACPI: LAPIC (acpi_id[0x0c] lapic_id[0x14] enabled) ACPI: LAPIC (acpi_id[0x0d] lapic_id[0x34] enabled) ACPI: LAPIC (acpi_id[0x0e] lapic_id[0x54] enabled) ACPI: LAPIC (acpi_id[0x0f] lapic_id[0x74] enabled) ACPI: LAPIC (acpi_id[0x10] lapic_id[0x02] enabled) ACPI: LAPIC (acpi_id[0x11] lapic_id[0x22] enabled) ACPI: LAPIC (acpi_id[0x12] lapic_id[0x42] enabled) ACPI: LAPIC (acpi_id[0x13] lapic_id[0x62] enabled) ACPI: LAPIC (acpi_id[0x14] lapic_id[0x12] enabled) ACPI: LAPIC (acpi_id[0x15] lapic_id[0x32] enabled) ACPI: LAPIC (acpi_id[0x16] lapic_id[0x52] enabled) ACPI: LAPIC (acpi_id[0x17] lapic_id[0x72] enabled) ACPI: LAPIC (acpi_id[0x18] lapic_id[0x06] enabled) ACPI: LAPIC (acpi_id[0x19] lapic_id[0x26] enabled) ACPI: LAPIC (acpi_id[0x1a] lapic_id[0x46] enabled) ACPI: LAPIC (acpi_id[0x1b] lapic_id[0x66] enabled) ACPI: LAPIC (acpi_id[0x1c] lapic_id[0x16] enabled) ACPI: LAPIC (acpi_id[0x1d] lapic_id[0x36] enabled) ACPI: LAPIC (acpi_id[0x1e] lapic_id[0x56] enabled) ACPI: LAPIC (acpi_id[0x1f] lapic_id[0x76] enabled) ACPI: LAPIC (acpi_id[0x20] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x21] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x22] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x23] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x24] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x25] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x26] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x27] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x28] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x29] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x2a] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x2b] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x2c] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x2d] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x2e] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x2f] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x30] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x31] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x32] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x33] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x34] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x35] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x36] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x37] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x38] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x39] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x3a] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x3b] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x3c] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x3d] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x3e] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x3f] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x40] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x41] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x42] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x43] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x44] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x45] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x46] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x47] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x48] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x49] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x4a] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x4b] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x4c] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x4d] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x4e] lapic_id[0xff] disabled) ACPI: LAPIC (acpi_id[0x4f] lapic_id[0xff] disabled) ACPI: LAPIC_NMI (acpi_id[0x00] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x01] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x02] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x03] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x04] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x05] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x06] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x07] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x08] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x09] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x0a] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x0b] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x0c] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x0d] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x0e] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x0f] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x10] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x11] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x12] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x13] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x14] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x15] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x16] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x17] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x18] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x19] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x1a] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x1b] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x1c] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x1d] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x1e] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x1f] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x20] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x21] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x22] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x23] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x24] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x25] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x26] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x27] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x28] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x29] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x2a] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x2b] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x2c] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x2d] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x2e] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x2f] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x30] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x31] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x32] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x33] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x34] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x35] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x36] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x37] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x38] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x39] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x3a] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x3b] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x3c] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x3d] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x3e] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x3f] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x40] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x41] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x42] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x43] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x44] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x45] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x46] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x47] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x48] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x49] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x4a] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x4b] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x4c] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x4d] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x4e] high level lint[0x1]) ACPI: LAPIC_NMI (acpi_id[0x4f] high level lint[0x1]) ACPI: IOAPIC (id[0x08] address[0xfec00000] gsi_base[0]) IOAPIC[0]: apic_id 8, version 32, address 0xfec00000, GSI 0-23 ACPI: IOAPIC (id[0x09] address[0xfec01000] gsi_base[24]) IOAPIC[1]: apic_id 9, version 32, address 0xfec01000, GSI 24-47 ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl) ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level) ACPI: IRQ0 used by override. ACPI: IRQ2 used by override. ACPI: IRQ9 used by override. Using ACPI (MADT) for SMP configuration information ACPI: HPET id: 0x8086a301 base: 0xfed00000 SMP: Allowing 80 CPUs, 48 hotplug CPUs nr_irqs_gsi: 48 PM: Registered nosave memory: 000000000009d000 - 00000000000a0000 PM: Registered nosave memory: 00000000000a0000 - 00000000000e0000 PM: Registered nosave memory: 00000000000e0000 - 0000000000100000 PM: Registered nosave memory: 0000000079b3b000 - 0000000079b88000 PM: Registered nosave memory: 000000007ed0a000 - 000000007ed0e000 PM: Registered nosave memory: 000000007ed0e000 - 000000007ed13000 PM: Registered nosave memory: 000000007ed13000 - 000000007ed14000 PM: Registered nosave memory: 000000007ed14000 - 000000007ed1f000 PM: Registered nosave memory: 000000007ed1f000 - 000000007ed2f000 PM: Registered nosave memory: 000000007ed2f000 - 000000007ed30000 PM: Registered nosave memory: 000000007ed30000 - 000000007ed31000 PM: Registered nosave memory: 000000007ed31000 - 000000007ed32000 PM: Registered nosave memory: 000000007ed32000 - 000000007ed52000 PM: Registered nosave memory: 000000007ed52000 - 000000007ed53000 PM: Registered nosave memory: 000000007ed59000 - 000000007f000000 PM: Registered nosave memory: 000000007f000000 - 0000000090000000 PM: Registered nosave memory: 0000000090000000 - 00000000fc000000 PM: Registered nosave memory: 00000000fc000000 - 00000000fd000000 PM: Registered nosave memory: 00000000fd000000 - 00000000fed1c000 PM: Registered nosave memory: 00000000fed1c000 - 00000000fed20000 PM: Registered nosave memory: 00000000fed20000 - 00000000ff800000 PM: Registered nosave memory: 00000000ff800000 - 0000000100000000 PM: Registered nosave memory: 000000107fe8f000 - 000000107fedf000 PM: Registered nosave memory: 000000107fedf000 - 000000107ffdf000 PM: Registered nosave memory: 000000107ffdf000 - 000000107ffff000 Allocating PCI resources starting at 90000000 (gap: 90000000:6c000000) Booting paravirtualized kernel on bare hardware NR_CPUS:320 nr_cpumask_bits:320 nr_cpu_ids:80 nr_node_ids:4 PERCPU: Embedded 27 pages/cpu @ffff880036600000 s79640 r8192 d22760 u262144 pcpu-alloc: s79640 r8192 d22760 u262144 alloc=1*2097152 pcpu-alloc: [0] 00 04 08 12 16 20 24 28 [0] 32 36 40 44 48 52 56 60 pcpu-alloc: [0] 64 68 72 76 -- -- -- -- [1] 01 05 09 13 17 21 25 29 pcpu-alloc: [1] 33 37 41 45 49 53 57 61 [1] 65 69 73 77 -- -- -- -- pcpu-alloc: [2] 02 06 10 14 18 22 26 30 [2] 34 38 42 46 50 54 58 62 pcpu-alloc: [2] 66 70 74 78 -- -- -- -- [3] 03 07 11 15 19 23 27 31 pcpu-alloc: [3] 35 39 43 47 51 55 59 63 [3] 67 71 75 79 -- -- -- -- Built 4 zonelists in Zone order, mobility grouping on. Total pages: 16535108 Policy zone: Normal Kernel command line: ro root=UUID=c897bbc4-3f40-4e1c-a0ce-1fa409531a01 rd_NO_LUKS rd_NO_LVM rd_NO_MD rd_NO_DM LANG=en_US.UTF-8 SYSFONT=latarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=us console=tty0 console=ttyS1,115200 crashkernel=192M rdloaddriver=ahci rdloaddriver=aacraid rdloaddriver=megaraid_sas cgroup_disable=memory selinux=0 nmi_watchdog=0 transparent_hugepage=never pciehp_disable tick_synchro=1 nointremap intel_iommu=off audit=0 Disabling memory control group subsystem Intel-IOMMU: disabled audit: disabled (until reboot) PID hash table entries: 4096 (order: 3, 32768 bytes) Tick synchro will use HPET. Checking aperture... No AGP bridge found PCI-DMA: Using software bounce buffering for IO (SWIOTLB) Placing 64MB software IO TLB between ffff880020000000 - ffff880024000000 software IO TLB at phys 0x20000000 - 0x24000000 Memory: 65876412k/69206016k available (4641k kernel code, 2118724k absent, 1210880k reserved, 6395k data, 784k init) Hierarchical RCU implementation. NR_IRQS:4352 nr_irqs:1456 Extended CMOS year: 2000 Console: colour VGA+ 80x25 console [tty0] enabled console [ttyS1] enabled HPET: enabling legacy interrupts Wrote HPET irq cfg 3 hpet clockevent registered HPET: 4 timers in total, 0 timers will be used for per-cpu timer Fast TSC calibration using PIT Detected 2266.842 MHz processor. Calibrating delay loop (skipped), value calculated using timer frequency.. 4533.68 BogoMIPS (lpj=22668420) pid_max: default: 81920 minimum: 640 Security Framework initialized SELinux: Disabled at boot. Dentry cache hash table entries: 8388608 (order: 14, 67108864 bytes) Inode-cache hash table entries: 4194304 (order: 13, 33554432 bytes) Mount-cache hash table entries: 256 Initializing cgroup subsys ns Initializing cgroup subsys cpuacct Initializing cgroup subsys memory Initializing cgroup subsys devices Initializing cgroup subsys freezer Initializing cgroup subsys net_cls Initializing cgroup subsys blkio CPU: Physical Processor ID: 0 CPU: Processor Core ID: 0 mce: CPU supports 22 MCE banks CPU0: Thermal monitoring enabled (TM1) using mwait in idle threads. ACPI: Core revision 20090903 ftrace: converting mcount calls to 0f 1f 44 00 00 ftrace: allocating 19294 entries in 76 pages Not enabling x2apic, Intr-remapping init failed. Setting APIC routing to physical flat ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1 CPU0: Intel(R) Xeon(R) CPU X7560 @ 2.27GHz stepping 06 Performance Events: PEBS fmt1+, Nehalem events, Intel PMU driver. ... version: 3 ... bit width: 48 ... generic registers: 4 ... value mask: 0000ffffffffffff ... max period: 000000007fffffff ... fixed-purpose events: 3 ... event mask: 000000070000000f Booting Node 1, Processors #1 Ok. Booting Node 2, Processors #2 Ok. Booting Node 3, Processors #3 Ok. Booting Node 0, Processors #4 Ok. Booting Node 1, Processors #5 Ok. Booting Node 2, Processors #6 Ok. Booting Node 3, Processors #7 Ok. Booting Node 0, Processors #8 Ok. Booting Node 1, Processors #9 Ok. Booting Node 2, Processors #10 Ok. Booting Node 3, Processors #11 Ok. Booting Node 0, Processors #12 Ok. Booting Node 1, Processors #13 Ok. Booting Node 2, Processors #14 Ok. Booting Node 3, Processors #15 Ok. Booting Node 0, Processors #16 Ok. Booting Node 1, Processors #17 Ok. Booting Node 2, Processors #18 Ok. Booting Node 3, Processors #19 Ok. Booting Node 0, Processors #20 Ok. Booting Node 1, Processors #21 Ok. Booting Node 2, Processors #22 Ok. Booting Node 3, Processors #23 Ok. Booting Node 0, Processors #24 Ok. Booting Node 1, Processors #25 Ok. Booting Node 2, Processors #26 Ok. Booting Node 3, Processors #27 Ok. Booting Node 0, Processors #28 Ok. Booting Node 1, Processors #29 Ok. Booting Node 2, Processors #30 Ok. Booting Node 3, Processors #31 Brought up 32 CPUs Total of 32 processors activated (145313.10 BogoMIPS). sizeof(vma)=200 bytes sizeof(page)=56 bytes sizeof(inode)=592 bytes sizeof(dentry)=192 bytes sizeof(ext3inode)=800 bytes sizeof(buffer_head)=104 bytes sizeof(skbuff)=232 bytes sizeof(task_struct)=1872 bytes devtmpfs: initialized regulator: core version 0.5 NET: Registered protocol family 16 ACPI FADT declares the system doesn't support PCIe ASPM, so disable it ACPI: bus type pci registered PCI: MCFG configuration 0: base 80000000 segment 0 buses 0 - 255 PCI: MCFG area at 80000000 reserved in E820 PCI: Using MMCONFIG at 80000000 - 8fffffff PCI: Using configuration type 1 for base access bio: create slab at 0 ACPI: EC: Look up EC in DSDT ACPI Error (psargs-0359): [WCB1] Namespace lookup failure, AE_NOT_FOUND ACPI Error (psparse-0537): Method parse/execution failed [\_SB_._OSC] (Node ffff88107e0803d0), AE_NOT_FOUND ACPI: Interpreter enabled ACPI: (supports S0 S5) ACPI: Using IOAPIC for interrupt routing ACPI: No dock devices found. ACPI: PCI Root Bridge [IOH0] (0000:00) I/O only node 4, pxm 4, CPU-mem node 0 pci 0000:00:00.0: PME# supported from D0 D3hot D3cold pci 0000:00:00.0: PME# disabled pci 0000:00:01.0: PME# supported from D0 D3hot D3cold pci 0000:00:01.0: PME# disabled pci 0000:00:03.0: PME# supported from D0 D3hot D3cold pci 0000:00:03.0: PME# disabled pci 0000:00:05.0: PME# supported from D0 D3hot D3cold pci 0000:00:05.0: PME# disabled pci 0000:00:07.0: PME# supported from D0 D3hot D3cold pci 0000:00:07.0: PME# disabled pci 0000:00:13.0: reg 10 32bit mmio: [0x91c24000-0x91c24fff] pci 0000:00:13.0: PME# supported from D0 D3hot D3cold pci 0000:00:13.0: PME# disabled pci 0000:00:16.0: reg 10 64bit mmio: [0x91c00000-0x91c03fff] pci 0000:00:16.1: reg 10 64bit mmio: [0x91c04000-0x91c07fff] pci 0000:00:16.2: reg 10 64bit mmio: [0x91c08000-0x91c0bfff] pci 0000:00:16.3: reg 10 64bit mmio: [0x91c0c000-0x91c0ffff] pci 0000:00:16.4: reg 10 64bit mmio: [0x91c10000-0x91c13fff] pci 0000:00:16.5: reg 10 64bit mmio: [0x91c14000-0x91c17fff] pci 0000:00:16.6: reg 10 64bit mmio: [0x91c18000-0x91c1bfff] pci 0000:00:16.7: reg 10 64bit mmio: [0x91c1c000-0x91c1ffff] pci 0000:00:1a.0: reg 20 io port: [0x20e0-0x20ff] pci 0000:00:1a.1: reg 20 io port: [0x20c0-0x20df] pci 0000:00:1a.2: reg 20 io port: [0x20a0-0x20bf] pci 0000:00:1a.7: reg 10 32bit mmio: [0x91c22000-0x91c223ff] pci 0000:00:1a.7: PME# supported from D0 D3hot D3cold pci 0000:00:1a.7: PME# disabled pci 0000:00:1d.0: reg 20 io port: [0x2080-0x209f] pci 0000:00:1d.1: reg 20 io port: [0x2060-0x207f] pci 0000:00:1d.2: reg 20 io port: [0x2040-0x205f] pci 0000:00:1d.7: reg 10 32bit mmio: [0x91c21000-0x91c213ff] pci 0000:00:1d.7: PME# supported from D0 D3hot D3cold pci 0000:00:1d.7: PME# disabled pci 0000:00:1f.2: reg 10 io port: [0x2108-0x210f] pci 0000:00:1f.2: reg 14 io port: [0x2114-0x2117] pci 0000:00:1f.2: reg 18 io port: [0x2100-0x2107] pci 0000:00:1f.2: reg 1c io port: [0x2110-0x2113] pci 0000:00:1f.2: reg 20 io port: [0x2020-0x203f] pci 0000:00:1f.2: reg 24 32bit mmio: [0x91c20000-0x91c207ff] pci 0000:00:1f.2: PME# supported from D3hot pci 0000:00:1f.2: PME# disabled pci 0000:00:1f.3: reg 10 64bit mmio: [0x91c23000-0x91c230ff] pci 0000:00:1f.3: reg 20 io port: [0x2000-0x201f] pci 0000:01:00.0: reg 10 32bit mmio: [0x91b20000-0x91b3ffff] pci 0000:01:00.0: reg 18 io port: [0x1020-0x103f] pci 0000:01:00.0: reg 1c 32bit mmio: [0x91b44000-0x91b47fff] pci 0000:01:00.0: PME# supported from D0 D3hot D3cold pci 0000:01:00.0: PME# disabled pci 0000:01:00.0: reg 184 64bit mmio pref: [0x91900000-0x91903fff] pci 0000:01:00.0: reg 190 64bit mmio pref: [0x91920000-0x91923fff] pci 0000:01:00.1: reg 10 32bit mmio: [0x91b00000-0x91b1ffff] pci 0000:01:00.1: reg 18 io port: [0x1000-0x101f] pci 0000:01:00.1: reg 1c 32bit mmio: [0x91b40000-0x91b43fff] pci 0000:01:00.1: PME# supported from D0 D3hot D3cold pci 0000:01:00.1: PME# disabled pci 0000:01:00.1: reg 184 64bit mmio pref: [0x91940000-0x91943fff] pci 0000:01:00.1: reg 190 64bit mmio pref: [0x91960000-0x91963fff] pci 0000:00:01.0: bridge io port: [0x1000-0x1fff] pci 0000:00:01.0: bridge 32bit mmio: [0x91b00000-0x91bfffff] pci 0000:00:01.0: bridge 64bit mmio pref: [0x91900000-0x919fffff] pci 0000:05:00.0: reg 10 64bit mmio: [0x91a00000-0x91afffff] pci 0000:05:00.0: reg 18 64bit mmio pref: [0x90000000-0x907fffff] pci 0000:00:05.0: bridge 32bit mmio: [0x91a00000-0x91afffff] pci 0000:00:05.0: bridge 64bit mmio pref: [0x90000000-0x907fffff] pci 0000:07:00.0: reg 10 32bit mmio pref: [0x90800000-0x90ffffff] pci 0000:07:00.0: reg 14 32bit mmio: [0x91800000-0x91803fff] pci 0000:07:00.0: reg 18 32bit mmio: [0x91000000-0x917fffff] pci 0000:00:1e.0: transparent bridge pci 0000:00:1e.0: bridge 32bit mmio: [0x91000000-0x918fffff] pci 0000:00:1e.0: bridge 64bit mmio pref: [0x90800000-0x90ffffff] pci_bus 0000:00: on NUMA node 4 (pxm 4) ACPI: PCI Interrupt Routing Table [\_SB_.IOH0._PRT] ACPI: PCI Interrupt Routing Table [\_SB_.IOH0.MRP1._PRT] ACPI: PCI Interrupt Routing Table [\_SB_.IOH0.MRP5._PRT] ACPI: PCI Interrupt Routing Table [\_SB_.IOH0.MRP7._PRT] ACPI: PCI Interrupt Routing Table [\_SB_.IOH0.IP2P._PRT] ACPI: PCI Root Bridge [PCI6] (0000:fc) ACPI: PCI Interrupt Routing Table [\_SB_.PCI6._PRT] ACPI: PCI Root Bridge [PCI5] (0000:fd) ACPI: PCI Interrupt Routing Table [\_SB_.PCI5._PRT] ACPI: PCI Root Bridge [PCI4] (0000:fe) ACPI: PCI Interrupt Routing Table [\_SB_.PCI4._PRT] ACPI: PCI Root Bridge [PCI3] (0000:ff) ACPI: PCI Interrupt Routing Table [\_SB_.PCI3._PRT] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 6 7 9 10 *11 12 14 15) ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 5 6 7 9 *10 11 12 14 15) ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 5 6 7 9 10 *11 12 14 15) ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 *5 6 7 9 10 11 12 14 15) ACPI: PCI Interrupt Link [LNKE] (IRQs 3 4 5 6 7 9 10 11 12 14 15) *0, disabled. ACPI: PCI Interrupt Link [LNKF] (IRQs 3 4 5 6 7 *9 10 11 12 14 15) ACPI: PCI Interrupt Link [LNKG] (IRQs 3 4 5 6 7 9 10 11 12 14 15) *0, disabled. ACPI: PCI Interrupt Link [LNKH] (IRQs 3 4 5 6 7 9 10 *11 12 14 15) HEST: HEST table parsing is initialized. vgaarb: device added: PCI:0000:07:00.0,decodes=io+mem,owns=io+mem,locks=none vgaarb: loaded SCSI subsystem initialized libata version 3.00 loaded. usbcore: registered new interface driver usbfs usbcore: registered new interface driver hub usbcore: registered new device driver usb PCI: Using ACPI for IRQ routing PCI: old code would have set cacheline size to 32 bytes, but clflush_size = 64 PCI: pci_cache_line_size set to 64 bytes NetLabel: Initializing NetLabel: domain hash size = 128 NetLabel: protocols = UNLABELED CIPSOv4 NetLabel: unlabeled traffic allowed by default hpet0: at MMIO 0xfed00000, IRQs 2, 8, 0, 0 hpet0: 4 comparators, 64-bit 14.318180 MHz counter Switching to clocksource tsc Note: Tick synchro is active, _NOT_ switching to hres timers Starting HPET test HPET TEST: OK pnp: PnP ACPI init ACPI: bus type pnp registered pnp: PnP ACPI: found 13 devices ACPI: ACPI bus type pnp unregistered system 00:07: ioport range 0x500-0x53f has been reserved system 00:07: ioport range 0x400-0x47f has been reserved system 00:07: ioport range 0x800-0x81f has been reserved system 00:07: iomem range 0xfed1c000-0xfed8bffe could not be reserved system 00:07: iomem range 0xff000000-0xffffffff could not be reserved system 00:07: iomem range 0xfee00000-0xfeefffff has been reserved system 00:07: iomem range 0xfed12000-0xfed1200f has been reserved system 00:07: iomem range 0xfed12010-0xfed1201f has been reserved system 00:07: iomem range 0xfed1b000-0xfed1bfff has been reserved pci 0000:01:00.0: reg 184 64bit mmio pref: [0x91900000-0x91903fff] pci 0000:01:00.0: reg 190 64bit mmio pref: [0x91920000-0x91923fff] pci 0000:01:00.0: reg 184 64bit mmio pref: [0x91900000-0x91903fff] pci 0000:01:00.1: reg 184 64bit mmio pref: [0x91940000-0x91943fff] pci 0000:01:00.0: reg 184 64bit mmio pref: [0x91900000-0x91903fff] pci 0000:01:00.0: reg 190 64bit mmio pref: [0x91920000-0x91923fff] pci 0000:01:00.1: reg 190 64bit mmio pref: [0x91960000-0x91963fff] pci 0000:01:00.0: reg 184 64bit mmio pref: [0x91900000-0x91903fff] pci 0000:01:00.0: reg 190 64bit mmio pref: [0x91920000-0x91923fff] pci 0000:01:00.1: reg 184 64bit mmio pref: [0x91940000-0x91943fff] pci 0000:01:00.0: reg 184 64bit mmio pref: [0x91900000-0x91903fff] pci 0000:01:00.0: reg 184 64bit mmio pref: [0x91900000-0x91903fff] pci 0000:01:00.0: reg 190 64bit mmio pref: [0x91920000-0x91923fff] pci 0000:01:00.0: reg 190 64bit mmio pref: [0x91920000-0x91923fff] pci 0000:01:00.1: reg 184 64bit mmio pref: [0x91940000-0x91943fff] pci 0000:01:00.1: reg 184 64bit mmio pref: [0x91940000-0x91943fff] pci 0000:01:00.1: reg 190 64bit mmio pref: [0x91960000-0x91963fff] pci 0000:01:00.1: reg 190 64bit mmio pref: [0x91960000-0x91963fff] pci 0000:00:01.0: PCI bridge to [bus 01-03] pci 0000:00:01.0: PCI bridge, secondary bus 0000:01 pci 0000:00:01.0: bridge window [0x1000-0x1fff] pci 0000:00:01.0: bridge window [0x91b00000-0x91bfffff] pci 0000:00:01.0: bridge window [0x91900000-0x919fffff] pci 0000:00:03.0: PCI bridge to [bus 04-04] pci 0000:00:03.0: PCI bridge, secondary bus 0000:04 pci 0000:00:03.0: bridge window [io disabled] pci 0000:00:03.0: bridge window [mem disabled] pci 0000:00:03.0: bridge window [mem pref disabled] pci 0000:00:05.0: PCI bridge to [bus 05-05] pci 0000:00:05.0: PCI bridge, secondary bus 0000:05 pci 0000:00:05.0: bridge window [io disabled] pci 0000:00:05.0: bridge window [0x91a00000-0x91afffff] pci 0000:00:05.0: bridge window [0x90000000-0x907fffff] pci 0000:00:07.0: PCI bridge to [bus 06-06] pci 0000:00:07.0: PCI bridge, secondary bus 0000:06 pci 0000:00:07.0: bridge window [io disabled] pci 0000:00:07.0: bridge window [mem disabled] pci 0000:00:07.0: bridge window [mem pref disabled] pci 0000:00:1e.0: PCI bridge to [bus 07-07] pci 0000:00:1e.0: PCI bridge, secondary bus 0000:07 pci 0000:00:1e.0: bridge window [io disabled] pci 0000:00:1e.0: bridge window [0x91000000-0x918fffff] pci 0000:00:1e.0: bridge window [0x90800000-0x90ffffff] pci 0000:00:01.0: PCIe errors handled by OS. pci 0000:00:01.0: ACPI _OSC control granted for 0x1c alloc irq_desc for 28 on node 0 alloc kstat_irqs on node 0 pci 0000:00:01.0: PCI INT A -> GSI 28 (level, low) -> IRQ 28 pci 0000:00:01.0: setting latency timer to 64 pci 0000:00:03.0: PCIe errors handled by OS. pci 0000:00:03.0: ACPI _OSC control granted for 0x1c alloc irq_desc for 24 on node 0 alloc kstat_irqs on node 0 pci 0000:00:03.0: PCI INT A -> GSI 24 (level, low) -> IRQ 24 pci 0000:00:03.0: setting latency timer to 64 pci 0000:00:05.0: PCIe errors handled by OS. pci 0000:00:05.0: ACPI _OSC control granted for 0x1c alloc irq_desc for 26 on node 0 alloc kstat_irqs on node 0 pci 0000:00:05.0: PCI INT A -> GSI 26 (level, low) -> IRQ 26 pci 0000:00:05.0: setting latency timer to 64 pci 0000:00:07.0: PCIe errors handled by OS. pci 0000:00:07.0: ACPI _OSC control granted for 0x1c alloc irq_desc for 30 on node 0 alloc kstat_irqs on node 0 pci 0000:00:07.0: PCI INT A -> GSI 30 (level, low) -> IRQ 30 pci 0000:00:07.0: setting latency timer to 64 pci 0000:00:1e.0: setting latency timer to 64 pci_bus 0000:00: resource 0 io: [0x00-0xffff] pci_bus 0000:00: resource 1 mem: [0x000000-0xffffffffffffffff] pci_bus 0000:01: resource 0 io: [0x1000-0x1fff] pci_bus 0000:01: resource 1 mem: [0x91b00000-0x91bfffff] pci_bus 0000:01: resource 2 pref mem [0x91900000-0x919fffff] pci_bus 0000:05: resource 1 mem: [0x91a00000-0x91afffff] pci_bus 0000:05: resource 2 pref mem [0x90000000-0x907fffff] pci_bus 0000:07: resource 1 mem: [0x91000000-0x918fffff] pci_bus 0000:07: resource 2 pref mem [0x90800000-0x90ffffff] pci_bus 0000:07: resource 3 io: [0x00-0xffff] pci_bus 0000:07: resource 4 mem: [0x000000-0xffffffffffffffff] pci_bus 0000:fc: resource 0 io: [0x00-0xffff] pci_bus 0000:fc: resource 1 mem: [0x000000-0xffffffffffffffff] pci_bus 0000:fd: resource 0 io: [0x00-0xffff] pci_bus 0000:fd: resource 1 mem: [0x000000-0xffffffffffffffff] pci_bus 0000:fe: resource 0 io: [0x00-0xffff] pci_bus 0000:fe: resource 1 mem: [0x000000-0xffffffffffffffff] pci_bus 0000:ff: resource 0 io: [0x00-0xffff] pci_bus 0000:ff: resource 1 mem: [0x000000-0xffffffffffffffff] NET: Registered protocol family 2 IP route cache hash table entries: 524288 (order: 10, 4194304 bytes) TCP established hash table entries: 524288 (order: 11, 8388608 bytes) TCP bind hash table entries: 65536 (order: 8, 1048576 bytes) TCP: Hash tables configured (established 524288 bind 65536) TCP reno registered NET: Registered protocol family 1 pci 0000:00:1a.7: EHCI: BIOS handoff failed (BIOS bug?) 01010001 pci 0000:00:1d.7: EHCI: BIOS handoff failed (BIOS bug?) 01010001 pci 0000:07:00.0: Boot video device Trying to unpack rootfs image as initramfs... Freeing initrd memory: 7770k freed HugeTLB registered 2 MB page size, pre-allocated 0 pages VFS: Disk quotas dquot_6.5.2 Dquot-cache hash table entries: 512 (order 0, 4096 bytes) msgmni has been set to 32768 alg: No test for stdrng (krng) ksign: Installing public key data Loading keyring - Added public key BC3C09616D927F63 - User ID: Red Hat, Inc. (Kernel Module GPG key) - Added public key D4A26C9CCD09BEDA - User ID: Red Hat Enterprise Linux Driver Update Program Block layer SCSI generic (bsg) driver version 0.4 loaded (major 253) io scheduler noop registered io scheduler anticipatory registered io scheduler deadline registered io scheduler cfq registered (default) pcieport 0000:00:01.0: setting latency timer to 64 alloc irq_desc for 48 on node 0 alloc kstat_irqs on node 0 pcieport 0000:00:01.0: irq 48 for MSI/MSI-X pcieport 0000:00:03.0: setting latency timer to 64 alloc irq_desc for 49 on node 0 alloc kstat_irqs on node 0 pcieport 0000:00:03.0: irq 49 for MSI/MSI-X pcieport 0000:00:05.0: setting latency timer to 64 alloc irq_desc for 50 on node 0 alloc kstat_irqs on node 0 pcieport 0000:00:05.0: irq 50 for MSI/MSI-X pcieport 0000:00:07.0: setting latency timer to 64 alloc irq_desc for 51 on node 0 alloc kstat_irqs on node 0 pcieport 0000:00:07.0: irq 51 for MSI/MSI-X aer 0000:00:01.0:pcieffffffc2: service driver aer loaded aer 0000:00:03.0:pcieffffffc2: service driver aer loaded aer 0000:00:05.0:pcieffffffc2: service driver aer loaded aer 0000:00:07.0:pcieffffffc2: service driver aer loaded pci_hotplug: PCI Hot Plug PCI Core version: 0.5 acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5 acpiphp: Slot [0] registered acpiphp: Slot [0-1] registered pci-stub: invalid id string "" intel_idle: MWAIT substates: 0x120 intel_idle: v0.4 model 0x2E intel_idle: lapic_timer_reliable_states 0x2 input: Sleep Button as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0C0E:00/input/input0 ACPI: Sleep Button [SLPB] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input1 ACPI: Power Button [PWRF] ACPI: acpi_idle yielding to intel_idle processor LNXCPU:00: registered as cooling_device0 processor LNXCPU:01: registered as cooling_device1 processor LNXCPU:02: registered as cooling_device2 processor LNXCPU:03: registered as cooling_device3 processor LNXCPU:04: registered as cooling_device4 processor LNXCPU:05: registered as cooling_device5 processor LNXCPU:06: registered as cooling_device6 processor LNXCPU:07: registered as cooling_device7 processor LNXCPU:08: registered as cooling_device8 processor LNXCPU:09: registered as cooling_device9 processor LNXCPU:0a: registered as cooling_device10 processor LNXCPU:0b: registered as cooling_device11 processor LNXCPU:0c: registered as cooling_device12 processor LNXCPU:0d: registered as cooling_device13 processor LNXCPU:0e: registered as cooling_device14 processor LNXCPU:0f: registered as cooling_device15 processor LNXCPU:10: registered as cooling_device16 processor LNXCPU:11: registered as cooling_device17 processor LNXCPU:12: registered as cooling_device18 processor LNXCPU:13: registered as cooling_device19 processor LNXCPU:14: registered as cooling_device20 processor LNXCPU:15: registered as cooling_device21 processor LNXCPU:16: registered as cooling_device22 processor LNXCPU:17: registered as cooling_device23 processor LNXCPU:18: registered as cooling_device24 processor LNXCPU:19: registered as cooling_device25 processor LNXCPU:1a: registered as cooling_device26 processor LNXCPU:1b: registered as cooling_device27 processor LNXCPU:1c: registered as cooling_device28 processor LNXCPU:1d: registered as cooling_device29 processor LNXCPU:1e: registered as cooling_device30 processor LNXCPU:1f: registered as cooling_device31 ERST: Error Record Serialization Table (ERST) support is initialized. Non-volatile memory driver v1.3 crash memory driver: version 1.1 Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled serial8250: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A serial8250: ttyS1 at I/O 0x2f8 (irq = 3) is a 16550A 00:08: ttyS1 at I/O 0x2f8 (irq = 3) is a 16550A brd: module loaded loop: module loaded Fixed MDIO Bus: probed PNP: No PS/2 controller found. Probing ports directly. mice: PS/2 mouse device common for all mice rtc_cmos 00:03: RTC can wake from S4 rtc_cmos 00:03: rtc core: registered rtc_cmos as rtc0 rtc0: alarms up to one month, y3k, 114 bytes nvram, hpet irqs cpuidle: using governor ladder cpuidle: using governor menu TCP cubic registered Initializing XFRM netlink socket NET: Registered protocol family 17 registered taskstats version 1 rtc_cmos 00:03: setting system clock to 2012-07-03 14:00:24 UTC (1341324024) Initalizing network drop monitor service Freeing unused kernel memory: 784k freed Write protecting the kernel read-only data: 10240k Freeing unused kernel memory: 1484k freed Freeing unused kernel memory: 2024k freed ahci 0000:00:1f.2: version 3.0 alloc irq_desc for 19 on node 0 alloc kstat_irqs on node 0 ahci 0000:00:1f.2: PCI INT A -> GSI 19 (level, low) -> IRQ 19 alloc irq_desc for 52 on node 0 alloc kstat_irqs on node 0 ahci 0000:00:1f.2: irq 52 for MSI/MSI-X ahci: SSS flag set, parallel bus scan disabled ahci 0000:00:1f.2: AHCI 0001.0200 32 slots 6 ports 3 Gbps 0x3f impl SATA mode ahci 0000:00:1f.2: flags: 64bit sntf ilck stag pm led clo pmp pio slum part ccc ems ahci 0000:00:1f.2: setting latency timer to 64 scsi0 : ahci scsi1 : ahci scsi2 : ahci scsi3 : ahci scsi4 : ahci scsi5 : ahci ata1: SATA max UDMA/133 abar m2048@0x91c20000 port 0x91c20100 irq 52 ata2: SATA max UDMA/133 abar m2048@0x91c20000 port 0x91c20180 irq 52 ata3: SATA max UDMA/133 abar m2048@0x91c20000 port 0x91c20200 irq 52 ata4: SATA max UDMA/133 abar m2048@0x91c20000 port 0x91c20280 irq 52 ata5: SATA max UDMA/133 abar m2048@0x91c20000 port 0x91c20300 irq 52 ata6: SATA max UDMA/133 abar m2048@0x91c20000 port 0x91c20380 irq 52 ata1: SATA link up 3.0 Gbps (SStatus 123 SControl 300) ata1.00: ATA-7: INTEL SSDSA2M160G2GC, 2CV102HA, max UDMA/133 ata1.00: 312581808 sectors, multi 1: LBA48 NCQ (depth 0/32) ata1.00: configured for UDMA/133 scsi 0:0:0:0: Direct-Access ATA INTEL SSDSA2M160 2CV1 PQ: 0 ANSI: 5 ata2: SATA link down (SStatus 0 SControl 300) ata3: SATA link down (SStatus 0 SControl 300) ata4: SATA link down (SStatus 0 SControl 300) ata5: SATA link down (SStatus 0 SControl 300) ata6: SATA link down (SStatus 0 SControl 300) dracut: FATAL: Module aacraid not found. megasas: 00.00.05.34-rc1 Thu. Feb. 24 17:00:00 PDT 2011 dracut: dracut-004-53.bl6 dracut: rd_NO_LUKS: removing cryptoluks activation dracut: rd_NO_LVM: removing LVM activation device-mapper: uevent: version 1.0.3 device-mapper: ioctl: 4.20.6-ioctl (2011-02-02) initialised: dm-devel@redhat.com udev: starting version 147 dracut: Starting plymouth daemon dracut: rd_NO_DM: removing DM RAID activation dracut: rd_NO_MD: removing MD RAID activation sd 0:0:0:0: [sda] 312581808 512-byte logical blocks: (160 GB/149 GiB) sd 0:0:0:0: [sda] Write Protect is off sd 0:0:0:0: [sda] Mode Sense: 00 3a 00 00 sd 0:0:0:0: [sda] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA sda: sda1 sda2 sda3 sda4 < sda5 > sd 0:0:0:0: [sda] Attached SCSI disk EXT4-fs (sda1): mounted filesystem with ordered data mode dracut: Mounted root filesystem /dev/sda1 dracut: Switching root udev: starting version 147 dca service started, version 1.12.1 Intel(R) Gigabit Ethernet Network Driver - version 3.0.6-k2 Copyright (c) 2007-2011 Intel Corporation. igb 0000:01:00.0: PCI INT A -> GSI 28 (level, low) -> IRQ 28 igb 0000:01:00.0: setting latency timer to 64 alloc irq_desc for 53 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 53 for MSI/MSI-X alloc irq_desc for 54 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 54 for MSI/MSI-X alloc irq_desc for 55 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 55 for MSI/MSI-X alloc irq_desc for 56 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 56 for MSI/MSI-X alloc irq_desc for 57 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 57 for MSI/MSI-X alloc irq_desc for 58 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 58 for MSI/MSI-X alloc irq_desc for 59 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 59 for MSI/MSI-X alloc irq_desc for 60 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 60 for MSI/MSI-X alloc irq_desc for 61 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.0: irq 61 for MSI/MSI-X igb 0000:01:00.0: 0 vfs allocated igb 0000:01:00.0: Intel(R) Gigabit Ethernet Network Connection igb 0000:01:00.0: eth0: (PCIe:2.5Gb/s:Width x4) 08:00:38:36:14:74 igb 0000:01:00.0: eth0: PBA No: FFFFFF-0FF igb 0000:01:00.0: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s) alloc irq_desc for 40 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: PCI INT B -> GSI 40 (level, low) -> IRQ 40 igb 0000:01:00.1: setting latency timer to 64 alloc irq_desc for 62 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 62 for MSI/MSI-X alloc irq_desc for 63 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 63 for MSI/MSI-X alloc irq_desc for 64 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 64 for MSI/MSI-X alloc irq_desc for 65 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 65 for MSI/MSI-X alloc irq_desc for 66 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 66 for MSI/MSI-X alloc irq_desc for 67 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 67 for MSI/MSI-X alloc irq_desc for 68 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 68 for MSI/MSI-X alloc irq_desc for 69 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 69 for MSI/MSI-X alloc irq_desc for 70 on node 0 alloc kstat_irqs on node 0 igb 0000:01:00.1: irq 70 for MSI/MSI-X igb 0000:01:00.1: 0 vfs allocated igb 0000:01:00.1: Intel(R) Gigabit Ethernet Network Connection igb 0000:01:00.1: eth1: (PCIe:2.5Gb/s:Width x4) 08:00:38:36:14:75 igb 0000:01:00.1: eth1: PBA No: FFFFFF-0FF igb 0000:01:00.1: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s) ioatdma: Intel(R) QuickData Technology Driver 4.00 alloc irq_desc for 43 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.0: PCI INT A -> GSI 43 (level, low) -> IRQ 43 ioatdma 0000:00:16.0: setting latency timer to 64 alloc irq_desc for 71 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.0: irq 71 for MSI/MSI-X alloc irq_desc for 44 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.1: PCI INT B -> GSI 44 (level, low) -> IRQ 44 ioatdma 0000:00:16.1: setting latency timer to 64 alloc irq_desc for 72 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.1: irq 72 for MSI/MSI-X alloc irq_desc for 45 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.2: PCI INT C -> GSI 45 (level, low) -> IRQ 45 ioatdma 0000:00:16.2: setting latency timer to 64 alloc irq_desc for 73 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.2: irq 73 for MSI/MSI-X alloc irq_desc for 46 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.3: PCI INT D -> GSI 46 (level, low) -> IRQ 46 ioatdma 0000:00:16.3: setting latency timer to 64 alloc irq_desc for 74 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.3: irq 74 for MSI/MSI-X ioatdma 0000:00:16.4: PCI INT A -> GSI 43 (level, low) -> IRQ 43 ioatdma 0000:00:16.4: setting latency timer to 64 alloc irq_desc for 75 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.4: irq 75 for MSI/MSI-X ioatdma 0000:00:16.5: PCI INT B -> GSI 44 (level, low) -> IRQ 44 ioatdma 0000:00:16.5: setting latency timer to 64 alloc irq_desc for 76 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.5: irq 76 for MSI/MSI-X ioatdma 0000:00:16.6: PCI INT C -> GSI 45 (level, low) -> IRQ 45 ioatdma 0000:00:16.6: setting latency timer to 64 alloc irq_desc for 77 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.6: irq 77 for MSI/MSI-X ioatdma 0000:00:16.7: PCI INT D -> GSI 46 (level, low) -> IRQ 46 ioatdma 0000:00:16.7: setting latency timer to 64 alloc irq_desc for 78 on node 0 alloc kstat_irqs on node 0 ioatdma 0000:00:16.7: irq 78 for MSI/MSI-X uhci_hcd: USB Universal Host Controller Interface driver alloc irq_desc for 16 on node 0 alloc kstat_irqs on node 0 uhci_hcd 0000:00:1a.0: PCI INT A -> GSI 16 (level, low) -> IRQ 16 uhci_hcd 0000:00:1a.0: setting latency timer to 64 uhci_hcd 0000:00:1a.0: UHCI Host Controller uhci_hcd 0000:00:1a.0: new USB bus registered, assigned bus number 1 uhci_hcd 0000:00:1a.0: irq 16, io base 0x000020e0 usb usb1: New USB device found, idVendor=1d6b, idProduct=0001 usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb1: Product: UHCI Host Controller usb usb1: Manufacturer: Linux 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 uhci_hcd usb usb1: SerialNumber: 0000:00:1a.0 usb usb1: configuration #1 chosen from 1 choice hub 1-0:1.0: USB hub found hub 1-0:1.0: 2 ports detected alloc irq_desc for 21 on node 0 alloc kstat_irqs on node 0 uhci_hcd 0000:00:1a.1: PCI INT B -> GSI 21 (level, low) -> IRQ 21 uhci_hcd 0000:00:1a.1: setting latency timer to 64 uhci_hcd 0000:00:1a.1: UHCI Host Controller uhci_hcd 0000:00:1a.1: new USB bus registered, assigned bus number 2 uhci_hcd 0000:00:1a.1: irq 21, io base 0x000020c0 usb usb2: New USB device found, idVendor=1d6b, idProduct=0001 usb usb2: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb2: Product: UHCI Host Controller usb usb2: Manufacturer: Linux 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 uhci_hcd usb usb2: SerialNumber: 0000:00:1a.1 usb usb2: configuration #1 chosen from 1 choice hub 2-0:1.0: USB hub found hub 2-0:1.0: 2 ports detected uhci_hcd 0000:00:1a.2: PCI INT D -> GSI 19 (level, low) -> IRQ 19 uhci_hcd 0000:00:1a.2: setting latency timer to 64 uhci_hcd 0000:00:1a.2: UHCI Host Controller uhci_hcd 0000:00:1a.2: new USB bus registered, assigned bus number 3 uhci_hcd 0000:00:1a.2: irq 19, io base 0x000020a0 usb usb3: New USB device found, idVendor=1d6b, idProduct=0001 usb usb3: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb3: Product: UHCI Host Controller usb usb3: Manufacturer: Linux 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 uhci_hcd usb usb3: SerialNumber: 0000:00:1a.2 usb usb3: configuration #1 chosen from 1 choice hub 3-0:1.0: USB hub found hub 3-0:1.0: 2 ports detected alloc irq_desc for 23 on node 0 alloc kstat_irqs on node 0 uhci_hcd 0000:00:1d.0: PCI INT A -> GSI 23 (level, low) -> IRQ 23 uhci_hcd 0000:00:1d.0: setting latency timer to 64 uhci_hcd 0000:00:1d.0: UHCI Host Controller uhci_hcd 0000:00:1d.0: new USB bus registered, assigned bus number 4 uhci_hcd 0000:00:1d.0: irq 23, io base 0x00002080 usb usb4: New USB device found, idVendor=1d6b, idProduct=0001 usb usb4: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb4: Product: UHCI Host Controller usb usb4: Manufacturer: Linux 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 uhci_hcd usb usb4: SerialNumber: 0000:00:1d.0 usb usb4: configuration #1 chosen from 1 choice hub 4-0:1.0: USB hub found hub 4-0:1.0: 2 ports detected uhci_hcd 0000:00:1d.1: PCI INT B -> GSI 19 (level, low) -> IRQ 19 uhci_hcd 0000:00:1d.1: setting latency timer to 64 uhci_hcd 0000:00:1d.1: UHCI Host Controller uhci_hcd 0000:00:1d.1: new USB bus registered, assigned bus number 5 uhci_hcd 0000:00:1d.1: irq 19, io base 0x00002060 usb usb5: New USB device found, idVendor=1d6b, idProduct=0001 usb usb5: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb5: Product: UHCI Host Controller usb usb5: Manufacturer: Linux 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 uhci_hcd usb usb5: SerialNumber: 0000:00:1d.1 usb usb5: configuration #1 chosen from 1 choice hub 5-0:1.0: USB hub found hub 5-0:1.0: 2 ports detected alloc irq_desc for 18 on node 0 alloc kstat_irqs on node 0 uhci_hcd 0000:00:1d.2: PCI INT C -> GSI 18 (level, low) -> IRQ 18 uhci_hcd 0000:00:1d.2: setting latency timer to 64 uhci_hcd 0000:00:1d.2: UHCI Host Controller uhci_hcd 0000:00:1d.2: new USB bus registered, assigned bus number 6 uhci_hcd 0000:00:1d.2: irq 18, io base 0x00002040 usb usb6: New USB device found, idVendor=1d6b, idProduct=0001 usb usb6: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb6: Product: UHCI Host Controller usb usb6: Manufacturer: Linux 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 uhci_hcd usb usb6: SerialNumber: 0000:00:1d.2 usb usb6: configuration #1 chosen from 1 choice hub 6-0:1.0: USB hub found hub 6-0:1.0: 2 ports detected ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver Warning! ehci_hcd should always be loaded before uhci_hcd and ohci_hcd, not after ehci_hcd 0000:00:1a.7: PCI INT C -> GSI 18 (level, low) -> IRQ 18 ehci_hcd 0000:00:1a.7: setting latency timer to 64 ehci_hcd 0000:00:1a.7: EHCI Host Controller ehci_hcd 0000:00:1a.7: new USB bus registered, assigned bus number 7 ehci_hcd 0000:00:1a.7: debug port 1 ehci_hcd 0000:00:1a.7: cache line size of 64 is not supported ehci_hcd 0000:00:1a.7: irq 18, io mem 0x91c22000 ehci_hcd 0000:00:1a.7: USB 2.0 started, EHCI 1.00 usb usb7: New USB device found, idVendor=1d6b, idProduct=0002 usb usb7: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb7: Product: EHCI Host Controller usb usb7: Manufacturer: Linux 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 ehci_hcd usb usb7: SerialNumber: 0000:00:1a.7 usb usb7: configuration #1 chosen from 1 choice hub 7-0:1.0: USB hub found hub 7-0:1.0: 6 ports detected ehci_hcd 0000:00:1d.7: PCI INT A -> GSI 23 (level, low) -> IRQ 23 ehci_hcd 0000:00:1d.7: setting latency timer to 64 ehci_hcd 0000:00:1d.7: EHCI Host Controller ehci_hcd 0000:00:1d.7: new USB bus registered, assigned bus number 8 ehci_hcd 0000:00:1d.7: debug port 1 ehci_hcd 0000:00:1d.7: cache line size of 64 is not supported ehci_hcd 0000:00:1d.7: irq 23, io mem 0x91c21000 usb 5-2: new full speed USB device using uhci_hcd and address 2 ehci_hcd 0000:00:1d.7: USB 2.0 started, EHCI 1.00 usb usb8: New USB device found, idVendor=1d6b, idProduct=0002 usb usb8: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb8: Product: EHCI Host Controller usb usb8: Manufacturer: Linux 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 ehci_hcd usb usb8: SerialNumber: 0000:00:1d.7 usb usb8: configuration #1 chosen from 1 choice hub 8-0:1.0: USB hub found hub 8-0:1.0: 6 ports detected iTCO_vendor_support: vendor-support=0 hub 5-0:1.0: unable to enumerate USB device on port 2 iTCO_wdt: Intel TCO WatchDog Timer Driver v1.05 iTCO_wdt: unable to reset NO_REBOOT flag, device disabled by hardware/BIOS i801_smbus 0000:00:1f.3: PCI INT B -> GSI 18 (level, low) -> IRQ 18 sd 0:0:0:0: Attached scsi generic sg0 type 0 usb 8-4: new high speed USB device using ehci_hcd and address 2 GHES: Generic hardware error source: 1 notified via NMI is not supported! usb 8-4: New USB device found, idVendor=14dd, idProduct=0002 usb 8-4: New USB device strings: Mfr=1, Product=2, SerialNumber=3 usb 8-4: Product: Multidevice usb 8-4: Manufacturer: Peppercon AG usb 8-4: SerialNumber: 0800383634462641C894DA0F1B6A30CC usb 8-4: configuration #1 chosen from 1 choice usbcore: registered new interface driver hiddev input: Peppercon AG Multidevice as /devices/pci0000:00/0000:00:1d.7/usb8/8-4/8-4:1.0/input/input2 generic-usb 0003:14DD:0002.0001: input,hidraw0: USB HID v1.01 Keyboard [Peppercon AG Multidevice] on usb-0000:00:1d.7-4/input0 input: Peppercon AG Multidevice as /devices/pci0000:00/0000:00:1d.7/usb8/8-4/8-4:1.1/input/input3 generic-usb 0003:14DD:0002.0002: input,hidraw1: USB HID v1.01 Mouse [Peppercon AG Multidevice] on usb-0000:00:1d.7-4/input1 usbcore: registered new interface driver usbhid usbhid: v2.6:USB HID core driver EXT4-fs (sda2): mounted filesystem with ordered data mode EXT4-fs (sda3): mounted filesystem with ordered data mode Adding 20477528k swap on /dev/sda5. Priority:-1 extents:1 across:20477528k SSD mlx4_core: Mellanox ConnectX core driver v1.0-ofed1.5.4 (November 10, 2011) mlx4_core: Initializing 0000:05:00.0 mlx4_core 0000:05:00.0: PCI INT A -> GSI 26 (level, low) -> IRQ 26 mlx4_core 0000:05:00.0: setting latency timer to 64 alloc irq_desc for 79 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 79 for MSI/MSI-X alloc irq_desc for 80 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 80 for MSI/MSI-X alloc irq_desc for 81 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 81 for MSI/MSI-X alloc irq_desc for 82 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 82 for MSI/MSI-X alloc irq_desc for 83 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 83 for MSI/MSI-X alloc irq_desc for 84 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 84 for MSI/MSI-X alloc irq_desc for 85 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 85 for MSI/MSI-X alloc irq_desc for 86 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 86 for MSI/MSI-X alloc irq_desc for 87 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 87 for MSI/MSI-X alloc irq_desc for 88 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 88 for MSI/MSI-X alloc irq_desc for 89 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 89 for MSI/MSI-X alloc irq_desc for 90 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 90 for MSI/MSI-X alloc irq_desc for 91 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 91 for MSI/MSI-X alloc irq_desc for 92 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 92 for MSI/MSI-X alloc irq_desc for 93 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 93 for MSI/MSI-X alloc irq_desc for 94 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 94 for MSI/MSI-X alloc irq_desc for 95 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 95 for MSI/MSI-X alloc irq_desc for 96 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 96 for MSI/MSI-X alloc irq_desc for 97 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 97 for MSI/MSI-X alloc irq_desc for 98 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 98 for MSI/MSI-X alloc irq_desc for 99 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 99 for MSI/MSI-X alloc irq_desc for 100 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 100 for MSI/MSI-X alloc irq_desc for 101 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 101 for MSI/MSI-X alloc irq_desc for 102 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 102 for MSI/MSI-X alloc irq_desc for 103 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 103 for MSI/MSI-X alloc irq_desc for 104 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 104 for MSI/MSI-X alloc irq_desc for 105 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 105 for MSI/MSI-X alloc irq_desc for 106 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 106 for MSI/MSI-X alloc irq_desc for 107 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 107 for MSI/MSI-X alloc irq_desc for 108 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 108 for MSI/MSI-X alloc irq_desc for 109 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 109 for MSI/MSI-X alloc irq_desc for 110 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 110 for MSI/MSI-X alloc irq_desc for 111 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 111 for MSI/MSI-X alloc irq_desc for 112 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 112 for MSI/MSI-X alloc irq_desc for 113 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 113 for MSI/MSI-X alloc irq_desc for 114 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 114 for MSI/MSI-X alloc irq_desc for 115 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 115 for MSI/MSI-X alloc irq_desc for 116 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 116 for MSI/MSI-X alloc irq_desc for 117 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 117 for MSI/MSI-X alloc irq_desc for 118 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 118 for MSI/MSI-X alloc irq_desc for 119 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 119 for MSI/MSI-X alloc irq_desc for 120 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 120 for MSI/MSI-X alloc irq_desc for 121 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 121 for MSI/MSI-X alloc irq_desc for 122 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 122 for MSI/MSI-X alloc irq_desc for 123 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 123 for MSI/MSI-X alloc irq_desc for 124 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 124 for MSI/MSI-X alloc irq_desc for 125 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 125 for MSI/MSI-X alloc irq_desc for 126 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 126 for MSI/MSI-X alloc irq_desc for 127 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 127 for MSI/MSI-X alloc irq_desc for 128 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 128 for MSI/MSI-X alloc irq_desc for 129 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 129 for MSI/MSI-X alloc irq_desc for 130 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 130 for MSI/MSI-X alloc irq_desc for 131 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 131 for MSI/MSI-X alloc irq_desc for 132 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 132 for MSI/MSI-X alloc irq_desc for 133 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 133 for MSI/MSI-X alloc irq_desc for 134 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 134 for MSI/MSI-X alloc irq_desc for 135 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 135 for MSI/MSI-X alloc irq_desc for 136 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 136 for MSI/MSI-X alloc irq_desc for 137 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 137 for MSI/MSI-X alloc irq_desc for 138 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 138 for MSI/MSI-X alloc irq_desc for 139 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 139 for MSI/MSI-X alloc irq_desc for 140 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 140 for MSI/MSI-X alloc irq_desc for 141 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 141 for MSI/MSI-X alloc irq_desc for 142 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 142 for MSI/MSI-X alloc irq_desc for 143 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 143 for MSI/MSI-X alloc irq_desc for 144 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 144 for MSI/MSI-X alloc irq_desc for 145 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 145 for MSI/MSI-X alloc irq_desc for 146 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 146 for MSI/MSI-X alloc irq_desc for 147 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 147 for MSI/MSI-X alloc irq_desc for 148 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 148 for MSI/MSI-X alloc irq_desc for 149 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 149 for MSI/MSI-X alloc irq_desc for 150 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 150 for MSI/MSI-X alloc irq_desc for 151 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 151 for MSI/MSI-X alloc irq_desc for 152 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 152 for MSI/MSI-X alloc irq_desc for 153 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 153 for MSI/MSI-X alloc irq_desc for 154 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 154 for MSI/MSI-X alloc irq_desc for 155 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 155 for MSI/MSI-X alloc irq_desc for 156 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 156 for MSI/MSI-X alloc irq_desc for 157 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 157 for MSI/MSI-X alloc irq_desc for 158 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 158 for MSI/MSI-X alloc irq_desc for 159 on node 0 alloc kstat_irqs on node 0 mlx4_core 0000:05:00.0: irq 159 for MSI/MSI-X mlx4_ib: Mellanox ConnectX InfiniBand driver v1.0-ofed1.5.4 (November 10, 2011) NET: Registered protocol family 10 lo: Disabled Privacy Extensions ib0: multicast join failed for ff12:401b:ffff:0000:0000:0000:ffff:ffff, status -22 ADDRCONF(NETDEV_UP): ib0: link is not ready ib0: multicast join failed for ff12:401b:ffff:0000:0000:0000:ffff:ffff, status -22 ib0: enabling connected mode will cause multicast packet drops ib0: mtu > 2044 will cause multicast packet drops. ib0: mtu > 2044 will cause multicast packet drops. ADDRCONF(NETDEV_UP): ib0.8182: link is not ready ib0: multicast join failed for ff12:401b:ffff:0000:0000:0000:ffff:ffff, status -22 ib0.8182: enabling connected mode will cause multicast packet drops ib0.8182: mtu > 2044 will cause multicast packet drops. ib0.8182: mtu > 2044 will cause multicast packet drops. igb: eth0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX/TX ib0: multicast join failed for ff12:401b:ffff:0000:0000:0000:ffff:ffff, status -22 ib0: multicast join failed for ff12:401b:ffff:0000:0000:0000:ffff:ffff, status -22 RPC: Registered udp transport module. RPC: Registered tcp transport module. RPC: Registered tcp NFSv4.1 backchannel transport module. ib0: multicast join failed for ff12:401b:ffff:0000:0000:0000:ffff:ffff, status -22 Slow work thread pool: Starting up Slow work thread pool: Ready FS-Cache: Loaded FS-Cache: Netfs 'nfs' registered for caching PAX: PMU arbitration service v1.0.1 has been started. sep3_4: PMU collection driver v3.4.2 has been loaded. sep3_4: IDT vector 0x40 will be used for handling PMU interrupts. ADDRCONF(NETDEV_CHANGE): ib0.8182: link becomes ready ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready Lustre: Lustre: Build Version: B-2_1_2_0-lustrebull-20120625122144-CHANGED-2.6.32-131.17.1.bl6.Bull.27.0.x86_64 Lustre: Added LNI JO.BOO.ZL.LOF@o2ib3 [8/64/0/180] Lustre: Added LNI JO.BOB.ZL.LOF@o2ib4 [8/64/0/180] Lustre: Lustre OSC module (ffffffffa08f8bc0). Lustre: Lustre LOV module (ffffffffa098b8e0). Lustre: Lustre client module (ffffffffa0a76600). LustreError: 152-6: Ignoring deprecated mount option 'acl'. Lustre: MGCJO.BOO.ZO.BW@o2ib3: Reactivating import Lustre: Mounted scratch-client LustreError: 152-6: Ignoring deprecated mount option 'acl'. Lustre: Mounted ptmp-client LustreError: 152-6: Ignoring deprecated mount option 'acl'. Lustre: Mounted cprot-client LustreError: 152-6: Ignoring deprecated mount option 'acl'. Lustre: MGCQQ.P.BBO.W@o2ib2: Reactivating import Lustre: Mounted cdep4-client Lustre: DEBUG MARKER: Tue Jul 3 16:10:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 16:15:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 16:20:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 16:25:02 2012 Lustre: DEBUG MARKER: Tue Jul 3 16:30:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 16:35:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 16:40:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 16:45:01 2012 nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a not responding, still trying nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK nfs: server SRVNFS2.baratins.a OK Lustre: DEBUG MARKER: Tue Jul 3 16:50:02 2012 Lustre: DEBUG MARKER: Tue Jul 3 16:55:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:00:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:05:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:10:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:15:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:20:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:25:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:30:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:35:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:40:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:45:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:50:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 17:55:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:00:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:05:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:10:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:15:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:20:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:25:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:30:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:35:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:40:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:45:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:50:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 18:55:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:00:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:05:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:10:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:15:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:20:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:25:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:30:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:35:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:40:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:45:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:50:02 2012 Lustre: DEBUG MARKER: Tue Jul 3 19:55:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:00:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:05:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:10:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:15:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:20:02 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:25:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:30:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:35:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:40:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:45:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:50:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 20:55:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:00:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:05:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:10:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:15:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:20:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:25:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:30:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:35:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:40:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:45:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:50:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 21:55:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:00:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:05:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:10:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:15:02 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:20:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:25:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:30:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:35:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:40:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:45:02 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:50:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 22:55:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:00:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:05:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:10:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:15:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:20:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:25:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:30:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:35:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:40:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:45:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:50:01 2012 Lustre: DEBUG MARKER: Tue Jul 3 23:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:10:02 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:40:02 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 00:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:10:02 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 01:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 02:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 03:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:40:02 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 04:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:00:01 2012 ipmi message handler version 39.2 IPMI System Interface driver. ipmi_si: Adding SMBIOS-specified kcs state machine ipmi_si: Adding ACPI-specified kcs state machine ipmi_si: Trying SMBIOS-specified kcs state machine at i/o address 0xca2, slave address 0x20, irq 0 ipmi: Found new BMC (man_id: 0x00006b, prod_id: 0x0004, dev_id: 0x30) IPMI kcs interface initialized ipmi device interface ip_tables: (C) 2000-2006 Netfilter Core Team Lustre: DEBUG MARKER: Wed Jul 4 05:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 05:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:00:01 2012 ipmi message handler version 39.2 IPMI System Interface driver. ipmi_si: Adding SMBIOS-specified kcs state machine ipmi_si: Adding ACPI-specified kcs state machine ipmi_si: Trying SMBIOS-specified kcs state machine at i/o address 0xca2, slave address 0x20, irq 0 ipmi: Found new BMC (man_id: 0x00006b, prod_id: 0x0004, dev_id: 0x30) IPMI kcs interface initialized ipmi device interface Lustre: DEBUG MARKER: Wed Jul 4 06:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 06:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 07:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 08:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 09:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:50:02 2012 Lustre: DEBUG MARKER: Wed Jul 4 10:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 11:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:00:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:05:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:10:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:20:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:30:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:40:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:45:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:50:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 12:55:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 13:00:01 2012 LustreError: 16821:0:(osc_lock.c:1109:osc_lock_enqueue_wait()) DEADLOCK POSSIBLE! LustreError: 16821:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lock@ffff8808717ceaf8[0 5 0 0 0 00000000] R(1):[0, 18446744073709551615]@[0x100080000:0x290ed44:0x0] { LustreError: 16821:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lovsub@ffff88075e353360: LustreError: 16821:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) osc@ffff8806fc6867b8: ffff8807117c8000 00101001 0x1fe0953368c65c55 4 (null) size: 0 mtime: 1341399775 atime: 1341399775 ctime: 1341399775 blocks: 0 LustreError: 16821:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) } lock@ffff8808717ceaf8 LustreError: 16821:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) queued. LustreError: 16821:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lock@ffff88087168ed98[2 1 0 1 1 00000000] W(2):[0, 18446744073709551615]@[0x100080000:0x290ed44:0x0] { LustreError: 16821:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lovsub@ffff8806a8ba24a0: [0 ffff880711657cf8 W(2):[0, 18446744073709551615]@[0x20020a124:0x90:0x0]] LustreError: 16821:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) osc@ffff88075e3e5a98: (null) 00000000 0x0 0 (null) size: 0 mtime: 0 atime: 0 ctime: 0 blocks: 0 LustreError: 16821:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) } lock@ffff88087168ed98 LustreError: 16821:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) queuing. Pid: 16821, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] osc_lock_enqueue+0x775/0xa30 [osc] [] ? cfs_waitq_broadcast+0x17/0x20 [libcfs] [] ? cl_lock_state_signal+0x81/0x160 [obdclass] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] lov_lock_enqueue+0x20b/0x7c0 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] cl_enqueue_locked+0x6d/0x200 [obdclass] [] cl_lock_request+0x7e/0x1c0 [obdclass] [] cl_io_lock+0x3d5/0x570 [obdclass] [] cl_io_loop+0xba/0x190 [obdclass] [] ll_file_io_generic+0x42b/0x560 [lustre] [] ll_file_aio_write+0x13c/0x2c0 [lustre] [] ll_file_write+0x169/0x2a0 [lustre] [] vfs_write+0xb8/0x1a0 [] sys_write+0x51/0x90 [] system_call_fastpath+0x16/0x1b Lustre: DEBUG MARKER: Wed Jul 4 13:05:01 2012 LustreError: 16818:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659457 LustreError: 16817:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659451 LustreError: 16817:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 1 previous similar message LustreError: 16812:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659474 LustreError: 16812:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 3 previous similar messages LustreError: 16821:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659488 LustreError: 16821:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 23 previous similar messages LustreError: 16825:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659521 LustreError: 16825:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 21 previous similar messages LustreError: 16820:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659526 LustreError: 16820:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 45 previous similar messages LustreError: 16824:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659551 LustreError: 16824:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 45 previous similar messages LustreError: 16819:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659583 LustreError: 16819:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 45 previous similar messages LustreError: 16828:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659635 LustreError: 16828:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 91 previous similar messages LustreError: 16814:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659727 LustreError: 16814:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 183 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 13:10:01 2012 LustreError: 16824:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541659886 LustreError: 16824:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 261 previous similar messages LustreError: 16830:0:(osc_lock.c:1109:osc_lock_enqueue_wait()) DEADLOCK POSSIBLE! LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lock@ffff880b2ac97518[1 3 0 1 1 00000000] W(2):[143360, 145919]@[0x102aa0000:0x4e90:0x0] { LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lovsub@ffff880b0618a9e0: [0 ffff880b596a6540 W(2):[432128, 454655]@[0x2103bd365:0x32e:0x0]] LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) osc@ffff880b2ad257b8: ffff880b2829f000 40100001 0x1fe0953368ca280c 3 ffff881078136190 size: 590610944 mtime: 1341400470 atime: 1341400471 ctime: 1341400470 blocks: 1139584 LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) } lock@ffff880b2ac97518 LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) queued. LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lock@ffff880b1b89b158[2 1 0 1 1 00000000] W(2):[145488, 145999]@[0x102aa0000:0x4e90:0x0] { LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lovsub@ffff880b08eebe60: [0 ffff880b711d64d8 W(2):[438352, 438863]@[0x2103bd365:0x32e:0x0]] LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) osc@ffff880b6b638b50: (null) 00000000 0x0 0 (null) size: 0 mtime: 0 atime: 0 ctime: 0 blocks: 0 LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) } lock@ffff880b1b89b158 LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) queuing. Pid: 16830, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] osc_lock_enqueue+0x775/0xa30 [osc] [] ? cfs_waitq_broadcast+0x17/0x20 [libcfs] [] ? cl_lock_state_signal+0x81/0x160 [obdclass] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] lov_lock_enqueue+0x20b/0x7c0 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] cl_enqueue_locked+0x6d/0x200 [obdclass] [] cl_lock_request+0x7e/0x1c0 [obdclass] [] cl_io_lock+0x3d5/0x570 [obdclass] [] cl_io_loop+0xba/0x190 [obdclass] [] ll_file_io_generic+0x42b/0x560 [lustre] [] ll_file_aio_write+0x13c/0x2c0 [lustre] [] ll_file_write+0x169/0x2a0 [lustre] [] vfs_write+0xb8/0x1a0 [] sys_write+0x51/0x90 [] system_call_fastpath+0x16/0x1b Lustre: DEBUG MARKER: Wed Jul 4 13:15:01 2012 LustreError: 16818:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541660274 LustreError: 16818:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 327 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 13:20:01 2012 LustreError: 16828:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541661069 LustreError: 16828:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 461 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 13:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 13:30:01 2012 LustreError: 16809:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541662043 LustreError: 16809:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 412 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 13:35:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 13:40:01 2012 LustreError: 16830:0:(osc_lock.c:1109:osc_lock_enqueue_wait()) DEADLOCK POSSIBLE! LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lock@ffff880b431eb4d8[1 3 0 1 1 00000000] W(2):[361472, 362111]@[0x102c40000:0x5307:0x0] { LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lovsub@ffff880bcfcee9a0: [0 ffff880bddb76130 W(2):[1086464, 1087103]@[0x2103bd57a:0xd:0x0]] [0 ffff880be5638540 R(1):[1086464, 1087103]@[0x2103bd57a:0xd:0x0]] LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) osc@ffff880be0037420: ffff880beaaa6000 40100001 0x1fe0953368d55617 3 ffff880460f98798 size: 1484783616 mtime: 1341402023 atime: 1341402023 ctime: 1341402023 blocks: 2853840 LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) } lock@ffff880b431eb4d8 LustreError: 16830:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) queued. LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lock@ffff880be0172c18[2 1 0 1 1 00000000] R(1):[361984, 362495]@[0x102c40000:0x5307:0x0] { LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lovsub@ffff880bddb4b5a0: [0 ffff880bf1ce7610 R(1):[1086976, 1087487]@[0x2103bd57a:0xd:0x0]] LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) osc@ffff880bd18ab420: (null) 00000000 0x0 0 (null) size: 0 mtime: 0 atime: 0 ctime: 0 blocks: 0 LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) } lock@ffff880be0172c18 LustreError: 16830:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) queuing. Pid: 16830, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] osc_lock_enqueue+0x775/0xa30 [osc] [] ? lov_sublock_env_get+0xd2/0x140 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] lov_lock_enqueue+0x20b/0x7c0 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] cl_enqueue_locked+0x6d/0x200 [obdclass] [] cl_lock_request+0x7e/0x1c0 [obdclass] [] cl_io_lock+0x3d5/0x570 [obdclass] [] cl_io_loop+0xba/0x190 [obdclass] [] ll_file_io_generic+0x3a7/0x560 [lustre] [] ll_file_aio_read+0x139/0x2c0 [lustre] [] ll_file_read+0x169/0x2a0 [lustre] [] vfs_read+0xb5/0x1a0 [] sys_read+0x51/0x90 [] system_call_fastpath+0x16/0x1b LustreError: 16829:0:(osc_lock.c:1109:osc_lock_enqueue_wait()) DEADLOCK POSSIBLE! LustreError: 16829:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lock@ffff880bf809c6d8[1 3 0 1 1 00000000] W(2):[386048, 387023]@[0x102910000:0x5006:0x0] { LustreError: 16829:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lovsub@ffff880c1896dea0: [0 ffff880bf90e12d0 W(2):[1158144, 1159119]@[0x2103bd57a:0xd:0x0]] [0 ffff880bf1ce7cf8 R(1):[1158144, 1159119]@[0x2103bd57a:0xd:0x0]] LustreError: 16829:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) osc@ffff880bd18ab870: ffff880bfdb16240 40100001 0x1fe0953368d60350 3 ffff880c73c49d20 size: 1585446912 mtime: 1341402161 atime: 1341402161 ctime: 1341402161 blocks: 3038568 LustreError: 16829:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) } lock@ffff880bf809c6d8 LustreError: 16829:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) queued. LustreError: 16829:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lock@ffff880bd39c37b8[2 1 0 1 1 00000000] R(1):[386560, 387071]@[0x102910000:0x5006:0x0] { LustreError: 16829:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lovsub@ffff880bf9017920: [0 ffff880bdaf709b8 R(1):[1158656, 1159167]@[0x2103bd57a:0xd:0x0]] LustreError: 16829:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) osc@ffff880c1896f420: (null) 00000000 0x0 0 (null) size: 0 mtime: 0 atime: 0 ctime: 0 blocks: 0 LustreError: 16829:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) } lock@ffff880bd39c37b8 LustreError: 16829:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) queuing. Pid: 16829, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] osc_lock_enqueue+0x775/0xa30 [osc] [] ? lov_sublock_env_get+0xd2/0x140 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] lov_lock_enqueue+0x20b/0x7c0 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] cl_enqueue_locked+0x6d/0x200 [obdclass] [] cl_lock_request+0x7e/0x1c0 [obdclass] [] cl_io_lock+0x3d5/0x570 [obdclass] [] cl_io_loop+0xba/0x190 [obdclass] [] ll_file_io_generic+0x3a7/0x560 [lustre] [] ll_file_aio_read+0x139/0x2c0 [lustre] [] ll_file_read+0x169/0x2a0 [lustre] [] vfs_read+0xb5/0x1a0 [] sys_read+0x51/0x90 [] system_call_fastpath+0x16/0x1b Lustre: DEBUG MARKER: Wed Jul 4 13:45:01 2012 LustreError: 16817:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541663228 LustreError: 16817:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 319 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 13:50:01 2012 LustreError: 16824:0:(osc_lock.c:1109:osc_lock_enqueue_wait()) DEADLOCK POSSIBLE! LustreError: 16824:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lock@ffff880c58fbea18[1 3 0 1 1 00000000] W(2):[18880, 20479]@[0x1001c0000:0x14a9f:0x0] { LustreError: 16824:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lovsub@ffff880c65ac2da0: [0 ffff880bc9c380c8 W(2):[2048, 18446744073709551615]@[0x2103bd57a:0x3:0x0]] LustreError: 16824:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) osc@ffff880c18936928: ffff880c5dfa9480 40120002 0x1fe0953368db0fda 3 ffff880c6f55cd60 size: 84279296 mtime: 1341402825 atime: 1341402825 ctime: 1341402825 blocks: 149688 LustreError: 16824:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) } lock@ffff880c58fbea18 LustreError: 16824:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) queued. LustreError: 16824:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lock@ffff880c627f1518[2 1 0 1 1 00000000] R(1):[18432, 18895]@[0x1001c0000:0x14a9f:0x0] { LustreError: 16824:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lovsub@ffff880c04aa2e20: [0 ffff880c4f8ccd60 R(1):[57344, 57807]@[0x2103bd57a:0x3:0x0]] LustreError: 16824:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) osc@ffff880c5b1de700: (null) 00000000 0x0 0 (null) size: 0 mtime: 0 atime: 0 ctime: 0 blocks: 0 LustreError: 16824:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) } lock@ffff880c627f1518 LustreError: 16824:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) queuing. Pid: 16824, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] osc_lock_enqueue+0x775/0xa30 [osc] [] ? cfs_waitq_broadcast+0x17/0x20 [libcfs] [] ? cl_lock_state_signal+0x81/0x160 [obdclass] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] lov_lock_enqueue+0x20b/0x7c0 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] cl_enqueue_locked+0x6d/0x200 [obdclass] [] cl_lock_request+0x7e/0x1c0 [obdclass] [] cl_io_lock+0x3d5/0x570 [obdclass] [] cl_io_loop+0xba/0x190 [obdclass] [] ll_file_io_generic+0x3a7/0x560 [lustre] [] ll_file_aio_read+0x139/0x2c0 [lustre] [] ll_file_read+0x169/0x2a0 [lustre] [] vfs_read+0xb5/0x1a0 [] sys_read+0x51/0x90 [] system_call_fastpath+0x16/0x1b Lustre: DEBUG MARKER: Wed Jul 4 13:55:01 2012 LustreError: 16809:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541664087 LustreError: 16809:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 835 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 14:00:01 2012 LustreError: 16819:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541664679 LustreError: 16819:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 353 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 14:05:02 2012 Lustre: DEBUG MARKER: Wed Jul 4 14:10:01 2012 LustreError: 16818:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541665245 LustreError: 16818:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 287 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 14:15:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 14:20:01 2012 LustreError: 16819:0:(file.c:2196:ll_inode_revalidate_fini()) failure -116 inode 144151064541665879 LustreError: 16819:0:(file.c:2196:ll_inode_revalidate_fini()) Skipped 229 previous similar messages Lustre: DEBUG MARKER: Wed Jul 4 14:25:01 2012 Lustre: DEBUG MARKER: Wed Jul 4 14:30:01 2012 LustreError: 16822:0:(osc_lock.c:1109:osc_lock_enqueue_wait()) DEADLOCK POSSIBLE! LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lock@ffff88059fbbe778[1 3 0 1 1 00000000] W(2):[810032, 818223]@[0x101140000:0xd1f6:0x0] { LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) lovsub@ffff88066f1d0de0: [0 ffff8804c6fe7470 W(2):[2431024, 2455599]@[0x2103bd57a:0xe:0x0]] [0 ffff8804c6fe72d0 R(1):[2431024, 2455599]@[0x2103bd57a:0xe:0x0]] LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) osc@ffff8805351917b8: ffff88060494d240 40100001 0x1fe0953368e5bb15 3 ffff880c73c49508 size: 3321161855 mtime: 1341405025 atime: 1341405025 ctime: 1341405025 blocks: 6363088 LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) } lock@ffff88059fbbe778 LustreError: 16822:0:(osc_lock.c:1110:osc_lock_enqueue_wait()) queued. LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lock@ffff88059fbbe158[2 1 0 1 1 00000000] R(1):[809984, 810495]@[0x101140000:0xd1f6:0x0] { LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) lovsub@ffff88066f1d08e0: [0 ffff8804c6fe70c8 R(1):[2430976, 2431487]@[0x2103bd57a:0xe:0x0]] LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) osc@ffff880535191a98: (null) 00000000 0x0 0 (null) size: 0 mtime: 0 atime: 0 ctime: 0 blocks: 0 LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) } lock@ffff88059fbbe158 LustreError: 16822:0:(osc_lock.c:1111:osc_lock_enqueue_wait()) queuing. Pid: 16822, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] osc_lock_enqueue+0x775/0xa30 [osc] [] ? cfs_waitq_broadcast+0x17/0x20 [libcfs] [] ? cl_lock_state_signal+0x81/0x160 [obdclass] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] lov_lock_enqueue+0x20b/0x7c0 [lov] [] cl_enqueue_try+0xf4/0x310 [obdclass] [] cl_enqueue_locked+0x6d/0x200 [obdclass] [] cl_lock_request+0x7e/0x1c0 [obdclass] [] cl_io_lock+0x3d5/0x570 [obdclass] [] cl_io_loop+0xba/0x190 [obdclass] [] ll_file_io_generic+0x3a7/0x560 [lustre] [] ll_file_aio_read+0x139/0x2c0 [lustre] [] ll_file_read+0x169/0x2a0 [lustre] [] vfs_read+0xb5/0x1a0 [] sys_read+0x51/0x90 [] system_call_fastpath+0x16/0x1b Lustre: DEBUG MARKER: Wed Jul 4 14:35:02 2012 LustreError: 11-0: an error occurred while communicating with QQ.P.BBO.BZ@o2ib2. The obd_ping operation failed with -107 Lustre: cdep4-OST02c4-osc-ffff88107812dc00: Connection to cdep4-OST02c4 (at QQ.P.BBO.BZ@o2ib2) was lost; in progress operations using this service will wait for recovery to complete LustreError: 167-0: This client was evicted by cdep4-OST02c4; in progress operations using this service will fail. LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) lock@ffff880c26e897b8[2 3 0 1 1 00000000] R(1):[787968, 18446744073709551615]@[0x102c40000:0x5307:0x0] { LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) lovsub@ffff880c5dfaafa0: [0 ffff880a2b418a20 R(1):[2364928, 18446744073709551615]@[0x2103bd57a:0xd:0x0]] LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) osc@ffff880a3834bee8: ffff8809d9539d80 40120002 0x1fe0953368e64d9e 3 ffff880c6f55c2f8 size: 3226075136 mtime: 1341405146 atime: 1341405146 ctime: 1341405146 blocks: 6190744 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) } lock@ffff880c26e897b8 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) dlmlock returned -5 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) lock@ffff880a166a9cb8[2 3 0 1 1 00000000] R(1):[787456, 18446744073709551615]@[0x102c40000:0x5307:0x0] { LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) lovsub@ffff880c5dfaa160: [0 ffff880a16d32268 R(1):[2364416, 18446744073709551615]@[0x2103bd57a:0xd:0x0]] LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) osc@ffff880a5fd3e088: ffff880a3e265000 40120002 0x1fe0953368e64da5 3 ffff881078136d20 size: 3226075136 mtime: 1341405146 atime: 1341405146 ctime: 1341405146 blocks: 6190744 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) } lock@ffff880a166a9cb8 LustreError: 22282:0:(osc_lock.c:816:osc_ldlm_completion_ast()) dlmlock returned -5 LustreError: 22282:0:(ldlm_resource.c:749:ldlm_resource_complain()) Namespace cdep4-OST02c4-osc-ffff88107812dc00 resource refcount nonzero (2) after lock cleanup; forcing cleanup. LustreError: 22282:0:(ldlm_resource.c:755:ldlm_resource_complain()) Resource: ffff880b2c57be40 (21255/0/0/0) (rc: 2) Lustre: cdep4-OST02c4-osc-ffff88107812dc00: Connection restored to cdep4-OST02c4 (at QQ.P.BBO.BZ@o2ib2) LustreError: 16825:0:(cl_lock.c:1412:cl_unuse_try()) result = -5, this is unlikely! LustreError: 16825:0:(cl_lock.c:1426:cl_unuse_locked()) lock@ffff880c26e89a58[1 3 0 1 0 00000000] R(1):[2364928, 18446744073709551615]@[0x2103bd57a:0xd:0x0] { LustreError: 16826:0:(cl_lock.c:1412:cl_unuse_try()) result = -5, this is unlikely! LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) lock@ffff880a0fb01a58[1 3 0 1 0 00000000] R(1):[2364416, 18446744073709551615]@[0x2103bd57a:0xd:0x0] { LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) vvp@ffff8809913bce20: LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) lov@ffff880a16d32268: 1 LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) 0 0: --- LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) } lock@ffff880a0fb01a58 LustreError: 16826:0:(cl_lock.c:1426:cl_unuse_locked()) unuse return -5 LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) lock@ffff880a0fb01a58[1 3 0 0 0 00000000] R(1):[2364416, 18446744073709551615]@[0x2103bd57a:0xd:0x0] { LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) vvp@ffff8809913bce20: LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) lov@ffff880a16d32268: 1 LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) 0 0: --- LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) } lock@ffff880a0fb01a58 LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) failed at lock->cll_state != CLS_HELD. LustreError: 16826:0:(cl_lock.c:906:cl_lock_hold_release()) LBUG Pid: 16826, comm: %%U1234_malh Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] lbug_with_loc+0x47/0xb0 [libcfs] [] cl_lock_hold_release+0x2a2/0x2b0 [obdclass] [] cl_lock_release+0x82/0x180 [obdclass] [] cl_lock_link_fini+0x68/0x160 [obdclass] [] cl_io_unlock+0x135/0x2e0 [obdclass] [] cl_io_loop+0xe5/0x190 [obdclass] [] ll_file_io_generic+0x3a7/0x560 [lustre] [] ll_file_aio_read+0x139/0x2c0 [lustre] [] ll_file_read+0x169/0x2a0 [lustre] [] vfs_read+0xb5/0x1a0 [] sys_read+0x51/0x90 [] system_call_fastpath+0x16/0x1b Kernel panic - not syncing: LBUG Pid: 16826, comm: %%U1234_malh Not tainted 2.6.32-131.17.1.bl6.Bull.27.0.x86_64 #1 Call Trace: [] ? panic+0x78/0x143 [] ? lbug_with_loc+0x9b/0xb0 [libcfs] [] ? cl_lock_hold_release+0x2a2/0x2b0 [obdclass] [] ? cl_lock_release+0x82/0x180 [obdclass] [] ? cl_lock_link_fini+0x68/0x160 [obdclass] [] ? cl_io_unlock+0x135/0x2e0 [obdclass] [] ? cl_io_loop+0xe5/0x190 [obdclass] [] ? ll_file_io_generic+0x3a7/0x560 [lustre] [] ? ll_file_aio_read+0x139/0x2c0 [lustre] [] ? ll_file_read+0x169/0x2a0 [lustre] [] ? vfs_read+0xb5/0x1a0 [] ? sys_read+0x51/0x90 [] ? system_call_fastpath+0x16/0x1b crash> mod -S MODULE NAME SIZE OBJECT FILE ffffffffa0008b40 ahci 40189 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/ata/ahci.ko ffffffffa000c320 hed 2306 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/acpi/hed.ko ffffffffa000f400 iTCO_vendor_support 3022 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/watchdog/iTCO_vendor_support.ko ffffffffa0020820 megaraid_sas 76847 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/scsi/megaraid/megaraid_sas.ko ffffffffa00296c0 ib_umad 12571 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/ib_umad.ko ffffffffa003ad00 dm_mod 75571 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/md/dm-mod.ko ffffffffa0047a60 uhci_hcd 26799 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/usb/host/uhci-hcd.ko ffffffffa004b2a0 crc_t10dif 1507 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/lib/crc-t10dif.ko ffffffffa004eb00 ghes 3893 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/acpi/apei/ghes.ko ffffffffa0059480 sd_mod 38196 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/scsi/sd_mod.ko mod: cannot find or load object file for pax module ffffffffa0060620 iptable_filter 2759 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/net/ipv4/netfilter/iptable_filter.ko ffffffffa0072220 jbd2 90114 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/fs/jbd2/jbd2.ko ffffffffa007d720 iTCO_wdt 12060 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/watchdog/iTCO_wdt.ko ffffffffa00823e0 i2c_i801 11133 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/i2c/busses/i2c-i801.ko ffffffffa00d0400 ext4 364482 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/fs/ext4/ext4.ko ffffffffa00ebea0 ioatdma 58384 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/dma/ioat/ioatdma.ko ffffffffa00f1f60 ib_addr 6319 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/ib_addr.ko ffffffffa00f4ec0 dca 7259 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/dca/dca.ko ffffffffa011d5c0 igb 157614 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/net/igb/igb.ko ffffffffa0130560 ehci_hcd 53598 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/usb/host/ehci-hcd.ko ffffffffa013a1e0 i2c_core 31498 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/i2c/i2c-core.ko ffffffffa0145500 sg 30026 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/scsi/sg.ko ffffffffa015a7a0 hid 81273 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/hid/hid.ko ffffffffa0168a80 usbhid 47705 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/hid/usbhid/usbhid.ko ffffffffa0171d60 dm_log 10152 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/md/dm-log.ko ffffffffa017aa60 dm_region_hash 12296 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/md/dm-region-hash.ko ffffffffa0180820 iw_cm 8898 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/iw_cm.ko ffffffffa0185f80 dm_mirror 14067 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/md/dm-mirror.ko ffffffffa01a52c0 mlx4_core 126431 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/net/mlx4/mlx4_core.ko ffffffffa01b6140 ib_uverbs 40307 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/ib_uverbs.ko ffffffffa01c52c0 ib_core 70602 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/ib_core.ko ffffffffa01d3b00 lvfs 36412 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/lvfs.ko ffffffffa01e05c0 ib_mad 40963 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/ib_mad.ko ffffffffa01f5ae0 mlx4_ib 78146 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/hw/mlx4/mlx4_ib.ko ffffffffa0244860 ipv6 322291 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/net/ipv6/ipv6.ko mod: cannot find or load object file for sep3_4 module ffffffffa026fe40 ib_sa 44741 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/ib_sa.ko ffffffffa027b9c0 ib_cm 37667 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/ib_cm.ko ffffffffa0291960 ib_ipoib 81709 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko ffffffffa02a3fe0 rdma_cm 36003 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/rdma_cm.ko ffffffffa02aba60 ip_tables 17733 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/net/ipv4/netfilter/ip_tables.ko ffffffffa02b0940 rdma_ucm 12825 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel-ib/kernel/drivers/infiniband/core/rdma_ucm.ko ffffffffa02b4640 freq_table 4015 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/drivers/cpufreq/freq_table.ko ffffffffa02bb460 acpi_cpufreq 6976 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.ko ffffffffa02ead00 sunrpc 243193 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/net/sunrpc/sunrpc.ko ffffffffa031cda0 auth_rpcgss 44957 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/net/sunrpc/auth_gss/auth_rpcgss.ko ffffffffa03295e0 nfs_acl 2613 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/fs/nfs_common/nfs_acl.ko ffffffffa0336a60 fscache 46985 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/fs/fscache/fscache.ko ffffffffa0353300 lockd 74556 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/fs/lockd/lockd.ko ffffffffa03af0e0 nfs 380135 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/kernel/fs/nfs/nfs.ko ffffffffa03e83c0 libcfs 304725 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/net/lustre/libcfs.ko ffffffffa0460280 lnet 354878 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/net/lustre/lnet.ko ffffffffa0528f80 obdclass 1111121 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/obdclass.ko ffffffffa06dd480 ptlrpc 1499329 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/ptlrpc.ko ffffffffa07e3240 ko2iblnd 256783 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/net/lustre/ko2iblnd.ko ffffffffa0816c20 fld 71830 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/fld.ko ffffffffa0828ca0 fid 62122 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/fid.ko ffffffffa0860e20 lquota 280023 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/lquota.ko ffffffffa08aac00 mdc 182083 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/mdc.ko ffffffffa08fc6c0 osc 298190 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/osc.ko ffffffffa09938e0 lov 617077 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/lov.ko ffffffffa0a7c740 lustre 867014 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/lustre.ko ffffffffa0af3500 mgc 71551 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/mgc.ko ffffffffa0b2f1a0 lmv 302181 /lib/modules/2.6.32-131.17.1.bl6.Bull.27.0.x86_64/extra/kernel/fs/lustre/lmv.ko crash> p/x *(struct cl_lock *)0xffff880a0fb01a58 $3 = { cll_ref = { counter = 0x1 }, cll_layers = { next = 0xffff8809913bce38, prev = 0xffff880a16d32280 }, cll_linkage = { next = 0xffff880c26e89c30, prev = 0xffff880c26e89a70 }, cll_descr = { cld_obj = 0xffff880b8d6cf408, cld_start = 0x241400, cld_end = 0xffffffffffffffff, cld_gid = 0x0, cld_mode = 0x1, cld_enq_flags = 0x0 }, cll_state = 0x3, cll_wq = { lock = { raw_lock = { slock = 0xc000c } }, task_list = { next = 0xffff880a0fb01ab8, prev = 0xffff880a0fb01ab8 } }, cll_guard = { count = { counter = 0x0 }, wait_lock = { raw_lock = { slock = 0x0 } }, wait_list = { next = 0xffff880a0fb01ad0, prev = 0xffff880a0fb01ad0 }, owner = 0xffff880c6fbb8000 }, cll_guarder = 0xffff880c782f27d0, cll_depth = 0x1, cll_intransit_owner = 0x0, cll_error = 0x0, cll_holds = 0x0, cll_users = 0x0, cll_flags = 0x0, cll_inclosure = { next = 0xffff880a0fb01b18, prev = 0xffff880a0fb01b18 }, cll_conflict = 0x0, cll_reference = {}, cll_holders = {}, cll_obj_ref = 0x0 } crash> crash> crash> foreach bt -r | grep ffff880a0fb01a ffff880c6fbbbc90: ffff880a0fb01a58 ffff880c6fbbbcb8 ffff880c6fbbbca0: cl_lock_hold_release+674 ffff880a0fb01a58 ffff880c6fbbbcd0: ffff880a0fb01a58 ffff880c783d7200 ffff880c6fbbbcf0: cl_lock_link_fini+104 ffff880a0fb01a58 crash> task | grep -w stack stack = 0xffff880c6fbb8000, crash>