2012-10-10 15:23:21.570490 {R6-llnl} [bgqadmin]{11}.0.0: Mounting Lustre File Systems 2012-10-10 15:25:24.467496 {R6-llnl} [bgqadmin]{11}.1.1: BUG: soft lockup - CPU#5 stuck for 67s! [ptlrpcd_rcv:3089] 2012-10-10 15:25:24.530006 {R6-llnl} [bgqadmin]{11}.1.1: BUG: soft lockup - CPU#0 stuck for 67s! [mount.lustre:3025] 2012-10-10 15:25:24.530310 {R6-llnl} [bgqadmin]{11}.1.1: Modules linked in: mgc(U) lustre(U) mdc(U) fid(U) fld(U) lov(U) osc(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) lvfs(U) libcfs(U) bgvrnic bgmudm 2012-10-10 15:25:24.530595 {R6-llnl} [bgqadmin]{11}.1.1: NIP: c00000000042e190 LR: 8000000003ef6af4 CTR: c00000000042e160 2012-10-10 15:25:24.530874 {R6-llnl} [bgqadmin]{11}.1.1: REGS: c0000003ea3ae990 TRAP: 0901 Not tainted (2.6.32-220.23.3.bgq.12llnl.V1R1M2.bgq62_7.ppc64) 2012-10-10 15:25:24.531155 {R6-llnl} [bgqadmin]{11}.1.1: MSR: 0000000080029000 CR: 84224444 XER: 20000000 2012-10-10 15:25:24.531435 {R6-llnl} [bgqadmin]{11}.1.1: TASK = c0000003e5011b80[3025] 'mount.lustre' THREAD: c0000003ea3ac000 CPU: 0 2012-10-10 15:25:24.531714 {R6-llnl} [bgqadmin]{11}.1.1: GPR00: 0000000080000000 c0000003ea3aec10 c0000000006de510 c0000003c1745a78 2012-10-10 15:25:24.531996 {R6-llnl} [bgqadmin]{11}.1.1: GPR04: 0000000000000001 0000000000001560 0000000000000000 0000000000000000 2012-10-10 15:25:24.532271 {R6-llnl} [bgqadmin]{11}.1.1: GPR08: 0000000000000000 0000000080000000 0000000000000000 c00000000042e160 2012-10-10 15:25:24.532548 {R6-llnl} [bgqadmin]{11}.1.1: GPR12: 8000000003f7c368 c000000000742500 2012-10-10 15:25:24.532824 {R6-llnl} [bgqadmin]{11}.1.1: NIP [c00000000042e190] ._spin_lock+0x30/0x44 2012-10-10 15:25:24.533099 {R6-llnl} [bgqadmin]{11}.1.1: LR [8000000003ef6af4] .ptlrpc_check_set+0x4f4/0x4e80 [ptlrpc] 2012-10-10 15:25:24.533375 {R6-llnl} [bgqadmin]{11}.1.1: Call Trace: 2012-10-10 15:25:24.533649 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3aec10] [8000000003ef6918] .ptlrpc_check_set+0x318/0x4e80 [ptlrpc] (unreliable) 2012-10-10 15:25:24.533927 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3aedb0] [8000000003efb96c] .ptlrpc_set_wait+0x4ec/0xcb0 [ptlrpc] 2012-10-10 15:25:24.534206 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3aef20] [8000000003efc884] .ptlrpc_queue_wait+0xd4/0x380 [ptlrpc] 2012-10-10 15:25:24.534507 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3aefe0] [8000000003ed2ab8] .ldlm_cli_enqueue+0x4d8/0x960 [ptlrpc] 2012-10-10 15:25:24.534783 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3af120] [80000000071c9758] .mgc_enqueue.clone.0+0x198/0x410 [mgc] 2012-10-10 15:25:24.535115 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3af240] [80000000071c9b58] .mgc_process_log+0x188/0x1680 [mgc] 2012-10-10 15:25:24.535391 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3af3f0] [80000000071cc08c] .do_config_log_add+0x49c/0x840 [mgc] 2012-10-10 15:25:24.535667 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3af4e0] [80000000071cd348] .mgc_process_config+0xcf8/0x1730 [mgc] 2012-10-10 15:25:24.535944 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3af600] [80000000028d2f8c] .lustre_process_log+0xbbc/0x1cf0 [obdclass] 2012-10-10 15:25:24.536224 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3af750] [8000000006e8f24c] .ll_fill_super+0xc7c/0xace0 [lustre] 2012-10-10 15:25:24.536557 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3af900] [80000000028f1ea4] .lustre_fill_super+0x4c4/0x8e0 [obdclass] 2012-10-10 15:25:24.536833 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3af9d0] [c0000000000d4aa0] .get_sb_nodev+0x84/0xe8 2012-10-10 15:25:24.537107 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3afa80] [80000000028cb3a8] .lustre_get_sb+0x28/0x40 [obdclass] 2012-10-10 15:25:24.537382 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3afb10] [c0000000000d3244] .vfs_kern_mount+0x80/0x114 2012-10-10 15:25:24.537656 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3afbc0] [c0000000000d3340] .do_kern_mount+0x58/0x130 2012-10-10 15:25:24.537930 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3afc80] [c0000000000f20fc] .do_mount+0x8c8/0x984 2012-10-10 15:25:24.538205 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3afd70] [c0000000000f2270] .SyS_mount+0xb8/0x124 2012-10-10 15:25:24.538543 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003ea3afe30] [c000000000000580] syscall_exit+0x0/0x2c 2012-10-10 15:25:24.538985 {R6-llnl} [bgqadmin]{11}.1.1: Instruction dump: 2012-10-10 15:25:24.539788 {R6-llnl} [bgqadmin]{11}.1.1: 38000000 980d0c94 812d0000 7c001829 2c000000 40c20010 7d20192d 40c2fff0 2012-10-10 15:25:24.540251 {R6-llnl} [bgqadmin]{11}.1.1: 4c00012c 2fa00000 4dfe0020 7c210b78 <80030000> 2fa00000 40defff4 7c421378 2012-10-10 15:25:24.540603 {R6-llnl} [bgqadmin]{11}.1.1: Modules linked in: mgc(U) lustre(U) mdc(U) fid(U) fld(U) lov(U) osc(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) lvfs(U) libcfs(U) bgvrnic bgmudm 2012-10-10 15:25:24.540943 {R6-llnl} [bgqadmin]{11}.1.1: NIP: c00000000042e18c LR: 8000000003ef6af4 CTR: c00000000042e160 2012-10-10 15:25:24.541218 {R6-llnl} [bgqadmin]{11}.1.1: REGS: c0000003e50e7900 TRAP: 0901 Not tainted (2.6.32-220.23.3.bgq.12llnl.V1R1M2.bgq62_7.ppc64) 2012-10-10 15:25:24.541500 {R6-llnl} [bgqadmin]{11}.1.1: MSR: 0000000080029000 CR: 88228484 XER: 20000000 2012-10-10 15:25:24.541780 {R6-llnl} [bgqadmin]{11}.1.1: TASK = c0000003c7b68dc0[3089] 'ptlrpcd_rcv' THREAD: c0000003e50e4000 CPU: 5 2012-10-10 15:25:24.542153 {R6-llnl} [bgqadmin]{11}.1.1: GPR00: 0000000080000000 c0000003e50e7b80 c0000000006de510 c0000003c1745a78 2012-10-10 15:25:24.542610 {R6-llnl} [bgqadmin]{11}.1.1: GPR04: 0000000000000001 0000000000000000 0000000000000000 0000000000000000 2012-10-10 15:25:24.543107 {R6-llnl} [bgqadmin]{11}.1.1: GPR08: c0000003c1745a60 0000000080000005 c0000003c1745a60 c00000000042e160 2012-10-10 15:25:24.543884 {R6-llnl} [bgqadmin]{11}.1.1: GPR12: 8000000003f7c368 c000000000746600 2012-10-10 15:25:24.544353 {R6-llnl} [bgqadmin]{11}.1.1: NIP [c00000000042e18c] ._spin_lock+0x2c/0x44 2012-10-10 15:25:24.544705 {R6-llnl} [bgqadmin]{11}.1.1: LR [8000000003ef6af4] .ptlrpc_check_set+0x4f4/0x4e80 [ptlrpc] 2012-10-10 15:25:24.545168 {R6-llnl} [bgqadmin]{11}.1.1: Call Trace: 2012-10-10 15:25:24.545439 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003e50e7b80] [8000000003ef6964] .ptlrpc_check_set+0x364/0x4e80 [ptlrpc] (unreliable) 2012-10-10 15:25:24.545713 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003e50e7d20] [8000000003f4d78c] .ptlrpcd_check+0x66c/0x8a0 [ptlrpc] 2012-10-10 15:25:24.545996 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003e50e7e40] [8000000003f4dd1c] .ptlrpcd+0x35c/0x510 [ptlrpc] 2012-10-10 15:25:24.546267 {R6-llnl} [bgqadmin]{11}.1.1: [c0000003e50e7f90] [c00000000001a9e0] .kernel_thread+0x54/0x70 2012-10-10 15:25:24.546578 {R6-llnl} [bgqadmin]{11}.1.1: Instruction dump: 2012-10-10 15:25:24.546931 {R6-llnl} [bgqadmin]{11}.1.1: 4bffffc8 38000000 980d0c94 812d0000 7c001829 2c000000 40c20010 7d20192d 2012-10-10 15:25:24.547300 {R6-llnl} [bgqadmin]{11}.1.1: 40c2fff0 4c00012c 2fa00000 4dfe0020 <7c210b78> 80030000 2fa00000 40defff4 2012-10-10 15:25:48.469171 {R6-llnl} [bgqadmin]{11}.1.0: BUG: soft lockup - CPU#4 stuck for 67s! [ll_ping:3158] 2012-10-10 15:25:48.477910 {R6-llnl} [bgqadmin]{11}.1.0: Modules linked in: mgc(U) lustre(U) mdc(U) fid(U) fld(U) lov(U) osc(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) lvfs(U) libcfs(U) bgvrnic bgmudm 2012-10-10 15:25:48.478324 {R6-llnl} [bgqadmin]{11}.1.0: NIP: c00000000042e18c LR: 8000000003f304ec CTR: c00000000042e160 2012-10-10 15:25:48.478720 {R6-llnl} [bgqadmin]{11}.1.0: REGS: c0000003ca50bba0 TRAP: 0901 Not tainted (2.6.32-220.23.3.bgq.12llnl.V1R1M2.bgq62_7.ppc64) 2012-10-10 15:25:48.479109 {R6-llnl} [bgqadmin]{11}.1.0: MSR: 0000000080029000 CR: 84228484 XER: 20000000 2012-10-10 15:25:48.479529 {R6-llnl} [bgqadmin]{11}.1.0: TASK = c0000003ca3b4e20[3158] 'll_ping' THREAD: c0000003ca508000 CPU: 4 2012-10-10 15:25:48.479908 {R6-llnl} [bgqadmin]{11}.1.0: GPR00: 0000000080000000 c0000003ca50be20 c0000000006de510 c0000003c1745a78 2012-10-10 15:25:48.480296 {R6-llnl} [bgqadmin]{11}.1.0: GPR04: 2222222222222222 0000000000000000 c0000003ca50bd38 c000000000008e10 2012-10-10 15:25:48.480704 {R6-llnl} [bgqadmin]{11}.1.0: GPR08: 0000000000000000 0000000080000004 c0000003c162f080 c00000000042e160 2012-10-10 15:25:48.481129 {R6-llnl} [bgqadmin]{11}.1.0: GPR12: 8000000003f7c368 c000000000745900 2012-10-10 15:25:48.481534 {R6-llnl} [bgqadmin]{11}.1.0: NIP [c00000000042e18c] ._spin_lock+0x2c/0x44 2012-10-10 15:25:48.481909 {R6-llnl} [bgqadmin]{11}.1.0: LR [8000000003f304ec] .ptlrpc_pinger_main+0x19c/0xcc0 [ptlrpc] 2012-10-10 15:25:48.482306 {R6-llnl} [bgqadmin]{11}.1.0: Call Trace: 2012-10-10 15:25:48.482823 {R6-llnl} [bgqadmin]{11}.1.0: [c0000003ca50be20] [8000000003f30460] .ptlrpc_pinger_main+0x110/0xcc0 [ptlrpc] (unreliable) 2012-10-10 15:25:48.483360 {R6-llnl} [bgqadmin]{11}.1.0: [c0000003ca50bf90] [c00000000001a9e0] .kernel_thread+0x54/0x70 2012-10-10 15:25:48.483892 {R6-llnl} [bgqadmin]{11}.1.0: Instruction dump: 2012-10-10 15:25:48.484429 {R6-llnl} [bgqadmin]{11}.1.0: 4bffffc8 38000000 980d0c94 812d0000 7c001829 2c000000 40c20010 7d20192d 2012-10-10 15:25:48.484965 {R6-llnl} [bgqadmin]{11}.1.0: 40c2fff0 4c00012c 2fa00000 4dfe0020 <7c210b78> 80030000 2fa00000 40defff4 2012-10-10 15:44:35.107184 {R6-llnl} [bgqadmin]{11}.0.0: Memory <- <0x0 2012-10-10 15:44:35.109164 {R6-llnl} [bgqadmin]{11}.0.0: 0x0 2012-10-10 15:44:35.111875 {R6-llnl} [bgqadmin]{11}.0.0: 0x3 2012-10-10 15:44:35.114044 {R6-llnl} [bgqadmin]{11}.0.0: 0xffff0000 2012-10-10 15:44:35.116364 {R6-llnl} [bgqadmin]{11}.0.0: > (16383MB)