crash> kmem -i PAGES TOTAL PERCENTAGE TOTAL MEM 65925432 251.5 GB ---- FREE 1222598 4.7 GB 1% of TOTAL MEM USED 64702834 246.8 GB 98% of TOTAL MEM SHARED 2588778 9.9 GB 3% of TOTAL MEM BUFFERS 2575811 9.8 GB 3% of TOTAL MEM CACHED 51432785 196.2 GB 78% of TOTAL MEM SLAB 2868124 10.9 GB 4% of TOTAL MEM TOTAL HUGE 0 0 ---- HUGE FREE 0 0 0% of TOTAL HUGE TOTAL SWAP 1048575 4 GB ---- SWAP USED 12 48 KB 0% of TOTAL SWAP SWAP FREE 1048563 4 GB 99% of TOTAL SWAP COMMIT LIMIT 34011291 129.7 GB ---- COMMITTED 133385 521 MB 0% of TOTAL LIMIT crash> ps | grep ">" > 0 0 0 ffffffff819f9480 RU 0.0 0 0 [swapper/0] > 0 0 1 ffff8820d3b58000 RU 0.0 0 0 [swapper/1] > 0 0 2 ffff88015317eeb0 RU 0.0 0 0 [swapper/2] > 0 0 3 ffff8820d3b5eeb0 RU 0.0 0 0 [swapper/3] > 0 0 4 ffff88015317af70 RU 0.0 0 0 [swapper/4] > 0 0 5 ffff8820d3b58fd0 RU 0.0 0 0 [swapper/5] > 0 0 6 ffff88015317dee0 RU 0.0 0 0 [swapper/6] > 0 0 7 ffff8820d3b5dee0 RU 0.0 0 0 [swapper/7] > 0 0 8 ffff88015317bf40 RU 0.0 0 0 [swapper/8] > 0 0 9 ffff8820d3b59fa0 RU 0.0 0 0 [swapper/9] > 0 0 11 ffff8820d3b5cf10 RU 0.0 0 0 [swapper/11] > 0 0 12 ffff8801531a0000 RU 0.0 0 0 [swapper/12] > 0 0 14 ffff8801531a6eb0 RU 0.0 0 0 [swapper/14] > 0 0 15 ffff8820d3b5bf40 RU 0.0 0 0 [swapper/15] > 0 0 16 ffff8801531a0fd0 RU 0.0 0 0 [swapper/16] > 0 0 17 ffff8820d3bc8000 RU 0.0 0 0 [swapper/17] > 0 0 18 ffff8801531a5ee0 RU 0.0 0 0 [swapper/18] > 0 0 19 ffff8820d3bceeb0 RU 0.0 0 0 [swapper/19] > 0 0 20 ffff8801531a1fa0 RU 0.0 0 0 [swapper/20] > 0 0 21 ffff8820d3bc8fd0 RU 0.0 0 0 [swapper/21] > 0 0 22 ffff8801531a4f10 RU 0.0 0 0 [swapper/22] > 0 0 23 ffff8820d3bcdee0 RU 0.0 0 0 [swapper/23] > 0 0 24 ffff8801531a2f70 RU 0.0 0 0 [swapper/24] > 0 0 25 ffff8820d3bc9fa0 RU 0.0 0 0 [swapper/25] > 0 0 26 ffff8801531a3f40 RU 0.0 0 0 [swapper/26] > 0 0 27 ffff8820d3bccf10 RU 0.0 0 0 [swapper/27] > 0 0 28 ffff8801531c8000 RU 0.0 0 0 [swapper/28] > 0 0 29 ffff8820d3bcaf70 RU 0.0 0 0 [swapper/29] > 0 0 30 ffff8801531ceeb0 RU 0.0 0 0 [swapper/30] > 0 0 31 ffff8820d3bcbf40 RU 0.0 0 0 [swapper/31] > 0 0 32 ffff8801531c8fd0 RU 0.0 0 0 [swapper/32] > 0 0 33 ffff8820d3008000 RU 0.0 0 0 [swapper/33] > 0 0 34 ffff8801531cdee0 RU 0.0 0 0 [swapper/34] > 0 0 35 ffff8820d300eeb0 RU 0.0 0 0 [swapper/35] > 0 0 36 ffff8801531c9fa0 RU 0.0 0 0 [swapper/36] > 0 0 38 ffff8801531ccf10 RU 0.0 0 0 [swapper/38] > 0 0 39 ffff8820d300dee0 RU 0.0 0 0 [swapper/39] > 0 0 40 ffff8801531caf70 RU 0.0 0 0 [swapper/40] > 0 0 41 ffff8820d3009fa0 RU 0.0 0 0 [swapper/41] > 0 0 42 ffff8801531cbf40 RU 0.0 0 0 [swapper/42] > 0 0 43 ffff8820d300cf10 RU 0.0 0 0 [swapper/43] > 0 0 44 ffff8801531f0000 RU 0.0 0 0 [swapper/44] > 0 0 45 ffff8820d300af70 RU 0.0 0 0 [swapper/45] > 0 0 46 ffff8801531f6eb0 RU 0.0 0 0 [swapper/46] > 0 0 47 ffff8820d300bf40 RU 0.0 0 0 [swapper/47] > 75 2 13 ffff8820d21d8fd0 RU 0.0 0 0 [ksoftirqd/13] > 263454 2 37 ffff881fec000000 RU 0.0 0 0 [md19_raid6] > 281433 2 10 ffff881c31258000 RU 0.0 0 0 [ll_ost00_045] crash> bt 75 PID: 75 TASK: ffff8820d21d8fd0 CPU: 13 COMMAND: "ksoftirqd/13" #0 [ffff883ffc785e48] crash_nmi_callback at ffffffff8104fd61 #1 [ffff883ffc785e58] nmi_handle at ffffffff816ad467 #2 [ffff883ffc785eb0] do_nmi at ffffffff816ad69d #3 [ffff883ffc785ef0] end_repeat_nmi at ffffffff816ac913 [exception RIP: __blk_recalc_rq_segments+151] RIP: ffffffff812ff157 RSP: ffff8820d21e3b90 RFLAGS: 00000246 RAX: 0000000000000000 RBX: 0000000000000000 RCX: ffff881e358dc030 RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000001e3037e RBP: ffff8820d21e3bd8 R8: ffff883fb6e8e9c0 R9: ffffea0000000000 R10: ffff8836ebec2000 R11: 0000001057ed1000 R12: ffff881e3034cfc0 R13: 000000000000002a R14: ffff881e3034ceb0 R15: 0000000000001000 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #4 [ffff8820d21e3b90] __blk_recalc_rq_segments at ffffffff812ff157 #5 [ffff8820d21e3be0] blk_recalc_rq_segments at ffffffff812ff848 #6 [ffff8820d21e3bf8] blk_update_request at ffffffff812f98c9 #7 [ffff8820d21e3c38] end_clone_bio at ffffffffc04b0d05 [dm_mod] #8 [ffff8820d21e3c68] bio_endio at ffffffff8123cac4 #9 [ffff8820d21e3c90] blk_update_request at ffffffff812f9750 #10 [ffff8820d21e3cd0] scsi_end_request at ffffffff8146e964 #11 [ffff8820d21e3d10] scsi_io_completion at ffffffff8146ecd8 #12 [ffff8820d21e3d70] scsi_finish_command at ffffffff81464985 #13 [ffff8820d21e3da0] scsi_softirq_done at ffffffff8146e222 #14 [ffff8820d21e3dd0] blk_done_softirq at ffffffff81300a70 #15 [ffff8820d21e3e10] __do_softirq at ffffffff81090b3f #16 [ffff8820d21e3e80] run_ksoftirqd at ffffffff81090d08 #17 [ffff8820d21e3e98] smpboot_thread_fn at ffffffff810b909f #18 [ffff8820d21e3ec8] kthread at ffffffff810b098f #19 [ffff8820d21e3f50] ret_from_fork at ffffffff816b4f58 crash> bt 263454 PID: 263454 TASK: ffff881fec000000 CPU: 37 COMMAND: "md19_raid6" #0 [ffff883ffca85e48] crash_nmi_callback at ffffffff8104fd61 #1 [ffff883ffca85e58] nmi_handle at ffffffff816ad467 #2 [ffff883ffca85eb0] do_nmi at ffffffff816ad69d #3 [ffff883ffca85ef0] end_repeat_nmi at ffffffff816ac913 [exception RIP: async_syndrome_val+467] RIP: ffffffffc0679883 RSP: ffff881f5f603950 RFLAGS: 00000246 RAX: 0000000000000000 RBX: 000000000000000a RCX: 0000000000000fff RDX: ffff88268653af88 RSI: ffff881f58007000 RDI: ffff883a2aac1000 RBP: ffff881f5f6039f8 R8: 00000000000000e7 R9: 0000000000000000 R10: 0000000000000008 R11: ffff883f1ca20078 R12: ffffea007d6001c0 R13: ffff883f1ca20060 R14: ffff883f1ca20058 R15: ffff881f5f603a10 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #4 [ffff881f5f603950] async_syndrome_val at ffffffffc0679883 [async_pq] #5 [ffff881f5f603a00] ops_run_check_pq at ffffffffc0f87686 [raid456] #6 [ffff881f5f603a70] raid_run_ops at ffffffffc0f8d0bf [raid456] #7 [ffff881f5f603b40] handle_stripe at ffffffffc0f91ab8 [raid456] #8 [ffff881f5f603cb0] handle_active_stripes at ffffffffc0f93a2d [raid456] #9 [ffff881f5f603d60] raid5d at ffffffffc0f940a8 [raid456] #10 [ffff881f5f603e50] md_thread at ffffffff8150e315 #11 [ffff881f5f603ec8] kthread at ffffffff810b098f #12 [ffff881f5f603f50] ret_from_fork at ffffffff816b4f58 crash> bt 281433 PID: 281433 TASK: ffff881c31258000 CPU: 10 COMMAND: "ll_ost00_045" #0 [ffff881c025d7a70] machine_kexec at ffffffff8105c4cb #1 [ffff881c025d7ad0] __crash_kexec at ffffffff81104a32 #2 [ffff881c025d7ba0] panic at ffffffff8169dc7f #3 [ffff881c025d7c20] lbug_with_loc at ffffffffc08088cb [libcfs] #4 [ffff881c025d7c88] tgt_grant_prepare_read at ffffffffc0bf82bb [ptlrpc] #5 [ffff881c025d7cc8] ofd_set_info_hdl at ffffffffc119df6d [ofd] #6 [ffff881c025d7d00] tgt_request_handle at ffffffffc0bda115 [ptlrpc] #7 [ffff881c025d7d48] ptlrpc_server_handle_request at ffffffffc0b82dd6 [ptlrpc] #8 [ffff881c025d7de8] ptlrpc_main at ffffffffc0b86512 [ptlrpc] #9 [ffff881c025d7ec8] kthread at ffffffff810b098f #10 [ffff881c025d7f50] ret_from_fork at ffffffff816b4f58