crash> ps | grep ">" > 0 0 0 ffffffff95c18480 RU 0.0 0 0 [swapper/0] > 0 0 2 ffff904829f40000 RU 0.0 0 0 [swapper/2] > 0 0 3 ffff905829438000 RU 0.0 0 0 [swapper/3] > 0 0 4 ffff9028a9bbe180 RU 0.0 0 0 [swapper/4] > 0 0 5 ffff903829e8a080 RU 0.0 0 0 [swapper/5] > 0 0 6 ffff904829f46180 RU 0.0 0 0 [swapper/6] > 0 0 7 ffff90582943e180 RU 0.0 0 0 [swapper/7] > 0 0 10 ffff904829f41040 RU 0.0 0 0 [swapper/10] > 0 0 12 ffff9028a9bbd140 RU 0.0 0 0 [swapper/12] > 0 0 13 ffff903829e8b0c0 RU 0.0 0 0 [swapper/13] > 0 0 14 ffff904829f45140 RU 0.0 0 0 [swapper/14] > 0 0 15 ffff90582943d140 RU 0.0 0 0 [swapper/15] > 0 0 16 ffff9028a9bbc100 RU 0.0 0 0 [swapper/16] > 0 0 17 ffff903829e8c100 RU 0.0 0 0 [swapper/17] > 0 0 18 ffff904829f42080 RU 0.0 0 0 [swapper/18] > 0 0 19 ffff90582943a080 RU 0.0 0 0 [swapper/19] > 0 0 20 ffff9028a9bf0000 RU 0.0 0 0 [swapper/20] > 0 0 21 ffff903829f10000 RU 0.0 0 0 [swapper/21] > 0 0 22 ffff904829f44100 RU 0.0 0 0 [swapper/22] > 0 0 23 ffff90582943c100 RU 0.0 0 0 [swapper/23] > 0 0 24 ffff9028a9bf6180 RU 0.0 0 0 [swapper/24] > 0 0 25 ffff903829f16180 RU 0.0 0 0 [swapper/25] > 0 0 26 ffff904829f430c0 RU 0.0 0 0 [swapper/26] > 0 0 27 ffff90582943b0c0 RU 0.0 0 0 [swapper/27] > 0 0 28 ffff9028a9bf1040 RU 0.0 0 0 [swapper/28] > 0 0 29 ffff903829f11040 RU 0.0 0 0 [swapper/29] > 0 0 30 ffff904829f70000 RU 0.0 0 0 [swapper/30] > 0 0 31 ffff905829458000 RU 0.0 0 0 [swapper/31] > 0 0 34 ffff904829f76180 RU 0.0 0 0 [swapper/34] > 0 0 35 ffff90582945e180 RU 0.0 0 0 [swapper/35] > 0 0 36 ffff9028a9bf2080 RU 0.0 0 0 [swapper/36] > 0 0 37 ffff903829f12080 RU 0.0 0 0 [swapper/37] > 0 0 38 ffff904829f71040 RU 0.0 0 0 [swapper/38] > 0 0 39 ffff905829459040 RU 0.0 0 0 [swapper/39] > 0 0 40 ffff9028a9bf4100 RU 0.0 0 0 [swapper/40] > 0 0 41 ffff903829f14100 RU 0.0 0 0 [swapper/41] > 0 0 42 ffff904829f75140 RU 0.0 0 0 [swapper/42] > 0 0 43 ffff90582945d140 RU 0.0 0 0 [swapper/43] > 0 0 45 ffff903829f130c0 RU 0.0 0 0 [swapper/45] > 0 0 46 ffff904829f72080 RU 0.0 0 0 [swapper/46] > 0 0 47 ffff90582945a080 RU 0.0 0 0 [swapper/47] > 19481 2 32 ffff903772676180 RU 0.0 0 0 [kiblnd_sd_00_03] > 19483 2 9 ffff903772670000 RU 0.0 0 0 [kiblnd_sd_01_01] > 19484 2 33 ffff903772671040 RU 0.0 0 0 [kiblnd_sd_01_02] > 20743 2 8 ffff9037724ae180 RU 0.0 0 0 [mdt00_009] > 21344 2 44 ffff903705aca080 RU 0.0 0 0 [mdt00_024] > 21612 2 1 ffff905758aca080 RU 0.0 0 0 [mdt_rdpg01_025] > 49210 2 11 ffff90477af56180 RU 0.0 0 0 [mdt03_089] crash> bt 19481 PID: 19481 TASK: ffff903772676180 CPU: 32 COMMAND: "kiblnd_sd_00_03" #0 [ffff90377f008e48] crash_nmi_callback at ffffffff95055d07 #1 [ffff90377f008e58] nmi_handle at ffffffff9576c8fc #2 [ffff90377f008eb0] do_nmi at ffffffff9576cb1d #3 [ffff90377f008ef0] end_repeat_nmi at ffffffff9576bd69 [exception RIP: __switch_to+373] RIP: ffffffff9502a645 RSP: ffff903779dbbcb0 RFLAGS: 00000046 RAX: 0000000000000000 RBX: ffff903772676180 RCX: 0000000000000000 RDX: 0000000000000000 RSI: 0000000000000020 RDI: ffff903772676180 RBP: ffff903779dbbcf8 R8: 0000000000001f81 R9: 0000000000000001 R10: 0000000000000000 R11: 0000000000000000 R12: ffff903772674100 R13: 0000000000000020 R14: 0000000000004000 R15: ffff903772674798 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #4 [ffff903779dbbcb0] __switch_to at ffffffff9502a645 #5 [ffff903779dbbd00] __schedule at ffffffff95767747 #6 [ffff903779dbbd98] schedule at ffffffff95767c49 #7 [ffff903779dbbda8] kiblnd_scheduler at ffffffffc0c8752c [ko2iblnd] #8 [ffff903779dbbec8] kthread at ffffffff950c1c31 #9 [ffff903779dbbf50] ret_from_fork_nospec_begin at ffffffff95774c24 crash> bt 19483 PID: 19483 TASK: ffff903772670000 CPU: 9 COMMAND: "kiblnd_sd_01_01" #0 [ffff90477f688e48] crash_nmi_callback at ffffffff95055d07 #1 [ffff90477f688e58] nmi_handle at ffffffff9576c8fc #2 [ffff90477f688eb0] do_nmi at ffffffff9576cb1d #3 [ffff90477f688ef0] end_repeat_nmi at ffffffff9576bd69 [exception RIP: __wake_up_common+49] RIP: ffffffff950cba71 RSP: ffff90377458fd30 RFLAGS: 00000086 RAX: ffff9037752a3e38 RBX: ffff90477c1f1a88 RCX: ffff9037733cbe38 RDX: 0000000000000001 RSI: 0000000000000003 RDI: ffff90477c1f1a88 RBP: ffff90377458fd68 R8: 0000000000000000 R9: 0000000000001000 R10: ffff904b88fd1600 R11: 0000000000000000 R12: ffff90477c1f1a90 R13: 0000000000000003 R14: 0000000000000000 R15: 0000000000000003 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #4 [ffff90377458fd30] __wake_up_common at ffffffff950cba71 #5 [ffff90377458fd70] __wake_up at ffffffff950cec59 #6 [ffff90377458fda8] kiblnd_scheduler at ffffffffc0c8779f [ko2iblnd] #7 [ffff90377458fec8] kthread at ffffffff950c1c31 #8 [ffff90377458ff50] ret_from_fork_nospec_begin at ffffffff95774c24 crash> bt 19484 PID: 19484 TASK: ffff903772671040 CPU: 33 COMMAND: "kiblnd_sd_01_02" #0 [ffff90477f808e48] crash_nmi_callback at ffffffff95055d07 #1 [ffff90477f808e58] nmi_handle at ffffffff9576c8fc #2 [ffff90477f808eb0] do_nmi at ffffffff9576cb1d #3 [ffff90477f808ef0] end_repeat_nmi at ffffffff9576bd69 [exception RIP: native_queued_spin_lock_slowpath+462] RIP: ffffffff9511228e RSP: ffff9037752a3d50 RFLAGS: 00000002 RAX: 0000000000000001 RBX: 0000000000000246 RCX: 0000000000000001 RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff90477c1f1a88 RBP: ffff9037752a3d50 R8: 0000000000000101 R9: 0000000000000001 R10: 0000000000000000 R11: 0000000000000000 R12: ffff9037752a3e20 R13: ffff90477c1f1a88 R14: ffff903772671040 R15: ffff90477c1f1aa0 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #4 [ffff9037752a3d50] native_queued_spin_lock_slowpath at ffffffff9511228e #5 [ffff9037752a3d58] queued_spin_lock_slowpath at ffffffff9575bfcb #6 [ffff9037752a3d68] _raw_spin_lock_irqsave at ffffffff9576a507 #7 [ffff9037752a3d80] remove_wait_queue at ffffffff950c28d9 #8 [ffff9037752a3da8] kiblnd_scheduler at ffffffffc0c8753b [ko2iblnd] #9 [ffff9037752a3ec8] kthread at ffffffff950c1c31 #10 [ffff9037752a3f50] ret_from_fork_nospec_begin at ffffffff95774c24 crash> bt 20743 PID: 20743 TASK: ffff9037724ae180 CPU: 8 COMMAND: "mdt00_009" #0 [ffff90377ee88e48] crash_nmi_callback at ffffffff95055d07 #1 [ffff90377ee88e58] nmi_handle at ffffffff9576c8fc #2 [ffff90377ee88eb0] do_nmi at ffffffff9576cb1d #3 [ffff90377ee88ef0] end_repeat_nmi at ffffffff9576bd69 [exception RIP: ldlm_grant_lock+53] RIP: ffffffffc0cc4aa5 RSP: ffff90377e9635f0 RFLAGS: 00000246 RAX: 0000000000000001 RBX: ffff902a4e4f1680 RCX: dead000000000200 RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffff902a4e4f1680 RBP: ffff90377e963608 R8: ffff902a4e4f16e0 R9: 0000202000000000 R10: ffff9047756f3000 R11: ffff903765b3af00 R12: ffff905383485bc0 R13: ffff90377e963684 R14: ffff90377e9637f0 R15: 0000000000000002 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #4 [ffff90377e9635f0] ldlm_grant_lock at ffffffffc0cc4aa5 [ptlrpc] #5 [ffff90377e963610] ldlm_process_inodebits_lock at ffffffffc0cf8336 [ptlrpc] #6 [ffff90377e963658] ldlm_lock_enqueue at ffffffffc0cc6c8b [ptlrpc] #7 [ffff90377e9636d0] ldlm_cli_enqueue_local at ffffffffc0ce0c4c [ptlrpc] #8 [ffff90377e963770] mdt_object_local_lock at ffffffffc13694bb [mdt] #9 [ffff90377e963830] mdt_object_lock_internal at ffffffffc1369b40 [mdt] #10 [ffff90377e963890] mdt_object_lock_try at ffffffffc1369f67 [mdt] #11 [ffff90377e9638b0] mdt_object_open_lock at ffffffffc138e938 [mdt] #12 [ffff90377e963928] mdt_open_by_fid_lock at ffffffffc139201c [mdt] #13 [ffff90377e963998] mdt_reint_open at ffffffffc1392cd0 [mdt] #14 [ffff90377e963a88] mdt_reint_rec at ffffffffc1386ed3 [mdt] #15 [ffff90377e963ab0] mdt_reint_internal at ffffffffc1365143 [mdt] #16 [ffff90377e963af0] mdt_intent_open at ffffffffc1371982 [mdt] #17 [ffff90377e963b30] mdt_intent_policy at ffffffffc136fa28 [mdt] #18 [ffff90377e963b98] ldlm_lock_enqueue at ffffffffc0cc6d46 [ptlrpc] #19 [ffff90377e963c10] ldlm_handle_enqueue0 at ffffffffc0cef707 [ptlrpc] #20 [ffff90377e963ca8] tgt_enqueue at ffffffffc0d766e2 [ptlrpc] #21 [ffff90377e963cc8] tgt_request_handle at ffffffffc0d7d73a [ptlrpc] #22 [ffff90377e963d50] ptlrpc_server_handle_request at ffffffffc0d21d0b [ptlrpc] #23 [ffff90377e963df0] ptlrpc_main at ffffffffc0d2563c [ptlrpc] #24 [ffff90377e963ec8] kthread at ffffffff950c1c31 #25 [ffff90377e963f50] ret_from_fork_nospec_begin at ffffffff95774c24 crash> bt 21344 PID: 21344 TASK: ffff903705aca080 CPU: 44 COMMAND: "mdt00_024" #0 [ffff90377f0c8e48] crash_nmi_callback at ffffffff95055d07 #1 [ffff90377f0c8e58] nmi_handle at ffffffff9576c8fc #2 [ffff90377f0c8eb0] do_nmi at ffffffff9576cb1d #3 [ffff90377f0c8ef0] end_repeat_nmi at ffffffff9576bd69 [exception RIP: llog_cat_id2handle+259] RIP: ffffffffc0a95e43 RSP: ffff903704ac7698 RFLAGS: 00000287 RAX: ffff90384c3b5300 RBX: ffff903704ac7784 RCX: 0000000000000001 RDX: 000000000001160c RSI: 000000000001509b RDI: ffffffff0001160c RBP: ffff903704ac76f8 R8: ffffffff0001160c R9: ffffffffc124f134 R10: ffffffff00000000 R11: 0000ffffffffffff R12: ffff90377ea69500 R13: 00000000ffffffff R14: ffff903704ac7c18 R15: ffff903704ac7728 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #4 [ffff903704ac7698] llog_cat_id2handle at ffffffffc0a95e43 [obdclass] #5 [ffff903704ac7700] llog_cat_cancel_records at ffffffffc0a97120 [obdclass] #6 [ffff903704ac7768] llog_changelog_cancel_cb at ffffffffc14dd9b4 [mdd] #7 [ffff903704ac77c8] llog_process_thread at ffffffffc0a91732 [obdclass] #8 [ffff903704ac78a0] llog_process_or_fork at ffffffffc0a924fc [obdclass] #9 [ffff903704ac78e8] llog_cat_process_cb at ffffffffc0a975f9 [obdclass] #10 [ffff903704ac7938] llog_process_thread at ffffffffc0a91732 [obdclass] #11 [ffff903704ac7a10] llog_process_or_fork at ffffffffc0a924fc [obdclass] #12 [ffff903704ac7a58] llog_cat_process_or_fork at ffffffffc0a9682f [obdclass] #13 [ffff903704ac7ad0] llog_cat_process at ffffffffc0a969de [obdclass] #14 [ffff903704ac7af0] llog_changelog_cancel at ffffffffc14dc9d4 [mdd] #15 [ffff903704ac7b30] mdd_changelog_llog_cancel at ffffffffc14dedd0 [mdd] #16 [ffff903704ac7b80] mdd_iocontrol at ffffffffc14e4d7e [mdd] #17 [ffff903704ac7bf0] mdt_iocontrol at ffffffffc136c8fc [mdt] #18 [ffff903704ac7c88] mdt_set_info at ffffffffc136d294 [mdt] #19 [ffff903704ac7cc8] tgt_request_handle at ffffffffc0d7d73a [ptlrpc] #20 [ffff903704ac7d50] ptlrpc_server_handle_request at ffffffffc0d21d0b [ptlrpc] #21 [ffff903704ac7df0] ptlrpc_main at ffffffffc0d2563c [ptlrpc] #22 [ffff903704ac7ec8] kthread at ffffffff950c1c31 #23 [ffff903704ac7f50] ret_from_fork_nospec_begin at ffffffff95774c24 crash> bt 21612 PID: 21612 TASK: ffff905758aca080 CPU: 1 COMMAND: "mdt_rdpg01_025" #0 [ffff90477f608e48] crash_nmi_callback at ffffffff95055d07 #1 [ffff90477f608e58] nmi_handle at ffffffff9576c8fc #2 [ffff90477f608eb0] do_nmi at ffffffff9576cb1d #3 [ffff90477f608ef0] end_repeat_nmi at ffffffff9576bd69 [exception RIP: lprocfs_counter_add+144] RIP: ffffffffc0abb8c0 RSP: ffff9047463ffd20 RFLAGS: 00000202 RAX: ffff9047650a06e0 RBX: ffff903779d7e000 RCX: 00000000000000dc RDX: ffff90373aa7e420 RSI: 000000000000002c RDI: ffff903779d7e000 RBP: ffff9047463ffd48 R8: 0000000000401ebd R9: 0000000000000030 R10: 0000000000000000 R11: ffff903b6e7202b8 R12: 000000000000002c R13: 0000000000000021 R14: ffff906776f6f800 R15: 15a85169004af150 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #4 [ffff9047463ffd20] lprocfs_counter_add at ffffffffc0abb8c0 [obdclass] #5 [ffff9047463ffd50] ptlrpc_server_handle_request at ffffffffc0d2219f [ptlrpc] #6 [ffff9047463ffdf0] ptlrpc_main at ffffffffc0d2563c [ptlrpc] #7 [ffff9047463ffec8] kthread at ffffffff950c1c31 #8 [ffff9047463fff50] ret_from_fork_nospec_begin at ffffffff95774c24 crash> bt 49210 PID: 49210 TASK: ffff90477af56180 CPU: 11 COMMAND: "mdt03_089" #0 [ffff905868b87550] machine_kexec at ffffffff95063674 #1 [ffff905868b875b0] __crash_kexec at ffffffff9511cef2 #2 [ffff905868b87680] panic at ffffffff9575b55b #3 [ffff905868b87700] lbug_with_loc at ffffffffc09588cb [libcfs] #4 [ffff905868b87720] osp_declare_create at ffffffffc154ecc5 [osp] #5 [ffff905868b87780] lod_sub_declare_create at ffffffffc149639f [lod] #6 [ffff905868b877e0] lod_qos_prep_create at ffffffffc1495904 [lod] #7 [ffff905868b878e8] lod_declare_instantiate_components at ffffffffc1476bba [lod] #8 [ffff905868b87920] lod_declare_layout_change at ffffffffc14894d5 [lod] #9 [ffff905868b87988] mdd_declare_layout_change at ffffffffc14fb102 [mdd] #10 [ffff905868b879b0] mdd_layout_change at ffffffffc1503e52 [mdd] #11 [ffff905868b87a28] mdt_layout_change at ffffffffc136a327 [mdt] #12 [ffff905868b87a78] mdt_intent_layout at ffffffffc137243e [mdt] #13 [ffff905868b87b30] mdt_intent_policy at ffffffffc136fa28 [mdt] #14 [ffff905868b87b98] ldlm_lock_enqueue at ffffffffc0cc6d46 [ptlrpc] #15 [ffff905868b87c10] ldlm_handle_enqueue0 at ffffffffc0cef707 [ptlrpc] #16 [ffff905868b87ca8] tgt_enqueue at ffffffffc0d766e2 [ptlrpc] #17 [ffff905868b87cc8] tgt_request_handle at ffffffffc0d7d73a [ptlrpc] #18 [ffff905868b87d50] ptlrpc_server_handle_request at ffffffffc0d21d0b [ptlrpc] #19 [ffff905868b87df0] ptlrpc_main at ffffffffc0d2563c [ptlrpc] #20 [ffff905868b87ec8] kthread at ffffffff950c1c31 #21 [ffff905868b87f50] ret_from_fork_nospec_begin at ffffffff95774c24