crash> bt -a
PID: 10869 TASK: ffff88044c582d10 CPU: 0 COMMAND: "kiblnd_sd_18"
#0 [ffff880036607d10] machine_kexec at ffffffff8102e66b
#1 [ffff880036607d70] crash_kexec at ffffffff810a9b08
#2 [ffff880036607e40] oops_end at ffffffff81456108
#3 [ffff880036607e70] die_nmi at ffffffff814562a9
#4 [ffff880036607ea0] do_nmi_callback at ffffffff81028f0b
#5 [ffff880036607f10] do_nmi at ffffffff81455e46
#6 [ffff880036607f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff880466a9fa10 RFLAGS: 00000283
RAX: 0000000000003f08 RBX: ffff880296799e00 RCX: 00000000000000c0
RDX: 0000000000003eeb RSI: ffff880296799e00 RDI: ffffffffa040d340
RBP: ffff880466a9fa10 R8: 0000000000001000 R9: 0000000000000000
R10: ffff880296799e00 R11: 0000000000000001 R12: 0000000000000000
R13: 0000000000000001 R14: ffff88047cd29280 R15: ffff880b468e0bb0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#7 [ffff880466a9fa10] _spin_lock at ffffffff81455011
#8 [ffff880466a9fa18] lnet_finalize at ffffffffa03f0218
#9 [ffff880466a9fa48] kiblnd_recv at ffffffffa0661b5a
#10 [ffff880466a9fb08] lnet_ni_recv at ffffffffa03f37a8
#11 [ffff880466a9fb98] lnet_recv_put at ffffffffa03f3b00
#12 [ffff880466a9fbe8] lnet_parse at ffffffffa03fa23a
#13 [ffff880466a9fce8] kiblnd_handle_rx at ffffffffa0662253
#14 [ffff880466a9fd78] kiblnd_rx_complete at ffffffffa0662e02
#15 [ffff880466a9fdf8] kiblnd_complete at ffffffffa0662fa2
#16 [ffff880466a9fe38] kiblnd_scheduler at ffffffffa066331c
#17 [ffff880466a9ff48] kernel_thread at ffffffff8100d1aa
PID: 10866 TASK: ffff8801006fab90 CPU: 1 COMMAND: "kiblnd_sd_15"
#0 [ffff88088e407e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88088e407e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88088e407ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88088e407ee0] notify_die at ffffffff810875de
#4 [ffff88088e407f10] do_nmi at ffffffff81455e1c
#5 [ffff88088e407f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff8802276f7cb0 RFLAGS: 00000297
RAX: 0000000000003efb RBX: ffff88088f80c400 RCX: ffff88043f99ecc0
RDX: 0000000000003ef5 RSI: ffff88088f80c400 RDI: ffffffffa040d340
RBP: ffff8802276f7cb0 R8: ffffc900177cb5e0 R9: 00049b3c447e0790
R10: 000000000000003f R11: 0000000000000064 R12: 0000000000000000
R13: ffff88088f80c400 R14: 0000000000000000 R15: 0000000000000000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8802276f7cb0] _spin_lock at ffffffff8145500e
#7 [ffff8802276f7cb8] lnet_finalize at ffffffffa03f0218
#8 [ffff8802276f7ce8] kiblnd_tx_done at ffffffffa065d482
#9 [ffff8802276f7d68] kiblnd_tx_complete at ffffffffa06615cf
#10 [ffff8802276f7df8] kiblnd_complete at ffffffffa0662f72
#11 [ffff8802276f7e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff8802276f7f48] kernel_thread at ffffffff8100d1aa
PID: 10882 TASK: ffff8804664deb50 CPU: 2 COMMAND: "kiblnd_sd_31"
#0 [ffff88048e407e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88048e407e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88048e407ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88048e407ee0] notify_die at ffffffff810875de
#4 [ffff88048e407f10] do_nmi at ffffffff81455e1c
#5 [ffff88048e407f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff880164f1bb90 RFLAGS: 00000297
RAX: 0000000000003f0a RBX: ffff880562f81800 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff880562f81800 RDI: ffffffffa040d340
RBP: ffff880164f1bb90 R8: 00000000000000c0 R9: 00000000000000c0
R10: 0000000000000004 R11: ffff880562f81800 R12: ffff880954e2ccc0
R13: 0000000000000000 R14: 00000000000000c0 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880164f1bb90] _spin_lock at ffffffff8145500e
#7 [ffff880164f1bb98] lnet_recv_put at ffffffffa03f3a5c
#8 [ffff880164f1bbe8] lnet_parse at ffffffffa03fa23a
#9 [ffff880164f1bce8] kiblnd_handle_rx at ffffffffa0662253
#10 [ffff880164f1bd78] kiblnd_rx_complete at ffffffffa0662e02
#11 [ffff880164f1bdf8] kiblnd_complete at ffffffffa0662fa2
#12 [ffff880164f1be38] kiblnd_scheduler at ffffffffa066331c
#13 [ffff880164f1bf48] kernel_thread at ffffffff8100d1aa
PID: 10853 TASK: ffff880349ef6f10 CPU: 3 COMMAND: "kiblnd_sd_02"
#0 [ffff880c8e407e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880c8e407e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880c8e407ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880c8e407ee0] notify_die at ffffffff810875de
#4 [ffff880c8e407f10] do_nmi at ffffffff81455e1c
#5 [ffff880c8e407f50] nmi at ffffffff81455710
[exception RIP: lnet_match_md+563]
RIP: ffffffffa03f4f33 RSP: ffff8804635b1b50 RFLAGS: 00000283
RAX: ffff88051285dac0 RBX: 0000000000000004 RCX: 000500080a6420a5
RDX: ffff88051285d840 RSI: 0000000000000000 RDI: 0000000000000004
RBP: ffff8804635b1be0 R8: 00000000000000c0 R9: 00000000000000c0
R10: 0000000000000004 R11: ffff88101d274c00 R12: 00000000000000c0
R13: ffff88051285dcc0 R14: 00000000000000c0 R15: 0000000000000001
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8804635b1b50] lnet_match_md at ffffffffa03f4f33
#7 [ffff8804635b1be8] lnet_parse at ffffffffa03f99cd
#8 [ffff8804635b1ce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff8804635b1d78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff8804635b1df8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff8804635b1e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff8804635b1f48] kernel_thread at ffffffff8100d1aa
PID: 10877 TASK: ffff880466ad0d90 CPU: 4 COMMAND: "kiblnd_sd_26"
#0 [ffff880036647e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880036647e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880036647ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880036647ee0] notify_die at ffffffff810875de
#4 [ffff880036647f10] do_nmi at ffffffff81455e1c
#5 [ffff880036647f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff88047c677be0 RFLAGS: 00000283
RAX: 0000000000003f01 RBX: ffff88014354e030 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: ffff88047e739160 RDI: ffffffffa040d340
RBP: ffff88047c677be0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801eeeaca00
R13: 0000000000000001 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff88047c677be0] _spin_lock at ffffffff81455011
#7 [ffff88047c677be8] lnet_parse at ffffffffa03f937a
#8 [ffff88047c677ce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff88047c677d78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff88047c677df8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff88047c677e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff88047c677f48] kernel_thread at ffffffff8100d1aa
PID: 10864 TASK: ffff88041e62c540 CPU: 5 COMMAND: "kiblnd_sd_13"
#0 [ffff88088e447e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88088e447e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88088e447ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88088e447ee0] notify_die at ffffffff810875de
#4 [ffff88088e447f10] do_nmi at ffffffff81455e1c
#5 [ffff88088e447f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff8803d375dbe0 RFLAGS: 00000287
RAX: 0000000000003f05 RBX: ffff8801e22f70e0 RCX: 000500080a6420a5
RDX: 0000000000003ef5 RSI: ffff88033ce45610 RDI: ffffffffa040d340
RBP: ffff8803d375dbe0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000001 R11: 0000000000000000 R12: ffff880b468eba00
R13: 0000000000000004 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8803d375dbe0] _spin_lock at ffffffff8145500e
#7 [ffff8803d375dbe8] lnet_parse at ffffffffa03f9982
#8 [ffff8803d375dce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff8803d375dd78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff8803d375ddf8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff8803d375de38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff8803d375df48] kernel_thread at ffffffff8100d1aa
PID: 10865 TASK: ffff88047d29ae50 CPU: 6 COMMAND: "kiblnd_sd_14"
#0 [ffff88048e447e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88048e447e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88048e447ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88048e447ee0] notify_die at ffffffff810875de
#4 [ffff88048e447f10] do_nmi at ffffffff81455e1c
#5 [ffff88048e447f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff88047cf55be0 RFLAGS: 00000283
RAX: 0000000000003f03 RBX: ffff8801e22f70e0 RCX: 000500080a64205b
RDX: 0000000000003ef5 RSI: ffff88033ce45170 RDI: ffffffffa040d340
RBP: ffff88047cf55be0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000001 R11: 0000000000000000 R12: ffff880562f9b800
R13: 0000000000000004 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff88047cf55be0] _spin_lock at ffffffff8145500e
#7 [ffff88047cf55be8] lnet_parse at ffffffffa03f9982
#8 [ffff88047cf55ce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff88047cf55d78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff88047cf55df8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff88047cf55e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff88047cf55f48] kernel_thread at ffffffff8100d1aa
PID: 10878 TASK: ffff880466ad15a0 CPU: 7 COMMAND: "kiblnd_sd_27"
#0 [ffff880c8e447e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880c8e447e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880c8e447ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880c8e447ee0] notify_die at ffffffff810875de
#4 [ffff880c8e447f10] do_nmi at ffffffff81455e1c
#5 [ffff880c8e447f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff880139ba1cb0 RFLAGS: 00000293
RAX: 0000000000003f06 RBX: ffff8809c6d8e400 RCX: ffff88043f99ecc0
RDX: 0000000000003ef5 RSI: ffff8809c6d8e400 RDI: ffffffffa040d340
RBP: ffff880139ba1cb0 R8: ffffc900177cb300 R9: 00049b380693fcf7
R10: 000000000000003c R11: 0000000000000064 R12: 0000000000000000
R13: ffff8809c6d8e400 R14: 0000000000000000 R15: 0000000000000000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880139ba1cb0] _spin_lock at ffffffff8145500e
#7 [ffff880139ba1cb8] lnet_finalize at ffffffffa03f0218
#8 [ffff880139ba1ce8] kiblnd_tx_done at ffffffffa065d482
#9 [ffff880139ba1d68] kiblnd_tx_complete at ffffffffa06615cf
#10 [ffff880139ba1df8] kiblnd_complete at ffffffffa0662f72
#11 [ffff880139ba1e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff880139ba1f48] kernel_thread at ffffffff8100d1aa
PID: 10884 TASK: ffff880456ec6080 CPU: 8 COMMAND: "router_checker"
#0 [ffff880036687e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880036687e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880036687ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880036687ee0] notify_die at ffffffff810875de
#4 [ffff880036687f10] do_nmi at ffffffff81455e1c
#5 [ffff880036687f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff8804614c7e70 RFLAGS: 00000293
RAX: 0000000000003efe RBX: ffff880436278140 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: 0000000000000000 RDI: ffffffffa040d340
RBP: ffff8804614c7e70 R8: ffff8804614c6000 R9: 0000000000000001
R10: 0000000000000000 R11: 00000000ffffffff R12: ffff880436278170
R13: 0000000000000000 R14: 00000001099f882e R15: 0000000000000015
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8804614c7e70] _spin_lock at ffffffff81455011
#7 [ffff8804614c7e78] lnet_router_checker at ffffffffa03fee6c
#8 [ffff8804614c7f48] kernel_thread at ffffffff8100d1aa
PID: 10881 TASK: ffff88047cdd7420 CPU: 9 COMMAND: "kiblnd_sd_30"
#0 [ffff88088e487e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88088e487e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88088e487ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88088e487ee0] notify_die at ffffffff810875de
#4 [ffff88088e487f10] do_nmi at ffffffff81455e1c
#5 [ffff88088e487f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff8801a129dcb0 RFLAGS: 00000293
RAX: 0000000000003f0c RBX: ffff880892a30800 RCX: ffff88043f99ecc0
RDX: 0000000000003ef5 RSI: ffff880892a30800 RDI: ffffffffa040d340
RBP: ffff8801a129dcb0 R8: ffffc900177cbae8 R9: 0000000000000000
R10: 000000000000003b R11: 0000000000000064 R12: 0000000000000000
R13: ffff880892a30800 R14: 0000000000000000 R15: 0000000000000000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8801a129dcb0] _spin_lock at ffffffff8145500e
#7 [ffff8801a129dcb8] lnet_finalize at ffffffffa03f0218
#8 [ffff8801a129dce8] kiblnd_tx_done at ffffffffa065d482
#9 [ffff8801a129dd68] kiblnd_tx_complete at ffffffffa06615cf
#10 [ffff8801a129ddf8] kiblnd_complete at ffffffffa0662f72
#11 [ffff8801a129de38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff8801a129df48] kernel_thread at ffffffff8100d1aa
PID: 10874 TASK: ffff880426806180 CPU: 10 COMMAND: "kiblnd_sd_23"
#0 [ffff88048e487e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88048e487e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88048e487ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88048e487ee0] notify_die at ffffffff810875de
#4 [ffff88048e487f10] do_nmi at ffffffff81455e1c
#5 [ffff88048e487f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff880452cf7b90 RFLAGS: 00000293
RAX: 0000000000003efd RBX: ffff880562fc3400 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff880562fc3400 RDI: ffffffffa040d340
RBP: ffff880452cf7b90 R8: 00000000000000c0 R9: 00000000000000c0
R10: 0000000000000004 R11: ffff880562fc3400 R12: ffff880b468eaac0
R13: 0000000000000000 R14: 00000000000000c0 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880452cf7b90] _spin_lock at ffffffff8145500e
#7 [ffff880452cf7b98] lnet_recv_put at ffffffffa03f3a5c
#8 [ffff880452cf7be8] lnet_parse at ffffffffa03fa23a
#9 [ffff880452cf7ce8] kiblnd_handle_rx at ffffffffa0662253
#10 [ffff880452cf7d78] kiblnd_rx_complete at ffffffffa0662e02
#11 [ffff880452cf7df8] kiblnd_complete at ffffffffa0662fa2
#12 [ffff880452cf7e38] kiblnd_scheduler at ffffffffa066331c
#13 [ffff880452cf7f48] kernel_thread at ffffffff8100d1aa
PID: 10879 TASK: ffff8801957c9260 CPU: 11 COMMAND: "kiblnd_sd_28"
#0 [ffff880c8e487e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880c8e487e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880c8e487ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880c8e487ee0] notify_die at ffffffff810875de
#4 [ffff880c8e487f10] do_nmi at ffffffff81455e1c
#5 [ffff880c8e487f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff880456c03be0 RFLAGS: 00000283
RAX: 0000000000003f10 RBX: ffff880354501030 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: ffff88107cc18340 RDI: ffffffffa040d340
RBP: ffff880456c03be0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: ffff880ccd6b3c00
R13: 0000000000000001 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880456c03be0] _spin_lock at ffffffff8145500e
#7 [ffff880456c03be8] lnet_parse at ffffffffa03f937a
#8 [ffff880456c03ce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff880456c03d78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff880456c03df8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff880456c03e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff880456c03f48] kernel_thread at ffffffff8100d1aa
PID: 10880 TASK: ffff8802b26d74a0 CPU: 12 COMMAND: "kiblnd_sd_29"
#0 [ffff8800366c7e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff8800366c7e90] notifier_call_chain at ffffffff81457d55
#2 [ffff8800366c7ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff8800366c7ee0] notify_die at ffffffff810875de
#4 [ffff8800366c7f10] do_nmi at ffffffff81455e1c
#5 [ffff8800366c7f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff8802d4dbdbe0 RFLAGS: 00000283
RAX: 0000000000003f12 RBX: ffff880deae06030 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: ffff88047e74c1c0 RDI: ffffffffa040d340
RBP: ffff8802d4dbdbe0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: ffff880160aefe00
R13: 0000000000000001 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8802d4dbdbe0] _spin_lock at ffffffff8145500e
#7 [ffff8802d4dbdbe8] lnet_parse at ffffffffa03f937a
#8 [ffff8802d4dbdce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff8802d4dbdd78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff8802d4dbddf8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff8802d4dbde38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff8802d4dbdf48] kernel_thread at ffffffff8100d1aa
PID: 10888 TASK: ffff88016c5e46c0 CPU: 13 COMMAND: "ptlrpcd"
#0 [ffff88088e4c7e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88088e4c7e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88088e4c7ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88088e4c7ee0] notify_die at ffffffff810875de
#4 [ffff88088e4c7f10] do_nmi at ffffffff81455e1c
#5 [ffff88088e4c7f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+24]
RIP: ffffffff81455008 RSP: ffff8804665f1b70 RFLAGS: 00000293
RAX: 0000000000003ef7 RBX: 000000000a317685 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: 0000000000000001 RDI: ffffffffa040d340
RBP: ffff8804665f1b70 R8: 0000000000000000 R9: 0000000000000000
R10: 0000000000000001 R11: 00000000fffffffc R12: ffff880533314864
R13: 0000000000000001 R14: ffff880533314864 R15: ffff880335beebe8
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8804665f1b70] _spin_lock at ffffffff81455008
#7 [ffff8804665f1b78] LNetMDUnlink at ffffffffa03f1c7e
#8 [ffff8804665f1c28] ptlrpc_unregister_reply at ffffffffa055002d
#9 [ffff8804665f1cf8] ptlrpc_check_set at ffffffffa0551f1b
#10 [ffff8804665f1dd8] ptlrpcd_check at ffffffffa0584840
#11 [ffff8804665f1e38] ptlrpcd at ffffffffa0584b57
#12 [ffff8804665f1f48] kernel_thread at ffffffff8100d1aa
PID: 10889 TASK: ffff88016c5e56e0 CPU: 14 COMMAND: "ptlrpcd-rcv"
#0 [ffff88048e4c7e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88048e4c7e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88048e4c7ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88048e4c7ee0] notify_die at ffffffff810875de
#4 [ffff88048e4c7f10] do_nmi at ffffffff81455e1c
#5 [ffff88048e4c7f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff88045512fb80 RFLAGS: 00000287
RAX: 0000000000003f11 RBX: 000000000d86c8c1 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: 0000000000000001 RDI: ffffffffa040d340
RBP: ffff88045512fb80 R8: 00000000fffffffa R9: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff880e689dd064
R13: 0000000000000001 R14: ffff880e689dd064 R15: 0000000000000000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff88045512fb80] _spin_lock at ffffffff8145500e
#7 [ffff88045512fb88] LNetMDUnlink at ffffffffa03f1c7e
#8 [ffff88045512fc38] ptlrpc_unregister_reply at ffffffffa055002d
#9 [ffff88045512fd08] ptlrpc_expire_one_request at ffffffffa0550872
#10 [ffff88045512fdc8] ptlrpc_expired_set at ffffffffa0550e94
#11 [ffff88045512fe38] ptlrpcd at ffffffffa0584c7d
#12 [ffff88045512ff48] kernel_thread at ffffffff8100d1aa
PID: 10868 TASK: ffff880466b64600 CPU: 15 COMMAND: "kiblnd_sd_17"
#0 [ffff880c8e4c7e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880c8e4c7e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880c8e4c7ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880c8e4c7ee0] notify_die at ffffffff810875de
#4 [ffff880c8e4c7f10] do_nmi at ffffffff81455e1c
#5 [ffff880c8e4c7f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff880465249b90 RFLAGS: 00000297
RAX: 0000000000003ef9 RBX: ffff880de2f09e00 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff880de2f09e00 RDI: ffffffffa040d340
RBP: ffff880465249b90 R8: 00000000000000c0 R9: 00000000000000c0
R10: 0000000000000004 R11: ffff880de2f09e00 R12: ffff880b468f7e40
R13: 0000000000000000 R14: 00000000000000c0 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880465249b90] _spin_lock at ffffffff8145500e
#7 [ffff880465249b98] lnet_recv_put at ffffffffa03f3a5c
#8 [ffff880465249be8] lnet_parse at ffffffffa03fa23a
#9 [ffff880465249ce8] kiblnd_handle_rx at ffffffffa0662253
#10 [ffff880465249d78] kiblnd_rx_complete at ffffffffa0662e02
#11 [ffff880465249df8] kiblnd_complete at ffffffffa0662fa2
#12 [ffff880465249e38] kiblnd_scheduler at ffffffffa066331c
#13 [ffff880465249f48] kernel_thread at ffffffff8100d1aa
PID: 10852 TASK: ffff8804638cc950 CPU: 16 COMMAND: "kiblnd_sd_01"
#0 [ffff880036707e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880036707e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880036707ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880036707ee0] notify_die at ffffffff810875de
#4 [ffff880036707f10] do_nmi at ffffffff81455e1c
#5 [ffff880036707f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff880464c59bb0 RFLAGS: 00000297
RAX: 0000000000003f0d RBX: ffff88088f81bc00 RCX: 0000000000000006
RDX: 0000000000003ef5 RSI: ffff880225e701f8 RDI: ffffffffa040d340
RBP: ffff880464c59bb0 R8: ffffc900177ccd98 R9: 00049b7572f54f6c
R10: 0000000000000001 R11: 0000000000000006 R12: ffff88047cd29280
R13: 0000000000000001 R14: ffff880436278500 R15: ffff88088f81bc00
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880464c59bb0] _spin_lock at ffffffff81455011
#7 [ffff880464c59bb8] lnet_post_send_locked at ffffffffa03f6159
#8 [ffff880464c59c18] lnet_return_credits_locked at ffffffffa03f6556
#9 [ffff880464c59c58] lnet_complete_msg_locked at ffffffffa03efe34
#10 [ffff880464c59cb8] lnet_finalize at ffffffffa03f0331
#11 [ffff880464c59ce8] kiblnd_tx_done at ffffffffa065d482
#12 [ffff880464c59d68] kiblnd_tx_complete at ffffffffa06615cf
#13 [ffff880464c59df8] kiblnd_complete at ffffffffa0662f72
#14 [ffff880464c59e38] kiblnd_scheduler at ffffffffa066331c
#15 [ffff880464c59f48] kernel_thread at ffffffff8100d1aa
PID: 10862 TASK: ffff880466a4d620 CPU: 17 COMMAND: "kiblnd_sd_11"
#0 [ffff88088e507e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88088e507e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88088e507ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88088e507ee0] notify_die at ffffffff810875de
#4 [ffff88088e507f10] do_nmi at ffffffff81455e1c
#5 [ffff88088e507f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff880464c79a10 RFLAGS: 00000297
RAX: 0000000000003ef6 RBX: ffff880b46969200 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff880b46969200 RDI: ffffffffa040d340
RBP: ffff880464c79a10 R8: 0000000000001000 R9: 0000000000000000
R10: ffff880b46969200 R11: 0000000000000001 R12: 0000000000000000
R13: 0000000000000001 R14: ffff88047cd29280 R15: ffff880b46964db0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880464c79a10] _spin_lock at ffffffff81455011
#7 [ffff880464c79a18] lnet_finalize at ffffffffa03f0218
#8 [ffff880464c79a48] kiblnd_recv at ffffffffa0661b5a
#9 [ffff880464c79b08] lnet_ni_recv at ffffffffa03f37a8
#10 [ffff880464c79b98] lnet_recv_put at ffffffffa03f3b00
#11 [ffff880464c79be8] lnet_parse at ffffffffa03fa23a
#12 [ffff880464c79ce8] kiblnd_handle_rx at ffffffffa0662253
#13 [ffff880464c79d78] kiblnd_rx_complete at ffffffffa0662e02
#14 [ffff880464c79df8] kiblnd_complete at ffffffffa0662fa2
#15 [ffff880464c79e38] kiblnd_scheduler at ffffffffa066331c
#16 [ffff880464c79f48] kernel_thread at ffffffff8100d1aa
PID: 10859 TASK: ffff8804638cd160 CPU: 18 COMMAND: "kiblnd_sd_08"
#0 [ffff88048e507e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88048e507e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88048e507ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88048e507ee0] notify_die at ffffffff810875de
#4 [ffff88048e507f10] do_nmi at ffffffff81455e1c
#5 [ffff88048e507f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff8803a783bb90 RFLAGS: 00000293
RAX: 0000000000003ef8 RBX: ffff8806c288f400 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff8806c288f400 RDI: ffffffffa040d340
RBP: ffff8803a783bb90 R8: 00000000000000c0 R9: 00000000000000c0
R10: 0000000000000004 R11: ffff8806c288f400 R12: ffff880bc7f83d40
R13: 0000000000000000 R14: 00000000000000c0 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8803a783bb90] _spin_lock at ffffffff81455011
#7 [ffff8803a783bb98] lnet_recv_put at ffffffffa03f3a5c
#8 [ffff8803a783bbe8] lnet_parse at ffffffffa03fa23a
#9 [ffff8803a783bce8] kiblnd_handle_rx at ffffffffa0662253
#10 [ffff8803a783bd78] kiblnd_rx_complete at ffffffffa0662e02
#11 [ffff8803a783bdf8] kiblnd_complete at ffffffffa0662fa2
#12 [ffff8803a783be38] kiblnd_scheduler at ffffffffa066331c
#13 [ffff8803a783bf48] kernel_thread at ffffffff8100d1aa
PID: 10857 TASK: ffff880463952740 CPU: 19 COMMAND: "kiblnd_sd_06"
#0 [ffff880c8e507e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880c8e507e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880c8e507ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880c8e507ee0] notify_die at ffffffff810875de
#4 [ffff880c8e507f10] do_nmi at ffffffff81455e1c
#5 [ffff880c8e507f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff880133d3bcb0 RFLAGS: 00000297
RAX: 0000000000003efc RBX: ffff880b46955200 RCX: ffff88043f99ecc0
RDX: 0000000000003ef5 RSI: ffff880b46955200 RDI: ffffffffa040d340
RBP: ffff880133d3bcb0 R8: ffffc900177cd4c8 R9: 0000000000000000
R10: 000000000000003e R11: 0000000000000064 R12: 0000000000000000
R13: ffff880b46955200 R14: 0000000000000000 R15: 0000000000000000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880133d3bcb0] _spin_lock at ffffffff8145500e
#7 [ffff880133d3bcb8] lnet_finalize at ffffffffa03f0218
#8 [ffff880133d3bce8] kiblnd_tx_done at ffffffffa065d482
#9 [ffff880133d3bd68] kiblnd_tx_complete at ffffffffa06615cf
#10 [ffff880133d3bdf8] kiblnd_complete at ffffffffa0662f72
#11 [ffff880133d3be38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff880133d3bf48] kernel_thread at ffffffff8100d1aa
PID: 10855 TASK: ffff880428772cd0 CPU: 20 COMMAND: "kiblnd_sd_04"
#0 [ffff880036747e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880036747e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880036747ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880036747ee0] notify_die at ffffffff810875de
#4 [ffff880036747f10] do_nmi at ffffffff81455e1c
#5 [ffff880036747f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff8803e10b7be0 RFLAGS: 00000297
RAX: 0000000000003f07 RBX: ffff8801e22f70e0 RCX: 000500080a6420bd
RDX: 0000000000003ef5 RSI: ffff88033ce45790 RDI: ffffffffa040d340
RBP: ffff8803e10b7be0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000001 R11: 0000000000000000 R12: ffff88020c56ca00
R13: 0000000000000004 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8803e10b7be0] _spin_lock at ffffffff8145500e
#7 [ffff8803e10b7be8] lnet_parse at ffffffffa03f9982
#8 [ffff8803e10b7ce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff8803e10b7d78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff8803e10b7df8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff8803e10b7e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff8803e10b7f48] kernel_thread at ffffffff8100d1aa
PID: 10860 TASK: ffff88047d2aaed0 CPU: 21 COMMAND: "kiblnd_sd_09"
#0 [ffff88088e547e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88088e547e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88088e547ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88088e547ee0] notify_die at ffffffff810875de
#4 [ffff88088e547f10] do_nmi at ffffffff81455e1c
#5 [ffff88088e547f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff88047d0e3a10 RFLAGS: 00000297
RAX: 0000000000003eff RBX: ffff880b468da200 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff880b468da200 RDI: ffffffffa040d340
RBP: ffff88047d0e3a10 R8: 0000000000001000 R9: 0000000000000000
R10: ffff880b468da200 R11: 0000000000000001 R12: 0000000000000000
R13: 0000000000000001 R14: ffff88047cd29280 R15: ffff880b468fe230
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff88047d0e3a10] _spin_lock at ffffffff8145500e
#7 [ffff88047d0e3a18] lnet_finalize at ffffffffa03f0218
#8 [ffff88047d0e3a48] kiblnd_recv at ffffffffa0661b5a
#9 [ffff88047d0e3b08] lnet_ni_recv at ffffffffa03f37a8
#10 [ffff88047d0e3b98] lnet_recv_put at ffffffffa03f3b00
#11 [ffff88047d0e3be8] lnet_parse at ffffffffa03fa23a
#12 [ffff88047d0e3ce8] kiblnd_handle_rx at ffffffffa0662253
#13 [ffff88047d0e3d78] kiblnd_rx_complete at ffffffffa0662e02
#14 [ffff88047d0e3df8] kiblnd_complete at ffffffffa0662fa2
#15 [ffff88047d0e3e38] kiblnd_scheduler at ffffffffa066331c
#16 [ffff88047d0e3f48] kernel_thread at ffffffff8100d1aa
PID: 10870 TASK: ffff88044c582500 CPU: 22 COMMAND: "kiblnd_sd_19"
#0 [ffff88048e547e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88048e547e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88048e547ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88048e547ee0] notify_die at ffffffff810875de
#4 [ffff88048e547f10] do_nmi at ffffffff81455e1c
#5 [ffff88048e547f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff8802b876bbe0 RFLAGS: 00000297
RAX: 0000000000003f0e RBX: ffff8801e22f70e0 RCX: 000500080a64206e
RDX: 0000000000003ef5 RSI: ffff88033ce452a0 RDI: ffffffffa040d340
RBP: ffff8802b876bbe0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000001 R11: 0000000000000000 R12: ffff880562fa6000
R13: 0000000000000004 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8802b876bbe0] _spin_lock at ffffffff81455011
#7 [ffff8802b876bbe8] lnet_parse at ffffffffa03f9982
#8 [ffff8802b876bce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff8802b876bd78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff8802b876bdf8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff8802b876be38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff8802b876bf48] kernel_thread at ffffffff8100d1aa
PID: 10871 TASK: ffff880456f535e0 CPU: 23 COMMAND: "kiblnd_sd_20"
#0 [ffff880c8e547e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880c8e547e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880c8e547ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880c8e547ee0] notify_die at ffffffff810875de
#4 [ffff880c8e547f10] do_nmi at ffffffff81455e1c
#5 [ffff880c8e547f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff88042dfa7bb0 RFLAGS: 00000293
RAX: 0000000000003efa RBX: ffff8809c6d95800 RCX: 0000000000000006
RDX: 0000000000003ef5 RSI: ffff880f119e30f8 RDI: ffffffffa040d340
RBP: ffff88042dfa7bb0 R8: ffffc900177cbae8 R9: 00049b3c447e7085
R10: 0000000000000001 R11: 0000000000000007 R12: ffff88047cd29280
R13: 0000000000000001 R14: ffff880436278d40 R15: ffff8809c6d95800
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff88042dfa7bb0] _spin_lock at ffffffff8145500e
#7 [ffff88042dfa7bb8] lnet_post_send_locked at ffffffffa03f6159
#8 [ffff88042dfa7c18] lnet_return_credits_locked at ffffffffa03f6556
#9 [ffff88042dfa7c58] lnet_complete_msg_locked at ffffffffa03efe34
#10 [ffff88042dfa7cb8] lnet_finalize at ffffffffa03f0331
#11 [ffff88042dfa7ce8] kiblnd_tx_done at ffffffffa065d482
#12 [ffff88042dfa7d68] kiblnd_tx_complete at ffffffffa06615cf
#13 [ffff88042dfa7df8] kiblnd_complete at ffffffffa0662f72
#14 [ffff88042dfa7e38] kiblnd_scheduler at ffffffffa066331c
#15 [ffff88042dfa7f48] kernel_thread at ffffffff8100d1aa
PID: 10873 TASK: ffff8801969637a0 CPU: 24 COMMAND: "kiblnd_sd_22"
#0 [ffff880036787e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880036787e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880036787ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880036787ee0] notify_die at ffffffff810875de
#4 [ffff880036787f10] do_nmi at ffffffff81455e1c
#5 [ffff880036787f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff880463933be0 RFLAGS: 00000287
RAX: 0000000000003f13 RBX: ffff88044d67e030 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: ffff88047e7742c0 RDI: ffffffffa040d340
RBP: ffff880463933be0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801fc415e00
R13: 0000000000000001 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880463933be0] _spin_lock at ffffffff81455011
#7 [ffff880463933be8] lnet_parse at ffffffffa03f937a
#8 [ffff880463933ce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff880463933d78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff880463933df8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff880463933e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff880463933f48] kernel_thread at ffffffff8100d1aa
PID: 10876 TASK: ffff880349ef6700 CPU: 25 COMMAND: "kiblnd_sd_25"
#0 [ffff88088e587e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88088e587e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88088e587ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88088e587ee0] notify_die at ffffffff810875de
#4 [ffff88088e587f10] do_nmi at ffffffff81455e1c
#5 [ffff88088e587f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff880466b41be0 RFLAGS: 00000293
RAX: 0000000000003f09 RBX: ffff88013c2b9030 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: ffff88047e77f320 RDI: ffffffffa040d340
RBP: ffff880466b41be0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: ffff880ade9ee200
R13: 0000000000000001 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880466b41be0] _spin_lock at ffffffff8145500e
#7 [ffff880466b41be8] lnet_parse at ffffffffa03f937a
#8 [ffff880466b41ce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff880466b41d78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff880466b41df8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff880466b41e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff880466b41f48] kernel_thread at ffffffff8100d1aa
PID: 10858 TASK: ffff88032aae5420 CPU: 26 COMMAND: "kiblnd_sd_07"
#0 [ffff88048e587e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88048e587e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88048e587ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88048e587ee0] notify_die at ffffffff810875de
#4 [ffff88048e587f10] do_nmi at ffffffff81455e1c
#5 [ffff88048e587f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff880349fb1b90 RFLAGS: 00000293
RAX: 0000000000003f0b RBX: ffff880562fb9400 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff880562fb9400 RDI: ffffffffa040d340
RBP: ffff880349fb1b90 R8: 00000000000000c0 R9: 00000000000000c0
R10: 0000000000000004 R11: ffff880562fb9400 R12: ffff880b46943640
R13: 0000000000000000 R14: 00000000000000c0 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff880349fb1b90] _spin_lock at ffffffff8145500e
#7 [ffff880349fb1b98] lnet_recv_put at ffffffffa03f3a5c
#8 [ffff880349fb1be8] lnet_parse at ffffffffa03fa23a
#9 [ffff880349fb1ce8] kiblnd_handle_rx at ffffffffa0662253
#10 [ffff880349fb1d78] kiblnd_rx_complete at ffffffffa0662e02
#11 [ffff880349fb1df8] kiblnd_complete at ffffffffa0662fa2
#12 [ffff880349fb1e38] kiblnd_scheduler at ffffffffa066331c
#13 [ffff880349fb1f48] kernel_thread at ffffffff8100d1aa
PID: 10863 TASK: ffff88042dfaa3c0 CPU: 27 COMMAND: "kiblnd_sd_12"
#0 [ffff880c8e587e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880c8e587e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880c8e587ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880c8e587ee0] notify_die at ffffffff810875de
#4 [ffff880c8e587f10] do_nmi at ffffffff81455e1c
#5 [ffff880c8e587f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff8804573c7b90 RFLAGS: 00000287
RAX: 0000000000003f14 RBX: ffff880de47fc000 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff880de47fc000 RDI: ffffffffa040d340
RBP: ffff8804573c7b90 R8: 00000000000000c0 R9: 00000000000000c0
R10: 0000000000000004 R11: ffff880de47fc000 R12: ffff880954e25dc0
R13: 0000000000000000 R14: 00000000000000c0 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8804573c7b90] _spin_lock at ffffffff8145500e
#7 [ffff8804573c7b98] lnet_recv_put at ffffffffa03f3a5c
#8 [ffff8804573c7be8] lnet_parse at ffffffffa03fa23a
#9 [ffff8804573c7ce8] kiblnd_handle_rx at ffffffffa0662253
#10 [ffff8804573c7d78] kiblnd_rx_complete at ffffffffa0662e02
#11 [ffff8804573c7df8] kiblnd_complete at ffffffffa0662fa2
#12 [ffff8804573c7e38] kiblnd_scheduler at ffffffffa066331c
#13 [ffff8804573c7f48] kernel_thread at ffffffff8100d1aa
PID: 10867 TASK: ffff8801006fa380 CPU: 28 COMMAND: "kiblnd_sd_16"
#0 [ffff8800367c7e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff8800367c7e90] notifier_call_chain at ffffffff81457d55
#2 [ffff8800367c7ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff8800367c7ee0] notify_die at ffffffff810875de
#4 [ffff8800367c7f10] do_nmi at ffffffff81455e1c
#5 [ffff8800367c7f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+24]
RIP: ffffffff81455008 RSP: ffff88038fbcbbe0 RFLAGS: 00000287
RAX: 0000000000003f00 RBX: ffff88044c24a030 RCX: 0000000000000000
RDX: 0000000000003ef5 RSI: ffff88047e789360 RDI: ffffffffa040d340
RBP: ffff88038fbcbbe0 R8: 0000000000000286 R9: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: ffff8802ff58fe00
R13: 0000000000000001 R14: ffff88047cd29280 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff88038fbcbbe0] _spin_lock at ffffffff81455008
#7 [ffff88038fbcbbe8] lnet_parse at ffffffffa03f937a
#8 [ffff88038fbcbce8] kiblnd_handle_rx at ffffffffa0662253
#9 [ffff88038fbcbd78] kiblnd_rx_complete at ffffffffa0662e02
#10 [ffff88038fbcbdf8] kiblnd_complete at ffffffffa0662fa2
#11 [ffff88038fbcbe38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff88038fbcbf48] kernel_thread at ffffffff8100d1aa
PID: 10872 TASK: ffff880196962f90 CPU: 29 COMMAND: "kiblnd_sd_21"
#0 [ffff88088e5c7e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88088e5c7e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88088e5c7ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88088e5c7ee0] notify_die at ffffffff810875de
#4 [ffff88088e5c7f10] do_nmi at ffffffff81455e1c
#5 [ffff88088e5c7f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff88047d6d9cb0 RFLAGS: 00000283
RAX: 0000000000003f04 RBX: ffff880b469a1e00 RCX: ffff88043f99ecc0
RDX: 0000000000003ef5 RSI: ffff880b469a1e00 RDI: ffffffffa040d340
RBP: ffff88047d6d9cb0 R8: ffffc900177cdb40 R9: 0000000000000000
R10: 000000000000003d R11: 0000000000000064 R12: 0000000000000000
R13: ffff880b469a1e00 R14: 0000000000000000 R15: 0000000000000000
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff88047d6d9cb0] _spin_lock at ffffffff81455011
#7 [ffff88047d6d9cb8] lnet_finalize at ffffffffa03f0218
#8 [ffff88047d6d9ce8] kiblnd_tx_done at ffffffffa065d482
#9 [ffff88047d6d9d68] kiblnd_tx_complete at ffffffffa06615cf
#10 [ffff88047d6d9df8] kiblnd_complete at ffffffffa0662f72
#11 [ffff88047d6d9e38] kiblnd_scheduler at ffffffffa066331c
#12 [ffff88047d6d9f48] kernel_thread at ffffffff8100d1aa
PID: 10856 TASK: ffff8804287734e0 CPU: 30 COMMAND: "kiblnd_sd_05"
#0 [ffff88048e5c7e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff88048e5c7e90] notifier_call_chain at ffffffff81457d55
#2 [ffff88048e5c7ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff88048e5c7ee0] notify_die at ffffffff810875de
#4 [ffff88048e5c7f10] do_nmi at ffffffff81455e1c
#5 [ffff88048e5c7f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+33]
RIP: ffffffff81455011 RSP: ffff8801891c3b90 RFLAGS: 00000287
RAX: 0000000000003f02 RBX: ffff8804a6431600 RCX: 00000000000000c0
RDX: 0000000000003ef5 RSI: ffff8804a6431600 RDI: ffffffffa040d340
RBP: ffff8801891c3b90 R8: 00000000000000c0 R9: 00000000000000c0
R10: 0000000000000004 R11: ffff8804a6431600 R12: ffff880b469573c0
R13: 0000000000000000 R14: 00000000000000c0 R15: 00000000000000c0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff8801891c3b90] _spin_lock at ffffffff81455011
#7 [ffff8801891c3b98] lnet_recv_put at ffffffffa03f3a5c
#8 [ffff8801891c3be8] lnet_parse at ffffffffa03fa23a
#9 [ffff8801891c3ce8] kiblnd_handle_rx at ffffffffa0662253
#10 [ffff8801891c3d78] kiblnd_rx_complete at ffffffffa0662e02
#11 [ffff8801891c3df8] kiblnd_complete at ffffffffa0662fa2
#12 [ffff8801891c3e38] kiblnd_scheduler at ffffffffa066331c
#13 [ffff8801891c3f48] kernel_thread at ffffffff8100d1aa
PID: 10854 TASK: ffff8801b15c2200 CPU: 31 COMMAND: "kiblnd_sd_03"
#0 [ffff880c8e5c7e80] crash_nmi_callback at ffffffff810266d6
#1 [ffff880c8e5c7e90] notifier_call_chain at ffffffff81457d55
#2 [ffff880c8e5c7ed0] atomic_notifier_call_chain at ffffffff81457dba
#3 [ffff880c8e5c7ee0] notify_die at ffffffff810875de
#4 [ffff880c8e5c7f10] do_nmi at ffffffff81455e1c
#5 [ffff880c8e5c7f50] nmi at ffffffff81455710
[exception RIP: _spin_lock+30]
RIP: ffffffff8145500e RSP: ffff88042de2fbb0 RFLAGS: 00000293
RAX: 0000000000003f0f RBX: ffff880b4696aa00 RCX: 0000000000000006
RDX: 0000000000003ef5 RSI: ffff880454c32238 RDI: ffffffffa040d340
RBP: ffff88042de2fbb0 R8: ffffc900177caf68 R9: 00049b3806920f5b
R10: 0000000000000001 R11: 0000000000000007 R12: ffff88047cd29280
R13: 0000000000000001 R14: ffff88087d1eac00 R15: ffff880b4696aa00
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#6 [ffff88042de2fbb0] _spin_lock at ffffffff8145500e
#7 [ffff88042de2fbb8] lnet_post_send_locked at ffffffffa03f6159
#8 [ffff88042de2fc18] lnet_return_credits_locked at ffffffffa03f6556
#9 [ffff88042de2fc58] lnet_complete_msg_locked at ffffffffa03efe34
#10 [ffff88042de2fcb8] lnet_finalize at ffffffffa03f0331
#11 [ffff88042de2fce8] kiblnd_tx_done at ffffffffa065d482
#12 [ffff88042de2fd68] kiblnd_tx_complete at ffffffffa06615cf
#13 [ffff88042de2fdf8] kiblnd_complete at ffffffffa0662f72
#14 [ffff88042de2fe38] kiblnd_scheduler at ffffffffa066331c
#15 [ffff88042de2ff48] kernel_thread at ffffffff8100d1aa
crash>