[131149.750675] lnet_ioctl+0x1d3/0x250 [lnet] [131149.750683] notifier_call_chain+0x47/0x70 [131149.750686] blocking_notifier_call_chain+0x3e/0x60 [131149.750694] libcfs_ioctl+0x24e/0x470 [libcfs] [131149.750703] ? security_capable+0x47/0x60 [131149.750710] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131149.750717] do_vfs_ioctl+0x90/0x5f0 [131149.750721] SyS_ioctl+0x74/0x80 [131149.750727] do_syscall_64+0x74/0x140 [131149.750730] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131149.750732] RIP: 0033:0x7fb52efa9587 [131149.750733] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131149.750735] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131149.750736] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131149.750737] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131149.750738] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131149.750739] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131149.750740] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131169.766927] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131169.766930] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131169.766935] Modules linked in: [131169.767584] Modules linked in: lnet(OEN) [131169.768126] lnet(OEN) libcfs(OEN) [131169.768128] libcfs(OEN) af_packet [131169.768130] af_packet iscsi_ibft [131169.768131] iscsi_ibft [131169.768133] iscsi_boot_sysfs [131169.768133] iscsi_boot_sysfs vmw_vsock_vmci_transport [131169.768135] vmw_vsock_vmci_transport [131169.768136] vsock [131169.768137] vsock sb_edac [131169.768138] sb_edac coretemp [131169.768139] coretemp crc32_pclmul [131169.768141] crc32_pclmul crc32c_intel [131169.768142] crc32c_intel ghash_clmulni_intel [131169.768144] ghash_clmulni_intel pcbc [131169.768145] pcbc aesni_intel [131169.768147] aesni_intel vmwgfx [131169.768148] vmwgfx aes_x86_64 [131169.768149] aes_x86_64 ttm [131169.768150] ttm crypto_simd [131169.768152] crypto_simd drm_kms_helper [131169.768153] drm_kms_helper glue_helper [131169.768155] glue_helper cryptd [131169.768156] cryptd [131169.768158] joydev [131169.768158] joydev drm [131169.768159] drm drm_panel_orientation_quirks [131169.768161] drm_panel_orientation_quirks ppdev [131169.768162] ppdev syscopyarea [131169.768164] syscopyarea vmw_balloon [131169.768165] vmw_balloon vmxnet3 [131169.768167] vmxnet3 sysfillrect [131169.768168] sysfillrect [131169.768170] shpchp [131169.768170] shpchp parport_pc [131169.768171] parport_pc parport [131169.768173] parport sysimgblt [131169.768174] sysimgblt i2c_piix4 [131169.768176] i2c_piix4 pcspkr [131169.768177] pcspkr ac [131169.768179] ac vmw_vmci [131169.768180] vmw_vmci fb_sys_fops [131169.768182] fb_sys_fops button [131169.768183] button ext4 [131169.768185] ext4 crc16 [131169.768186] crc16 jbd2 [131169.768188] jbd2 mbcache [131169.768189] mbcache sr_mod [131169.768191] sr_mod cdrom [131169.768192] cdrom ata_generic [131169.768194] ata_generic sd_mod [131169.768195] sd_mod ata_piix [131169.768197] ata_piix ahci [131169.768198] ahci libahci [131169.768199] libahci serio_raw [131169.768201] serio_raw mptspi [131169.768202] mptspi libata [131169.768203] libata scsi_transport_spi [131169.768205] scsi_transport_spi mptscsih [131169.768206] mptscsih mptbase [131169.768207] mptbase floppy [131169.768208] floppy sg [131169.768210] sg scsi_mod [131169.768211] scsi_mod autofs4 [131169.768213] autofs4 [131169.768215] Supported: No, Unsupported modules are loaded [131169.768216] Supported: No, Unsupported modules are loaded [131169.768222] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131169.768224] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131169.768225] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131169.768226] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131169.768229] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131169.768230] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131169.768280] RIP: 0010:lnet_check_routers+0x6f/0x5e0 [lnet] [131169.768290] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131169.768292] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131169.768293] RSP: 0018:ffffabaa436ebe20 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff10 [131169.768295] ORIG_RAX: ffffffffffffff10 [131169.768296] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131169.768297] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131169.768298] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131169.768299] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131169.768301] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131169.768302] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131169.768304] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131169.768304] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131169.768306] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131169.768306] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131169.768309] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131169.768310] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131169.768312] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131169.768312] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131169.768314] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131169.768315] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131169.768323] Call Trace: [131169.768323] Call Trace: [131169.768341] _raw_spin_lock+0x1d/0x20 [131169.768353] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131169.768358] ? __switch_to_asm+0x34/0x70 [131169.768373] lnet_peer_discovery+0xee/0x1160 [lnet] [131169.768381] ? wait_woken+0x80/0x80 [131169.768386] ? __schedule+0x247/0x860 [131169.768396] ? lnet_send_ping+0x190/0x190 [lnet] [131169.768404] kthread+0x11a/0x130 [131169.768406] ? wait_woken+0x80/0x80 [131169.768418] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131169.768419] ? kthread_create_on_node+0x40/0x40 [131169.768422] kthread+0x11a/0x130 [131169.768424] ret_from_fork+0x35/0x40 [131169.768426] ? kthread_create_on_node+0x40/0x40 [131169.768427] Code: [131169.768429] ret_from_fork+0x35/0x40 [131169.768430] 8b 2d a6 ec 02 [131169.768432] Code: 66 [131169.768433] 00 [131169.768434] 2e [131169.768435] 49 [131169.768436] 0f [131169.768436] 81 [131169.768437] 1f [131169.768438] fd [131169.768438] 84 [131169.768439] f0 [131169.768440] 00 [131169.768440] 90 [131169.768441] 00 [131169.768442] 7a [131169.768443] 00 [131169.768443] c0 [131169.768444] 00 [131169.768445] 0f [131169.768445] 00 [131169.768446] 84 [131169.768447] 66 [131169.768447] 34 [131169.768448] 66 [131169.768449] 01 [131169.768450] 66 [131169.768450] 00 [131169.768451] 66 [131169.768451] 00 [131169.768452] 90 [131169.768453] 41 [131169.768454] 66 [131169.768454] bc [131169.768455] 66 [131169.768456] 01 [131169.768457] 66 [131169.768457] 00 00 [131169.768458] 66 90 [131169.768460] 00 e8 [131169.768461] ba 01 [131169.768462] 1e [131169.768463] 00 [131169.768464] 99 98 [131169.768465] 00 [131169.768466] c8 [131169.768467] 00 [131169.768467] 49 8d [131169.768469] 8b 07 [131169.768470] 6d [131169.768471] 85 [131169.768471] d0 49 [131169.768472] c0 75 [131169.768474] 89 c6 [131169.768475] 0a f0 [131169.768476] 41 [131169.768477] 0f [131169.768477] 8b [131169.768478] b1 [131169.768479] 75 [131169.768480] 17 [131169.768480] 18 eb [131169.768481] 85 c0 [131169.768482] 27 <48> [131169.768483] 75 f2 [131169.768484] 39 [131169.768485] f3 [131169.768486] c3 [131169.768487] c3 [131169.768487] 0f [131169.768488] f3 [131169.768489] 84 [131169.768490] 90 [131169.768490] 3f [131169.768491] [131169.768492] 01 [131169.768493] ec [131169.768493] 00 00 [131169.768494] 81 fe [131169.768495] 8b 78 [131169.768497] 00 01 [131169.768498] 28 [131169.768499] 00 [131169.768499] 48 [131169.768500] 00 [131169.768500] 85 [131169.768501] 0f [131169.768502] db 48 [131169.768503] 84 91 [131169.768504] 0f 44 [131169.768505] 00 00 [131169.768507] d8 [131169.768508] 00 [131169.768508] 45 31 [131169.768509] 41 [131169.768510] b8 01 01 00 00 b9 [131177.773868] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131177.774716] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131177.774776] Supported: No, Unsupported modules are loaded [131177.774782] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131177.774783] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131177.774787] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131177.774801] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131177.774803] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131177.774805] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131177.774806] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131177.774807] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131177.774808] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131177.774810] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131177.774813] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131177.774814] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131177.774815] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131177.774824] Call Trace: [131177.774841] _raw_spin_lock+0x1d/0x20 [131177.774861] cfs_percpt_lock+0x69/0xf0 [libcfs] [131177.774892] lnet_add_route+0x172/0xcb0 [lnet] [131177.774904] LNetCtl+0x6cf/0x11a0 [lnet] [131177.774914] ? LNetNIInit+0x66/0xb70 [lnet] [131177.774925] lnet_ioctl+0x1d3/0x250 [lnet] [131177.774932] notifier_call_chain+0x47/0x70 [131177.774935] blocking_notifier_call_chain+0x3e/0x60 [131177.774943] libcfs_ioctl+0x24e/0x470 [libcfs] [131177.774950] ? security_capable+0x47/0x60 [131177.774956] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131177.774961] do_vfs_ioctl+0x90/0x5f0 [131177.774965] SyS_ioctl+0x74/0x80 [131177.774973] do_syscall_64+0x74/0x140 [131177.774976] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131177.774978] RIP: 0033:0x7fb52efa9587 [131177.774979] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131177.774981] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131177.774982] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131177.774982] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131177.774983] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131177.774984] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131177.774986] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131197.791225] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131197.791228] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131197.791233] Modules linked in: [131197.791958] Modules linked in: [131197.792487] lnet(OEN) [131197.792489] lnet(OEN) [131197.792490] libcfs(OEN) [131197.792491] libcfs(OEN) af_packet [131197.792492] af_packet iscsi_ibft [131197.792494] iscsi_ibft iscsi_boot_sysfs [131197.792496] iscsi_boot_sysfs vmw_vsock_vmci_transport [131197.792498] vmw_vsock_vmci_transport vsock [131197.792499] vsock sb_edac [131197.792501] sb_edac coretemp [131197.792502] coretemp crc32_pclmul [131197.792504] crc32_pclmul crc32c_intel [131197.792505] crc32c_intel [131197.792506] ghash_clmulni_intel [131197.792507] ghash_clmulni_intel pcbc [131197.792508] pcbc aesni_intel [131197.792510] aesni_intel vmwgfx [131197.792511] vmwgfx aes_x86_64 [131197.792512] aes_x86_64 ttm [131197.792514] ttm crypto_simd [131197.792515] crypto_simd drm_kms_helper [131197.792517] drm_kms_helper glue_helper [131197.792518] glue_helper cryptd [131197.792519] cryptd joydev [131197.792521] joydev drm [131197.792522] drm drm_panel_orientation_quirks [131197.792524] drm_panel_orientation_quirks ppdev [131197.792525] ppdev syscopyarea [131197.792526] syscopyarea vmw_balloon [131197.792528] vmw_balloon vmxnet3 [131197.792529] vmxnet3 sysfillrect [131197.792531] sysfillrect shpchp [131197.792532] shpchp parport_pc [131197.792533] parport_pc parport [131197.792535] parport sysimgblt [131197.792536] sysimgblt i2c_piix4 [131197.792538] i2c_piix4 pcspkr [131197.792539] pcspkr ac [131197.792540] ac vmw_vmci [131197.792542] vmw_vmci fb_sys_fops [131197.792543] fb_sys_fops button [131197.792545] button ext4 [131197.792546] ext4 crc16 [131197.792548] crc16 jbd2 [131197.792549] jbd2 mbcache [131197.792550] mbcache sr_mod [131197.792552] sr_mod cdrom [131197.792553] cdrom ata_generic [131197.792555] ata_generic sd_mod [131197.792556] sd_mod ata_piix [131197.792557] ata_piix ahci [131197.792559] ahci libahci [131197.792560] libahci serio_raw [131197.792562] serio_raw mptspi [131197.792563] mptspi libata [131197.792564] libata scsi_transport_spi [131197.792565] scsi_transport_spi mptscsih [131197.792567] mptscsih mptbase [131197.792568] mptbase floppy [131197.792569] floppy [131197.792571] sg [131197.792571] sg [131197.792572] scsi_mod [131197.792573] scsi_mod [131197.792573] autofs4 [131197.792574] autofs4 [131197.792576] Supported: No, Unsupported modules are loaded [131197.792577] Supported: No, Unsupported modules are loaded [131197.792583] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131197.792586] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131197.792587] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131197.792588] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131197.792591] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131197.792592] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131197.792642] RIP: 0010:lnet_check_routers+0x7b/0x5e0 [lnet] [131197.792653] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131197.792654] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131197.792656] RSP: 0018:ffffabaa436ebe20 EFLAGS: 00000287 [131197.792658] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131197.792659] ORIG_RAX: ffffffffffffff10 [131197.792660] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131197.792661] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131197.792662] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131197.792663] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131197.792665] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131197.792666] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131197.792667] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131197.792668] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131197.792669] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131197.792671] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131197.792672] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131197.792673] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131197.792674] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131197.792675] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131197.792676] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131197.792685] Call Trace: [131197.792685] Call Trace: [131197.792702] _raw_spin_lock+0x1d/0x20 [131197.792714] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131197.792719] ? __switch_to_asm+0x34/0x70 [131197.792734] lnet_peer_discovery+0xee/0x1160 [lnet] [131197.792742] ? wait_woken+0x80/0x80 [131197.792747] ? __schedule+0x247/0x860 [131197.792749] ? wait_woken+0x80/0x80 [131197.792759] ? lnet_send_ping+0x190/0x190 [lnet] [131197.792767] kthread+0x11a/0x130 [131197.792779] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131197.792781] ? kthread_create_on_node+0x40/0x40 [131197.792784] kthread+0x11a/0x130 [131197.792785] ret_from_fork+0x35/0x40 [131197.792788] ? kthread_create_on_node+0x40/0x40 [131197.792788] Code: [131197.792790] ret_from_fork+0x35/0x40 [131197.792791] c0 0f 84 [131197.792793] Code: 66 [131197.792794] 34 [131197.792795] 2e [131197.792795] 01 [131197.792796] 0f [131197.792797] 00 [131197.792798] 1f [131197.792798] 00 [131197.792799] 84 [131197.792800] 41 bc [131197.792801] 00 00 [131197.792802] 01 [131197.792803] 00 [131197.792803] 00 [131197.792804] 00 [131197.792804] 00 [131197.792805] 00 [131197.792806] 00 e8 [131197.792807] 66 [131197.792808] 1e [131197.792808] 66 66 [131197.792810] 99 98 [131197.792811] 66 90 [131197.792812] c8 [131197.792813] 66 [131197.792813] 49 [131197.792814] 66 [131197.792815] 8d [131197.792816] 66 [131197.792816] 6d [131197.792817] 66 [131197.792818] d0 [131197.792818] 90 [131197.792819] 49 [131197.792820] ba [131197.792820] 89 c6 [131197.792821] 01 00 [131197.792823] 41 [131197.792823] 00 [131197.792824] 8b 75 [131197.792825] 00 8b [131197.792826] 18 eb [131197.792827] 07 85 [131197.792828] 27 [131197.792829] c0 [131197.792830] 48 [131197.792831] 75 [131197.792831] 39 [131197.792832] 0a [131197.792833] c3 [131197.792834] f0 [131197.792834] 0f 84 [131197.792835] 0f b1 [131197.792836] 3f [131197.792837] 17 [131197.792838] 01 00 [131197.792839] 85 c0 [131197.792840] 00 [131197.792841] 75 [131197.792842] 8b 78 [131197.792843] f2 f3 [131197.792844] 28 [131197.792845] c3 [131197.792845] <48> [131197.792846] f3 [131197.792847] 85 [131197.792848] 90 [131197.792848] db 48 [131197.792849] ec [131197.792851] 0f [131197.792851] 81 [131197.792852] 44 [131197.792853] fe [131197.792853] d8 [131197.792854] 00 [131197.792855] 45 31 [131197.792856] 01 00 [131197.792857] e4 e8 [131197.792858] 00 0f [131197.792859] 46 [131197.792860] 84 [131197.792861] b3 [131197.792862] 91 [131197.792862] fd [131197.792863] 00 [131197.792864] ff [131197.792864] 00 [131197.792865] 85 [131197.792873] 00 [131197.792873] c0 41 [131197.792875] 41 b8 [131197.792876] 8b [131197.792877] 01 [131197.792877] 77 [131197.792878] 01 [131197.792878] 28 [131197.792879] 00 [131197.792880] 00 b9 [131205.796759] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131205.797605] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131205.797666] Supported: No, Unsupported modules are loaded [131205.797673] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131205.797674] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131205.797677] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131205.797691] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131205.797692] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131205.797695] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131205.797696] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131205.797698] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131205.797698] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131205.797700] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131205.797703] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131205.797704] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131205.797705] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131205.797714] Call Trace: [131205.797732] _raw_spin_lock+0x1d/0x20 [131205.797751] cfs_percpt_lock+0x69/0xf0 [libcfs] [131205.797779] lnet_add_route+0x172/0xcb0 [lnet] [131205.797792] LNetCtl+0x6cf/0x11a0 [lnet] [131205.797802] ? LNetNIInit+0x66/0xb70 [lnet] [131205.797813] lnet_ioctl+0x1d3/0x250 [lnet] [131205.797823] notifier_call_chain+0x47/0x70 [131205.797825] blocking_notifier_call_chain+0x3e/0x60 [131205.797833] libcfs_ioctl+0x24e/0x470 [libcfs] [131205.797840] ? security_capable+0x47/0x60 [131205.797847] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131205.797854] do_vfs_ioctl+0x90/0x5f0 [131205.797858] SyS_ioctl+0x74/0x80 [131205.797866] do_syscall_64+0x74/0x140 [131205.797869] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131205.797872] RIP: 0033:0x7fb52efa9587 [131205.797872] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131205.797874] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131205.797875] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131205.797876] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131205.797877] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131205.797878] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131205.797879] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131225.797249] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131225.797252] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131225.797258] Modules linked in: [131225.797847] Modules linked in: lnet(OEN) [131225.798310] lnet(OEN) libcfs(OEN) [131225.798312] libcfs(OEN) af_packet [131225.798314] af_packet iscsi_ibft [131225.798316] iscsi_ibft iscsi_boot_sysfs [131225.798318] iscsi_boot_sysfs vmw_vsock_vmci_transport [131225.798320] vmw_vsock_vmci_transport vsock [131225.798321] vsock sb_edac [131225.798322] sb_edac coretemp [131225.798324] coretemp crc32_pclmul [131225.798325] crc32_pclmul crc32c_intel [131225.798327] crc32c_intel ghash_clmulni_intel [131225.798329] ghash_clmulni_intel pcbc [131225.798330] pcbc aesni_intel [131225.798331] aesni_intel vmwgfx [131225.798333] vmwgfx aes_x86_64 [131225.798334] aes_x86_64 ttm [131225.798336] ttm crypto_simd [131225.798337] crypto_simd drm_kms_helper [131225.798339] drm_kms_helper glue_helper [131225.798340] glue_helper cryptd [131225.798342] cryptd joydev [131225.798343] joydev drm [131225.798344] drm drm_panel_orientation_quirks [131225.798346] drm_panel_orientation_quirks ppdev [131225.798347] ppdev syscopyarea [131225.798349] syscopyarea vmw_balloon [131225.798351] vmw_balloon vmxnet3 [131225.798352] vmxnet3 sysfillrect [131225.798354] sysfillrect shpchp [131225.798355] shpchp parport_pc [131225.798357] parport_pc parport [131225.798358] parport sysimgblt [131225.798360] sysimgblt i2c_piix4 [131225.798361] i2c_piix4 pcspkr [131225.798363] pcspkr ac [131225.798364] ac vmw_vmci [131225.798365] vmw_vmci fb_sys_fops [131225.798367] fb_sys_fops button [131225.798369] button ext4 [131225.798370] ext4 crc16 [131225.798372] crc16 jbd2 [131225.798373] jbd2 mbcache [131225.798375] mbcache sr_mod [131225.798376] sr_mod cdrom [131225.798378] cdrom ata_generic [131225.798379] ata_generic sd_mod [131225.798381] sd_mod ata_piix [131225.798383] ata_piix ahci [131225.798384] ahci libahci [131225.798386] libahci serio_raw [131225.798387] serio_raw mptspi [131225.798389] mptspi libata [131225.798390] libata scsi_transport_spi [131225.798392] scsi_transport_spi mptscsih [131225.798393] mptscsih mptbase [131225.798394] mptbase floppy [131225.798396] floppy sg [131225.798397] sg scsi_mod [131225.798399] scsi_mod autofs4 [131225.798400] autofs4 [131225.798403] Supported: No, Unsupported modules are loaded [131225.798404] Supported: No, Unsupported modules are loaded [131225.798410] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131225.798412] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131225.798413] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131225.798414] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131225.798418] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131225.798419] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131225.798468] RIP: 0010:lnet_check_routers+0x7b/0x5e0 [lnet] [131225.798478] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131225.798479] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131225.798482] RSP: 0018:ffffabaa436ebe20 EFLAGS: 00000287 [131225.798483] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131225.798484] ORIG_RAX: ffffffffffffff10 [131225.798485] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131225.798486] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131225.798488] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131225.798489] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131225.798490] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131225.798491] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131225.798492] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131225.798493] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131225.798495] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131225.798496] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131225.798498] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131225.798499] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131225.798500] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131225.798502] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131225.798503] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131225.798510] Call Trace: [131225.798511] Call Trace: [131225.798528] _raw_spin_lock+0x1d/0x20 [131225.798540] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131225.798546] ? __switch_to_asm+0x34/0x70 [131225.798561] lnet_peer_discovery+0xee/0x1160 [lnet] [131225.798565] ? wait_woken+0x80/0x80 [131225.798571] ? __schedule+0x247/0x860 [131225.798573] ? wait_woken+0x80/0x80 [131225.798583] ? lnet_send_ping+0x190/0x190 [lnet] [131225.798590] kthread+0x11a/0x130 [131225.798601] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131225.798603] kthread+0x11a/0x130 [131225.798607] ? kthread_create_on_node+0x40/0x40 [131225.798608] ? kthread_create_on_node+0x40/0x40 [131225.798611] ret_from_fork+0x35/0x40 [131225.798612] ret_from_fork+0x35/0x40 [131225.798614] Code: [131225.798615] Code: [131225.798617] 66 [131225.798617] c0 [131225.798619] 2e [131225.798619] 0f 84 [131225.798620] 0f [131225.798621] 34 [131225.798622] 1f [131225.798623] 01 [131225.798624] 84 00 00 [131225.798626] 00 [131225.798627] 00 [131225.798627] 00 [131225.798628] 00 [131225.798629] 41 [131225.798629] 00 [131225.798630] bc [131225.798631] 66 [131225.798631] 01 [131225.798632] 66 [131225.798633] 00 [131225.798634] 66 [131225.798634] 00 [131225.798635] 66 [131225.798636] 00 [131225.798637] 90 [131225.798637] e8 [131225.798638] 66 [131225.798639] 1e [131225.798640] 66 [131225.798640] 99 [131225.798641] 66 [131225.798641] 98 [131225.798642] 66 [131225.798643] c8 [131225.798644] 90 [131225.798644] 49 [131225.798645] ba [131225.798646] 8d 6d d0 [131225.798648] 01 [131225.798649] 49 [131225.798649] 00 [131225.798650] 89 [131225.798651] 00 [131225.798652] c6 [131225.798652] 00 [131225.798653] 41 [131225.798654] 8b [131225.798655] 8b [131225.798655] 07 85 c0 [131225.798657] 75 [131225.798658] 75 [131225.798658] 18 [131225.798659] 0a [131225.798660] eb [131225.798661] f0 [131225.798661] 27 [131225.798662] 0f [131225.798663] 48 [131225.798664] b1 [131225.798664] 39 [131225.798665] 17 [131225.798666] c3 [131225.798667] 85 [131225.798667] 0f 84 3f [131225.798669] c0 [131225.798670] 01 [131225.798671] 75 [131225.798672] 00 [131225.798672] f2 [131225.798673] 00 [131225.798673] f3 [131225.798674] 8b [131225.798675] c3 [131225.798676] 78 [131225.798676] f3 [131225.798677] 28 [131225.798678] 90 [131225.798679] <48> [131225.798680] [131225.798681] 85 [131225.798681] ec 81 fe [131225.798683] db [131225.798684] 00 [131225.798685] 48 [131225.798685] 01 [131225.798686] 0f [131225.798687] 00 [131225.798687] 44 [131225.798688] 00 [131225.798689] d8 [131225.798690] 0f [131225.798690] 45 [131225.798691] 84 [131225.798692] 31 [131225.798693] 91 [131225.798693] e4 [131225.798694] 00 [131225.798695] e8 [131225.798696] 00 [131225.798696] 46 [131225.798697] 00 [131225.798698] b3 [131225.798699] 41 b8 [131225.798700] fd [131225.798701] 01 [131225.798701] ff 85 c0 [131225.798703] 01 [131225.798704] 41 [131225.798705] 00 00 [131225.798707] 8b [131225.798707] b9 [131225.798708] 77 [131225.798709] 28 [131233.797793] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131233.798569] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131233.798629] Supported: No, Unsupported modules are loaded [131233.798635] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131233.798636] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131233.798639] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131233.798654] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131233.798656] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131233.798658] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131233.798659] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131233.798660] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131233.798661] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131233.798662] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131233.798664] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131233.798665] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131233.798666] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131233.798675] Call Trace: [131233.798692] _raw_spin_lock+0x1d/0x20 [131233.798712] cfs_percpt_lock+0x69/0xf0 [libcfs] [131233.798742] lnet_add_route+0x172/0xcb0 [lnet] [131233.798754] LNetCtl+0x6cf/0x11a0 [lnet] [131233.798764] ? LNetNIInit+0x66/0xb70 [lnet] [131233.798775] lnet_ioctl+0x1d3/0x250 [lnet] [131233.798782] notifier_call_chain+0x47/0x70 [131233.798785] blocking_notifier_call_chain+0x3e/0x60 [131233.798793] libcfs_ioctl+0x24e/0x470 [libcfs] [131233.798799] ? security_capable+0x47/0x60 [131233.798806] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131233.798811] do_vfs_ioctl+0x90/0x5f0 [131233.798815] SyS_ioctl+0x74/0x80 [131233.798823] do_syscall_64+0x74/0x140 [131233.798826] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131233.798828] RIP: 0033:0x7fb52efa9587 [131233.798829] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131233.798830] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131233.798831] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131233.798832] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131233.798833] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131233.798834] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131233.798835] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131253.799160] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131253.799163] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131253.799168] Modules linked in: [131253.799750] Modules linked in: [131253.800214] lnet(OEN) [131253.800215] lnet(OEN) libcfs(OEN) [131253.800217] libcfs(OEN) af_packet [131253.800219] af_packet iscsi_ibft [131253.800221] iscsi_ibft [131253.800222] iscsi_boot_sysfs [131253.800223] iscsi_boot_sysfs vmw_vsock_vmci_transport [131253.800225] vmw_vsock_vmci_transport [131253.800226] vsock [131253.800227] vsock sb_edac [131253.800228] sb_edac coretemp [131253.800230] coretemp crc32_pclmul [131253.800231] crc32_pclmul crc32c_intel [131253.800233] crc32c_intel ghash_clmulni_intel [131253.800234] ghash_clmulni_intel pcbc [131253.800236] pcbc aesni_intel [131253.800237] aesni_intel vmwgfx [131253.800238] vmwgfx aes_x86_64 [131253.800239] aes_x86_64 ttm [131253.800241] ttm crypto_simd [131253.800242] crypto_simd drm_kms_helper [131253.800244] drm_kms_helper glue_helper [131253.800245] glue_helper cryptd [131253.800247] cryptd joydev [131253.800248] joydev drm [131253.800249] drm drm_panel_orientation_quirks [131253.800251] drm_panel_orientation_quirks ppdev [131253.800252] ppdev syscopyarea [131253.800254] syscopyarea vmw_balloon [131253.800255] vmw_balloon vmxnet3 [131253.800257] vmxnet3 sysfillrect [131253.800258] sysfillrect shpchp [131253.800260] shpchp parport_pc [131253.800261] parport_pc parport [131253.800263] parport sysimgblt [131253.800264] sysimgblt i2c_piix4 [131253.800265] i2c_piix4 pcspkr [131253.800267] pcspkr ac [131253.800268] ac vmw_vmci [131253.800269] vmw_vmci fb_sys_fops [131253.800271] fb_sys_fops button [131253.800273] button ext4 [131253.800274] ext4 crc16 [131253.800275] crc16 jbd2 [131253.800277] jbd2 mbcache [131253.800278] mbcache sr_mod [131253.800280] sr_mod cdrom [131253.800281] cdrom ata_generic [131253.800283] ata_generic sd_mod [131253.800284] sd_mod [131253.800285] ata_piix [131253.800286] ata_piix ahci [131253.800287] ahci libahci [131253.800288] libahci serio_raw [131253.800290] serio_raw mptspi [131253.800291] mptspi libata [131253.800293] libata scsi_transport_spi [131253.800294] scsi_transport_spi mptscsih [131253.800295] mptscsih mptbase [131253.800296] mptbase floppy [131253.800298] floppy sg [131253.800299] sg [131253.800300] scsi_mod [131253.800301] scsi_mod autofs4 [131253.800302] autofs4 [131253.800305] Supported: No, Unsupported modules are loaded [131253.800306] Supported: No, Unsupported modules are loaded [131253.800313] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131253.800315] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131253.800317] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131253.800318] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131253.800322] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131253.800323] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131253.800338] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131253.800339] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131253.800385] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131253.800387] ORIG_RAX: ffffffffffffff10 [131253.800388] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131253.800389] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131253.800391] ORIG_RAX: ffffffffffffff10 [131253.800391] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131253.800393] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131253.800394] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131253.800396] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131253.800397] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131253.800398] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131253.800399] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131253.800400] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131253.800401] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131253.800402] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131253.800404] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131253.800405] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131253.800406] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131253.800408] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131253.800409] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131253.800416] Call Trace: [131253.800417] Call Trace: [131253.800434] _raw_spin_lock+0x1d/0x20 [131253.800445] lnet_check_routers+0x9e/0x5e0 [lnet] [131253.800463] lnet_peer_discovery+0xee/0x1160 [lnet] [131253.800474] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131253.800479] ? __switch_to_asm+0x34/0x70 [131253.800484] ? __schedule+0x247/0x860 [131253.800489] ? wait_woken+0x80/0x80 [131253.800491] ? wait_woken+0x80/0x80 [131253.800501] ? lnet_send_ping+0x190/0x190 [lnet] [131253.800512] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131253.800520] kthread+0x11a/0x130 [131253.800522] kthread+0x11a/0x130 [131253.800525] ? kthread_create_on_node+0x40/0x40 [131253.800527] ? kthread_create_on_node+0x40/0x40 [131253.800529] ret_from_fork+0x35/0x40 [131253.800531] ret_from_fork+0x35/0x40 [131253.800533] Code: [131253.800534] Code: 66 [131253.800536] ef 31 c0 [131253.800538] 2e [131253.800539] eb [131253.800540] 0f [131253.800541] cc [131253.800541] 1f [131253.800542] f3 [131253.800543] 84 [131253.800544] c3 [131253.800544] 00 [131253.800545] 66 [131253.800546] 00 [131253.800547] 0f [131253.800547] 00 [131253.800548] 1f [131253.800549] 00 [131253.800550] 84 [131253.800550] 00 [131253.800551] 00 [131253.800552] 66 [131253.800552] 00 [131253.800553] 66 [131253.800554] 00 [131253.800554] 66 [131253.800555] 00 [131253.800556] 66 [131253.800557] 00 [131253.800557] 90 [131253.800558] 66 [131253.800559] 66 [131253.800560] 66 [131253.800560] 66 [131253.800561] 66 [131253.800562] 66 [131253.800563] 66 [131253.800563] 66 [131253.800564] 90 [131253.800564] 90 ba [131253.800566] 85 f6 [131253.800567] 01 [131253.800568] 48 [131253.800569] 00 [131253.800569] 8d [131253.800570] 00 00 [131253.800571] 57 10 [131253.800572] 8b [131253.800573] 48 [131253.800574] 07 [131253.800575] 8b [131253.800576] 85 [131253.800576] 47 [131253.800577] c0 [131253.800578] 10 [131253.800578] 75 [131253.800579] 74 [131253.800580] 0a [131253.800581] 29 [131253.800582] f0 [131253.800582] 48 [131253.800583] 0f [131253.800584] 39 [131253.800584] b1 [131253.800585] c2 [131253.800586] 17 [131253.800587] 74 [131253.800587] 85 [131253.800588] 14 [131253.800589] c0 [131253.800590] 3b [131253.800590] 75 [131253.800591] 70 [131253.800592] f2 [131253.800593] 28 [131253.800593] f3 [131253.800594] 75 [131253.800595] c3 [131253.800596] 07 [131253.800596] f3 [131253.800597] [131253.800598] 90 [131253.800599] 11 [131253.800599] [131253.800600] 39 [131253.800601] ec 81 [131253.800602] 70 [131253.800603] fe [131253.800604] 28 74 [131253.800605] 00 [131253.800606] 0c [131253.800606] 01 [131253.800607] 48 [131253.800608] 00 [131253.800609] 8b [131253.800609] 00 [131253.800610] 00 [131253.800611] 0f [131253.800612] 48 [131253.800612] 84 [131253.800613] 39 [131253.800614] 91 00 [131253.800615] c2 [131253.800616] 00 [131253.800616] 75 [131253.800617] 00 [131253.800618] f3 31 [131253.800619] 41 [131253.800620] c0 [131253.800620] b8 [131253.800621] f3 [131253.800622] 01 [131253.800623] c3 [131253.800623] 01 [131253.800624] 48 [131253.800625] 00 [131253.800626] 8b [131253.800626] 00 [131253.800628] b9 [131261.799703] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [lnetctl:15141] [131261.800502] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131261.800575] Supported: No, Unsupported modules are loaded [131261.800581] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131261.800582] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131261.800586] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131261.800601] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131261.800602] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131261.800604] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131261.800605] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131261.800606] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131261.800607] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131261.800608] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131261.800610] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131261.800611] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131261.800613] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131261.800621] Call Trace: [131261.800639] _raw_spin_lock+0x1d/0x20 [131261.800659] cfs_percpt_lock+0x69/0xf0 [libcfs] [131261.800688] lnet_add_route+0x172/0xcb0 [lnet] [131261.800701] LNetCtl+0x6cf/0x11a0 [lnet] [131261.800710] ? LNetNIInit+0x66/0xb70 [lnet] [131261.800721] lnet_ioctl+0x1d3/0x250 [lnet] [131261.800731] notifier_call_chain+0x47/0x70 [131261.800734] blocking_notifier_call_chain+0x3e/0x60 [131261.800741] libcfs_ioctl+0x24e/0x470 [libcfs] [131261.800750] ? security_capable+0x47/0x60 [131261.800757] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131261.800764] do_vfs_ioctl+0x90/0x5f0 [131261.800768] SyS_ioctl+0x74/0x80 [131261.800776] do_syscall_64+0x74/0x140 [131261.800779] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131261.800781] RIP: 0033:0x7fb52efa9587 [131261.800782] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131261.800784] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131261.800785] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131261.800786] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131261.800787] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131261.800788] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131261.800789] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131269.716233] INFO: rcu_sched self-detected stall on CPU [131269.716238] INFO: rcu_sched self-detected stall on CPU [131269.717276] 5-...: (240013 ticks this GP) idle=4fa/140000000000001/0 softirq=3607039/3607039 fqs=120000 [131269.717280] 4-...: (240014 ticks this GP) idle=62a/140000000000001/0 softirq=126578/126578 fqs=120000 [131269.717281] [131269.718180] [131269.719165] (t=240016 jiffies g=1359077 c=1359076 q=47290) [131269.719654] (t=240016 jiffies g=1359077 c=1359076 q=47290) [131269.720148] NMI backtrace for cpu 4 [131269.720155] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131269.720157] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131269.720158] Call Trace: [131269.720167] [131269.720189] dump_stack+0x5c/0x82 [131269.720196] nmi_cpu_backtrace+0x89/0x90 [131269.720210] ? irq_force_complete_move+0x140/0x140 [131269.720212] nmi_trigger_cpumask_backtrace+0xd1/0x100 [131269.720221] rcu_dump_cpu_stacks+0x9a/0xd2 [131269.720227] rcu_check_callbacks+0x714/0x980 [131269.720234] ? tick_sched_do_timer+0x60/0x60 [131269.720242] update_process_times+0x28/0x50 [131269.720245] tick_sched_handle.isra.19+0x1f/0x60 [131269.720248] ? tick_sched_do_timer+0x60/0x60 [131269.720249] tick_sched_timer+0x38/0x70 [131269.720252] __hrtimer_run_queues+0xce/0x210 [131269.720254] hrtimer_interrupt+0xa6/0x1f0 [131269.720268] smp_apic_timer_interrupt+0x3a/0x60 [131269.720272] apic_timer_interrupt+0x87/0x90 [131269.720274] [131269.720281] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131269.720283] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131269.720285] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131269.720286] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131269.720287] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131269.720288] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131269.720289] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131269.720339] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131269.720344] _raw_spin_lock+0x1d/0x20 [131269.720356] lnet_peer_discovery+0xee/0x1160 [lnet] [131269.720363] ? __schedule+0x247/0x860 [131269.720366] ? wait_woken+0x80/0x80 [131269.720377] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131269.720382] kthread+0x11a/0x130 [131269.720385] ? kthread_create_on_node+0x40/0x40 [131269.720387] ret_from_fork+0x35/0x40 [131269.720411] Sending NMI from CPU 4 to CPUs 5: [131269.721416] NMI backtrace for cpu 5 [131269.721417] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131269.721418] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131269.721418] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131269.721419] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131269.721419] RSP: 0018:ffff9e9e37d43e58 EFLAGS: 00000002 [131269.721420] RAX: 0000000000000001 RBX: 0000000000000087 RCX: 0000000000000006 [131269.721421] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffffff8a0b68c0 [131269.721421] RBP: ffffffff8a0b68c0 R08: 000000000000276a R09: 0000000000000004 [131269.721430] R10: ffffffff89e538b2 R11: 0000000000000001 R12: ffffffff8a22bc60 [131269.721431] R13: ffffffff8a0b66c0 R14: ffffffff8a22bc60 R15: ffffffff8a0b66c0 [131269.721431] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131269.721432] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131269.721432] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131269.721433] Call Trace: [131269.721433] [131269.721434] _raw_spin_lock_irqsave+0x32/0x40 [131269.721434] rcu_dump_cpu_stacks+0x48/0xd2 [131269.721434] rcu_check_callbacks+0x714/0x980 [131269.721435] ? update_wall_time+0x47b/0x6f0 [131269.721435] ? tick_sched_do_timer+0x60/0x60 [131269.721436] update_process_times+0x28/0x50 [131269.721436] tick_sched_handle.isra.19+0x1f/0x60 [131269.721437] tick_sched_timer+0x38/0x70 [131269.721437] __hrtimer_run_queues+0xce/0x210 [131269.721437] hrtimer_interrupt+0xa6/0x1f0 [131269.721438] smp_apic_timer_interrupt+0x3a/0x60 [131269.721438] apic_timer_interrupt+0x87/0x90 [131269.721438] [131269.721439] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131269.721439] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 [131269.721440] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131269.721440] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131269.721441] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131269.721441] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131269.721442] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131269.721442] lnet_check_routers+0x9e/0x5e0 [lnet] [131269.721443] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131269.721443] ? __switch_to_asm+0x34/0x70 [131269.721443] ? wait_woken+0x80/0x80 [131269.721444] ? lnet_send_ping+0x190/0x190 [lnet] [131269.721444] kthread+0x11a/0x130 [131269.721444] ? kthread_create_on_node+0x40/0x40 [131269.721445] ret_from_fork+0x35/0x40 [131269.721445] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131269.721462] Sending NMI from CPU 5 to CPUs 4: [131269.721669] NMI backtrace for cpu 4 [131269.721670] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131269.721671] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131269.721671] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131269.721672] RIP: 0010:__default_send_IPI_dest_field+0x12/0x70 [131269.721672] RSP: 0018:ffff9e9e37d03d20 EFLAGS: 00010002 [131269.721673] RAX: 000000000000f144 RBX: 0000000000000047 RCX: ffff9ea0c719c100 [131269.721673] RDX: 0000000000000000 RSI: 00000000000000fd RDI: 0000000000000013 [131269.721674] RBP: ffff9ea237cc0000 R08: 0000000000000013 R09: 0000000000000004 [131269.721674] R10: ffff9e9e37d03d28 R11: 0000000000000001 R12: ffff9e9e36ff4f0c [131269.721675] R13: 0000000000000000 R14: 0000000000000046 R15: 0000000000022dc0 [131269.721675] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131269.721676] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131269.721676] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131269.721676] Call Trace: [131269.721677] [131269.721677] ? sched_clock+0x5/0x10 [131269.721677] ? sched_clock_cpu+0xc/0xa0 [131269.721678] default_send_IPI_single_phys+0x34/0x40 [131269.721678] try_to_wake_up+0x380/0x430 [131269.721678] __wake_up_common+0x74/0x120 [131269.721679] ep_poll_callback+0x107/0x270 [131269.721679] __wake_up_common+0x74/0x120 [131269.721680] __wake_up_common_lock+0x6c/0x90 [131269.721680] irq_work_run_list+0x49/0x70 [131269.721680] ? tick_sched_do_timer+0x60/0x60 [131269.721681] update_process_times+0x3b/0x50 [131269.721681] tick_sched_handle.isra.19+0x1f/0x60 [131269.721681] ? tick_sched_do_timer+0x60/0x60 [131269.721682] tick_sched_timer+0x38/0x70 [131269.721682] __hrtimer_run_queues+0xce/0x210 [131269.721682] hrtimer_interrupt+0xa6/0x1f0 [131269.721683] smp_apic_timer_interrupt+0x3a/0x60 [131269.721684] apic_timer_interrupt+0x87/0x90 [131269.721684] [131269.721685] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131269.721685] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131269.721686] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131269.721686] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131269.721687] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131269.721688] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131269.721688] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131269.721689] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131269.721689] _raw_spin_lock+0x1d/0x20 [131269.721689] lnet_peer_discovery+0xee/0x1160 [lnet] [131269.721690] ? __schedule+0x247/0x860 [131269.721690] ? wait_woken+0x80/0x80 [131269.721692] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131269.721693] kthread+0x11a/0x130 [131269.721693] ? kthread_create_on_node+0x40/0x40 [131269.721693] ret_from_fork+0x35/0x40 [131269.721694] Code: 04 83 fe 02 0f 44 c2 89 04 25 00 c3 5f ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 48 83 ec 10 83 fe 02 75 04 eb 31 f3 90 <8b> 04 25 00 c3 5f ff f6 c4 10 75 f2 c1 e7 18 89 3c 25 10 c3 5f [131269.722467] NMI backtrace for cpu 5 [131269.722470] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131269.722471] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131269.722471] Call Trace: [131269.722473] [131269.722475] dump_stack+0x5c/0x82 [131269.722478] nmi_cpu_backtrace+0x89/0x90 [131269.722481] ? irq_force_complete_move+0x140/0x140 [131269.722483] nmi_trigger_cpumask_backtrace+0xd1/0x100 [131269.722486] rcu_dump_cpu_stacks+0x9a/0xd2 [131269.722487] rcu_check_callbacks+0x714/0x980 [131269.722489] ? update_wall_time+0x47b/0x6f0 [131269.722491] ? tick_sched_do_timer+0x60/0x60 [131269.722493] update_process_times+0x28/0x50 [131269.722495] tick_sched_handle.isra.19+0x1f/0x60 [131269.722497] tick_sched_timer+0x38/0x70 [131269.722498] __hrtimer_run_queues+0xce/0x210 [131269.722500] hrtimer_interrupt+0xa6/0x1f0 [131269.722502] smp_apic_timer_interrupt+0x3a/0x60 [131269.722504] apic_timer_interrupt+0x87/0x90 [131269.722505] [131269.722517] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131269.722518] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 [131269.722520] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131269.722521] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131269.722522] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131269.722522] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131269.722523] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131269.722534] lnet_check_routers+0x9e/0x5e0 [lnet] [131269.722545] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131269.722547] ? __switch_to_asm+0x34/0x70 [131269.722549] ? wait_woken+0x80/0x80 [131269.722558] ? lnet_send_ping+0x190/0x190 [lnet] [131269.722560] kthread+0x11a/0x130 [131269.722562] ? kthread_create_on_node+0x40/0x40 [131269.722564] ret_from_fork+0x35/0x40 [131289.799014] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [lnetctl:15141] [131289.799858] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131289.799928] Supported: No, Unsupported modules are loaded [131289.799935] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131289.799936] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131289.799939] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131289.799954] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131289.799955] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131289.799958] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131289.799959] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131289.799960] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131289.799961] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131289.799962] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131289.799965] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131289.799966] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131289.799967] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131289.799975] Call Trace: [131289.799993] _raw_spin_lock+0x1d/0x20 [131289.800011] cfs_percpt_lock+0x69/0xf0 [libcfs] [131289.800041] lnet_add_route+0x172/0xcb0 [lnet] [131289.800053] LNetCtl+0x6cf/0x11a0 [lnet] [131289.800062] ? LNetNIInit+0x66/0xb70 [lnet] [131289.800073] lnet_ioctl+0x1d3/0x250 [lnet] [131289.800081] notifier_call_chain+0x47/0x70 [131289.800084] blocking_notifier_call_chain+0x3e/0x60 [131289.800092] libcfs_ioctl+0x24e/0x470 [libcfs] [131289.800099] ? security_capable+0x47/0x60 [131289.800105] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131289.800110] do_vfs_ioctl+0x90/0x5f0 [131289.800114] SyS_ioctl+0x74/0x80 [131289.800119] do_syscall_64+0x74/0x140 [131289.800122] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131289.800124] RIP: 0033:0x7fb52efa9587 [131289.800125] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131289.800127] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131289.800127] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131289.800128] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131289.800129] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131289.800130] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131289.800132] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131293.798489] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131293.798493] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131293.798498] Modules linked in: [131293.799157] Modules linked in: lnet(OEN) [131293.799706] lnet(OEN) libcfs(OEN) [131293.799708] libcfs(OEN) af_packet [131293.799710] af_packet iscsi_ibft [131293.799712] iscsi_ibft iscsi_boot_sysfs [131293.799714] iscsi_boot_sysfs vmw_vsock_vmci_transport [131293.799715] vmw_vsock_vmci_transport vsock [131293.799717] vsock sb_edac [131293.799718] sb_edac coretemp [131293.799720] coretemp crc32_pclmul [131293.799721] crc32_pclmul crc32c_intel [131293.799723] crc32c_intel ghash_clmulni_intel [131293.799724] ghash_clmulni_intel pcbc [131293.799726] pcbc aesni_intel [131293.799727] aesni_intel vmwgfx [131293.799728] vmwgfx aes_x86_64 [131293.799730] aes_x86_64 ttm [131293.799731] ttm crypto_simd [131293.799733] crypto_simd drm_kms_helper [131293.799734] drm_kms_helper glue_helper [131293.799736] glue_helper cryptd [131293.799737] cryptd joydev [131293.799739] joydev drm [131293.799740] drm drm_panel_orientation_quirks [131293.799742] drm_panel_orientation_quirks ppdev [131293.799743] ppdev syscopyarea [131293.799744] syscopyarea vmw_balloon [131293.799746] vmw_balloon vmxnet3 [131293.799748] vmxnet3 sysfillrect [131293.799749] sysfillrect shpchp [131293.799751] shpchp parport_pc [131293.799752] parport_pc parport [131293.799754] parport sysimgblt [131293.799755] sysimgblt i2c_piix4 [131293.799757] i2c_piix4 pcspkr [131293.799758] pcspkr ac [131293.799759] ac vmw_vmci [131293.799760] vmw_vmci fb_sys_fops [131293.799762] fb_sys_fops button [131293.799763] button ext4 [131293.799765] ext4 crc16 [131293.799766] crc16 jbd2 [131293.799768] jbd2 mbcache [131293.799769] mbcache sr_mod [131293.799771] sr_mod cdrom [131293.799772] cdrom ata_generic [131293.799774] ata_generic sd_mod [131293.799775] sd_mod ata_piix [131293.799776] ata_piix ahci [131293.799778] ahci libahci [131293.799779] libahci serio_raw [131293.799781] serio_raw mptspi [131293.799782] mptspi libata [131293.799783] libata scsi_transport_spi [131293.799785] scsi_transport_spi mptscsih [131293.799786] mptscsih mptbase [131293.799787] mptbase floppy [131293.799789] floppy sg [131293.799790] sg scsi_mod [131293.799792] scsi_mod autofs4 [131293.799793] autofs4 [131293.799796] Supported: No, Unsupported modules are loaded [131293.799797] Supported: No, Unsupported modules are loaded [131293.799803] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131293.799805] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131293.799806] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131293.799807] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131293.799810] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131293.799812] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131293.799826] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131293.799878] RIP: 0010:lnet_check_routers+0x96/0x5e0 [lnet] [131293.799879] RSP: 0018:ffffabaa436ebe20 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 [131293.799882] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131293.799884] RAX: 0000000000000000 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131293.799885] ORIG_RAX: ffffffffffffff10 [131293.799886] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131293.799887] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131293.799888] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131293.799889] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131293.799891] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131293.799892] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131293.799893] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131293.799894] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131293.799895] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131293.799896] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131293.799898] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131293.799899] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131293.799900] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131293.799902] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131293.799903] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131293.799910] Call Trace: [131293.799911] Call Trace: [131293.799929] _raw_spin_lock+0x1d/0x20 [131293.799941] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131293.799946] ? __switch_to_asm+0x34/0x70 [131293.799962] lnet_peer_discovery+0xee/0x1160 [lnet] [131293.799966] ? wait_woken+0x80/0x80 [131293.799971] ? __schedule+0x247/0x860 [131293.799973] ? wait_woken+0x80/0x80 [131293.799983] ? lnet_send_ping+0x190/0x190 [lnet] [131293.799990] kthread+0x11a/0x130 [131293.800001] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131293.800004] ? kthread_create_on_node+0x40/0x40 [131293.800007] kthread+0x11a/0x130 [131293.800008] ret_from_fork+0x35/0x40 [131293.800011] ? kthread_create_on_node+0x40/0x40 [131293.800011] Code: [131293.800014] ret_from_fork+0x35/0x40 [131293.800014] 75 18 eb 27 [131293.800016] Code: 66 [131293.800018] 48 [131293.800019] 2e [131293.800019] 39 [131293.800020] 0f [131293.800021] c3 [131293.800021] 1f [131293.800022] 0f [131293.800023] 84 [131293.800023] 84 3f [131293.800025] 00 [131293.800025] 01 [131293.800026] 00 [131293.800027] 00 [131293.800027] 00 [131293.800028] 00 [131293.800029] 00 [131293.800030] 8b [131293.800030] 00 [131293.800031] 78 [131293.800031] 66 [131293.800032] 28 [131293.800033] 66 [131293.800034] 48 [131293.800034] 66 [131293.800035] 85 [131293.800036] 66 90 [131293.800037] db [131293.800038] 66 [131293.800038] 48 [131293.800039] 66 [131293.800040] 0f [131293.800041] 66 [131293.800041] 44 [131293.800042] 66 [131293.800042] d8 [131293.800043] 90 [131293.800044] 45 [131293.800045] ba [131293.800045] 31 e4 [131293.800046] 01 [131293.800047] e8 [131293.800048] 00 00 [131293.800049] 46 [131293.800050] 00 [131293.800050] b3 [131293.800051] 8b [131293.800052] fd [131293.800053] 07 [131293.800053] ff 85 c0 [131293.800055] 85 [131293.800056] 41 [131293.800056] c0 [131293.800057] 8b [131293.800058] 75 [131293.800059] 77 [131293.800059] 0a [131293.800060] 28 [131293.800061] f0 [131293.800062] 0f [131293.800062] 0f [131293.800063] 85 [131293.800063] b1 [131293.800064] 2a [131293.800065] 17 [131293.800066] 01 [131293.800066] 85 [131293.800067] 00 [131293.800068] c0 [131293.800069] 00 [131293.800069] 75 [131293.800070] <48> [131293.800071] f2 [131293.800072] 89 [131293.800072] f3 [131293.800073] ef [131293.800074] c3 [131293.800074] e8 [131293.800075] f3 [131293.800076] 82 [131293.800076] 90 [131293.800077] 6b [131293.800078] [131293.800079] 00 [131293.800079] ec [131293.800080] 00 [131293.800081] 81 [131293.800081] 48 [131293.800082] fe [131293.800083] 85 [131293.800083] 00 [131293.800084] c0 [131293.800085] 01 [131293.800086] 49 [131293.800086] 00 00 [131293.800088] 89 [131293.800088] 0f [131293.800089] c7 [131293.800090] 84 [131293.800090] 75 c9 [131293.800092] 91 [131293.800093] 48 [131293.800093] 00 [131293.800094] b8 [131293.800095] 00 [131293.800095] 00 [131293.800096] 00 [131293.800097] 04 [131293.800097] 41 b8 [131293.800099] 00 [131293.800100] 01 [131293.800101] 01 00 00 b9 [131317.795361] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [lnetctl:15141] [131317.796214] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131317.796273] Supported: No, Unsupported modules are loaded [131317.796279] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131317.796281] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131317.796284] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131317.796298] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131317.796299] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131317.796302] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131317.796303] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131317.796303] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131317.796304] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131317.796305] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131317.796308] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131317.796309] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131317.796310] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131317.796319] Call Trace: [131317.796337] _raw_spin_lock+0x1d/0x20 [131317.796356] cfs_percpt_lock+0x69/0xf0 [libcfs] [131317.796385] lnet_add_route+0x172/0xcb0 [lnet] [131317.796399] LNetCtl+0x6cf/0x11a0 [lnet] [131317.796409] ? LNetNIInit+0x66/0xb70 [lnet] [131317.796420] lnet_ioctl+0x1d3/0x250 [lnet] [131317.796426] notifier_call_chain+0x47/0x70 [131317.796429] blocking_notifier_call_chain+0x3e/0x60 [131317.796437] libcfs_ioctl+0x24e/0x470 [libcfs] [131317.796444] ? security_capable+0x47/0x60 [131317.796451] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131317.796456] do_vfs_ioctl+0x90/0x5f0 [131317.796460] SyS_ioctl+0x74/0x80 [131317.796467] do_syscall_64+0x74/0x140 [131317.796471] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131317.796474] RIP: 0033:0x7fb52efa9587 [131317.796475] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131317.796476] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131317.796477] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131317.796478] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131317.796479] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131317.796480] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131317.796481] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131321.794834] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131321.794837] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131321.794843] Modules linked in: [131321.795504] Modules linked in: [131321.796048] lnet(OEN) [131321.796048] lnet(OEN) libcfs(OEN) [131321.796050] libcfs(OEN) af_packet [131321.796052] af_packet iscsi_ibft [131321.796053] iscsi_ibft iscsi_boot_sysfs [131321.796055] iscsi_boot_sysfs vmw_vsock_vmci_transport [131321.796057] vmw_vsock_vmci_transport vsock [131321.796058] vsock sb_edac [131321.796060] sb_edac coretemp [131321.796061] coretemp crc32_pclmul [131321.796063] crc32_pclmul crc32c_intel [131321.796064] crc32c_intel ghash_clmulni_intel [131321.796066] ghash_clmulni_intel pcbc [131321.796067] pcbc aesni_intel [131321.796069] aesni_intel vmwgfx [131321.796070] vmwgfx aes_x86_64 [131321.796071] aes_x86_64 ttm [131321.796073] ttm crypto_simd [131321.796074] crypto_simd drm_kms_helper [131321.796076] drm_kms_helper glue_helper [131321.796077] glue_helper cryptd [131321.796079] cryptd joydev [131321.796080] joydev drm [131321.796081] drm drm_panel_orientation_quirks [131321.796083] drm_panel_orientation_quirks ppdev [131321.796084] ppdev syscopyarea [131321.796086] syscopyarea vmw_balloon [131321.796087] vmw_balloon vmxnet3 [131321.796089] vmxnet3 sysfillrect [131321.796090] sysfillrect shpchp [131321.796092] shpchp parport_pc [131321.796093] parport_pc parport [131321.796095] parport sysimgblt [131321.796096] sysimgblt i2c_piix4 [131321.796097] i2c_piix4 pcspkr [131321.796099] pcspkr ac [131321.796100] ac vmw_vmci [131321.796101] vmw_vmci fb_sys_fops [131321.796103] fb_sys_fops button [131321.796104] button ext4 [131321.796106] ext4 crc16 [131321.796107] crc16 jbd2 [131321.796108] jbd2 mbcache [131321.796110] mbcache sr_mod [131321.796111] sr_mod cdrom [131321.796113] cdrom ata_generic [131321.796114] ata_generic sd_mod [131321.796116] sd_mod ata_piix [131321.796117] ata_piix ahci [131321.796118] ahci libahci [131321.796120] libahci serio_raw [131321.796121] serio_raw mptspi [131321.796122] mptspi libata [131321.796124] libata scsi_transport_spi [131321.796125] scsi_transport_spi mptscsih [131321.796126] mptscsih mptbase [131321.796128] mptbase floppy [131321.796129] floppy sg [131321.796131] sg scsi_mod [131321.796132] scsi_mod autofs4 [131321.796134] autofs4 [131321.796136] Supported: No, Unsupported modules are loaded [131321.796137] Supported: No, Unsupported modules are loaded [131321.796143] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131321.796145] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131321.796146] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131321.796147] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131321.796150] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131321.796151] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131321.796199] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131321.796210] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131321.796212] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131321.796213] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131321.796214] ORIG_RAX: ffffffffffffff10 [131321.796215] ORIG_RAX: ffffffffffffff10 [131321.796216] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131321.796217] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131321.796219] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131321.796220] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131321.796221] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131321.796222] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131321.796223] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131321.796224] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131321.796225] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131321.796226] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131321.796228] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131321.796229] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131321.796230] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131321.796231] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131321.796233] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131321.796234] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131321.796242] Call Trace: [131321.796242] Call Trace: [131321.796260] _raw_spin_lock+0x1d/0x20 [131321.796272] lnet_check_routers+0x9e/0x5e0 [lnet] [131321.796290] lnet_peer_discovery+0xee/0x1160 [lnet] [131321.796301] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131321.796307] ? __schedule+0x247/0x860 [131321.796313] ? wait_woken+0x80/0x80 [131321.796316] ? __switch_to_asm+0x34/0x70 [131321.796318] ? wait_woken+0x80/0x80 [131321.796329] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131321.796337] kthread+0x11a/0x130 [131321.796347] ? lnet_send_ping+0x190/0x190 [lnet] [131321.796350] kthread+0x11a/0x130 [131321.796353] ? kthread_create_on_node+0x40/0x40 [131321.796355] ? kthread_create_on_node+0x40/0x40 [131321.796357] ret_from_fork+0x35/0x40 [131321.796359] ret_from_fork+0x35/0x40 [131321.796361] Code: [131321.796362] Code: [131321.796363] ef [131321.796364] 66 [131321.796365] 31 [131321.796366] 2e [131321.796367] c0 [131321.796367] 0f [131321.796368] eb [131321.796369] 1f [131321.796369] cc [131321.796370] 84 [131321.796371] f3 [131321.796371] 00 [131321.796372] c3 [131321.796373] 00 [131321.796374] 66 [131321.796374] 00 [131321.796375] 0f [131321.796376] 00 [131321.796377] 1f [131321.796377] 00 [131321.796378] 84 [131321.796379] 66 [131321.796379] 00 [131321.796380] 66 [131321.796381] 00 [131321.796381] 66 66 90 [131321.796383] 00 [131321.796384] 66 [131321.796385] 00 00 [131321.796386] 66 [131321.796387] 66 [131321.796388] 66 66 [131321.796389] 66 66 [131321.796390] 90 [131321.796391] 66 [131321.796391] ba [131321.796392] 90 [131321.796393] 01 [131321.796393] 85 [131321.796394] 00 [131321.796395] f6 [131321.796395] 00 [131321.796396] 48 [131321.796397] 00 [131321.796398] 8d [131321.796398] 8b [131321.796399] 57 [131321.796400] 07 85 [131321.796401] 10 48 [131321.796402] c0 [131321.796403] 8b [131321.796403] 75 [131321.796404] 47 [131321.796405] 0a [131321.796406] 10 [131321.796406] f0 [131321.796407] 74 [131321.796408] 0f [131321.796409] 29 [131321.796409] b1 17 [131321.796411] 48 39 [131321.796412] 85 [131321.796412] c2 [131321.796413] c0 [131321.796414] 74 [131321.796414] 75 [131321.796415] 14 [131321.796416] f2 f3 [131321.796417] 3b [131321.796418] c3 [131321.796418] 70 28 [131321.796420] f3 [131321.796420] 75 [131321.796421] 90 [131321.796422] 07 [131321.796422] [131321.796423] [131321.796424] ec [131321.796425] 11 [131321.796426] 81 fe [131321.796427] 39 70 [131321.796428] 00 [131321.796429] 28 [131321.796429] 01 [131321.796430] 74 [131321.796431] 00 [131321.796432] 0c [131321.796432] 00 [131321.796433] 48 [131321.796434] 0f [131321.796435] 8b [131321.796435] 84 [131321.796436] 00 [131321.796437] 91 [131321.796437] 48 [131321.796438] 00 [131321.796439] 39 [131321.796439] 00 00 [131321.796441] c2 75 [131321.796442] 41 [131321.796443] f3 [131321.796443] b8 [131321.796444] 31 [131321.796445] 01 [131321.796445] c0 [131321.796446] 01 [131321.796447] f3 [131321.796447] 00 [131321.796448] c3 [131321.796449] 00 [131321.796450] 48 [131321.796450] b9 [131321.796451] 8b [131345.788679] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [lnetctl:15141] [131345.789466] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131345.789526] Supported: No, Unsupported modules are loaded [131345.789533] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131345.789534] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131345.789537] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131345.789551] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131345.789553] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131345.789555] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131345.789556] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131345.789557] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131345.789558] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131345.789558] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131345.789561] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131345.789562] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131345.789563] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131345.789572] Call Trace: [131345.789589] _raw_spin_lock+0x1d/0x20 [131345.789609] cfs_percpt_lock+0x69/0xf0 [libcfs] [131345.789638] lnet_add_route+0x172/0xcb0 [lnet] [131345.789650] LNetCtl+0x6cf/0x11a0 [lnet] [131345.789661] ? LNetNIInit+0x66/0xb70 [lnet] [131345.789671] lnet_ioctl+0x1d3/0x250 [lnet] [131345.789682] notifier_call_chain+0x47/0x70 [131345.789685] blocking_notifier_call_chain+0x3e/0x60 [131345.789693] libcfs_ioctl+0x24e/0x470 [libcfs] [131345.789700] ? security_capable+0x47/0x60 [131345.789706] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131345.789714] do_vfs_ioctl+0x90/0x5f0 [131345.789718] SyS_ioctl+0x74/0x80 [131345.789725] do_syscall_64+0x74/0x140 [131345.789729] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131345.789731] RIP: 0033:0x7fb52efa9587 [131345.789732] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131345.789734] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131345.789735] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131345.789736] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131345.789737] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131345.789738] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131345.789739] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131349.784962] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131349.784965] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131349.784971] Modules linked in: [131349.785577] Modules linked in: [131349.786073] lnet(OEN) [131349.786074] lnet(OEN) libcfs(OEN) [131349.786076] libcfs(OEN) [131349.786077] af_packet [131349.786078] af_packet iscsi_ibft [131349.786080] iscsi_ibft iscsi_boot_sysfs [131349.786082] iscsi_boot_sysfs vmw_vsock_vmci_transport [131349.786084] vmw_vsock_vmci_transport vsock [131349.786085] vsock sb_edac [131349.786086] sb_edac coretemp [131349.786088] coretemp crc32_pclmul [131349.786089] crc32_pclmul crc32c_intel [131349.786091] crc32c_intel ghash_clmulni_intel [131349.786092] ghash_clmulni_intel pcbc [131349.786094] pcbc aesni_intel [131349.786095] aesni_intel vmwgfx [131349.786096] vmwgfx aes_x86_64 [131349.786098] aes_x86_64 ttm [131349.786099] ttm crypto_simd [131349.786100] crypto_simd drm_kms_helper [131349.786102] drm_kms_helper glue_helper [131349.786104] glue_helper cryptd [131349.786105] cryptd joydev [131349.786107] joydev drm [131349.786108] drm drm_panel_orientation_quirks [131349.786109] drm_panel_orientation_quirks ppdev [131349.786111] ppdev syscopyarea [131349.786112] syscopyarea vmw_balloon [131349.786114] vmw_balloon vmxnet3 [131349.786115] vmxnet3 sysfillrect [131349.786117] sysfillrect shpchp [131349.786118] shpchp parport_pc [131349.786120] parport_pc parport [131349.786121] parport sysimgblt [131349.786122] sysimgblt i2c_piix4 [131349.786124] i2c_piix4 pcspkr [131349.786125] pcspkr ac [131349.786126] ac vmw_vmci [131349.786128] vmw_vmci fb_sys_fops [131349.786129] fb_sys_fops button [131349.786130] button ext4 [131349.786132] ext4 crc16 [131349.786133] crc16 jbd2 [131349.786135] jbd2 mbcache [131349.786136] mbcache sr_mod [131349.786137] sr_mod cdrom [131349.786139] cdrom ata_generic [131349.786140] ata_generic sd_mod [131349.786142] sd_mod ata_piix [131349.786143] ata_piix ahci [131349.786144] ahci libahci [131349.786146] libahci serio_raw [131349.786147] serio_raw mptspi [131349.786149] mptspi libata [131349.786150] libata scsi_transport_spi [131349.786151] scsi_transport_spi mptscsih [131349.786152] mptscsih mptbase [131349.786154] mptbase floppy [131349.786156] floppy sg [131349.786157] sg scsi_mod [131349.786158] scsi_mod autofs4 [131349.786160] autofs4 [131349.786162] Supported: No, Unsupported modules are loaded [131349.786163] Supported: No, Unsupported modules are loaded [131349.786169] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131349.786171] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131349.786172] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131349.786173] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131349.786176] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131349.786178] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131349.786225] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131349.786236] RIP: 0010:native_queued_spin_lock_slowpath+0xf/0x190 [131349.786237] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131349.786239] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131349.786240] ORIG_RAX: ffffffffffffff10 [131349.786241] ORIG_RAX: ffffffffffffff10 [131349.786242] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131349.786243] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131349.786245] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131349.786246] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131349.786247] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131349.786248] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131349.786249] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131349.786250] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131349.786251] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131349.786252] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131349.786254] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131349.786255] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131349.786257] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131349.786258] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131349.786259] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131349.786260] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131349.786268] Call Trace: [131349.786269] Call Trace: [131349.786287] lnet_check_routers+0x9e/0x5e0 [lnet] [131349.786296] _raw_spin_lock+0x1d/0x20 [131349.786314] lnet_peer_discovery+0xee/0x1160 [lnet] [131349.786325] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131349.786330] ? __switch_to_asm+0x34/0x70 [131349.786336] ? __schedule+0x247/0x860 [131349.786339] ? wait_woken+0x80/0x80 [131349.786342] ? wait_woken+0x80/0x80 [131349.786353] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131349.786362] kthread+0x11a/0x130 [131349.786372] ? lnet_send_ping+0x190/0x190 [lnet] [131349.786374] kthread+0x11a/0x130 [131349.786376] ? kthread_create_on_node+0x40/0x40 [131349.786378] ret_from_fork+0x35/0x40 [131349.786380] ? kthread_create_on_node+0x40/0x40 [131349.786383] ret_from_fork+0x35/0x40 [131349.786383] Code: [131349.786384] Code: [131349.786386] 04 [131349.786386] ef [131349.786387] 24 c6 [131349.786389] 31 [131349.786390] 03 [131349.786390] c0 eb [131349.786391] 01 [131349.786392] cc [131349.786393] 48 [131349.786394] f3 [131349.786394] c7 [131349.786395] c3 [131349.786396] 00 00 [131349.786397] 66 0f [131349.786398] 00 [131349.786399] 1f [131349.786400] 00 [131349.786401] 84 [131349.786401] 00 [131349.786402] 00 [131349.786403] e9 [131349.786404] 00 [131349.786404] 89 [131349.786405] 00 [131349.786406] fe ff [131349.786407] 00 00 [131349.786408] ff [131349.786409] 66 [131349.786409] 66 66 [131349.786411] 66 [131349.786411] 2e [131349.786412] 66 [131349.786413] 0f [131349.786414] 66 [131349.786414] 1f [131349.786415] 90 [131349.786416] 84 [131349.786417] 85 [131349.786417] 00 00 [131349.786419] f6 48 [131349.786420] 00 [131349.786420] 8d [131349.786421] 00 00 [131349.786422] 57 10 [131349.786423] 66 [131349.786424] 48 [131349.786425] 66 [131349.786426] 8b [131349.786426] 66 [131349.786427] 47 [131349.786427] 66 [131349.786428] 10 [131349.786429] 90 [131349.786430] 74 [131349.786430] 66 [131349.786431] 29 [131349.786432] 66 [131349.786433] 48 [131349.786433] 66 [131349.786434] 39 [131349.786434] 66 [131349.786435] c2 [131349.786436] 90 [131349.786437] 74 [131349.786437] ba [131349.786438] 14 [131349.786439] 01 [131349.786440] 3b [131349.786440] 00 [131349.786441] 70 [131349.786442] 00 [131349.786443] 28 [131349.786443] 00 [131349.786444] 75 [131349.786445] <8b> [131349.786446] 07 [131349.786446] 07 [131349.786447] [131349.786448] 85 c0 [131349.786449] 11 [131349.786450] 75 [131349.786451] 39 [131349.786451] 0a f0 [131349.786453] 70 28 [131349.786454] 0f b1 [131349.786455] 74 0c [131349.786456] 17 [131349.786457] 48 [131349.786458] 85 [131349.786458] 8b [131349.786459] c0 75 [131349.786460] 00 48 [131349.786461] f2 [131349.786462] 39 [131349.786463] f3 c3 [131349.786464] c2 [131349.786465] f3 [131349.786466] 75 [131349.786466] 90 [131349.786467] f3 [131349.786467] eb [131349.786468] 31 [131349.786469] ec [131349.786470] c0 [131349.786470] 81 [131349.786471] f3 [131349.786473] c3 48 8b [131373.762643] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131373.763418] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131373.763477] Supported: No, Unsupported modules are loaded [131373.763483] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131373.763484] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131373.763487] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131373.763501] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131373.763502] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131373.763505] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131373.763506] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131373.763506] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131373.763507] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131373.763508] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131373.763510] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131373.763511] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131373.763512] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131373.763522] Call Trace: [131373.763539] _raw_spin_lock+0x1d/0x20 [131373.763559] cfs_percpt_lock+0x69/0xf0 [libcfs] [131373.763589] lnet_add_route+0x172/0xcb0 [lnet] [131373.763601] LNetCtl+0x6cf/0x11a0 [lnet] [131373.763610] ? LNetNIInit+0x66/0xb70 [lnet] [131373.763621] lnet_ioctl+0x1d3/0x250 [lnet] [131373.763629] notifier_call_chain+0x47/0x70 [131373.763632] blocking_notifier_call_chain+0x3e/0x60 [131373.763639] libcfs_ioctl+0x24e/0x470 [libcfs] [131373.763646] ? security_capable+0x47/0x60 [131373.763653] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131373.763658] do_vfs_ioctl+0x90/0x5f0 [131373.763662] SyS_ioctl+0x74/0x80 [131373.763667] do_syscall_64+0x74/0x140 [131373.763671] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131373.763673] RIP: 0033:0x7fb52efa9587 [131373.763674] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131373.763676] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131373.763677] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131373.763678] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131373.763679] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131373.763680] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131373.763681] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131377.758915] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131377.758918] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131377.758924] Modules linked in: [131377.760410] Modules linked in: lnet(OEN) [131377.760892] lnet(OEN) libcfs(OEN) [131377.760894] libcfs(OEN) af_packet [131377.760896] af_packet [131377.760897] iscsi_ibft [131377.760898] iscsi_ibft iscsi_boot_sysfs [131377.760900] iscsi_boot_sysfs vmw_vsock_vmci_transport [131377.760901] vmw_vsock_vmci_transport vsock [131377.760903] vsock sb_edac [131377.760904] sb_edac coretemp [131377.760905] coretemp crc32_pclmul [131377.760907] crc32_pclmul crc32c_intel [131377.760909] crc32c_intel ghash_clmulni_intel [131377.760910] ghash_clmulni_intel pcbc [131377.760912] pcbc aesni_intel [131377.760913] aesni_intel vmwgfx [131377.760914] vmwgfx aes_x86_64 [131377.760916] aes_x86_64 [131377.760916] ttm [131377.760917] ttm [131377.760918] crypto_simd [131377.760919] crypto_simd [131377.760919] drm_kms_helper [131377.760920] drm_kms_helper [131377.760921] glue_helper [131377.760922] glue_helper [131377.760922] cryptd [131377.760923] cryptd [131377.760924] joydev [131377.760924] joydev [131377.760925] drm [131377.760926] drm [131377.760926] drm_panel_orientation_quirks [131377.760927] drm_panel_orientation_quirks [131377.760928] ppdev [131377.760929] ppdev [131377.760929] syscopyarea [131377.760930] syscopyarea [131377.760931] vmw_balloon [131377.760932] vmw_balloon [131377.760944] vmxnet3 [131377.760945] vmxnet3 sysfillrect [131377.760946] sysfillrect shpchp [131377.760948] shpchp parport_pc [131377.760949] parport_pc parport [131377.760951] parport sysimgblt [131377.760952] sysimgblt i2c_piix4 [131377.760954] i2c_piix4 pcspkr [131377.760955] pcspkr ac [131377.760956] ac vmw_vmci [131377.760957] vmw_vmci fb_sys_fops [131377.760959] fb_sys_fops button [131377.760961] button ext4 [131377.760962] ext4 crc16 [131377.760963] crc16 jbd2 [131377.760965] jbd2 mbcache [131377.760966] mbcache sr_mod [131377.760968] sr_mod cdrom [131377.760969] cdrom ata_generic [131377.760970] ata_generic sd_mod [131377.760972] sd_mod ata_piix [131377.760973] ata_piix ahci [131377.760974] ahci libahci [131377.760976] libahci serio_raw [131377.760977] serio_raw mptspi [131377.760978] mptspi [131377.760983] libata [131377.760984] libata scsi_transport_spi [131377.760985] scsi_transport_spi mptscsih [131377.760986] mptscsih mptbase [131377.760987] mptbase floppy [131377.760989] floppy sg [131377.760991] sg scsi_mod [131377.760992] scsi_mod autofs4 [131377.760993] autofs4 [131377.760996] Supported: No, Unsupported modules are loaded [131377.760997] Supported: No, Unsupported modules are loaded [131377.761004] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131377.761006] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131377.761007] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131377.761008] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131377.761011] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131377.761013] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131377.761063] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131377.761073] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131377.761074] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131377.761076] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131377.761077] ORIG_RAX: ffffffffffffff10 [131377.761078] ORIG_RAX: ffffffffffffff10 [131377.761079] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131377.761080] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131377.761082] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131377.761083] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131377.761084] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131377.761085] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131377.761086] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131377.761087] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131377.761088] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131377.761089] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131377.761091] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131377.761093] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131377.761094] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131377.761095] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131377.761096] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131377.761097] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131377.761104] Call Trace: [131377.761105] Call Trace: [131377.761125] lnet_check_routers+0x9e/0x5e0 [lnet] [131377.761136] _raw_spin_lock+0x1d/0x20 [131377.761150] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131377.761166] lnet_peer_discovery+0xee/0x1160 [lnet] [131377.761170] ? __switch_to_asm+0x34/0x70 [131377.761175] ? __schedule+0x247/0x860 [131377.761180] ? wait_woken+0x80/0x80 [131377.761182] ? wait_woken+0x80/0x80 [131377.761192] ? lnet_send_ping+0x190/0x190 [lnet] [131377.761203] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131377.761211] kthread+0x11a/0x130 [131377.761214] kthread+0x11a/0x130 [131377.761217] ? kthread_create_on_node+0x40/0x40 [131377.761219] ? kthread_create_on_node+0x40/0x40 [131377.761221] ret_from_fork+0x35/0x40 [131377.761223] ret_from_fork+0x35/0x40 [131377.761228] Code: [131377.761228] Code: [131377.761230] 66 2e [131377.761231] ef [131377.761232] 0f [131377.761233] 31 [131377.761234] 1f [131377.761235] c0 [131377.761235] 84 [131377.761236] eb [131377.761237] 00 [131377.761237] cc [131377.761238] 00 [131377.761239] f3 [131377.761240] 00 [131377.761240] c3 [131377.761241] 00 [131377.761241] 66 [131377.761242] 00 [131377.761243] 0f [131377.761244] 66 [131377.761244] 1f [131377.761245] 66 [131377.761246] 84 [131377.761246] 66 [131377.761247] 00 [131377.761248] 66 [131377.761248] 00 [131377.761249] 90 [131377.761250] 00 [131377.761251] 66 [131377.761251] 00 [131377.761252] 66 [131377.761253] 00 [131377.761254] 66 [131377.761254] 66 [131377.761255] 66 [131377.761255] 66 [131377.761256] 90 [131377.761257] 66 66 [131377.761258] ba [131377.761259] 90 [131377.761260] 01 [131377.761260] 85 [131377.761261] 00 [131377.761262] f6 [131377.761262] 00 [131377.761263] 48 [131377.761264] 00 [131377.761265] 8d [131377.761265] 8b [131377.761266] 57 [131377.761266] 07 [131377.761267] 10 [131377.761268] 85 [131377.761269] 48 [131377.761269] c0 [131377.761270] 8b [131377.761271] 75 [131377.761272] 47 [131377.761272] 0a [131377.761273] 10 [131377.761273] f0 [131377.761274] 74 [131377.761275] 0f [131377.761276] 29 [131377.761276] b1 [131377.761277] 48 [131377.761277] 17 [131377.761278] 39 [131377.761279] 85 [131377.761280] c2 [131377.761280] c0 [131377.761281] 74 [131377.761282] 75 [131377.761282] 14 [131377.761283] f2 [131377.761284] 3b [131377.761284] f3 [131377.761285] 70 [131377.761286] c3 [131377.761287] 28 [131377.761287] f3 [131377.761288] 75 [131377.761288] 90 [131377.761289] 07 [131377.761290] [131377.761291] [131377.761291] ec [131377.761292] 11 [131377.761293] 81 [131377.761294] 39 [131377.761294] fe [131377.761295] 70 [131377.761296] 00 [131377.761296] 28 [131377.761297] 01 [131377.761298] 74 [131377.761298] 00 [131377.761299] 0c [131377.761300] 00 [131377.761301] 48 [131377.761301] 0f 84 91 [131377.761303] 8b 00 48 [131377.761305] 00 [131377.761306] 39 [131377.761307] 00 [131377.761307] c2 [131377.761308] 00 [131377.761309] 75 [131377.761309] 41 [131377.761310] f3 [131377.761311] b8 [131377.761312] 31 [131377.761312] 01 [131377.761313] c0 [131377.761314] 01 [131377.761314] f3 [131377.761315] 00 [131377.761316] c3 [131377.761316] 00 [131377.761317] 48 [131377.761318] b9 [131377.761319] 8b [131401.738837] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131401.739575] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131401.739637] Supported: No, Unsupported modules are loaded [131401.739643] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131401.739645] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131401.739647] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131401.739661] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131401.739663] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131401.739665] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131401.739666] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131401.739667] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131401.739668] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131401.739669] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131401.739671] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131401.739672] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131401.739673] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131401.739682] Call Trace: [131401.739700] _raw_spin_lock+0x1d/0x20 [131401.739719] cfs_percpt_lock+0x69/0xf0 [libcfs] [131401.739750] lnet_add_route+0x172/0xcb0 [lnet] [131401.739762] LNetCtl+0x6cf/0x11a0 [lnet] [131401.739772] ? LNetNIInit+0x66/0xb70 [lnet] [131401.739783] lnet_ioctl+0x1d3/0x250 [lnet] [131401.739790] notifier_call_chain+0x47/0x70 [131401.739793] blocking_notifier_call_chain+0x3e/0x60 [131401.739801] libcfs_ioctl+0x24e/0x470 [libcfs] [131401.739808] ? security_capable+0x47/0x60 [131401.739814] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131401.739819] do_vfs_ioctl+0x90/0x5f0 [131401.739842] SyS_ioctl+0x74/0x80 [131401.739849] do_syscall_64+0x74/0x140 [131401.739855] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131401.739858] RIP: 0033:0x7fb52efa9587 [131401.739859] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131401.739861] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131401.739861] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131401.739862] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131401.739863] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131401.739864] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131401.739865] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131405.738543] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131405.738546] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131405.738552] Modules linked in: [131405.739166] Modules linked in: [131405.739643] lnet(OEN) [131405.739644] lnet(OEN) libcfs(OEN) [131405.739646] libcfs(OEN) [131405.739647] af_packet [131405.739648] af_packet [131405.739649] iscsi_ibft [131405.739649] iscsi_ibft iscsi_boot_sysfs [131405.739651] iscsi_boot_sysfs vmw_vsock_vmci_transport [131405.739653] vmw_vsock_vmci_transport vsock [131405.739654] vsock sb_edac [131405.739656] sb_edac coretemp [131405.739657] coretemp crc32_pclmul [131405.739659] crc32_pclmul crc32c_intel [131405.739661] crc32c_intel ghash_clmulni_intel [131405.739662] ghash_clmulni_intel pcbc [131405.739664] pcbc aesni_intel [131405.739665] aesni_intel vmwgfx [131405.739666] vmwgfx aes_x86_64 [131405.739668] aes_x86_64 ttm [131405.739669] ttm crypto_simd [131405.739670] crypto_simd drm_kms_helper [131405.739672] drm_kms_helper glue_helper [131405.739674] glue_helper cryptd [131405.739675] cryptd joydev [131405.739677] joydev drm [131405.739678] drm drm_panel_orientation_quirks [131405.739679] drm_panel_orientation_quirks ppdev [131405.739681] ppdev syscopyarea [131405.739682] syscopyarea vmw_balloon [131405.739684] vmw_balloon vmxnet3 [131405.739686] vmxnet3 sysfillrect [131405.739687] sysfillrect shpchp [131405.739689] shpchp parport_pc [131405.739690] parport_pc parport [131405.739692] parport sysimgblt [131405.739693] sysimgblt i2c_piix4 [131405.739695] i2c_piix4 pcspkr [131405.739696] pcspkr ac [131405.739698] ac vmw_vmci [131405.739699] vmw_vmci [131405.739700] fb_sys_fops [131405.739701] fb_sys_fops [131405.739702] button [131405.739703] button ext4 [131405.739704] ext4 crc16 [131405.739705] crc16 jbd2 [131405.739707] jbd2 mbcache [131405.739708] mbcache sr_mod [131405.739710] sr_mod cdrom [131405.739711] cdrom ata_generic [131405.739713] ata_generic sd_mod [131405.739714] sd_mod ata_piix [131405.739716] ata_piix ahci [131405.739717] ahci libahci [131405.739718] libahci serio_raw [131405.739720] serio_raw mptspi [131405.739721] mptspi libata [131405.739723] libata scsi_transport_spi [131405.739724] scsi_transport_spi mptscsih [131405.739726] mptscsih mptbase [131405.739727] mptbase floppy [131405.739729] floppy sg [131405.739730] sg scsi_mod [131405.739732] scsi_mod autofs4 [131405.739733] autofs4 [131405.739736] Supported: No, Unsupported modules are loaded [131405.739737] Supported: No, Unsupported modules are loaded [131405.739743] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131405.739745] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131405.739747] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131405.739748] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131405.739751] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131405.739752] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131405.739767] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131405.739811] RIP: 0010:lnet_check_routers+0x85/0x5e0 [lnet] [131405.739812] RSP: 0018:ffffabaa436ebe20 EFLAGS: 00000246 [131405.739813] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131405.739815] ORIG_RAX: ffffffffffffff10 [131405.739815] ORIG_RAX: ffffffffffffff10 [131405.739817] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131405.739818] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131405.739819] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131405.739820] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131405.739821] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131405.739823] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131405.739824] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131405.739825] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131405.739826] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131405.739827] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131405.739829] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131405.739830] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131405.739832] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131405.739833] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131405.739834] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131405.739836] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131405.739842] Call Trace: [131405.739843] Call Trace: [131405.739860] _raw_spin_lock+0x1d/0x20 [131405.739889] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131405.739895] ? __switch_to_asm+0x34/0x70 [131405.739911] lnet_peer_discovery+0xee/0x1160 [lnet] [131405.739920] ? wait_woken+0x80/0x80 [131405.739925] ? __schedule+0x247/0x860 [131405.739935] ? lnet_send_ping+0x190/0x190 [lnet] [131405.739944] kthread+0x11a/0x130 [131405.739946] ? wait_woken+0x80/0x80 [131405.739958] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131405.739960] ? kthread_create_on_node+0x40/0x40 [131405.739963] kthread+0x11a/0x130 [131405.739964] ret_from_fork+0x35/0x40 [131405.739967] ? kthread_create_on_node+0x40/0x40 [131405.739967] Code: [131405.739970] ret_from_fork+0x35/0x40 [131405.739970] 00 00 00 [131405.739972] Code: 66 [131405.739973] e8 1e [131405.739975] 2e [131405.739976] 99 [131405.739976] 0f [131405.739977] 98 [131405.739978] 1f [131405.739979] c8 [131405.739979] 84 [131405.739980] 49 [131405.739980] 00 [131405.739981] 8d [131405.739982] 00 [131405.739983] 6d [131405.739983] 00 [131405.739984] d0 [131405.739985] 00 [131405.739986] 49 [131405.739986] 00 [131405.739987] 89 [131405.739988] 66 [131405.739988] c6 [131405.739989] 66 [131405.739990] 41 [131405.739990] 66 [131405.739991] 8b [131405.739992] 66 [131405.739993] 75 [131405.739993] 90 [131405.739994] 18 [131405.739995] 66 [131405.739995] eb [131405.739996] 66 [131405.739997] 27 [131405.739997] 66 [131405.739998] 48 [131405.739999] 66 [131405.740000] 39 [131405.740000] 90 [131405.740001] c3 [131405.740002] ba [131405.740003] 0f [131405.740003] 01 [131405.740004] 84 [131405.740004] 00 [131405.740005] 3f [131405.740006] 00 [131405.740007] 01 [131405.740007] 00 [131405.740008] 00 [131405.740009] 8b [131405.740010] 00 [131405.740010] 07 [131405.740011] 8b [131405.740012] 85 [131405.740012] 78 [131405.740013] c0 [131405.740014] 28 [131405.740014] 75 [131405.740015] 48 [131405.740016] 0a [131405.740017] 85 [131405.740017] f0 [131405.740018] db [131405.740019] 0f [131405.740019] 48 [131405.740020] b1 [131405.740021] 0f [131405.740021] 17 [131405.740022] 44 [131405.740023] 85 [131405.740024] d8 [131405.740024] c0 [131405.740025] 45 [131405.740026] 75 [131405.740026] 31 [131405.740027] f2 [131405.740028] e4 [131405.740028] f3 [131405.740029] [131405.740030] c3 [131405.740031] 46 [131405.740032] f3 [131405.740032] b3 [131405.740033] 90 [131405.740034] fd [131405.740035] ec [131405.740036] ff 85 [131405.740037] 81 [131405.740038] c0 [131405.740038] fe [131405.740039] 41 [131405.740040] 00 [131405.740041] 8b [131405.740041] 01 [131405.740042] 77 [131405.740043] 00 [131405.740044] 28 [131405.740044] 00 [131405.740045] 0f [131405.740046] 0f [131405.740046] 85 [131405.740047] 84 [131405.740048] 2a [131405.740048] 91 [131405.740049] 01 [131405.740050] 00 [131405.740051] 00 [131405.740051] 00 [131405.740052] 00 [131405.740053] 00 [131405.740054] 48 [131405.740054] 41 [131405.740055] 89 [131405.740056] b8 [131405.740057] ef [131405.740057] 01 [131405.740058] e8 [131405.740059] 01 [131405.740060] 00 00 b9 [131429.735114] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131429.735873] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131429.735934] Supported: No, Unsupported modules are loaded [131429.735940] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131429.735940] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131429.735943] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131429.735957] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131429.735958] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131429.735960] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131429.735961] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131429.735962] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131429.735963] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131429.735964] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131429.735967] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131429.735968] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131429.735969] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131429.735978] Call Trace: [131429.735996] _raw_spin_lock+0x1d/0x20 [131429.736016] cfs_percpt_lock+0x69/0xf0 [libcfs] [131429.736046] lnet_add_route+0x172/0xcb0 [lnet] [131429.736059] LNetCtl+0x6cf/0x11a0 [lnet] [131429.736069] ? LNetNIInit+0x66/0xb70 [lnet] [131429.736080] lnet_ioctl+0x1d3/0x250 [lnet] [131429.736086] notifier_call_chain+0x47/0x70 [131429.736089] blocking_notifier_call_chain+0x3e/0x60 [131429.736097] libcfs_ioctl+0x24e/0x470 [libcfs] [131429.736104] ? security_capable+0x47/0x60 [131429.736110] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131429.736115] do_vfs_ioctl+0x90/0x5f0 [131429.736119] SyS_ioctl+0x74/0x80 [131429.736127] do_syscall_64+0x74/0x140 [131429.736131] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131429.736133] RIP: 0033:0x7fb52efa9587 [131429.736134] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131429.736136] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131429.736137] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131429.736137] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131429.736138] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131429.736139] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131429.736141] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131433.734530] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131433.734533] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131433.734539] Modules linked in: [131433.735182] Modules linked in: [131433.735754] lnet(OEN) [131433.735755] lnet(OEN) [131433.735757] libcfs(OEN) [131433.735757] libcfs(OEN) af_packet [131433.735759] af_packet iscsi_ibft [131433.735761] iscsi_ibft iscsi_boot_sysfs [131433.735763] iscsi_boot_sysfs vmw_vsock_vmci_transport [131433.735765] vmw_vsock_vmci_transport vsock [131433.735766] vsock sb_edac [131433.735768] sb_edac coretemp [131433.735769] coretemp crc32_pclmul [131433.735771] crc32_pclmul crc32c_intel [131433.735772] crc32c_intel ghash_clmulni_intel [131433.735773] ghash_clmulni_intel pcbc [131433.735775] pcbc aesni_intel [131433.735776] aesni_intel vmwgfx [131433.735777] vmwgfx aes_x86_64 [131433.735779] aes_x86_64 ttm [131433.735780] ttm crypto_simd [131433.735782] crypto_simd drm_kms_helper [131433.735783] drm_kms_helper glue_helper [131433.735785] glue_helper cryptd [131433.735786] cryptd joydev [131433.735788] joydev drm [131433.735789] drm drm_panel_orientation_quirks [131433.735790] drm_panel_orientation_quirks ppdev [131433.735792] ppdev syscopyarea [131433.735793] syscopyarea vmw_balloon [131433.735795] vmw_balloon vmxnet3 [131433.735797] vmxnet3 sysfillrect [131433.735798] sysfillrect shpchp [131433.735800] shpchp parport_pc [131433.735801] parport_pc [131433.735802] parport [131433.735803] parport [131433.735804] sysimgblt [131433.735805] sysimgblt i2c_piix4 [131433.735806] i2c_piix4 pcspkr [131433.735807] pcspkr ac [131433.735809] ac vmw_vmci [131433.735810] vmw_vmci fb_sys_fops [131433.735812] fb_sys_fops button [131433.735813] button ext4 [131433.735815] ext4 crc16 [131433.735816] crc16 jbd2 [131433.735817] jbd2 mbcache [131433.735819] mbcache sr_mod [131433.735820] sr_mod cdrom [131433.735822] cdrom ata_generic [131433.735823] ata_generic sd_mod [131433.735825] sd_mod ata_piix [131433.735826] ata_piix ahci [131433.735827] ahci libahci [131433.735829] libahci serio_raw [131433.735830] serio_raw mptspi [131433.735832] mptspi libata [131433.735833] libata scsi_transport_spi [131433.735835] scsi_transport_spi mptscsih [131433.735836] mptscsih mptbase [131433.735837] mptbase floppy [131433.735839] floppy sg [131433.735840] sg scsi_mod [131433.735842] scsi_mod autofs4 [131433.735843] autofs4 [131433.735846] Supported: No, Unsupported modules are loaded [131433.735847] Supported: No, Unsupported modules are loaded [131433.735853] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131433.735855] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131433.735857] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131433.735858] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131433.735861] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131433.735862] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131433.735883] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131433.735927] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131433.735928] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131433.735930] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131433.735931] ORIG_RAX: ffffffffffffff10 [131433.735931] ORIG_RAX: ffffffffffffff10 [131433.735933] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131433.735934] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131433.735935] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131433.735936] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131433.735937] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131433.735938] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131433.735940] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131433.735940] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131433.735942] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131433.735942] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131433.735945] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131433.735946] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131433.735947] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131433.735948] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131433.735950] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131433.735951] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131433.735958] Call Trace: [131433.735958] Call Trace: [131433.735975] _raw_spin_lock+0x1d/0x20 [131433.735986] lnet_check_routers+0x9e/0x5e0 [lnet] [131433.736005] lnet_peer_discovery+0xee/0x1160 [lnet] [131433.736015] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131433.736020] ? __switch_to_asm+0x34/0x70 [131433.736025] ? __schedule+0x247/0x860 [131433.736030] ? wait_woken+0x80/0x80 [131433.736032] ? wait_woken+0x80/0x80 [131433.736042] ? lnet_send_ping+0x190/0x190 [lnet] [131433.736053] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131433.736061] kthread+0x11a/0x130 [131433.736063] kthread+0x11a/0x130 [131433.736066] ? kthread_create_on_node+0x40/0x40 [131433.736068] ? kthread_create_on_node+0x40/0x40 [131433.736070] ret_from_fork+0x35/0x40 [131433.736072] ret_from_fork+0x35/0x40 [131433.736074] Code: [131433.736075] Code: [131433.736077] ef [131433.736077] 66 [131433.736078] 31 [131433.736079] 2e [131433.736080] c0 [131433.736080] 0f [131433.736081] eb [131433.736082] 1f [131433.736083] cc [131433.736083] 84 [131433.736084] f3 [131433.736085] 00 00 [131433.736086] c3 66 [131433.736087] 00 [131433.736088] 0f [131433.736089] 00 [131433.736089] 1f [131433.736090] 00 [131433.736091] 84 [131433.736091] 66 [131433.736092] 00 [131433.736093] 66 [131433.736094] 00 [131433.736094] 66 [131433.736095] 00 [131433.736096] 66 [131433.736097] 00 [131433.736097] 90 [131433.736098] 00 [131433.736099] 66 [131433.736099] 66 [131433.736100] 66 [131433.736101] 66 [131433.736101] 66 [131433.736102] 66 [131433.736103] 66 [131433.736104] 66 [131433.736104] 90 ba [131433.736105] 90 85 [131433.736107] 01 [131433.736108] f6 [131433.736108] 00 [131433.736109] 48 [131433.736110] 00 [131433.736110] 8d [131433.736111] 00 [131433.736112] 57 [131433.736112] 8b [131433.736113] 10 [131433.736114] 07 [131433.736115] 48 [131433.736115] 85 [131433.736116] 8b [131433.736117] c0 [131433.736118] 47 [131433.736118] 75 [131433.736119] 10 [131433.736119] 0a [131433.736120] 74 [131433.736121] f0 [131433.736122] 29 [131433.736122] 0f b1 [131433.736124] 48 39 [131433.736125] 17 [131433.736126] c2 [131433.736126] 85 [131433.736127] 74 [131433.736127] c0 [131433.736128] 14 [131433.736129] 75 [131433.736130] 3b [131433.736130] f2 [131433.736131] 70 [131433.736132] f3 [131433.736132] 28 [131433.736133] c3 [131433.736134] 75 [131433.736134] f3 [131433.736135] 07 [131433.736136] 90 [131433.736137] [131433.736137] [131433.736138] 11 [131433.736139] ec [131433.736140] 39 [131433.736140] 81 [131433.736141] 70 [131433.736142] fe [131433.736142] 28 [131433.736143] 00 [131433.736144] 74 [131433.736144] 01 [131433.736145] 0c [131433.736146] 00 [131433.736147] 48 [131433.736147] 00 [131433.736148] 8b [131433.736149] 0f [131433.736150] 00 [131433.736150] 84 [131433.736151] 48 [131433.736152] 91 [131433.736152] 39 [131433.736153] 00 [131433.736154] c2 [131433.736154] 00 [131433.736155] 75 [131433.736156] 00 [131433.736157] f3 [131433.736157] 41 [131433.736158] 31 [131433.736159] b8 [131433.736160] c0 [131433.736160] 01 [131433.736161] f3 [131433.736162] 01 [131433.736163] c3 [131433.736163] 00 [131433.736164] 48 [131433.736165] 00 [131433.736165] 8b [131433.736166] b9 [131449.660206] INFO: rcu_sched self-detected stall on CPU [131449.660211] INFO: rcu_sched self-detected stall on CPU [131449.661266] 5-...: (285016 ticks this GP) idle=4fa/140000000000001/0 softirq=3607039/3607039 fqs=142501 [131449.661269] 4-...: (285017 ticks this GP) idle=62a/140000000000001/0 softirq=126578/126578 fqs=142501 [131449.661271] [131449.662204] (t=285019 jiffies g=1359077 c=1359076 q=56292) [131449.663230] (t=285019 jiffies g=1359077 c=1359076 q=56292) [131449.663739] Sending NMI from CPU 5 to CPUs 4: [131449.665358] NMI backtrace for cpu 4 [131449.665359] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131449.665360] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131449.665360] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131449.665361] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131449.665362] RSP: 0018:ffff9e9e37d03e58 EFLAGS: 00000002 [131449.665363] RAX: 0000000000000001 RBX: 0000000000000087 RCX: 0000000000000006 [131449.665363] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffffff8a0b68c0 [131449.665364] RBP: ffffffff8a0b68c0 R08: 0000000000000000 R09: 0000000000aaaaaa [131449.665364] R10: ffffffff89e538b2 R11: 00000000ffffffff R12: ffffffff8a22bc60 [131449.665365] R13: ffffffff8a0b66c0 R14: ffffffff8a22bc60 R15: ffffffff8a0b66c0 [131449.665365] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131449.665366] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131449.665366] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131449.665367] Call Trace: [131449.665367] [131449.665367] _raw_spin_lock_irqsave+0x32/0x40 [131449.665368] rcu_dump_cpu_stacks+0x48/0xd2 [131449.665368] rcu_check_callbacks+0x714/0x980 [131449.665368] ? tick_sched_do_timer+0x60/0x60 [131449.665369] update_process_times+0x28/0x50 [131449.665369] tick_sched_handle.isra.19+0x1f/0x60 [131449.665370] ? tick_sched_do_timer+0x60/0x60 [131449.665370] tick_sched_timer+0x38/0x70 [131449.665370] __hrtimer_run_queues+0xce/0x210 [131449.665371] hrtimer_interrupt+0xa6/0x1f0 [131449.665371] smp_apic_timer_interrupt+0x3a/0x60 [131449.665371] apic_timer_interrupt+0x87/0x90 [131449.665372] [131449.665372] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131449.665373] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131449.665373] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131449.665374] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131449.665374] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131449.665375] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131449.665375] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131449.665376] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131449.665376] _raw_spin_lock+0x1d/0x20 [131449.665376] lnet_peer_discovery+0xee/0x1160 [lnet] [131449.665377] ? __schedule+0x247/0x860 [131449.665377] ? wait_woken+0x80/0x80 [131449.665378] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131449.665378] kthread+0x11a/0x130 [131449.665378] ? kthread_create_on_node+0x40/0x40 [131449.665379] ret_from_fork+0x35/0x40 [131449.665379] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131449.665408] NMI backtrace for cpu 5 [131449.665411] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131449.665411] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131449.665412] Call Trace: [131449.665426] [131449.665436] dump_stack+0x5c/0x82 [131449.665448] nmi_cpu_backtrace+0x89/0x90 [131449.665461] ? irq_force_complete_move+0x140/0x140 [131449.665464] nmi_trigger_cpumask_backtrace+0xd1/0x100 [131449.665467] rcu_dump_cpu_stacks+0x9a/0xd2 [131449.665469] rcu_check_callbacks+0x714/0x980 [131449.665471] ? update_wall_time+0x47b/0x6f0 [131449.665473] ? tick_sched_do_timer+0x60/0x60 [131449.665475] update_process_times+0x28/0x50 [131449.665477] tick_sched_handle.isra.19+0x1f/0x60 [131449.665479] tick_sched_timer+0x38/0x70 [131449.665481] __hrtimer_run_queues+0xce/0x210 [131449.665482] hrtimer_interrupt+0xa6/0x1f0 [131449.665486] smp_apic_timer_interrupt+0x3a/0x60 [131449.665488] apic_timer_interrupt+0x87/0x90 [131449.665489] [131449.665506] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131449.665508] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 [131449.665509] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131449.665511] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131449.665512] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131449.665513] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131449.665514] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131449.665527] lnet_check_routers+0x9e/0x5e0 [lnet] [131449.665540] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131449.665543] ? __switch_to_asm+0x34/0x70 [131449.665546] ? wait_woken+0x80/0x80 [131449.665555] ? lnet_send_ping+0x190/0x190 [lnet] [131449.665558] kthread+0x11a/0x130 [131449.665560] ? kthread_create_on_node+0x40/0x40 [131449.665562] ret_from_fork+0x35/0x40 [131449.665570] NMI backtrace for cpu 4 [131449.665572] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131449.665573] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131449.665574] Call Trace: [131449.665575] [131449.665578] dump_stack+0x5c/0x82 [131449.665581] nmi_cpu_backtrace+0x89/0x90 [131449.665584] ? irq_force_complete_move+0x140/0x140 [131449.665586] nmi_trigger_cpumask_backtrace+0xd1/0x100 [131449.665589] rcu_dump_cpu_stacks+0x9a/0xd2 [131449.665590] rcu_check_callbacks+0x714/0x980 [131449.665592] ? tick_sched_do_timer+0x60/0x60 [131449.665595] update_process_times+0x28/0x50 [131449.665597] tick_sched_handle.isra.19+0x1f/0x60 [131449.665598] ? tick_sched_do_timer+0x60/0x60 [131449.665600] tick_sched_timer+0x38/0x70 [131449.665601] __hrtimer_run_queues+0xce/0x210 [131449.665603] hrtimer_interrupt+0xa6/0x1f0 [131449.665605] smp_apic_timer_interrupt+0x3a/0x60 [131449.665608] apic_timer_interrupt+0x87/0x90 [131449.665608] [131449.665610] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131449.665611] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131449.665613] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131449.665614] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131449.665615] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131449.665615] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131449.665616] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131449.665628] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131449.665630] _raw_spin_lock+0x1d/0x20 [131449.665640] lnet_peer_discovery+0xee/0x1160 [lnet] [131449.665643] ? __schedule+0x247/0x860 [131449.665645] ? wait_woken+0x80/0x80 [131449.665655] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131449.665657] kthread+0x11a/0x130 [131449.665659] ? kthread_create_on_node+0x40/0x40 [131449.665661] ret_from_fork+0x35/0x40 [131449.665672] Sending NMI from CPU 4 to CPUs 5: [131449.666449] NMI backtrace for cpu 5 [131449.666451] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131449.666452] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131449.666452] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131449.666453] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131449.666454] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131449.666455] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131449.666456] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131449.666456] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131449.666456] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131449.666457] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131449.666457] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131449.666458] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131449.666458] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131449.666459] Call Trace: [131449.666459] lnet_check_routers+0x9e/0x5e0 [lnet] [131449.666459] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131449.666460] ? __switch_to_asm+0x34/0x70 [131449.666460] ? wait_woken+0x80/0x80 [131449.666461] ? lnet_send_ping+0x190/0x190 [lnet] [131449.666462] kthread+0x11a/0x130 [131449.666462] ? kthread_create_on_node+0x40/0x40 [131449.666463] ret_from_fork+0x35/0x40 [131449.666463] Code: ef 31 c0 eb cc f3 c3 66 0f 1f 84 00 00 00 00 00 66 66 66 66 90 85 f6 48 8d 57 10 48 8b 47 10 74 29 48 39 c2 74 14 3b 70 28 75 07 11 39 70 28 74 0c 48 8b 00 48 39 c2 75 f3 31 c0 f3 c3 48 8b [131457.731040] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131457.731895] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131457.731954] Supported: No, Unsupported modules are loaded [131457.731961] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131457.731962] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131457.731966] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131457.731980] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131457.731981] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131457.731983] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131457.731984] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131457.731985] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131457.731986] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131457.731987] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131457.731989] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131457.731990] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131457.731991] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131457.732000] Call Trace: [131457.732019] _raw_spin_lock+0x1d/0x20 [131457.732039] cfs_percpt_lock+0x69/0xf0 [libcfs] [131457.732070] lnet_add_route+0x172/0xcb0 [lnet] [131457.732082] LNetCtl+0x6cf/0x11a0 [lnet] [131457.732093] ? LNetNIInit+0x66/0xb70 [lnet] [131457.732104] lnet_ioctl+0x1d3/0x250 [lnet] [131457.732111] notifier_call_chain+0x47/0x70 [131457.732113] blocking_notifier_call_chain+0x3e/0x60 [131457.732121] libcfs_ioctl+0x24e/0x470 [libcfs] [131457.732128] ? security_capable+0x47/0x60 [131457.732135] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131457.732141] do_vfs_ioctl+0x90/0x5f0 [131457.732145] SyS_ioctl+0x74/0x80 [131457.732153] do_syscall_64+0x74/0x140 [131457.732156] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131457.732159] RIP: 0033:0x7fb52efa9587 [131457.732160] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131457.732162] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131457.732162] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131457.732163] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131457.732164] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131457.732165] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131457.732166] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131473.725914] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131473.725917] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131473.725924] Modules linked in: [131473.726588] Modules linked in: [131473.727150] lnet(OEN) [131473.727150] lnet(OEN) [131473.727152] libcfs(OEN) [131473.727152] libcfs(OEN) af_packet [131473.727154] af_packet iscsi_ibft [131473.727156] iscsi_ibft iscsi_boot_sysfs [131473.727158] iscsi_boot_sysfs vmw_vsock_vmci_transport [131473.727159] vmw_vsock_vmci_transport vsock [131473.727161] vsock sb_edac [131473.727162] sb_edac coretemp [131473.727163] coretemp crc32_pclmul [131473.727165] crc32_pclmul crc32c_intel [131473.727166] crc32c_intel ghash_clmulni_intel [131473.727168] ghash_clmulni_intel pcbc [131473.727169] pcbc aesni_intel [131473.727171] aesni_intel vmwgfx [131473.727172] vmwgfx aes_x86_64 [131473.727173] aes_x86_64 ttm [131473.727175] ttm crypto_simd [131473.727176] crypto_simd drm_kms_helper [131473.727178] drm_kms_helper glue_helper [131473.727179] glue_helper cryptd [131473.727181] cryptd joydev [131473.727182] joydev drm [131473.727183] drm drm_panel_orientation_quirks [131473.727185] drm_panel_orientation_quirks ppdev [131473.727186] ppdev syscopyarea [131473.727187] syscopyarea vmw_balloon [131473.727189] vmw_balloon vmxnet3 [131473.727191] vmxnet3 sysfillrect [131473.727192] sysfillrect shpchp [131473.727194] shpchp parport_pc [131473.727195] parport_pc parport [131473.727196] parport sysimgblt [131473.727198] sysimgblt i2c_piix4 [131473.727199] i2c_piix4 pcspkr [131473.727201] pcspkr ac [131473.727202] ac vmw_vmci [131473.727203] vmw_vmci fb_sys_fops [131473.727205] fb_sys_fops button [131473.727206] button ext4 [131473.727208] ext4 crc16 [131473.727209] crc16 jbd2 [131473.727211] jbd2 mbcache [131473.727212] mbcache sr_mod [131473.727214] sr_mod cdrom [131473.727215] cdrom ata_generic [131473.727217] ata_generic sd_mod [131473.727218] sd_mod ata_piix [131473.727220] ata_piix ahci [131473.727221] ahci libahci [131473.727222] libahci serio_raw [131473.727224] serio_raw mptspi [131473.727225] mptspi libata [131473.727227] libata scsi_transport_spi [131473.727228] scsi_transport_spi mptscsih [131473.727230] mptscsih mptbase [131473.727231] mptbase floppy [131473.727232] floppy sg [131473.727234] sg scsi_mod [131473.727235] scsi_mod autofs4 [131473.727237] autofs4 [131473.727239] Supported: No, Unsupported modules are loaded [131473.727240] Supported: No, Unsupported modules are loaded [131473.727246] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131473.727248] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131473.727249] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131473.727250] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131473.727253] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131473.727254] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131473.727302] RIP: 0010:lnet_check_routers+0x9e/0x5e0 [lnet] [131473.727313] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131473.727314] RSP: 0018:ffffabaa436ebe20 EFLAGS: 00000287 [131473.727316] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131473.727317] ORIG_RAX: ffffffffffffff10 [131473.727317] ORIG_RAX: ffffffffffffff10 [131473.727319] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131473.727320] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131473.727321] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131473.727322] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131473.727323] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131473.727324] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131473.727326] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131473.727327] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131473.727328] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131473.727329] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131473.727332] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131473.727333] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131473.727334] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131473.727336] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131473.727337] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131473.727338] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131473.727346] Call Trace: [131473.727347] Call Trace: [131473.727364] _raw_spin_lock+0x1d/0x20 [131473.727375] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131473.727381] ? __switch_to_asm+0x34/0x70 [131473.727397] lnet_peer_discovery+0xee/0x1160 [lnet] [131473.727404] ? wait_woken+0x80/0x80 [131473.727410] ? __schedule+0x247/0x860 [131473.727412] ? wait_woken+0x80/0x80 [131473.727422] ? lnet_send_ping+0x190/0x190 [lnet] [131473.727429] kthread+0x11a/0x130 [131473.727440] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131473.727442] ? kthread_create_on_node+0x40/0x40 [131473.727445] kthread+0x11a/0x130 [131473.727447] ret_from_fork+0x35/0x40 [131473.727449] ? kthread_create_on_node+0x40/0x40 [131473.727450] Code: [131473.727452] ret_from_fork+0x35/0x40 [131473.727452] 84 3f 01 [131473.727454] Code: 66 [131473.727456] 00 [131473.727457] 2e [131473.727457] 00 [131473.727458] 0f [131473.727459] 8b [131473.727459] 1f [131473.727460] 78 [131473.727461] 84 [131473.727461] 28 [131473.727462] 00 [131473.727463] 48 85 [131473.727464] 00 [131473.727465] db [131473.727466] 00 [131473.727466] 48 [131473.727467] 00 [131473.727468] 0f [131473.727468] 00 66 [131473.727470] 44 [131473.727471] 66 [131473.727471] d8 [131473.727472] 66 [131473.727473] 45 [131473.727474] 66 90 [131473.727475] 31 [131473.727476] 66 [131473.727476] e4 [131473.727477] 66 [131473.727478] e8 [131473.727479] 66 [131473.727479] 46 [131473.727480] 66 [131473.727480] b3 [131473.727481] 90 [131473.727482] fd [131473.727483] ba [131473.727483] ff [131473.727484] 01 [131473.727485] 85 [131473.727486] 00 [131473.727486] c0 [131473.727487] 00 [131473.727487] 41 8b [131473.727489] 00 [131473.727490] 77 [131473.727490] 8b [131473.727491] 28 [131473.727492] 07 [131473.727492] 0f [131473.727493] 85 [131473.727494] 85 [131473.727494] c0 [131473.727495] 2a [131473.727496] 75 [131473.727497] 01 [131473.727497] 0a [131473.727498] 00 [131473.727499] f0 [131473.727500] 00 [131473.727500] 0f [131473.727501] 48 [131473.727502] b1 [131473.727503] 89 [131473.727503] 17 [131473.727504] ef [131473.727505] 85 [131473.727505] e8 [131473.727506] c0 [131473.727507] 82 [131473.727508] 75 [131473.727508] 6b [131473.727509] f2 [131473.727510] 00 [131473.727510] f3 [131473.727511] 00 [131473.727512] c3 [131473.727513] <48> [131473.727513] f3 [131473.727514] 85 [131473.727515] 90 [131473.727516] c0 [131473.727516] [131473.727517] 49 [131473.727518] ec [131473.727518] 89 [131473.727519] 81 [131473.727520] c7 [131473.727520] fe [131473.727521] 75 [131473.727522] 00 [131473.727523] c9 [131473.727523] 01 [131473.727524] 48 [131473.727525] 00 [131473.727526] b8 [131473.727526] 00 [131473.727527] 00 [131473.727528] 0f [131473.727529] 04 [131473.727529] 84 91 [131473.727530] 00 [131473.727531] 00 [131473.727532] 00 [131473.727533] 00 [131473.727533] 98 [131473.727534] 00 [131473.727535] 03 [131473.727535] 41 [131473.727536] 00 [131473.727537] b8 [131473.727537] 00 [131473.727538] 01 [131473.727539] c7 [131473.727540] 01 [131473.727540] 05 [131473.727541] 00 [131473.727542] 5e [131473.727542] 00 [131473.727544] b9 [131485.712572] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131485.713402] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131485.713463] Supported: No, Unsupported modules are loaded [131485.713469] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131485.713470] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131485.713473] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131485.713487] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131485.713488] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131485.713490] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131485.713491] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131485.713492] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131485.713493] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131485.713494] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131485.713497] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131485.713498] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131485.713499] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131485.713508] Call Trace: [131485.713525] _raw_spin_lock+0x1d/0x20 [131485.713545] cfs_percpt_lock+0x69/0xf0 [libcfs] [131485.713572] lnet_add_route+0x172/0xcb0 [lnet] [131485.713584] LNetCtl+0x6cf/0x11a0 [lnet] [131485.713594] ? LNetNIInit+0x66/0xb70 [lnet] [131485.713605] lnet_ioctl+0x1d3/0x250 [lnet] [131485.713616] notifier_call_chain+0x47/0x70 [131485.713619] blocking_notifier_call_chain+0x3e/0x60 [131485.713627] libcfs_ioctl+0x24e/0x470 [libcfs] [131485.713634] ? security_capable+0x47/0x60 [131485.713641] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131485.713648] do_vfs_ioctl+0x90/0x5f0 [131485.713652] SyS_ioctl+0x74/0x80 [131485.713660] do_syscall_64+0x74/0x140 [131485.713663] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131485.713666] RIP: 0033:0x7fb52efa9587 [131485.713666] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131485.713668] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131485.713669] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131485.713670] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131485.713671] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131485.713672] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131485.713673] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131501.694796] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131501.694800] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131501.694806] Modules linked in: [131501.695447] Modules linked in: [131501.695993] lnet(OEN) [131501.695994] lnet(OEN) libcfs(OEN) [131501.695996] libcfs(OEN) [131501.695998] af_packet [131501.695998] af_packet iscsi_ibft [131501.696000] iscsi_ibft iscsi_boot_sysfs [131501.696002] iscsi_boot_sysfs vmw_vsock_vmci_transport [131501.696004] vmw_vsock_vmci_transport [131501.696005] vsock [131501.696006] vsock [131501.696006] sb_edac coretemp [131501.696008] sb_edac coretemp [131501.696009] crc32_pclmul [131501.696010] crc32_pclmul crc32c_intel [131501.696011] crc32c_intel ghash_clmulni_intel [131501.696013] ghash_clmulni_intel pcbc [131501.696014] pcbc aesni_intel [131501.696016] aesni_intel vmwgfx [131501.696017] vmwgfx aes_x86_64 [131501.696018] aes_x86_64 ttm [131501.696020] ttm crypto_simd [131501.696021] crypto_simd drm_kms_helper [131501.696023] drm_kms_helper glue_helper [131501.696024] glue_helper cryptd [131501.696026] cryptd joydev [131501.696027] joydev drm [131501.696028] drm drm_panel_orientation_quirks [131501.696030] drm_panel_orientation_quirks ppdev [131501.696031] ppdev syscopyarea [131501.696033] syscopyarea vmw_balloon [131501.696034] vmw_balloon vmxnet3 [131501.696037] vmxnet3 sysfillrect [131501.696038] sysfillrect shpchp [131501.696040] shpchp parport_pc [131501.696041] parport_pc parport [131501.696042] parport sysimgblt [131501.696044] sysimgblt i2c_piix4 [131501.696045] i2c_piix4 pcspkr [131501.696046] pcspkr ac [131501.696048] ac vmw_vmci [131501.696049] vmw_vmci fb_sys_fops [131501.696050] fb_sys_fops button [131501.696052] button ext4 [131501.696053] ext4 crc16 [131501.696055] crc16 jbd2 [131501.696056] jbd2 mbcache [131501.696057] mbcache sr_mod [131501.696059] sr_mod cdrom [131501.696061] cdrom ata_generic [131501.696062] ata_generic sd_mod [131501.696063] sd_mod ata_piix [131501.696065] ata_piix ahci [131501.696066] ahci libahci [131501.696068] libahci serio_raw [131501.696069] serio_raw mptspi [131501.696070] mptspi libata [131501.696072] libata scsi_transport_spi [131501.696074] scsi_transport_spi mptscsih [131501.696075] mptscsih mptbase [131501.696076] mptbase floppy [131501.696077] floppy sg [131501.696079] sg scsi_mod [131501.696080] scsi_mod autofs4 [131501.696082] autofs4 [131501.696085] Supported: No, Unsupported modules are loaded [131501.696086] Supported: No, Unsupported modules are loaded [131501.696091] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131501.696094] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131501.696095] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131501.696095] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131501.696099] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131501.696100] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131501.696151] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131501.696162] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131501.696163] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131501.696165] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131501.696165] ORIG_RAX: ffffffffffffff10 [131501.696167] ORIG_RAX: ffffffffffffff10 [131501.696168] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131501.696169] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131501.696170] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131501.696171] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131501.696172] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131501.696174] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131501.696175] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131501.696176] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131501.696177] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131501.696178] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131501.696180] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131501.696181] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131501.696183] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131501.696184] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131501.696185] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131501.696186] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131501.696194] Call Trace: [131501.696194] Call Trace: [131501.696212] _raw_spin_lock+0x1d/0x20 [131501.696224] lnet_check_routers+0x9e/0x5e0 [lnet] [131501.696242] lnet_peer_discovery+0xee/0x1160 [lnet] [131501.696253] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131501.696258] ? __switch_to_asm+0x34/0x70 [131501.696263] ? __schedule+0x247/0x860 [131501.696268] ? wait_woken+0x80/0x80 [131501.696270] ? wait_woken+0x80/0x80 [131501.696280] ? lnet_send_ping+0x190/0x190 [lnet] [131501.696291] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131501.696300] kthread+0x11a/0x130 [131501.696303] kthread+0x11a/0x130 [131501.696305] ? kthread_create_on_node+0x40/0x40 [131501.696307] ret_from_fork+0x35/0x40 [131501.696309] ? kthread_create_on_node+0x40/0x40 [131501.696311] ret_from_fork+0x35/0x40 [131501.696312] Code: [131501.696313] Code: [131501.696314] 66 2e [131501.696316] ef 31 [131501.696317] 0f [131501.696318] c0 [131501.696319] 1f [131501.696319] eb [131501.696320] 84 [131501.696321] cc [131501.696322] 00 [131501.696322] f3 [131501.696323] 00 [131501.696324] c3 [131501.696325] 00 [131501.696325] 66 [131501.696326] 00 [131501.696327] 0f [131501.696328] 00 [131501.696328] 1f [131501.696329] 66 [131501.696330] 84 [131501.696330] 66 [131501.696331] 00 [131501.696332] 66 [131501.696332] 00 [131501.696333] 66 [131501.696334] 00 [131501.696335] 90 [131501.696335] 00 [131501.696336] 66 [131501.696337] 00 [131501.696338] 66 [131501.696338] 66 [131501.696339] 66 [131501.696340] 66 [131501.696341] 66 [131501.696341] 66 [131501.696342] 90 [131501.696343] 66 [131501.696343] ba [131501.696344] 90 [131501.696345] 01 [131501.696345] 85 [131501.696346] 00 [131501.696347] f6 [131501.696348] 00 [131501.696348] 48 [131501.696349] 00 [131501.696350] 8d [131501.696351] 8b [131501.696351] 57 [131501.696352] 07 [131501.696353] 10 [131501.696354] 85 [131501.696354] 48 [131501.696355] c0 [131501.696356] 8b [131501.696357] 75 [131501.696357] 47 [131501.696358] 0a f0 [131501.696359] 10 [131501.696360] 0f [131501.696361] 74 [131501.696362] b1 [131501.696362] 29 48 [131501.696364] 17 85 [131501.696365] 39 c2 [131501.696366] c0 75 [131501.696368] 74 [131501.696368] f2 [131501.696369] 14 [131501.696370] f3 [131501.696370] 3b [131501.696371] c3 [131501.696372] 70 [131501.696373] f3 [131501.696373] 28 [131501.696374] 90 [131501.696375] 75 07 [131501.696376] ec [131501.696377] [131501.696378] 81 [131501.696379] 11 [131501.696380] fe [131501.696380] 39 [131501.696381] 00 [131501.696382] 70 28 [131501.696383] 01 00 [131501.696384] 74 0c [131501.696385] 00 0f [131501.696387] 48 [131501.696387] 84 [131501.696388] 8b [131501.696389] 91 [131501.696390] 00 [131501.696390] 00 [131501.696391] 48 [131501.696392] 00 [131501.696393] 39 [131501.696393] 00 [131501.696394] c2 [131501.696395] 41 [131501.696395] 75 [131501.696396] b8 [131501.696397] f3 [131501.696398] 01 [131501.696398] 31 [131501.696399] 01 [131501.696400] c0 [131501.696401] 00 [131501.696401] f3 [131501.696402] 00 [131501.696403] c3 [131501.696403] b9 [131501.696404] 48 [131501.696405] 8b [131513.681451] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131513.682280] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131513.682339] Supported: No, Unsupported modules are loaded [131513.682345] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131513.682346] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131513.682349] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131513.682362] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131513.682363] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131513.682366] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131513.682368] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131513.682369] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131513.682370] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131513.682370] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131513.682374] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131513.682375] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131513.682376] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131513.682385] Call Trace: [131513.682402] _raw_spin_lock+0x1d/0x20 [131513.682422] cfs_percpt_lock+0x69/0xf0 [libcfs] [131513.682453] lnet_add_route+0x172/0xcb0 [lnet] [131513.682465] LNetCtl+0x6cf/0x11a0 [lnet] [131513.682475] ? LNetNIInit+0x66/0xb70 [lnet] [131513.682486] lnet_ioctl+0x1d3/0x250 [lnet] [131513.682493] notifier_call_chain+0x47/0x70 [131513.682496] blocking_notifier_call_chain+0x3e/0x60 [131513.682504] libcfs_ioctl+0x24e/0x470 [libcfs] [131513.682514] ? security_capable+0x47/0x60 [131513.682521] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131513.682528] do_vfs_ioctl+0x90/0x5f0 [131513.682532] SyS_ioctl+0x74/0x80 [131513.682539] do_syscall_64+0x74/0x140 [131513.682543] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131513.682545] RIP: 0033:0x7fb52efa9587 [131513.682546] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131513.682548] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131513.682549] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131513.682550] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131513.682550] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131513.682551] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131513.682553] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131529.676589] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131529.676592] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131529.676598] Modules linked in: [131529.677202] Modules linked in: lnet(OEN) [131529.677699] lnet(OEN) libcfs(OEN) [131529.677701] libcfs(OEN) af_packet [131529.677703] af_packet iscsi_ibft [131529.677705] iscsi_ibft iscsi_boot_sysfs [131529.677707] iscsi_boot_sysfs vmw_vsock_vmci_transport [131529.677709] vmw_vsock_vmci_transport vsock [131529.677710] vsock sb_edac [131529.677712] sb_edac coretemp [131529.677713] coretemp crc32_pclmul [131529.677715] crc32_pclmul crc32c_intel [131529.677716] crc32c_intel ghash_clmulni_intel [131529.677718] ghash_clmulni_intel pcbc [131529.677719] pcbc aesni_intel [131529.677721] aesni_intel vmwgfx [131529.677722] vmwgfx aes_x86_64 [131529.677723] aes_x86_64 ttm [131529.677724] ttm crypto_simd [131529.677726] crypto_simd drm_kms_helper [131529.677727] drm_kms_helper glue_helper [131529.677729] glue_helper cryptd [131529.677730] cryptd joydev [131529.677732] joydev drm [131529.677733] drm drm_panel_orientation_quirks [131529.677735] drm_panel_orientation_quirks ppdev [131529.677736] ppdev syscopyarea [131529.677737] syscopyarea vmw_balloon [131529.677739] vmw_balloon vmxnet3 [131529.677741] vmxnet3 sysfillrect [131529.677743] sysfillrect shpchp [131529.677744] shpchp parport_pc [131529.677745] parport_pc parport [131529.677747] parport sysimgblt [131529.677748] sysimgblt i2c_piix4 [131529.677749] i2c_piix4 pcspkr [131529.677751] pcspkr ac [131529.677753] ac vmw_vmci [131529.677754] vmw_vmci fb_sys_fops [131529.677755] fb_sys_fops button [131529.677757] button ext4 [131529.677758] ext4 crc16 [131529.677760] crc16 jbd2 [131529.677761] jbd2 mbcache [131529.677762] mbcache sr_mod [131529.677764] sr_mod cdrom [131529.677766] cdrom ata_generic [131529.677767] ata_generic sd_mod [131529.677769] sd_mod ata_piix [131529.677770] ata_piix ahci [131529.677772] ahci libahci [131529.677773] libahci serio_raw [131529.677775] serio_raw mptspi [131529.677776] mptspi libata [131529.677778] libata scsi_transport_spi [131529.677779] scsi_transport_spi mptscsih [131529.677780] mptscsih mptbase [131529.677782] mptbase floppy [131529.677783] floppy sg [131529.677785] sg scsi_mod [131529.677786] scsi_mod autofs4 [131529.677787] autofs4 [131529.677791] Supported: No, Unsupported modules are loaded [131529.677791] Supported: No, Unsupported modules are loaded [131529.677797] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131529.677800] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131529.677801] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131529.677802] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131529.677805] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131529.677806] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131529.677855] RIP: 0010:lnet_get_next_peer_net_locked+0x46/0x50 [lnet] [131529.677865] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131529.677873] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131529.677875] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000287 [131529.677877] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131529.677878] ORIG_RAX: ffffffffffffff10 [131529.677879] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131529.677880] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131529.677881] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131529.677882] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131529.677883] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131529.677884] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131529.677886] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131529.677886] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131529.677888] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131529.677889] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131529.677891] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131529.677892] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131529.677893] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131529.677895] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131529.677896] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131529.677903] Call Trace: [131529.677903] Call Trace: [131529.677920] _raw_spin_lock+0x1d/0x20 [131529.677932] lnet_check_routers+0x9e/0x5e0 [lnet] [131529.677950] lnet_peer_discovery+0xee/0x1160 [lnet] [131529.677961] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131529.677966] ? __switch_to_asm+0x34/0x70 [131529.677971] ? __schedule+0x247/0x860 [131529.677976] ? wait_woken+0x80/0x80 [131529.677978] ? wait_woken+0x80/0x80 [131529.677988] ? lnet_send_ping+0x190/0x190 [lnet] [131529.677999] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131529.678007] kthread+0x11a/0x130 [131529.678010] kthread+0x11a/0x130 [131529.678013] ? kthread_create_on_node+0x40/0x40 [131529.678015] ? kthread_create_on_node+0x40/0x40 [131529.678017] ret_from_fork+0x35/0x40 [131529.678018] ret_from_fork+0x35/0x40 [131529.678021] Code: [131529.678021] Code: [131529.678023] eb [131529.678024] 66 [131529.678025] 11 [131529.678026] 2e [131529.678027] 39 [131529.678027] 0f [131529.678028] 70 [131529.678029] 1f [131529.678030] 28 [131529.678030] 84 [131529.678031] 74 [131529.678032] 00 [131529.678033] 0c [131529.678033] 00 [131529.678034] 48 [131529.678035] 00 [131529.678036] 8b [131529.678036] 00 [131529.678037] 00 [131529.678038] 00 66 [131529.678039] 48 [131529.678040] 66 [131529.678040] 39 [131529.678041] 66 [131529.678042] c2 [131529.678043] 66 [131529.678043] 75 [131529.678044] 90 [131529.678045] f3 [131529.678046] 66 [131529.678046] 31 [131529.678047] 66 [131529.678047] c0 [131529.678048] 66 [131529.678049] f3 [131529.678050] 66 [131529.678050] c3 [131529.678051] 90 [131529.678052] 48 [131529.678053] ba [131529.678053] 8b [131529.678054] 01 [131529.678055] 00 [131529.678055] 00 [131529.678056] 48 [131529.678057] 00 [131529.678057] 39 [131529.678058] 00 [131529.678059] d0 [131529.678060] 8b [131529.678060] 75 [131529.678061] 07 [131529.678061] f6 [131529.678062] 85 [131529.678063] 48 [131529.678064] c0 [131529.678064] 8b [131529.678065] 75 [131529.678066] 47 [131529.678067] 0a [131529.678067] 10 48 39 [131529.678069] f0 [131529.678070] c2 [131529.678070] 0f [131529.678071] ba [131529.678072] b1 [131529.678073] 00 [131529.678073] 17 [131529.678074] 00 [131529.678075] 85 c0 75 [131529.678077] 00 [131529.678077] f2 [131529.678078] 00 [131529.678079] f3 [131529.678079] 48 [131529.678080] c3 [131529.678081] 0f [131529.678082] f3 [131529.678082] 44 [131529.678083] 90 [131529.678084] c2 [131529.678085] [131529.678085] 66 [131529.678087] ec [131529.678087] 0f [131529.678088] 81 [131529.678089] 1f [131529.678089] fe [131529.678090] 84 [131529.678091] 00 [131529.678092] 00 [131529.678092] 01 [131529.678093] 00 [131529.678094] 00 [131529.678095] 00 [131529.678095] 00 [131529.678096] 00 [131529.678097] 0f [131529.678097] 00 [131529.678098] 84 [131529.678099] 66 [131529.678099] 91 [131529.678100] 66 [131529.678101] 00 [131529.678102] 66 [131529.678102] 00 [131529.678103] 66 [131529.678104] 00 [131529.678104] 90 [131529.678105] 41 [131529.678106] 48 [131529.678106] b8 [131529.678107] 85 [131529.678108] 01 [131529.678109] d2 [131529.678109] 01 [131529.678110] 74 [131529.678111] 00 [131529.678112] 28 [131529.678112] 00 [131529.678113] 48 [131529.678113] b9 [131541.690514] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131541.691311] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131541.691373] Supported: No, Unsupported modules are loaded [131541.691379] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131541.691381] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131541.691383] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131541.691397] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131541.691399] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131541.691401] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131541.691402] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131541.691403] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131541.691403] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131541.691404] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131541.691407] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131541.691408] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131541.691409] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131541.691419] Call Trace: [131541.691437] _raw_spin_lock+0x1d/0x20 [131541.691456] cfs_percpt_lock+0x69/0xf0 [libcfs] [131541.691488] lnet_add_route+0x172/0xcb0 [lnet] [131541.691500] LNetCtl+0x6cf/0x11a0 [lnet] [131541.691510] ? LNetNIInit+0x66/0xb70 [lnet] [131541.691522] lnet_ioctl+0x1d3/0x250 [lnet] [131541.691530] notifier_call_chain+0x47/0x70 [131541.691533] blocking_notifier_call_chain+0x3e/0x60 [131541.691540] libcfs_ioctl+0x24e/0x470 [libcfs] [131541.691548] ? security_capable+0x47/0x60 [131541.691554] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131541.691561] do_vfs_ioctl+0x90/0x5f0 [131541.691565] SyS_ioctl+0x74/0x80 [131541.691572] do_syscall_64+0x74/0x140 [131541.691576] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131541.691578] RIP: 0033:0x7fb52efa9587 [131541.691579] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131541.691581] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131541.691582] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131541.691583] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131541.691584] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131541.691585] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131541.691586] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131557.713992] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131557.713995] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131557.714001] Modules linked in: [131557.714601] Modules linked in: lnet(OEN) [131557.715082] lnet(OEN) [131557.715083] libcfs(OEN) [131557.715084] libcfs(OEN) [131557.715085] af_packet [131557.715085] af_packet iscsi_ibft [131557.715087] iscsi_ibft iscsi_boot_sysfs [131557.715089] iscsi_boot_sysfs vmw_vsock_vmci_transport [131557.715091] vmw_vsock_vmci_transport vsock [131557.715092] vsock sb_edac [131557.715094] sb_edac coretemp [131557.715095] coretemp [131557.715096] crc32_pclmul [131557.715097] crc32_pclmul [131557.715097] crc32c_intel [131557.715098] crc32c_intel [131557.715099] ghash_clmulni_intel [131557.715099] ghash_clmulni_intel [131557.715100] pcbc [131557.715101] pcbc aesni_intel [131557.715102] aesni_intel vmwgfx [131557.715103] vmwgfx aes_x86_64 [131557.715104] aes_x86_64 ttm [131557.715106] ttm crypto_simd [131557.715107] crypto_simd drm_kms_helper [131557.715108] drm_kms_helper glue_helper [131557.715110] glue_helper cryptd [131557.715111] cryptd joydev [131557.715112] joydev drm [131557.715114] drm drm_panel_orientation_quirks [131557.715115] drm_panel_orientation_quirks ppdev [131557.715116] ppdev syscopyarea [131557.715118] syscopyarea vmw_balloon [131557.715120] vmw_balloon vmxnet3 [131557.715121] vmxnet3 sysfillrect [131557.715123] sysfillrect shpchp [131557.715124] shpchp parport_pc [131557.715125] parport_pc parport [131557.715127] parport sysimgblt [131557.715128] sysimgblt i2c_piix4 [131557.715129] i2c_piix4 pcspkr [131557.715130] pcspkr ac [131557.715132] ac vmw_vmci [131557.715133] vmw_vmci fb_sys_fops [131557.715134] fb_sys_fops [131557.715135] button [131557.715136] button ext4 [131557.715137] ext4 crc16 [131557.715138] crc16 jbd2 [131557.715140] jbd2 mbcache [131557.715141] mbcache sr_mod [131557.715142] sr_mod cdrom [131557.715144] cdrom ata_generic [131557.715145] ata_generic sd_mod [131557.715146] sd_mod ata_piix [131557.715148] ata_piix ahci [131557.715149] ahci libahci [131557.715150] libahci serio_raw [131557.715152] serio_raw mptspi [131557.715153] mptspi libata [131557.715155] libata scsi_transport_spi [131557.715157] scsi_transport_spi mptscsih [131557.715158] mptscsih mptbase [131557.715159] mptbase floppy [131557.715161] floppy [131557.715161] sg [131557.715162] sg [131557.715163] scsi_mod [131557.715164] scsi_mod [131557.715164] autofs4 [131557.715165] autofs4 [131557.715167] Supported: No, Unsupported modules are loaded [131557.715168] Supported: No, Unsupported modules are loaded [131557.715173] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131557.715175] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131557.715176] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131557.715177] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131557.715180] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131557.715181] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131557.715230] RIP: 0010:lnet_check_routers+0x8a/0x5e0 [lnet] [131557.715241] RIP: 0010:native_queued_spin_lock_slowpath+0x1f/0x190 [131557.715243] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131557.715244] RSP: 0018:ffffabaa436ebe20 EFLAGS: 00000246 [131557.715245] ORIG_RAX: ffffffffffffff10 [131557.715246] ORIG_RAX: ffffffffffffff10 [131557.715247] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131557.715248] RAX: 0000000000000000 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131557.715250] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131557.715251] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131557.715252] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131557.715253] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131557.715254] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131557.715255] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131557.715256] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131557.715257] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131557.715260] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131557.715261] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131557.715262] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131557.715263] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131557.715265] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131557.715266] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131557.715278] Call Trace: [131557.715279] Call Trace: [131557.715295] _raw_spin_lock+0x1d/0x20 [131557.715306] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131557.715311] ? __switch_to_asm+0x34/0x70 [131557.715326] lnet_peer_discovery+0xee/0x1160 [lnet] [131557.715337] ? wait_woken+0x80/0x80 [131557.715342] ? __schedule+0x247/0x860 [131557.715345] ? wait_woken+0x80/0x80 [131557.715355] ? lnet_send_ping+0x190/0x190 [lnet] [131557.715363] kthread+0x11a/0x130 [131557.715374] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131557.715376] kthread+0x11a/0x130 [131557.715378] ? kthread_create_on_node+0x40/0x40 [131557.715380] ret_from_fork+0x35/0x40 [131557.715382] ? kthread_create_on_node+0x40/0x40 [131557.715384] ret_from_fork+0x35/0x40 [131557.715385] Code: [131557.715386] Code: [131557.715388] 99 [131557.715388] ff [131557.715389] 98 [131557.715390] 66 [131557.715391] c8 49 [131557.715392] 66 2e [131557.715393] 8d [131557.715394] 0f [131557.715394] 6d [131557.715395] 1f [131557.715396] d0 [131557.715397] 84 [131557.715397] 49 [131557.715398] 00 [131557.715399] 89 [131557.715400] 00 [131557.715400] c6 41 [131557.715401] 00 00 [131557.715402] 8b [131557.715403] 00 [131557.715404] 75 [131557.715404] 66 [131557.715405] 18 eb [131557.715406] 66 66 [131557.715407] 27 [131557.715408] 66 [131557.715409] 48 39 [131557.715410] 90 66 [131557.715411] c3 [131557.715412] 66 [131557.715412] 0f 84 [131557.715413] 66 66 [131557.715414] 3f [131557.715415] 90 [131557.715416] 01 [131557.715417] ba [131557.715417] 00 [131557.715418] 01 [131557.715419] 00 [131557.715420] 00 [131557.715420] 8b [131557.715421] 00 [131557.715421] 78 28 [131557.715423] 00 8b [131557.715424] 48 85 [131557.715425] 07 85 [131557.715426] db 48 [131557.715427] c0 75 [131557.715428] 0f [131557.715429] 0a [131557.715430] 44 d8 [131557.715431] f0 0f [131557.715432] 45 [131557.715433] b1 [131557.715433] 31 [131557.715434] 17 [131557.715435] e4 e8 [131557.715436] 85 c0 [131557.715437] 46 [131557.715438] 75 [131557.715438] b3 [131557.715439] f2 [131557.715440] fd [131557.715441] f3 [131557.715441] ff <85> [131557.715443] c3 [131557.715444] c0 [131557.715444] 90 [131557.715445] 41 8b [131557.715446] eb ec [131557.715447] 77 [131557.715448] 81 [131557.715449] 28 0f [131557.715450] fe 00 [131557.715451] 85 [131557.715452] 01 [131557.715452] 2a [131557.715453] 00 [131557.715454] 01 [131557.715454] 00 [131557.715455] 00 [131557.715456] 0f [131557.715456] 00 [131557.715457] 84 [131557.715458] 48 89 [131557.715459] 91 00 [131557.715460] ef [131557.715461] 00 [131557.715461] e8 [131557.715462] 00 [131557.715462] 82 6b [131557.715464] 41 b8 [131557.715465] 00 00 [131557.715466] 01 01 [131557.715467] 48 [131557.715468] 00 [131569.713121] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131569.713876] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131569.713937] Supported: No, Unsupported modules are loaded [131569.713943] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131569.713944] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131569.713947] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131569.713960] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131569.713961] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131569.713964] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131569.713965] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131569.713966] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131569.713966] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131569.713967] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131569.713970] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131569.713971] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131569.713972] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131569.713980] Call Trace: [131569.713998] _raw_spin_lock+0x1d/0x20 [131569.714017] cfs_percpt_lock+0x69/0xf0 [libcfs] [131569.714047] lnet_add_route+0x172/0xcb0 [lnet] [131569.714059] LNetCtl+0x6cf/0x11a0 [lnet] [131569.714069] ? LNetNIInit+0x66/0xb70 [lnet] [131569.714080] lnet_ioctl+0x1d3/0x250 [lnet] [131569.714089] notifier_call_chain+0x47/0x70 [131569.714091] blocking_notifier_call_chain+0x3e/0x60 [131569.714099] libcfs_ioctl+0x24e/0x470 [libcfs] [131569.714106] ? security_capable+0x47/0x60 [131569.714113] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131569.714118] do_vfs_ioctl+0x90/0x5f0 [131569.714122] SyS_ioctl+0x74/0x80 [131569.714128] do_syscall_64+0x74/0x140 [131569.714131] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131569.714134] RIP: 0033:0x7fb52efa9587 [131569.714135] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131569.714137] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131569.714137] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131569.714138] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131569.714139] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131569.714140] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131569.714141] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131585.711960] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131585.711963] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131585.711970] Modules linked in: [131585.712557] Modules linked in: lnet(OEN) [131585.713049] lnet(OEN) [131585.713050] libcfs(OEN) af_packet [131585.713051] libcfs(OEN) af_packet [131585.713053] iscsi_ibft [131585.713054] iscsi_ibft [131585.713055] iscsi_boot_sysfs [131585.713056] iscsi_boot_sysfs vmw_vsock_vmci_transport [131585.713057] vmw_vsock_vmci_transport vsock [131585.713058] vsock sb_edac [131585.713060] sb_edac coretemp [131585.713061] coretemp crc32_pclmul [131585.713063] crc32_pclmul crc32c_intel [131585.713064] crc32c_intel ghash_clmulni_intel [131585.713065] ghash_clmulni_intel pcbc [131585.713067] pcbc aesni_intel [131585.713068] aesni_intel vmwgfx [131585.713069] vmwgfx aes_x86_64 [131585.713070] aes_x86_64 ttm [131585.713072] ttm crypto_simd [131585.713073] crypto_simd drm_kms_helper [131585.713075] drm_kms_helper [131585.713075] glue_helper [131585.713076] glue_helper [131585.713077] cryptd [131585.713078] cryptd [131585.713078] joydev [131585.713079] joydev [131585.713079] drm drm_panel_orientation_quirks [131585.713081] drm drm_panel_orientation_quirks [131585.713082] ppdev [131585.713083] ppdev syscopyarea [131585.713084] syscopyarea vmw_balloon [131585.713086] vmw_balloon vmxnet3 [131585.713087] vmxnet3 sysfillrect [131585.713089] sysfillrect shpchp [131585.713090] shpchp parport_pc [131585.713091] parport_pc parport [131585.713093] parport sysimgblt [131585.713094] sysimgblt i2c_piix4 [131585.713095] i2c_piix4 [131585.713096] pcspkr [131585.713097] pcspkr ac [131585.713098] ac vmw_vmci [131585.713099] vmw_vmci fb_sys_fops [131585.713100] fb_sys_fops button [131585.713102] button ext4 [131585.713103] ext4 crc16 [131585.713105] crc16 jbd2 [131585.713106] jbd2 mbcache [131585.713107] mbcache sr_mod [131585.713109] sr_mod cdrom [131585.713110] cdrom ata_generic [131585.713112] ata_generic sd_mod [131585.713113] sd_mod ata_piix [131585.713114] ata_piix ahci [131585.713116] ahci libahci [131585.713117] libahci serio_raw [131585.713119] serio_raw mptspi [131585.713120] mptspi libata [131585.713121] libata scsi_transport_spi [131585.713123] scsi_transport_spi mptscsih [131585.713124] mptscsih mptbase [131585.713125] mptbase floppy [131585.713127] floppy sg [131585.713128] sg scsi_mod [131585.713129] scsi_mod autofs4 [131585.713131] autofs4 [131585.713133] Supported: No, Unsupported modules are loaded [131585.713134] Supported: No, Unsupported modules are loaded [131585.713140] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131585.713142] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131585.713144] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131585.713145] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131585.713148] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131585.713149] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131585.713198] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131585.713209] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131585.713210] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131585.713212] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 [131585.713215] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131585.713216] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131585.713217] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131585.713218] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131585.713219] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131585.713220] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131585.713221] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131585.713222] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131585.713223] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131585.713224] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131585.713227] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131585.713228] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131585.713229] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131585.713230] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131585.713232] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131585.713233] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131585.713240] Call Trace: [131585.713241] Call Trace: [131585.713257] _raw_spin_lock+0x1d/0x20 [131585.713269] lnet_check_routers+0x9e/0x5e0 [lnet] [131585.713287] lnet_peer_discovery+0xee/0x1160 [lnet] [131585.713298] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131585.713303] ? __switch_to_asm+0x34/0x70 [131585.713308] ? __schedule+0x247/0x860 [131585.713314] ? wait_woken+0x80/0x80 [131585.713317] ? wait_woken+0x80/0x80 [131585.713327] ? lnet_send_ping+0x190/0x190 [lnet] [131585.713338] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131585.713344] kthread+0x11a/0x130 [131585.713346] kthread+0x11a/0x130 [131585.713349] ? kthread_create_on_node+0x40/0x40 [131585.713351] ? kthread_create_on_node+0x40/0x40 [131585.713353] ret_from_fork+0x35/0x40 [131585.713355] ret_from_fork+0x35/0x40 [131585.713358] Code: [131585.713358] Code: 66 [131585.713360] ef 31 [131585.713362] 2e [131585.713363] c0 [131585.713363] 0f [131585.713364] eb [131585.713364] 1f [131585.713365] cc [131585.713366] 84 00 [131585.713367] f3 [131585.713368] 00 [131585.713368] c3 [131585.713369] 00 [131585.713370] 66 [131585.713371] 00 [131585.713371] 0f [131585.713372] 00 [131585.713372] 1f 84 [131585.713374] 66 [131585.713374] 00 [131585.713375] 66 66 [131585.713376] 00 [131585.713377] 66 [131585.713377] 00 [131585.713378] 90 [131585.713379] 00 [131585.713380] 66 [131585.713380] 00 [131585.713381] 66 [131585.713382] 66 66 [131585.713383] 66 [131585.713383] 66 [131585.713384] 66 [131585.713385] 66 [131585.713385] 90 ba [131585.713386] 90 85 [131585.713388] 01 [131585.713388] f6 [131585.713389] 00 [131585.713390] 48 [131585.713390] 00 [131585.713391] 8d [131585.713392] 00 8b [131585.713393] 57 [131585.713394] 07 [131585.713394] 10 [131585.713395] 85 [131585.713395] 48 [131585.713396] c0 [131585.713397] 8b [131585.713397] 75 [131585.713398] 47 [131585.713399] 0a [131585.713399] 10 [131585.713400] f0 [131585.713401] 74 [131585.713402] 0f [131585.713402] 29 [131585.713403] b1 [131585.713403] 48 [131585.713404] 17 [131585.713405] 39 [131585.713406] 85 [131585.713406] c2 [131585.713407] c0 [131585.713407] 74 [131585.713408] 75 [131585.713409] 14 [131585.713410] f2 [131585.713410] 3b [131585.713411] f3 [131585.713411] 70 [131585.713412] c3 [131585.713413] 28 [131585.713413] f3 [131585.713414] 75 [131585.713415] 90 [131585.713415] 07 [131585.713416] [131585.713417] [131585.713418] ec [131585.713419] 11 [131585.713419] 81 [131585.713420] 39 [131585.713421] fe [131585.713421] 70 [131585.713422] 00 [131585.713423] 28 [131585.713423] 01 [131585.713424] 74 [131585.713425] 00 [131585.713425] 0c [131585.713426] 00 [131585.713426] 48 [131585.713427] 0f [131585.713428] 8b [131585.713429] 84 [131585.713429] 00 [131585.713430] 91 [131585.713430] 48 [131585.713431] 00 [131585.713432] 39 [131585.713432] 00 [131585.713433] c2 [131585.713434] 00 [131585.713434] 75 [131585.713435] 41 [131585.713436] f3 [131585.713436] b8 [131585.713437] 31 [131585.713438] 01 [131585.713438] c0 [131585.713439] 01 [131585.713439] f3 c3 [131585.713441] 00 [131585.713441] 48 [131585.713442] 00 [131585.713443] 8b [131585.713443] b9 [131597.711085] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131597.711838] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131597.711900] Supported: No, Unsupported modules are loaded [131597.711906] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131597.711907] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131597.711910] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131597.711924] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131597.711925] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131597.711927] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131597.711928] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131597.711929] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131597.711930] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131597.711931] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131597.711933] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131597.711934] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131597.711935] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131597.711944] Call Trace: [131597.711962] _raw_spin_lock+0x1d/0x20 [131597.711983] cfs_percpt_lock+0x69/0xf0 [libcfs] [131597.712015] lnet_add_route+0x172/0xcb0 [lnet] [131597.712027] LNetCtl+0x6cf/0x11a0 [lnet] [131597.712037] ? LNetNIInit+0x66/0xb70 [lnet] [131597.712048] lnet_ioctl+0x1d3/0x250 [lnet] [131597.712056] notifier_call_chain+0x47/0x70 [131597.712059] blocking_notifier_call_chain+0x3e/0x60 [131597.712067] libcfs_ioctl+0x24e/0x470 [libcfs] [131597.712075] ? security_capable+0x47/0x60 [131597.712081] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131597.712086] do_vfs_ioctl+0x90/0x5f0 [131597.712090] SyS_ioctl+0x74/0x80 [131597.712096] do_syscall_64+0x74/0x140 [131597.712099] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131597.712101] RIP: 0033:0x7fb52efa9587 [131597.712102] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131597.712104] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131597.712105] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131597.712106] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131597.712107] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131597.712108] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131597.712109] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131613.712112] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [lnet_discovery:15010] [131613.712115] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [monitor_thread:15011] [131613.712122] Modules linked in: [131613.712808] Modules linked in: [131613.713297] lnet(OEN) [131613.713298] lnet(OEN) [131613.713300] libcfs(OEN) [131613.713300] libcfs(OEN) af_packet [131613.713302] af_packet iscsi_ibft [131613.713304] iscsi_ibft iscsi_boot_sysfs [131613.713305] iscsi_boot_sysfs [131613.713307] vmw_vsock_vmci_transport [131613.713307] vmw_vsock_vmci_transport [131613.713308] vsock [131613.713309] vsock [131613.713310] sb_edac [131613.713311] sb_edac coretemp [131613.713312] coretemp crc32_pclmul [131613.713314] crc32_pclmul crc32c_intel [131613.713315] crc32c_intel ghash_clmulni_intel [131613.713317] ghash_clmulni_intel pcbc [131613.713318] pcbc aesni_intel [131613.713320] aesni_intel vmwgfx [131613.713321] vmwgfx aes_x86_64 [131613.713322] aes_x86_64 ttm [131613.713324] ttm crypto_simd [131613.713325] crypto_simd drm_kms_helper [131613.713326] drm_kms_helper glue_helper [131613.713328] glue_helper [131613.713329] cryptd [131613.713330] cryptd [131613.713330] joydev [131613.713331] joydev [131613.713332] drm [131613.713333] drm [131613.713334] drm_panel_orientation_quirks [131613.713335] drm_panel_orientation_quirks [131613.713336] ppdev [131613.713336] ppdev syscopyarea [131613.713338] syscopyarea vmw_balloon [131613.713339] vmw_balloon vmxnet3 [131613.713341] vmxnet3 sysfillrect [131613.713343] sysfillrect shpchp [131613.713344] shpchp parport_pc [131613.713346] parport_pc parport [131613.713347] parport sysimgblt [131613.713348] sysimgblt i2c_piix4 [131613.713350] i2c_piix4 pcspkr [131613.713351] pcspkr ac [131613.713352] ac vmw_vmci [131613.713354] vmw_vmci fb_sys_fops [131613.713355] fb_sys_fops button [131613.713357] button ext4 [131613.713359] ext4 crc16 [131613.713360] crc16 jbd2 [131613.713361] jbd2 mbcache [131613.713363] mbcache sr_mod [131613.713364] sr_mod cdrom [131613.713366] cdrom ata_generic [131613.713368] ata_generic sd_mod [131613.713369] sd_mod ata_piix [131613.713371] ata_piix ahci [131613.713372] ahci libahci [131613.713373] libahci serio_raw [131613.713375] serio_raw mptspi [131613.713376] mptspi libata [131613.713378] libata scsi_transport_spi [131613.713379] scsi_transport_spi mptscsih [131613.713380] mptscsih mptbase [131613.713382] mptbase floppy [131613.713383] floppy sg [131613.713384] sg scsi_mod [131613.713386] scsi_mod autofs4 [131613.713388] autofs4 [131613.713390] Supported: No, Unsupported modules are loaded [131613.713391] Supported: No, Unsupported modules are loaded [131613.713397] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131613.713400] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131613.713400] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131613.713403] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131613.713404] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131613.713405] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131613.713419] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131613.713421] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131613.713465] RIP: 0010:lnet_islocalnet_locked+0xc/0x40 [lnet] [131613.713466] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131613.713467] ORIG_RAX: ffffffffffffff10 [131613.713469] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131613.713469] ORIG_RAX: ffffffffffffff10 [131613.713470] RAX: ffff9ea037045d80 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131613.713471] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131613.713473] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131613.713474] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131613.713475] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131613.713476] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131613.713478] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131613.713479] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131613.713480] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131613.713481] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131613.713482] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131613.713484] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131613.713485] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131613.713486] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131613.713487] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131613.713494] Call Trace: [131613.713494] Call Trace: [131613.713514] lnet_check_routers+0x8a/0x5e0 [lnet] [131613.713524] _raw_spin_lock+0x1d/0x20 [131613.713542] lnet_peer_discovery+0xee/0x1160 [lnet] [131613.713553] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131613.713558] ? __switch_to_asm+0x34/0x70 [131613.713563] ? __schedule+0x247/0x860 [131613.713568] ? wait_woken+0x80/0x80 [131613.713570] ? wait_woken+0x80/0x80 [131613.713580] ? lnet_send_ping+0x190/0x190 [lnet] [131613.713591] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131613.713600] kthread+0x11a/0x130 [131613.713603] kthread+0x11a/0x130 [131613.713606] ? kthread_create_on_node+0x40/0x40 [131613.713608] ? kthread_create_on_node+0x40/0x40 [131613.713610] ret_from_fork+0x35/0x40 [131613.713612] ret_from_fork+0x35/0x40 [131613.713614] Code: [131613.713615] Code: [131613.713617] 66 [131613.713617] 89 e6 [131613.713619] 2e [131613.713619] 89 44 [131613.713621] 0f 1f [131613.713622] 24 [131613.713623] 84 00 [131613.713624] 04 e8 [131613.713625] 00 00 [131613.713626] f4 aa [131613.713627] 00 00 [131613.713628] f9 ff [131613.713630] 66 66 [131613.713631] 8b 44 [131613.713632] 66 66 [131613.713633] 24 04 [131613.713634] 90 66 [131613.713636] 48 [131613.713637] 66 [131613.713637] 83 [131613.713638] 66 [131613.713639] c4 08 [131613.713640] 66 90 [131613.713641] 5b [131613.713642] ba [131613.713643] 5d [131613.713644] 01 00 00 [131613.713646] 41 5c [131613.713647] 00 8b [131613.713648] c3 31 [131613.713650] 07 85 [131613.713651] c0 c3 [131613.713652] c0 75 [131613.713653] 0f 1f [131613.713654] 0a f0 [131613.713655] 40 00 [131613.713657] 0f b1 [131613.713658] 66 66 [131613.713659] 17 85 [131613.713660] 66 66 [131613.713661] c0 75 [131613.713663] 90 48 [131613.713664] f2 f3 [131613.713665] 8b 05 [131613.713666] c3 f3 [131613.713667] c4 38 [131613.713668] 90 [131613.713670] 05 00 [131613.713671] ec 81 [131613.713672] <48> 3d [131613.713673] fe 00 [131613.713674] a0 90 [131613.713675] 01 00 [131613.713677] 7a c0 [131613.713678] 00 0f [131613.713679] 75 0d [131613.713680] 84 91 [131613.713681] eb 1a [131613.713682] 00 00 [131613.713684] 48 [131613.713684] 00 [131613.713685] 8b [131613.713686] 41 b8 [131613.713687] 00 48 [131613.713688] 01 01 [131613.713689] 3d a0 [131613.713690] 00 00 [131613.713692] 90 7a [131613.713693] b9 [131613.713694] c0 [131613.713695] 74 0f [131625.712314] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [lnetctl:15141] [131625.713087] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131625.713147] Supported: No, Unsupported modules are loaded [131625.713153] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131625.713154] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131625.713158] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131625.713171] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131625.713173] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131625.713175] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131625.713176] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131625.713177] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131625.713178] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131625.713179] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131625.713182] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131625.713183] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131625.713184] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131625.713193] Call Trace: [131625.713210] _raw_spin_lock+0x1d/0x20 [131625.713230] cfs_percpt_lock+0x69/0xf0 [libcfs] [131625.713262] lnet_add_route+0x172/0xcb0 [lnet] [131625.713274] LNetCtl+0x6cf/0x11a0 [lnet] [131625.713284] ? LNetNIInit+0x66/0xb70 [lnet] [131625.713295] lnet_ioctl+0x1d3/0x250 [lnet] [131625.713305] notifier_call_chain+0x47/0x70 [131625.713308] blocking_notifier_call_chain+0x3e/0x60 [131625.713316] libcfs_ioctl+0x24e/0x470 [libcfs] [131625.713323] ? security_capable+0x47/0x60 [131625.713329] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131625.713337] do_vfs_ioctl+0x90/0x5f0 [131625.713341] SyS_ioctl+0x74/0x80 [131625.713349] do_syscall_64+0x74/0x140 [131625.713352] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131625.713354] RIP: 0033:0x7fb52efa9587 [131625.713355] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131625.713357] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131625.713358] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131625.713359] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131625.713360] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131625.713360] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131625.713362] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131629.656019] INFO: rcu_sched self-detected stall on CPU [131629.656024] INFO: rcu_sched self-detected stall on CPU [131629.657111] 4-...: (330021 ticks this GP) idle=62a/140000000000001/0 softirq=126578/126578 fqs=164993 [131629.657114] 5-...: (330020 ticks this GP) idle=4fa/140000000000001/0 softirq=3607039/3607039 fqs=164993 [131629.657116] [131629.658079] [131629.659117] (t=330023 jiffies g=1359077 c=1359076 q=65144) [131629.659635] (t=330023 jiffies g=1359077 c=1359076 q=65144) [131629.660178] Sending NMI from CPU 5 to CPUs 4: [131629.661182] NMI backtrace for cpu 4 [131629.661183] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131629.661184] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131629.661185] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131629.661185] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131629.661186] RSP: 0018:ffff9e9e37d03e58 EFLAGS: 00000002 [131629.661187] RAX: 0000000000000001 RBX: 0000000000000087 RCX: 0000000000000006 [131629.661187] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffffff8a0b68c0 [131629.661188] RBP: ffffffff8a0b68c0 R08: 0000000000003583 R09: 0000000000000004 [131629.661188] R10: ffffffff89e538b2 R11: 0000000000000001 R12: ffffffff8a22bc60 [131629.661189] R13: ffffffff8a0b66c0 R14: ffffffff8a22bc60 R15: ffffffff8a0b66c0 [131629.661189] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131629.661190] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131629.661191] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131629.661191] Call Trace: [131629.661191] [131629.661192] _raw_spin_lock_irqsave+0x32/0x40 [131629.661192] rcu_dump_cpu_stacks+0x48/0xd2 [131629.661193] rcu_check_callbacks+0x714/0x980 [131629.661193] ? tick_sched_do_timer+0x60/0x60 [131629.661194] update_process_times+0x28/0x50 [131629.661194] tick_sched_handle.isra.19+0x1f/0x60 [131629.661195] ? tick_sched_do_timer+0x60/0x60 [131629.661195] tick_sched_timer+0x38/0x70 [131629.661195] __hrtimer_run_queues+0xce/0x210 [131629.661196] hrtimer_interrupt+0xa6/0x1f0 [131629.661196] smp_apic_timer_interrupt+0x3a/0x60 [131629.661197] apic_timer_interrupt+0x87/0x90 [131629.661197] [131629.661197] RIP: 0010:native_queued_spin_lock_slowpath+0xf/0x190 [131629.661198] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131629.661199] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131629.661199] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131629.661200] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131629.661200] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131629.661201] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131629.661201] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131629.661201] _raw_spin_lock+0x1d/0x20 [131629.661202] lnet_peer_discovery+0xee/0x1160 [lnet] [131629.661202] ? __schedule+0x247/0x860 [131629.661202] ? wait_woken+0x80/0x80 [131629.661203] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131629.661203] kthread+0x11a/0x130 [131629.661204] ? kthread_create_on_node+0x40/0x40 [131629.661204] ret_from_fork+0x35/0x40 [131629.661204] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131629.661233] NMI backtrace for cpu 5 [131629.661236] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131629.661237] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131629.661238] Call Trace: [131629.661241] [131629.661251] dump_stack+0x5c/0x82 [131629.661258] nmi_cpu_backtrace+0x89/0x90 [131629.661269] ? irq_force_complete_move+0x140/0x140 [131629.661272] nmi_trigger_cpumask_backtrace+0xd1/0x100 [131629.661275] rcu_dump_cpu_stacks+0x9a/0xd2 [131629.661277] rcu_check_callbacks+0x714/0x980 [131629.661279] ? update_wall_time+0x47b/0x6f0 [131629.661282] ? tick_sched_do_timer+0x60/0x60 [131629.661284] update_process_times+0x28/0x50 [131629.661286] tick_sched_handle.isra.19+0x1f/0x60 [131629.661288] tick_sched_timer+0x38/0x70 [131629.661289] __hrtimer_run_queues+0xce/0x210 [131629.661291] hrtimer_interrupt+0xa6/0x1f0 [131629.661295] smp_apic_timer_interrupt+0x3a/0x60 [131629.661297] apic_timer_interrupt+0x87/0x90 [131629.661298] [131629.661315] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131629.661316] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 [131629.661318] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131629.661319] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131629.661320] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131629.661320] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131629.661321] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131629.661336] lnet_check_routers+0x9e/0x5e0 [lnet] [131629.661349] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131629.661352] ? __switch_to_asm+0x34/0x70 [131629.661354] ? wait_woken+0x80/0x80 [131629.661364] ? lnet_send_ping+0x190/0x190 [lnet] [131629.661366] kthread+0x11a/0x130 [131629.661369] ? kthread_create_on_node+0x40/0x40 [131629.661370] ret_from_fork+0x35/0x40 [131629.661377] NMI backtrace for cpu 4 [131629.661380] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131629.661381] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131629.661381] Call Trace: [131629.661383] [131629.661385] dump_stack+0x5c/0x82 [131629.661388] nmi_cpu_backtrace+0x89/0x90 [131629.661391] ? irq_force_complete_move+0x140/0x140 [131629.661394] nmi_trigger_cpumask_backtrace+0xd1/0x100 [131629.661396] rcu_dump_cpu_stacks+0x9a/0xd2 [131629.661398] rcu_check_callbacks+0x714/0x980 [131629.661400] ? tick_sched_do_timer+0x60/0x60 [131629.661402] update_process_times+0x28/0x50 [131629.661404] tick_sched_handle.isra.19+0x1f/0x60 [131629.661406] ? tick_sched_do_timer+0x60/0x60 [131629.661407] tick_sched_timer+0x38/0x70 [131629.661409] __hrtimer_run_queues+0xce/0x210 [131629.661411] hrtimer_interrupt+0xa6/0x1f0 [131629.661413] smp_apic_timer_interrupt+0x3a/0x60 [131629.661415] apic_timer_interrupt+0x87/0x90 [131629.661417] [131629.661418] RIP: 0010:native_queued_spin_lock_slowpath+0xf/0x190 [131629.661419] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131629.661421] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131629.661422] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131629.661423] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131629.661424] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131629.661425] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131629.661436] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131629.661439] _raw_spin_lock+0x1d/0x20 [131629.661450] lnet_peer_discovery+0xee/0x1160 [lnet] [131629.661455] ? __schedule+0x247/0x860 [131629.661456] ? wait_woken+0x80/0x80 [131629.661467] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131629.661469] kthread+0x11a/0x130 [131629.661471] ? kthread_create_on_node+0x40/0x40 [131629.661473] ret_from_fork+0x35/0x40 [131629.661483] Sending NMI from CPU 4 to CPUs 5: [131629.663982] NMI backtrace for cpu 5 [131629.663983] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131629.663984] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131629.663985] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131629.663986] RIP: 0010:lnet_get_next_peer_net_locked+0x1b/0x50 [lnet] [131629.663986] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 [131629.663988] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131629.663988] RDX: ffff9ea2379d6d10 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131629.663988] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131629.663989] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131629.663989] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131629.663990] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131629.663990] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131629.663991] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131629.663991] Call Trace: [131629.663991] lnet_check_routers+0x9e/0x5e0 [lnet] [131629.663992] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131629.663992] ? __switch_to_asm+0x34/0x70 [131629.663993] ? wait_woken+0x80/0x80 [131629.663993] ? lnet_send_ping+0x190/0x190 [lnet] [131629.663993] kthread+0x11a/0x130 [131629.663994] ? kthread_create_on_node+0x40/0x40 [131629.663994] ret_from_fork+0x35/0x40 [131629.663994] Code: ef 31 c0 eb cc f3 c3 66 0f 1f 84 00 00 00 00 00 66 66 66 66 90 85 f6 48 8d 57 10 48 8b 47 10 74 29 48 39 c2 74 14 3b 70 28 75 07 11 39 70 28 74 0c 48 8b 00 48 39 c2 75 f3 31 c0 f3 c3 48 8b [131653.710257] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [lnetctl:15141] [131653.710270] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131653.710278] Modules linked in: [131653.710282] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131653.710283] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131653.710347] Supported: No, Unsupported modules are loaded [131653.710353] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131653.710354] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131653.710357] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131653.710375] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131653.710376] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131653.710378] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131653.710379] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131653.710380] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131653.710381] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131653.710382] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131653.710384] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131653.710385] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131653.710386] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131653.710395] Call Trace: [131653.710413] _raw_spin_lock+0x1d/0x20 [131653.710476] lnet_peer_discovery+0xee/0x1160 [lnet] [131653.710485] ? __schedule+0x247/0x860 [131653.710489] ? wait_woken+0x80/0x80 [131653.710501] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131653.710509] kthread+0x11a/0x130 [131653.710513] ? kthread_create_on_node+0x40/0x40 [131653.710515] ret_from_fork+0x35/0x40 [131653.710517] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131653.711098] Modules linked in: [131653.711905] lnet(OEN) libcfs(OEN) af_packet [131653.712655] lnet(OEN) libcfs(OEN) [131653.712661] iscsi_ibft iscsi_boot_sysfs [131653.712666] af_packet [131653.712670] vmw_vsock_vmci_transport [131653.712672] iscsi_ibft [131653.712676] vsock [131653.712678] iscsi_boot_sysfs vmw_vsock_vmci_transport [131653.712683] sb_edac [131653.712687] vsock [131653.712690] coretemp crc32_pclmul [131653.712694] sb_edac coretemp [131653.712701] crc32c_intel [131653.712704] crc32_pclmul [131653.712708] ghash_clmulni_intel [131653.712710] crc32c_intel ghash_clmulni_intel [131653.712717] pcbc aesni_intel [131653.712722] pcbc [131653.712724] vmwgfx aes_x86_64 [131653.712729] aesni_intel vmwgfx [131653.712735] ttm crypto_simd [131653.712740] aes_x86_64 [131653.712743] drm_kms_helper glue_helper [131653.712746] ttm [131653.712750] cryptd [131653.712752] crypto_simd [131653.712755] joydev [131653.712756] drm_kms_helper [131653.712760] drm [131653.712762] glue_helper cryptd [131653.712768] drm_panel_orientation_quirks ppdev syscopyarea [131653.712775] joydev drm [131653.712780] vmw_balloon vmxnet3 [131653.712784] drm_panel_orientation_quirks ppdev [131653.712791] sysfillrect [131653.712797] syscopyarea [131653.712799] shpchp parport_pc [131653.712804] vmw_balloon vmxnet3 [131653.712809] parport sysimgblt [131653.712815] sysfillrect shpchp [131653.712819] i2c_piix4 pcspkr [131653.712824] parport_pc parport [131653.712829] ac vmw_vmci [131653.712834] sysimgblt [131653.712839] fb_sys_fops button [131653.712844] i2c_piix4 [131653.712848] ext4 [131653.712852] pcspkr [131653.712854] crc16 jbd2 [131653.712861] ac [131653.712864] mbcache [131653.712875] vmw_vmci [131653.712879] sr_mod [131653.712882] fb_sys_fops [131653.712886] cdrom ata_generic sd_mod [131653.712891] button [131653.712895] ata_piix [131653.712898] ext4 [131653.712901] ahci [131653.712905] crc16 [131653.712908] libahci serio_raw [131653.712913] jbd2 mbcache [131653.712918] mptspi [131653.712921] sr_mod [131653.712924] libata [131653.712928] cdrom [131653.712932] scsi_transport_spi [131653.712934] ata_generic sd_mod [131653.712939] mptscsih mptbase floppy [131653.712946] ata_piix [131653.712950] sg [131653.712953] ahci [131653.712955] scsi_mod autofs4 [131653.712958] libahci serio_raw [131653.712966] Supported: No, Unsupported modules are loaded [131653.712970] mptspi [131653.712974] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131653.712977] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131653.712979] libata [131653.712984] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131653.712985] scsi_transport_spi mptscsih mptbase [131653.713008] RIP: 0010:lnet_get_next_peer_net_locked+0x46/0x50 [lnet] [131653.713012] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000287 [131653.713014] floppy [131653.713019] ORIG_RAX: ffffffffffffff10 [131653.713020] sg scsi_mod [131653.713027] RAX: ffff9ea42c5c5580 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131653.713030] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: ffff9ea2379d6d00 [131653.713032] autofs4 [131653.713035] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131653.713039] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131653.713043] Supported: No, Unsupported modules are loaded [131653.713046] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131653.713050] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131653.713053] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131653.713054] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131653.713059] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131653.713062] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131653.713066] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131653.713068] Call Trace: [131653.713081] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131653.713084] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131653.713100] lnet_check_routers+0x9e/0x5e0 [lnet] [131653.713105] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131653.713108] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131653.713109] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131653.713121] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131653.713125] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131653.713129] ? __switch_to_asm+0x34/0x70 [131653.713134] ? wait_woken+0x80/0x80 [131653.713137] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131653.713150] ? lnet_send_ping+0x190/0x190 [lnet] [131653.713154] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131653.713158] kthread+0x11a/0x130 [131653.713161] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131653.713164] ? kthread_create_on_node+0x40/0x40 [131653.713169] ret_from_fork+0x35/0x40 [131653.713173] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131653.713176] Code: eb 11 39 70 28 74 0c 48 8b 00 [131653.713183] Call Trace: [131653.713185] 48 39 c2 75 f3 31 c0 f3 c3 48 8b 00 48 39 [131653.713203] _raw_spin_lock+0x1d/0x20 [131653.713204] d0 75 f6 48 8b 47 10 48 39 c2 [131653.713226] cfs_percpt_lock+0x69/0xf0 [libcfs] [131653.713228] ba 00 00 00 00 48 [131653.713276] lnet_add_route+0x172/0xcb0 [lnet] [131653.713278] 0f 44 c2 [131653.713290] LNetCtl+0x6cf/0x11a0 [lnet] [131653.713293] [131653.713304] ? LNetNIInit+0x66/0xb70 [lnet] [131653.713318] lnet_ioctl+0x1d3/0x250 [lnet] [131653.713321] 66 [131653.713330] notifier_call_chain+0x47/0x70 [131653.713332] 0f [131653.713335] blocking_notifier_call_chain+0x3e/0x60 [131653.713337] 1f [131653.713346] libcfs_ioctl+0x24e/0x470 [libcfs] [131653.713348] 84 00 00 [131653.713358] ? security_capable+0x47/0x60 [131653.713362] 00 [131653.713371] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131653.713379] do_vfs_ioctl+0x90/0x5f0 [131653.713380] 00 00 66 66 66 [131653.713390] SyS_ioctl+0x74/0x80 [131653.713393] 66 90 [131653.713402] do_syscall_64+0x74/0x140 [131653.713404] 48 [131653.713409] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131653.713412] 85 [131653.713417] RIP: 0033:0x7fb52efa9587 [131653.713419] d2 [131653.713423] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 [131653.713426] 74 28 [131653.713431] ORIG_RAX: 0000000000000010 [131653.713437] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131653.713438] 48 [131653.713446] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131653.713449] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131653.713450] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131653.713453] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131653.713454] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131681.708175] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131681.708178] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131681.708191] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [lnetctl:15141] [131681.708193] Modules linked in: [131681.708197] Modules linked in: [131681.708203] lnet(OEN) [131681.708207] lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131681.708335] Supported: No, Unsupported modules are loaded [131681.708341] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131681.708343] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131681.708347] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131681.708361] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131681.708364] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131681.708369] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131681.708370] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131681.708371] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131681.708373] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131681.708376] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131681.708379] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131681.708382] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131681.708383] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131681.708392] Call Trace: [131681.708410] _raw_spin_lock+0x1d/0x20 [131681.708430] cfs_percpt_lock+0x69/0xf0 [libcfs] [131681.708458] lnet_add_route+0x172/0xcb0 [lnet] [131681.708472] LNetCtl+0x6cf/0x11a0 [lnet] [131681.708487] ? LNetNIInit+0x66/0xb70 [lnet] [131681.708502] lnet_ioctl+0x1d3/0x250 [lnet] [131681.708509] notifier_call_chain+0x47/0x70 [131681.708513] blocking_notifier_call_chain+0x3e/0x60 [131681.708522] libcfs_ioctl+0x24e/0x470 [libcfs] [131681.708533] ? security_capable+0x47/0x60 [131681.708540] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131681.708549] do_vfs_ioctl+0x90/0x5f0 [131681.708554] SyS_ioctl+0x74/0x80 [131681.708560] do_syscall_64+0x74/0x140 [131681.708564] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131681.708568] RIP: 0033:0x7fb52efa9587 [131681.708570] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131681.708575] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131681.708577] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131681.708580] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131681.708581] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131681.708583] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131681.708585] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131681.708924] Modules linked in: lnet(OEN) [131681.709446] libcfs(OEN) [131681.709937] libcfs(OEN) af_packet [131681.709939] af_packet iscsi_ibft [131681.709941] iscsi_ibft [131681.709942] iscsi_boot_sysfs [131681.709943] iscsi_boot_sysfs [131681.709944] vmw_vsock_vmci_transport [131681.709945] vmw_vsock_vmci_transport [131681.709946] vsock [131681.709947] vsock [131681.709947] sb_edac [131681.709948] sb_edac [131681.709949] coretemp [131681.709949] coretemp [131681.709950] crc32_pclmul [131681.709951] crc32_pclmul [131681.709952] crc32c_intel [131681.709953] crc32c_intel [131681.709954] ghash_clmulni_intel [131681.709955] ghash_clmulni_intel [131681.709956] pcbc [131681.709956] pcbc [131681.709957] aesni_intel vmwgfx [131681.709958] aesni_intel vmwgfx [131681.709960] aes_x86_64 [131681.709960] aes_x86_64 [131681.709961] ttm [131681.709962] ttm [131681.709963] crypto_simd [131681.709963] crypto_simd [131681.709964] drm_kms_helper [131681.709965] drm_kms_helper [131681.709966] glue_helper [131681.709967] glue_helper [131681.709967] cryptd [131681.709968] cryptd [131681.709969] joydev [131681.709970] joydev [131681.709970] drm [131681.709971] drm [131681.709972] drm_panel_orientation_quirks [131681.709973] drm_panel_orientation_quirks [131681.709973] ppdev [131681.709974] ppdev syscopyarea [131681.709976] syscopyarea [131681.709977] vmw_balloon [131681.709977] vmw_balloon [131681.709978] vmxnet3 [131681.709979] vmxnet3 [131681.709980] sysfillrect [131681.709981] sysfillrect [131681.709982] shpchp [131681.709983] shpchp [131681.709983] parport_pc [131681.709984] parport_pc [131681.709985] parport [131681.709986] parport [131681.709987] sysimgblt [131681.709987] sysimgblt [131681.709989] i2c_piix4 [131681.709989] i2c_piix4 [131681.709990] pcspkr [131681.709991] pcspkr [131681.709992] ac [131681.709993] ac vmw_vmci [131681.709994] vmw_vmci [131681.709995] fb_sys_fops [131681.709995] fb_sys_fops [131681.709997] button [131681.709997] button ext4 [131681.709999] ext4 crc16 [131681.710000] crc16 jbd2 [131681.710002] jbd2 [131681.710003] mbcache [131681.710003] mbcache [131681.710005] sr_mod [131681.710005] sr_mod [131681.710006] cdrom [131681.710007] cdrom [131681.710008] ata_generic [131681.710009] ata_generic [131681.710010] sd_mod [131681.710011] sd_mod [131681.710012] ata_piix [131681.710013] ata_piix [131681.710014] ahci [131681.710014] ahci [131681.710015] libahci [131681.710016] libahci [131681.710017] serio_raw [131681.710018] serio_raw [131681.710019] mptspi [131681.710019] mptspi [131681.710020] libata [131681.710021] libata [131681.710022] scsi_transport_spi [131681.710023] scsi_transport_spi [131681.710024] mptscsih [131681.710025] mptscsih [131681.710025] mptbase [131681.710026] mptbase [131681.710027] floppy [131681.710028] floppy [131681.710028] sg [131681.710029] sg [131681.710030] scsi_mod [131681.710031] scsi_mod [131681.710031] autofs4 [131681.710033] autofs4 [131681.710035] Supported: No, Unsupported modules are loaded [131681.710035] Supported: No, Unsupported modules are loaded [131681.710042] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131681.710044] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131681.710045] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131681.710046] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131681.710049] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131681.710050] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131681.710098] RIP: 0010:lnet_islocalnet_locked+0x16/0x40 [lnet] [131681.710108] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131681.710110] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131681.710111] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131681.710113] ORIG_RAX: ffffffffffffff10 [131681.710114] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131681.710115] RAX: ffff9ea037045d80 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131681.710116] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131681.710117] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131681.710118] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131681.710119] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131681.710120] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131681.710121] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131681.710122] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131681.710123] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131681.710125] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131681.710127] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131681.710128] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131681.710129] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131681.710130] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131681.710131] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131681.710137] Call Trace: [131681.710138] Call Trace: [131681.710155] _raw_spin_lock+0x1d/0x20 [131681.710166] lnet_check_routers+0x8a/0x5e0 [lnet] [131681.710182] lnet_peer_discovery+0xee/0x1160 [lnet] [131681.710193] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131681.710198] ? __switch_to_asm+0x34/0x70 [131681.710203] ? __schedule+0x247/0x860 [131681.710207] ? wait_woken+0x80/0x80 [131681.710209] ? wait_woken+0x80/0x80 [131681.710219] ? lnet_send_ping+0x190/0x190 [lnet] [131681.710230] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131681.710238] kthread+0x11a/0x130 [131681.710240] kthread+0x11a/0x130 [131681.710243] ? kthread_create_on_node+0x40/0x40 [131681.710245] ? kthread_create_on_node+0x40/0x40 [131681.710247] ret_from_fork+0x35/0x40 [131681.710249] ret_from_fork+0x35/0x40 [131681.710251] Code: [131681.710252] Code: 66 [131681.710254] ff 8b [131681.710255] 2e 0f [131681.710256] 44 24 [131681.710258] 1f 84 00 [131681.710260] 04 [131681.710261] 00 [131681.710261] 48 [131681.710262] 00 00 [131681.710263] 83 c4 [131681.710264] 00 66 [131681.710266] 08 5b [131681.710267] 66 66 [131681.710268] 5d 41 [131681.710269] 66 90 [131681.710270] 5c c3 [131681.710271] 66 66 [131681.710272] 31 c0 [131681.710274] 66 66 [131681.710275] c3 0f [131681.710276] 90 ba [131681.710277] 1f 40 [131681.710278] 01 00 [131681.710280] 00 66 [131681.710281] 00 00 [131681.710282] 66 66 [131681.710283] 8b 07 [131681.710284] 66 90 [131681.710286] 85 c0 [131681.710287] 48 [131681.710288] 75 [131681.710288] 8b [131681.710289] 0a [131681.710290] 05 c4 [131681.710291] f0 0f [131681.710292] 38 [131681.710293] b1 [131681.710294] 05 00 [131681.710295] 17 85 [131681.710296] 48 [131681.710297] c0 [131681.710298] 3d a0 [131681.710299] 75 f2 [131681.710300] 90 7a [131681.710301] f3 c3 [131681.710302] c0 75 [131681.710303] f3 90 [131681.710305] 0d eb [131681.710306] ec [131681.710307] 1a <48> [131681.710308] 81 fe [131681.710310] 8b [131681.710310] 00 01 [131681.710311] 00 48 [131681.710313] 00 00 [131681.710314] 3d a0 [131681.710315] 0f 84 [131681.710316] 90 7a [131681.710317] 91 00 [131681.710318] c0 74 [131681.710319] 00 00 [131681.710321] 0f 3b [131681.710322] 41 b8 [131681.710323] 78 10 [131681.710324] 01 01 [131681.710325] 75 f0 [131681.710326] 00 00 [131681.710328] 48 85 [131681.710329] b9 [131681.710330] c0 0f 95 [131709.706069] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131709.706082] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [lnetctl:15141] [131709.706091] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131709.706093] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131709.706208] Supported: No, Unsupported modules are loaded [131709.706214] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131709.706217] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131709.706220] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131709.706234] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131709.706237] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131709.706242] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131709.706244] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131709.706245] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131709.706247] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131709.706249] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131709.706252] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131709.706254] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131709.706256] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131709.706265] Call Trace: [131709.706286] _raw_spin_lock+0x1d/0x20 [131709.706311] cfs_percpt_lock+0x69/0xf0 [libcfs] [131709.706355] lnet_add_route+0x172/0xcb0 [lnet] [131709.706372] LNetCtl+0x6cf/0x11a0 [lnet] [131709.706383] ? LNetNIInit+0x66/0xb70 [lnet] [131709.706395] lnet_ioctl+0x1d3/0x250 [lnet] [131709.706406] notifier_call_chain+0x47/0x70 [131709.706410] blocking_notifier_call_chain+0x3e/0x60 [131709.706421] libcfs_ioctl+0x24e/0x470 [libcfs] [131709.706434] ? security_capable+0x47/0x60 [131709.706444] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131709.706454] do_vfs_ioctl+0x90/0x5f0 [131709.706459] SyS_ioctl+0x74/0x80 [131709.706466] do_syscall_64+0x74/0x140 [131709.706471] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131709.706475] RIP: 0033:0x7fb52efa9587 [131709.706477] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131709.706482] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131709.706485] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131709.706487] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131709.706490] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131709.706492] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131709.706495] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9 [131709.706769] Modules linked in: lnet(OEN) [131709.707268] Modules linked in: [131709.707729] libcfs(OEN) [131709.707730] lnet(OEN) libcfs(OEN) [131709.707732] af_packet [131709.707733] af_packet [131709.707734] iscsi_ibft [131709.707735] iscsi_ibft [131709.707735] iscsi_boot_sysfs [131709.707736] iscsi_boot_sysfs [131709.707737] vmw_vsock_vmci_transport [131709.707738] vmw_vsock_vmci_transport [131709.707738] vsock [131709.707739] vsock [131709.707740] sb_edac coretemp [131709.707741] sb_edac coretemp crc32_pclmul [131709.707743] crc32_pclmul [131709.707744] crc32c_intel [131709.707745] crc32c_intel [131709.707746] ghash_clmulni_intel [131709.707746] ghash_clmulni_intel [131709.707747] pcbc [131709.707748] pcbc [131709.707749] aesni_intel [131709.707750] aesni_intel [131709.707750] vmwgfx [131709.707751] vmwgfx aes_x86_64 [131709.707752] aes_x86_64 [131709.707753] ttm [131709.707754] ttm [131709.707755] crypto_simd [131709.707756] crypto_simd [131709.707756] drm_kms_helper [131709.707757] drm_kms_helper [131709.707758] glue_helper [131709.707758] glue_helper [131709.707759] cryptd [131709.707760] cryptd [131709.707760] joydev [131709.707761] joydev [131709.707762] drm [131709.707763] drm [131709.707763] drm_panel_orientation_quirks [131709.707764] drm_panel_orientation_quirks [131709.707765] ppdev [131709.707766] ppdev [131709.707766] syscopyarea [131709.707767] syscopyarea [131709.707768] vmw_balloon [131709.707768] vmw_balloon [131709.707769] vmxnet3 [131709.707770] vmxnet3 [131709.707770] sysfillrect [131709.707771] sysfillrect [131709.707772] shpchp parport_pc [131709.707773] shpchp parport_pc parport [131709.707775] parport [131709.707776] sysimgblt [131709.707776] sysimgblt [131709.707778] i2c_piix4 [131709.707778] i2c_piix4 pcspkr [131709.707779] pcspkr [131709.707780] ac [131709.707781] ac [131709.707782] vmw_vmci [131709.707783] vmw_vmci [131709.707783] fb_sys_fops [131709.707784] fb_sys_fops [131709.707785] button [131709.707786] button [131709.707786] ext4 [131709.707787] ext4 [131709.707788] crc16 [131709.707788] crc16 [131709.707789] jbd2 [131709.707790] jbd2 [131709.707790] mbcache [131709.707791] mbcache [131709.707792] sr_mod [131709.707793] sr_mod [131709.707793] cdrom [131709.707794] cdrom [131709.707795] ata_generic [131709.707796] ata_generic [131709.707796] sd_mod [131709.707797] sd_mod [131709.707798] ata_piix [131709.707798] ata_piix [131709.707799] ahci [131709.707800] ahci [131709.707800] libahci [131709.707801] libahci [131709.707802] serio_raw [131709.707803] serio_raw [131709.707803] mptspi [131709.707804] mptspi [131709.707805] libata [131709.707806] libata [131709.707806] scsi_transport_spi mptscsih [131709.707808] scsi_transport_spi mptscsih [131709.707809] mptbase [131709.707810] mptbase [131709.707811] floppy [131709.707812] floppy [131709.707812] sg [131709.707813] sg [131709.707814] scsi_mod [131709.707815] scsi_mod [131709.707815] autofs4 [131709.707816] autofs4 [131709.707819] Supported: No, Unsupported modules are loaded [131709.707819] Supported: No, Unsupported modules are loaded [131709.707826] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131709.707828] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131709.707829] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131709.707830] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131709.707834] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131709.707835] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131709.707883] RIP: 0010:lnet_islocalnet_locked+0x16/0x40 [lnet] [131709.707884] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000202 [131709.707895] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131709.707896] ORIG_RAX: ffffffffffffff10 [131709.707897] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 [131709.707899] RAX: ffff9ea037045d80 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131709.707900] ORIG_RAX: ffffffffffffff10 [131709.707901] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131709.707902] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131709.707903] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131709.707904] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131709.707906] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131709.707907] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131709.707908] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131709.707909] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131709.707910] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131709.707913] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131709.707914] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131709.707915] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131709.707917] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131709.707918] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131709.707919] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131709.707926] Call Trace: [131709.707927] Call Trace: [131709.707946] lnet_check_routers+0x8a/0x5e0 [lnet] [131709.707956] _raw_spin_lock+0x1d/0x20 [131709.707972] lnet_peer_discovery+0xee/0x1160 [lnet] [131709.707983] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131709.707988] ? __switch_to_asm+0x34/0x70 [131709.707993] ? __schedule+0x247/0x860 [131709.707997] ? wait_woken+0x80/0x80 [131709.707999] ? wait_woken+0x80/0x80 [131709.708009] ? lnet_send_ping+0x190/0x190 [lnet] [131709.708020] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131709.708028] kthread+0x11a/0x130 [131709.708030] kthread+0x11a/0x130 [131709.708033] ? kthread_create_on_node+0x40/0x40 [131709.708035] ? kthread_create_on_node+0x40/0x40 [131709.708037] ret_from_fork+0x35/0x40 [131709.708039] ret_from_fork+0x35/0x40 [131709.708041] Code: [131709.708042] Code: [131709.708043] 66 2e [131709.708045] ff [131709.708045] 0f [131709.708046] 8b [131709.708047] 1f [131709.708048] 44 [131709.708048] 84 [131709.708049] 24 [131709.708050] 00 [131709.708051] 04 [131709.708051] 00 [131709.708052] 48 [131709.708053] 00 [131709.708054] 83 [131709.708054] 00 00 [131709.708056] c4 08 [131709.708057] 66 [131709.708058] 5b [131709.708058] 66 [131709.708059] 5d [131709.708060] 66 [131709.708061] 41 5c [131709.708062] 66 [131709.708063] c3 [131709.708063] 90 [131709.708064] 31 [131709.708065] 66 66 [131709.708066] c0 c3 [131709.708067] 66 [131709.708068] 0f [131709.708069] 66 90 [131709.708070] 1f 40 [131709.708071] ba [131709.708072] 00 [131709.708073] 01 [131709.708074] 66 [131709.708074] 00 [131709.708075] 66 [131709.708076] 00 [131709.708077] 66 [131709.708077] 00 [131709.708078] 66 [131709.708079] 8b [131709.708080] 90 [131709.708080] 07 85 [131709.708082] 48 8b [131709.708083] c0 [131709.708084] 05 [131709.708084] 75 [131709.708085] c4 [131709.708086] 0a [131709.708087] 38 [131709.708087] f0 [131709.708088] 05 [131709.708089] 0f [131709.708090] 00 [131709.708090] b1 17 [131709.708092] 48 3d [131709.708093] 85 c0 [131709.708094] a0 [131709.708095] 75 [131709.708096] 90 7a [131709.708097] f2 [131709.708098] c0 [131709.708099] f3 [131709.708100] 75 [131709.708100] c3 [131709.708101] 0d [131709.708102] f3 [131709.708102] eb [131709.708103] 90 [131709.708104] 1a [131709.708105] [131709.708106] <48> [131709.708106] ec [131709.708107] 8b [131709.708108] 81 [131709.708109] 00 [131709.708109] fe [131709.708110] 48 [131709.708111] 00 01 [131709.708112] 3d a0 [131709.708113] 00 [131709.708114] 90 [131709.708115] 00 [131709.708116] 7a [131709.708116] 0f [131709.708117] c0 [131709.708118] 84 91 [131709.708119] 74 0f [131709.708120] 00 [131709.708121] 3b [131709.708122] 00 00 [131709.708123] 78 10 [131709.708124] 41 [131709.708125] 75 [131709.708126] b8 [131709.708127] f0 [131709.708127] 01 [131709.708128] 48 [131709.708129] 01 00 [131709.708130] 85 [131709.708131] 00 [131709.708131] c0 [131709.708132] b9 [131709.708133] 0f [131709.708135] 95 [131737.703986] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [lnetctl:15141] [131737.704000] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [monitor_thread:15011] [131737.704010] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [lnet_discovery:15010] [131737.704011] Modules linked in: lnet(OEN) libcfs(OEN) af_packet iscsi_ibft iscsi_boot_sysfs vmw_vsock_vmci_transport vsock sb_edac coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel pcbc aesni_intel vmwgfx aes_x86_64 ttm crypto_simd drm_kms_helper glue_helper cryptd joydev drm drm_panel_orientation_quirks ppdev syscopyarea vmw_balloon vmxnet3 sysfillrect shpchp parport_pc parport sysimgblt i2c_piix4 pcspkr ac vmw_vmci fb_sys_fops button ext4 crc16 jbd2 mbcache sr_mod cdrom ata_generic sd_mod ata_piix ahci libahci serio_raw mptspi libata scsi_transport_spi mptscsih mptbase floppy sg scsi_mod autofs4 [131737.704090] Supported: No, Unsupported modules are loaded [131737.704097] CPU: 5 PID: 15011 Comm: monitor_thread Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131737.704098] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131737.704101] task: ffff9e9e34d09080 task.stack: ffffabaa436e8000 [131737.704176] RIP: 0010:lnet_islocalnet_locked+0xc/0x40 [lnet] [131737.704178] RSP: 0018:ffffabaa436ebe18 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 [131737.704181] RAX: ffff9ea037045d80 RBX: ffff9ea037cfb640 RCX: 0000000000000000 [131737.704182] RDX: 0000000000000000 RSI: 00000000000d0004 RDI: 00000000000d0004 [131737.704184] RBP: ffff9ea2379d6d00 R08: ffffffffc07b90ff R09: ffff9e9e33e86ef7 [131737.704185] R10: ffffabaa436ebe68 R11: 656e206c61636f6c R12: 0000000000000000 [131737.704187] R13: ffff9ea2379d6d30 R14: 000000005cffdc03 R15: ffff9ea42c5c5580 [131737.704190] FS: 0000000000000000(0000) GS:ffff9e9e37d40000(0000) knlGS:0000000000000000 [131737.704192] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131737.704193] CR2: 00005633f92b9000 CR3: 000000062d00a000 CR4: 00000000000406e0 [131737.704202] Call Trace: [131737.704225] lnet_check_routers+0x8a/0x5e0 [lnet] [131737.704241] lnet_monitor_thread+0x4ab/0x1170 [lnet] [131737.704252] ? __switch_to_asm+0x34/0x70 [131737.704262] ? wait_woken+0x80/0x80 [131737.704272] ? lnet_send_ping+0x190/0x190 [lnet] [131737.704280] kthread+0x11a/0x130 [131737.704284] ? kthread_create_on_node+0x40/0x40 [131737.704287] ret_from_fork+0x35/0x40 [131737.704289] Code: 89 e6 89 44 24 04 e8 f4 aa f9 ff 8b 44 24 04 48 83 c4 08 5b 5d 41 5c c3 31 c0 c3 0f 1f 40 00 66 66 66 66 90 48 8b 05 c4 38 05 00 <48> 3d a0 90 7a c0 75 0d eb 1a 48 8b 00 48 3d a0 90 7a c0 74 0f [131737.704808] Modules linked in: lnet(OEN) [131737.705561] Modules linked in: [131737.706206] libcfs(OEN) [131737.706209] lnet(OEN) libcfs(OEN) [131737.706216] af_packet iscsi_ibft [131737.706220] af_packet iscsi_ibft [131737.706226] iscsi_boot_sysfs vmw_vsock_vmci_transport [131737.706233] iscsi_boot_sysfs vmw_vsock_vmci_transport [131737.706238] vsock sb_edac [131737.706242] vsock sb_edac [131737.706248] coretemp crc32_pclmul [131737.706254] coretemp crc32_pclmul [131737.706258] crc32c_intel [131737.706262] crc32c_intel ghash_clmulni_intel pcbc [131737.706270] ghash_clmulni_intel pcbc [131737.706274] aesni_intel vmwgfx [131737.706280] aesni_intel [131737.706282] aes_x86_64 [131737.706286] vmwgfx [131737.706288] ttm crypto_simd [131737.706292] aes_x86_64 ttm [131737.706299] drm_kms_helper glue_helper cryptd [131737.706308] crypto_simd [131737.706310] joydev drm [131737.706315] drm_kms_helper glue_helper [131737.706321] drm_panel_orientation_quirks ppdev [131737.706329] cryptd [131737.706332] syscopyarea vmw_balloon [131737.706335] joydev drm [131737.706341] vmxnet3 sysfillrect [131737.706347] drm_panel_orientation_quirks [131737.706350] shpchp [131737.706352] ppdev syscopyarea [131737.706358] parport_pc parport sysimgblt [131737.706364] vmw_balloon [131737.706368] i2c_piix4 [131737.706372] vmxnet3 [131737.706374] pcspkr ac [131737.706379] sysfillrect [131737.706381] vmw_vmci fb_sys_fops [131737.706385] shpchp parport_pc [131737.706390] button [131737.706394] parport [131737.706396] ext4 crc16 jbd2 [131737.706403] sysimgblt i2c_piix4 [131737.706408] mbcache sr_mod [131737.706413] pcspkr [131737.706416] cdrom [131737.706418] ac vmw_vmci [131737.706424] ata_generic sd_mod [131737.706429] fb_sys_fops button [131737.706434] ata_piix ahci [131737.706439] ext4 [131737.706442] libahci [131737.706444] crc16 jbd2 [131737.706450] serio_raw mptspi [131737.706458] mbcache [131737.706460] libata [131737.706464] sr_mod [131737.706465] scsi_transport_spi mptscsih [131737.706471] cdrom [131737.706474] mptbase [131737.706476] ata_generic sd_mod [131737.706481] floppy sg [131737.706486] ata_piix ahci [131737.706491] scsi_mod autofs4 [131737.706495] libahci serio_raw [131737.706504] Supported: No, Unsupported modules are loaded [131737.706506] mptspi libata [131737.706513] CPU: 4 PID: 15010 Comm: lnet_discovery Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131737.706514] scsi_transport_spi [131737.706519] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131737.706522] mptscsih [131737.706526] task: ffff9e9e2af49800 task.stack: ffffabaa467b8000 [131737.706530] mptbase [131737.706539] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131737.706542] RSP: 0018:ffffabaa467bbe90 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff10 [131737.706548] floppy [131737.706551] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000004 [131737.706554] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131737.706557] sg [131737.706560] RBP: ffff9ea0379687c0 R08: 000076840ff8a800 R09: 0000768679410700 [131737.706563] R10: ffffabaa467bbea0 R11: 00000000003d0900 R12: ffffabaa467bbeb8 [131737.706564] scsi_mod autofs4 [131737.706570] R13: ffff9e9e2af49800 R14: 0000000000000000 R15: ffffffffc07862e0 [131737.706574] FS: 0000000000000000(0000) GS:ffff9e9e37d00000(0000) knlGS:0000000000000000 [131737.706577] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131737.706582] CR2: 00007f232509c2e0 CR3: 000000062d00a000 CR4: 00000000000406e0 [131737.706585] Supported: No, Unsupported modules are loaded [131737.706593] CPU: 8 PID: 15141 Comm: lnetctl Tainted: G OEL 4.12.14-150.14-default #1 SLE15 [131737.706595] Call Trace: [131737.706599] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013 [131737.706604] _raw_spin_lock+0x1d/0x20 [131737.706609] task: ffff9ea037474880 task.stack: ffffabaa44cec000 [131737.706626] lnet_peer_discovery+0xee/0x1160 [lnet] [131737.706641] RIP: 0010:native_queued_spin_lock_slowpath+0x21/0x190 [131737.706648] ? __schedule+0x247/0x860 [131737.706653] ? wait_woken+0x80/0x80 [131737.706657] RSP: 0018:ffffabaa44cefcd0 EFLAGS: 00000202 [131737.706671] ? lnet_peer_merge_data+0xea0/0xea0 [lnet] [131737.706675] kthread+0x11a/0x130 [131737.706678] ORIG_RAX: ffffffffffffff10 [131737.706682] RAX: 0000000000000001 RBX: ffff9ea037569600 RCX: 0000000000000000 [131737.706686] ? kthread_create_on_node+0x40/0x40 [131737.706690] ret_from_fork+0x35/0x40 [131737.706693] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e9e3782d440 [131737.706696] RBP: 0000000000000000 R08: 000000000002cbb8 R09: 0000000000000000 [131737.706697] Code: 66 2e [131737.706705] R10: ffff9ea02df606c0 R11: ffff9ea0379f43d3 R12: 0000000000000004 [131737.706706] 0f 1f [131737.706712] R13: ffff9e9ec7c37f00 R14: 0000000000020000 R15: ffff9ea02df606c0 [131737.706714] 84 00 00 00 00 [131737.706725] FS: 00007fb52fc04740(0000) GS:ffff9ea037c00000(0000) knlGS:0000000000000000 [131737.706728] 00 [131737.706732] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [131737.706734] 66 [131737.706737] CR2: 00007fb52f022f03 CR3: 0000000436eb2000 CR4: 00000000000406e0 [131737.706739] 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b [131737.706749] Call Trace: [131737.706750] 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 [131737.706771] _raw_spin_lock+0x1d/0x20 [131737.706773] ec 81 fe 00 01 00 00 0f 84 91 00 [131737.706797] cfs_percpt_lock+0x69/0xf0 [libcfs] [131737.706799] 00 00 41 b8 01 01 00 [131737.706844] lnet_add_route+0x172/0xcb0 [lnet] [131737.706847] 00 b9 [131737.706862] LNetCtl+0x6cf/0x11a0 [lnet] [131737.706873] ? LNetNIInit+0x66/0xb70 [lnet] [131737.706885] lnet_ioctl+0x1d3/0x250 [lnet] [131737.706893] notifier_call_chain+0x47/0x70 [131737.706896] blocking_notifier_call_chain+0x3e/0x60 [131737.706904] libcfs_ioctl+0x24e/0x470 [libcfs] [131737.706911] ? security_capable+0x47/0x60 [131737.706917] libcfs_psdev_ioctl+0xbe/0xd0 [libcfs] [131737.706923] do_vfs_ioctl+0x90/0x5f0 [131737.706928] SyS_ioctl+0x74/0x80 [131737.706934] do_syscall_64+0x74/0x140 [131737.706936] entry_SYSCALL_64_after_hwframe+0x3d/0xa2 [131737.706939] RIP: 0033:0x7fb52efa9587 [131737.706940] RSP: 002b:00007fff4182dad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [131737.706942] RAX: ffffffffffffffda RBX: 000000000000007e RCX: 00007fb52efa9587 [131737.706943] RDX: 00007fff4182dbc0 RSI: 00000000c0b86551 RDI: 0000000000000003 [131737.706944] RBP: 00007fff4182db40 R08: 00000000017a4310 R09: 00000000017a42f0 [131737.706945] R10: 000000000000055b R11: 0000000000000206 R12: 00000000ffffffff [131737.706946] R13: 00007fff4182dbc0 R14: 000000000000007e R15: 000000000000007e [131737.706947] Code: 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 66 66 66 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 f3 c3 f3 90 ec 81 fe 00 01 00 00 0f 84 91 00 00 00 41 b8 01 01 00 00 b9