[55672.315672] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55672.556661] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55672.628172] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55672.664919] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55672.702778] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55672.728346] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55672.756226] RSP: 0018:ffff8808262efa98 EFLAGS: 00000202 [55672.774651] RAX: 0000000000005040 RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55672.799044] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55672.823474] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55672.847895] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55672.872284] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55672.896678] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55672.924255] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55672.944079] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55672.968478] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55672.992866] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55673.017282] Stack: [55673.024812] ffff8808262efac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55673.050123] ffff88083fb5e000 ffff8808262efb80 ffffffffa092c9b5 00000000000001a8 [55673.075420] ffffc9000e331730 ffff8808262efb68 ffffffffa0ebd587 ffffffffffffffff [55673.100723] Call Trace: [55673.109714] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55673.131844] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55673.154831] [] lnet_select_pathway+0x235/0x1140 [lnet] [55673.178084] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55673.200517] [] lnet_send+0x51/0x180 [lnet] [55673.220342] [] lnet_finalize+0x3f1/0x690 [lnet] [55673.241628] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55673.263446] [] ? __slab_free+0x81/0x2f0 [55673.282432] [] lnet_ni_recv+0xc3/0x320 [lnet] [55673.303143] [] lnet_parse+0x945/0xf90 [lnet] [55673.323545] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55673.346831] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55673.368126] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55673.391662] [] ? dequeue_task_fair+0x41e/0x660 [55673.412669] [] ? wake_up_state+0x20/0x20 [55673.431946] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55673.454909] [] kthread+0xcf/0xe0 [55673.471879] [] ? kthread_create_on_node+0x140/0x140 [55673.494263] [] ret_from_fork+0x58/0x90 [55673.512887] [] ? kthread_create_on_node+0x140/0x140 [55673.535238] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55675.078306] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [55675.106055] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55675.346518] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55675.417829] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55675.454489] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55675.492309] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55675.517835] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55675.545722] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [55675.564164] RAX: 0000000000006b7b RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55675.588601] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55675.613012] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55675.637450] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55675.661862] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55675.686302] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55675.713859] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55675.733680] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55675.758122] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55675.782531] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55675.806965] Stack: [55675.814479] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55675.839829] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [55675.865169] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [55675.890511] Call Trace: [55675.899501] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55675.921646] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55675.944671] [] lnet_select_pathway+0x235/0x1140 [lnet] [55675.967978] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55675.990395] [] lnet_send+0x51/0x180 [lnet] [55676.010280] [] lnet_finalize+0x3f1/0x690 [lnet] [55676.031590] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55676.053444] [] ? __slab_free+0x81/0x2f0 [55676.072462] [] lnet_ni_recv+0xc3/0x320 [lnet] [55676.093182] [] lnet_parse+0x945/0xf90 [lnet] [55676.113575] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55676.136571] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55676.159861] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55676.181143] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55676.204711] [] ? dequeue_task_fair+0x41e/0x660 [55676.225701] [] ? wake_up_state+0x20/0x20 [55676.244980] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55676.267923] [] kthread+0xcf/0xe0 [55676.284846] [] ? kthread_create_on_node+0x140/0x140 [55676.307215] [] ret_from_fork+0x58/0x90 [55676.325854] [] ? kthread_create_on_node+0x140/0x140 [55676.348187] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55679.077894] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [lctl:36467] [55679.077896] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [kworker/3:2:27192] [55679.077916] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55679.077921] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55679.077922] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55679.077923] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55679.077925] Workqueue: ib_cm cm_work_handler [ib_cm] [55679.077925] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55679.077927] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55679.077927] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000202 [55679.077928] RAX: 0000000000007e2a RBX: 0000000000000000 RCX: 0000000000000a1e [55679.077928] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55679.077929] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55679.077930] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55679.077930] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55679.077931] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55679.077932] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55679.077932] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55679.077933] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55679.077933] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55679.077933] Stack: [55679.077934] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [55679.077935] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [55679.077936] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [55679.077937] Call Trace: [55679.077942] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55679.077947] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55679.077951] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55679.077953] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55679.077957] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55679.077959] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55679.077961] [] cma_req_handler+0x227/0x570 [rdma_cm] [55679.077963] [] cm_process_work+0x25/0x140 [ib_cm] [55679.077964] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55679.077966] [] cm_work_handler+0x185/0x1484 [ib_cm] [55679.077967] [] process_one_work+0x17b/0x470 [55679.077968] [] worker_thread+0x126/0x410 [55679.077969] [] ? rescuer_thread+0x460/0x460 [55679.077971] [] kthread+0xcf/0xe0 [55679.077972] [] ? kthread_create_on_node+0x140/0x140 [55679.077974] [] ret_from_fork+0x58/0x90 [55679.077975] [] ? kthread_create_on_node+0x140/0x140 [55679.077985] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55680.369373] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55680.609952] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55680.681299] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55680.714823] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55680.752627] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55680.778161] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55680.806001] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [55680.824367] RAX: 00000000000045f2 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55680.848777] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55680.873193] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55680.897595] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55680.921966] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55680.946366] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55680.973878] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55680.993664] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55681.018069] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55681.042460] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55681.066868] Stack: [55681.074349] ffff8807f639fd88 ffffffffa08aa831 0000000200000002 ffff8807f63a2980 [55681.099632] 0000000000000000 ffff8807f639fdc0 ffffffffa0942e30 00000000c0086540 [55681.124956] ffff8807f63a2900 0000000000000002 0000000000650aa0 0000000000000000 [55681.150276] Call Trace: [55681.159208] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55681.181358] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55681.202910] [] LNetCtl+0x116/0x800 [lnet] [55681.222476] [] lnet_ioctl+0x60/0x250 [lnet] [55681.242608] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55681.264135] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55681.286801] [] do_vfs_ioctl+0x2d5/0x4b0 [55681.305742] [] ? __fd_install+0x47/0x60 [55681.324701] [] SyS_ioctl+0xa1/0xc0 [55681.342257] [] system_call_fastpath+0x16/0x1b [55681.362961] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55683.077482] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [kiblnd_sd_01_00:2900] [55683.077483] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kiblnd_sd_01_02:2902] [55683.077503] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55683.077508] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55683.077509] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55683.077509] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55683.077510] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55683.077512] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55683.077512] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [55683.077513] RAX: 0000000000002db1 RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55683.077513] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55683.077514] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55683.077514] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55683.077515] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55683.077516] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55683.077516] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55683.077517] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55683.077517] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55683.077518] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55683.077518] Stack: [55683.077519] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [55683.077520] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [55683.077521] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [55683.077521] Call Trace: [55683.077526] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55683.077531] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55683.077536] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55683.077541] [] lnet_finalize+0x1e9/0x690 [lnet] [55683.077545] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55683.077548] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55683.077550] [] ? dequeue_task_fair+0x41e/0x660 [55683.077551] [] ? wake_up_state+0x20/0x20 [55683.077555] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55683.077556] [] kthread+0xcf/0xe0 [55683.077558] [] ? kthread_create_on_node+0x140/0x140 [55683.077559] [] ret_from_fork+0x58/0x90 [55683.077561] [] ? kthread_create_on_node+0x140/0x140 [55683.077571] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55684.268891] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55684.509993] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55684.581616] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55684.618343] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55684.656243] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55684.681824] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55684.709685] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55684.728071] RAX: 00000000000034ef RBX: 00000201e39c5101 RCX: 0000000000000a1e [55684.752495] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55684.776906] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55684.801310] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55684.825675] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55684.850046] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55684.877587] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55684.897399] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55684.921816] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55684.946205] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55684.970585] Stack: [55684.978077] ffff8808217bbcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba3200 [55685.003399] 0000000000000001 ffff8808217bbd08 ffffffffa09217bb ffff8804c1ba3200 [55685.028710] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff8808217bbd70 [55685.054013] Call Trace: [55685.062981] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55685.085095] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55685.108068] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55685.130471] [] lnet_finalize+0x1e9/0x690 [lnet] [55685.151730] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55685.174423] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55685.197993] [] ? dequeue_task_fair+0x41e/0x660 [55685.218992] [] ? wake_up_state+0x20/0x20 [55685.238237] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55685.261215] [] kthread+0xcf/0xe0 [55685.278204] [] ? kthread_create_on_node+0x140/0x140 [55685.300627] [] ret_from_fork+0x58/0x90 [55685.319297] [] ? kthread_create_on_node+0x140/0x140 [55685.341697] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55687.077070] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55687.077071] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [55687.077090] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55687.077095] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55687.077096] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55687.077097] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55687.077097] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55687.077099] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55687.077100] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [55687.077100] RAX: 0000000000003cdc RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55687.077101] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55687.077101] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55687.077102] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55687.077102] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55687.077103] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55687.077104] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55687.077104] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55687.077105] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55687.077105] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55687.077105] Stack: [55687.077106] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [55687.077107] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [55687.077108] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [55687.077109] Call Trace: [55687.077114] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55687.077119] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55687.077125] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55687.077130] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55687.077136] [] lnet_finalize+0x1e9/0x690 [lnet] [55687.077139] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55687.077143] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55687.077144] [] ? dequeue_task_fair+0x41e/0x660 [55687.077146] [] ? wake_up_state+0x20/0x20 [55687.077149] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55687.077151] [] kthread+0xcf/0xe0 [55687.077152] [] ? kthread_create_on_node+0x140/0x140 [55687.077154] [] ret_from_fork+0x58/0x90 [55687.077155] [] ? kthread_create_on_node+0x140/0x140 [55687.077165] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55688.292638] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55688.533666] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55688.605188] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55688.641655] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55688.679552] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55688.705148] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55688.733041] RSP: 0018:ffff880820b03c00 EFLAGS: 00000206 [55688.751463] RAX: 0000000000004288 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55688.775855] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55688.800278] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55688.824681] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55688.849066] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55688.873454] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55688.900986] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55688.920839] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55688.945271] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55688.969660] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55688.994051] Stack: [55689.001548] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [55689.026857] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [55689.052189] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [55689.077485] Call Trace: [55689.086458] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55689.108590] [] lnet_post_send_locked+0x446/0x880 [lnet] [55689.132141] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55689.157993] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55689.180157] [] lnet_finalize+0x1e9/0x690 [lnet] [55689.201414] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55689.224145] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55689.247701] [] ? dequeue_task_fair+0x41e/0x660 [55689.268691] [] ? sched_clock_cpu+0x85/0xc0 [55689.288519] [] ? wake_up_state+0x20/0x20 [55689.307775] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55689.330757] [] kthread+0xcf/0xe0 [55689.347739] [] ? kthread_create_on_node+0x140/0x140 [55689.370163] [] ret_from_fork+0x58/0x90 [55689.388852] [] ? kthread_create_on_node+0x140/0x140 [55689.411286] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55691.076658] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55691.076659] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55691.076678] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55691.076684] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55691.076685] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55691.076686] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55691.076686] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55691.076688] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55691.076689] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [55691.076689] RAX: 0000000000007e6b RBX: 0000000100000000 RCX: 0000000000000a1e [55691.076690] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55691.076690] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55691.076691] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55691.076692] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55691.076692] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55691.076693] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55691.076694] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55691.076694] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55691.076695] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55691.076695] Stack: [55691.076696] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [55691.076697] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [55691.076698] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [55691.076698] Call Trace: [55691.076703] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55691.076709] [] lnet_finalize+0xf6/0x690 [lnet] [55691.076713] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55691.076715] [] ? enqueue_task_fair+0x208/0x6c0 [55691.076716] [] ? sched_clock_cpu+0x85/0xc0 [55691.076722] [] lnet_ni_recv+0xc3/0x320 [lnet] [55691.076728] [] lnet_parse+0x945/0xf90 [lnet] [55691.076731] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55691.076732] [] ? __wake_up+0x44/0x50 [55691.076736] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55691.076737] [] ? dequeue_task_fair+0x41e/0x660 [55691.076738] [] ? wake_up_state+0x20/0x20 [55691.076742] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55691.076743] [] kthread+0xcf/0xe0 [55691.076745] [] ? kthread_create_on_node+0x140/0x140 [55691.076747] [] ret_from_fork+0x58/0x90 [55691.076748] [] ? kthread_create_on_node+0x140/0x140 [55691.076758] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55692.342479] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55692.583013] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55692.654310] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55692.690672] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55692.728467] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55692.754002] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55692.781843] RSP: 0018:ffff88082626bce0 EFLAGS: 00000206 [55692.800191] RAX: 00000000000075f5 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55692.824592] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55692.848962] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55692.873340] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55692.897762] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55692.922147] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55692.949677] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55692.969491] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55692.993918] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55693.018355] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55693.042748] Stack: [55693.050285] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [55693.075583] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [55693.100916] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [55693.126206] Call Trace: [55693.135155] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55693.157277] [] lnet_finalize+0xf6/0x690 [lnet] [55693.178285] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55693.201000] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55693.224541] [] ? dequeue_task_fair+0x41e/0x660 [55693.245527] [] ? wake_up_state+0x20/0x20 [55693.264794] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55693.287808] [] kthread+0xcf/0xe0 [55693.304797] [] ? kthread_create_on_node+0x140/0x140 [55693.327245] [] ret_from_fork+0x58/0x90 [55693.345962] [] ? kthread_create_on_node+0x140/0x140 [55693.368387] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55699.075835] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55699.075837] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [55699.075856] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55699.075861] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55699.075862] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55699.075863] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55699.075864] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55699.075865] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55699.075866] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55699.075867] RAX: 00000000000060d2 RBX: 0000000000016c40 RCX: 0000000000000a1e [55699.075867] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55699.075868] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55699.075868] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55699.075869] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55699.075869] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55699.075870] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55699.075871] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55699.075871] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55699.075872] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55699.075872] Stack: [55699.075873] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55699.075874] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55699.075875] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55699.075875] Call Trace: [55699.075881] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55699.075886] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55699.075889] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55699.075892] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55699.075894] [] ? lock_timer_base.isra.33+0x2b/0x50 [55699.075895] [] ? try_to_del_timer_sync+0x5e/0x90 [55699.075899] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55699.075900] [] ? schedule_timeout+0x17d/0x2d0 [55699.075904] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55699.075905] [] ? wake_up_state+0x20/0x20 [55699.075909] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55699.075910] [] kthread+0xcf/0xe0 [55699.075912] [] ? kthread_create_on_node+0x140/0x140 [55699.075913] [] ret_from_fork+0x58/0x90 [55699.075915] [] ? kthread_create_on_node+0x140/0x140 [55699.075925] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55700.314801] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55700.555826] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55700.627335] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55700.664080] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55700.701959] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55700.727540] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55700.755392] RSP: 0018:ffff8808262efa98 EFLAGS: 00000202 [55700.773780] RAX: 000000000000235e RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55700.798173] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55700.822597] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55700.846991] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55700.871379] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55700.895767] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55700.923326] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55700.943185] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55700.967616] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55700.992047] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55701.016465] Stack: [55701.024011] ffff8808262efac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55701.049349] ffff88083fb5e000 ffff8808262efb80 ffffffffa092c9b5 00000000000001a8 [55701.074650] ffffc9000e331730 ffff8808262efb68 ffffffffa0ebd587 ffffffffffffffff [55701.099988] Call Trace: [55701.108954] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55701.131108] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55701.154098] [] lnet_select_pathway+0x235/0x1140 [lnet] [55701.177376] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55701.199822] [] lnet_send+0x51/0x180 [lnet] [55701.219687] [] lnet_finalize+0x3f1/0x690 [lnet] [55701.240974] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55701.262804] [] ? __slab_free+0x81/0x2f0 [55701.281789] [] lnet_ni_recv+0xc3/0x320 [lnet] [55701.302486] [] lnet_parse+0x945/0xf90 [lnet] [55701.322897] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55701.346193] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55701.367476] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55701.391037] [] ? dequeue_task_fair+0x41e/0x660 [55701.412009] [] ? wake_up_state+0x20/0x20 [55701.431282] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55701.454284] [] kthread+0xcf/0xe0 [55701.471237] [] ? kthread_create_on_node+0x140/0x140 [55701.493628] [] ret_from_fork+0x58/0x90 [55701.512250] [] ? kthread_create_on_node+0x140/0x140 [55701.534610] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55703.075422] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [55703.103197] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55703.343693] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55703.415019] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55703.451715] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55703.489538] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55703.515111] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55703.542986] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000202 [55703.561427] RAX: 0000000000007640 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55703.585832] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55703.610272] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55703.634681] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55703.659076] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55703.683523] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55703.711116] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55703.730977] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55703.755428] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55703.779869] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55703.804301] Stack: [55703.811842] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55703.837154] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [55703.862439] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [55703.887745] Call Trace: [55703.896726] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55703.918898] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55703.941927] [] lnet_select_pathway+0x235/0x1140 [lnet] [55703.965201] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55703.987660] [] lnet_send+0x51/0x180 [lnet] [55704.007526] [] lnet_finalize+0x3f1/0x690 [lnet] [55704.028801] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55704.050675] [] ? __slab_free+0x81/0x2f0 [55704.069662] [] lnet_ni_recv+0xc3/0x320 [lnet] [55704.090378] [] lnet_parse+0x945/0xf90 [lnet] [55704.110827] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55704.133829] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55704.157103] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55704.178410] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55704.201984] [] ? dequeue_task_fair+0x41e/0x660 [55704.222987] [] ? wake_up_state+0x20/0x20 [55704.242243] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55704.265210] [] kthread+0xcf/0xe0 [55704.282157] [] ? kthread_create_on_node+0x140/0x140 [55704.304511] [] ret_from_fork+0x58/0x90 [55704.323129] [] ? kthread_create_on_node+0x140/0x140 [55704.345449] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55707.075010] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [lctl:36467] [55707.075012] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [kworker/3:2:27192] [55707.075031] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55707.075037] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55707.075038] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55707.075038] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55707.075040] Workqueue: ib_cm cm_work_handler [ib_cm] [55707.075041] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55707.075043] RIP: 0010:[] [] _raw_spin_lock+0x30/0x50 [55707.075043] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000216 [55707.075044] RAX: 00000000000028b6 RBX: 0000000000000000 RCX: 0000000000000a1e [55707.075044] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55707.075045] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55707.075045] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55707.075046] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55707.075047] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55707.075047] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55707.075048] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55707.075048] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55707.075049] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55707.075049] Stack: [55707.075050] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [55707.075051] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [55707.075052] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [55707.075052] Call Trace: [55707.075057] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55707.075062] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55707.075066] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55707.075068] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55707.075072] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55707.075074] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55707.075076] [] cma_req_handler+0x227/0x570 [rdma_cm] [55707.075078] [] cm_process_work+0x25/0x140 [ib_cm] [55707.075079] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55707.075081] [] cm_work_handler+0x185/0x1484 [ib_cm] [55707.075082] [] process_one_work+0x17b/0x470 [55707.075083] [] worker_thread+0x126/0x410 [55707.075084] [] ? rescuer_thread+0x460/0x460 [55707.075086] [] kthread+0xcf/0xe0 [55707.075087] [] ? kthread_create_on_node+0x140/0x140 [55707.075089] [] ret_from_fork+0x58/0x90 [55707.075090] [] ? kthread_create_on_node+0x140/0x140 [55707.075100] Code: b8 00 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 90 83 e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 [55708.366604] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55708.607195] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55708.678546] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55708.712079] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55708.749882] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55708.775385] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55708.803238] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [55708.821597] RAX: 00000000000028ee RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55708.845973] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55708.870380] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55708.894792] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55708.919169] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55708.943562] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55708.971087] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55708.990888] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55709.015268] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55709.039659] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55709.064060] Stack: [55709.071582] ffff8807f639fd88 ffffffffa08aa831 0000000200000002 ffff8807f63a2980 [55709.096902] 0000000000000000 ffff8807f639fdc0 ffffffffa0942e30 00000000c0086540 [55709.122223] ffff8807f63a2900 0000000000000002 0000000000650aa0 0000000000000000 [55709.147508] Call Trace: [55709.156449] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55709.178562] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55709.200092] [] LNetCtl+0x116/0x800 [lnet] [55709.219617] [] lnet_ioctl+0x60/0x250 [lnet] [55709.239685] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55709.261232] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55709.283894] [] do_vfs_ioctl+0x2d5/0x4b0 [55709.302876] [] ? __fd_install+0x47/0x60 [55709.321839] [] SyS_ioctl+0xa1/0xc0 [55709.339376] [] system_call_fastpath+0x16/0x1b [55709.360059] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55711.074598] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kiblnd_sd_01_02:2902] [55711.074600] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [kiblnd_sd_01_00:2900] [55711.074621] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55711.074627] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55711.074628] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55711.074628] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55711.074629] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55711.074631] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55711.074632] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55711.074632] RAX: 00000000000028e1 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55711.074633] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55711.074633] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55711.074634] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55711.074634] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55711.074635] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55711.074636] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55711.074637] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55711.074637] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55711.074638] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55711.074638] Stack: [55711.074639] ffff8808217bbcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba3200 [55711.074640] 0000000000000001 ffff8808217bbd08 ffffffffa09217bb ffff8804c1ba3200 [55711.074641] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff8808217bbd70 [55711.074641] Call Trace: [55711.074647] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55711.074651] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55711.074657] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55711.074662] [] lnet_finalize+0x1e9/0x690 [lnet] [55711.074666] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55711.074669] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55711.074671] [] ? dequeue_task_fair+0x41e/0x660 [55711.074672] [] ? wake_up_state+0x20/0x20 [55711.074676] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55711.074678] [] kthread+0xcf/0xe0 [55711.074679] [] ? kthread_create_on_node+0x140/0x140 [55711.074681] [] ret_from_fork+0x58/0x90 [55711.074682] [] ? kthread_create_on_node+0x140/0x140 [55711.074692] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55712.266201] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55712.507193] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55712.578746] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55712.615476] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55712.653343] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55712.678897] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55712.706755] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000206 [55712.725140] RAX: 0000000000006648 RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55712.749557] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55712.773971] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55712.798340] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55712.822709] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55712.847077] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55712.874642] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55712.894448] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55712.918836] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55712.943255] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55712.967629] Stack: [55712.975148] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [55713.000434] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [55713.025701] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [55713.051002] Call Trace: [55713.059952] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55713.082078] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55713.105051] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55713.127470] [] lnet_finalize+0x1e9/0x690 [lnet] [55713.148753] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55713.171455] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55713.194989] [] ? dequeue_task_fair+0x41e/0x660 [55713.215948] [] ? wake_up_state+0x20/0x20 [55713.235231] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55713.258205] [] kthread+0xcf/0xe0 [55713.275153] [] ? kthread_create_on_node+0x140/0x140 [55713.297571] [] ret_from_fork+0x58/0x90 [55713.316262] [] ? kthread_create_on_node+0x140/0x140 [55713.338679] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55715.074187] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55715.074188] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [55715.074206] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55715.074211] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55715.074212] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55715.074213] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55715.074214] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55715.074215] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55715.074216] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000206 [55715.074216] RAX: 0000000000007393 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55715.074217] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55715.074217] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55715.074218] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55715.074218] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55715.074219] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55715.074220] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55715.074220] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55715.074221] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55715.074222] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55715.074222] Stack: [55715.074223] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [55715.074224] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [55715.074225] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [55715.074225] Call Trace: [55715.074230] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55715.074235] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55715.074241] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55715.074247] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55715.074252] [] lnet_finalize+0x1e9/0x690 [lnet] [55715.074256] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55715.074259] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55715.074260] [] ? dequeue_task_fair+0x41e/0x660 [55715.074262] [] ? wake_up_state+0x20/0x20 [55715.074266] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55715.074267] [] kthread+0xcf/0xe0 [55715.074268] [] ? kthread_create_on_node+0x140/0x140 [55715.074270] [] ret_from_fork+0x58/0x90 [55715.074271] [] ? kthread_create_on_node+0x140/0x140 [55715.074281] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55716.290000] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55716.530966] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55716.602503] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55716.638977] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55716.676865] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55716.702423] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55716.730273] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [55716.748694] RAX: 0000000000005ce3 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55716.773120] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55716.797535] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55716.821919] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55716.846300] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55716.870683] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55716.898216] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55716.918064] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55716.942461] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55716.966881] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55716.991256] Stack: [55716.998790] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [55717.024121] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [55717.049418] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [55717.074751] Call Trace: [55717.083736] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55717.105869] [] lnet_post_send_locked+0x446/0x880 [lnet] [55717.129420] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55717.155302] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55717.177454] [] lnet_finalize+0x1e9/0x690 [lnet] [55717.198712] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55717.221429] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55717.244989] [] ? dequeue_task_fair+0x41e/0x660 [55717.265986] [] ? sched_clock_cpu+0x85/0xc0 [55717.285853] [] ? wake_up_state+0x20/0x20 [55717.305117] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55717.328095] [] kthread+0xcf/0xe0 [55717.345064] [] ? kthread_create_on_node+0x140/0x140 [55717.367487] [] ret_from_fork+0x58/0x90 [55717.386173] [] ? kthread_create_on_node+0x140/0x140 [55717.408615] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55719.073775] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55719.073776] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55719.073794] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55719.073800] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55719.073801] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55719.073801] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55719.073802] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55719.073803] RIP: 0010:[] [] _raw_spin_lock+0x30/0x50 [55719.073804] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [55719.073804] RAX: 0000000000007680 RBX: 0000000100000000 RCX: 0000000000000a1e [55719.073805] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55719.073805] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55719.073806] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55719.073806] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55719.073807] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55719.073808] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55719.073808] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55719.073809] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55719.073810] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55719.073810] Stack: [55719.073811] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [55719.073812] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [55719.073813] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [55719.073813] Call Trace: [55719.073818] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55719.073824] [] lnet_finalize+0xf6/0x690 [lnet] [55719.073828] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55719.073829] [] ? enqueue_task_fair+0x208/0x6c0 [55719.073831] [] ? sched_clock_cpu+0x85/0xc0 [55719.073837] [] lnet_ni_recv+0xc3/0x320 [lnet] [55719.073842] [] lnet_parse+0x945/0xf90 [lnet] [55719.073846] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55719.073847] [] ? __wake_up+0x44/0x50 [55719.073850] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55719.073852] [] ? dequeue_task_fair+0x41e/0x660 [55719.073853] [] ? wake_up_state+0x20/0x20 [55719.073857] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55719.073858] [] kthread+0xcf/0xe0 [55719.073860] [] ? kthread_create_on_node+0x140/0x140 [55719.073861] [] ret_from_fork+0x58/0x90 [55719.073863] [] ? kthread_create_on_node+0x140/0x140 [55719.073873] Code: b8 00 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 90 83 e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 [55720.339681] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55720.580213] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55720.651519] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55720.687879] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55720.725703] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55720.751229] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55720.779026] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55720.797373] RAX: 00000000000006e2 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55720.821775] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55720.846143] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55720.870546] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55720.894964] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55720.919383] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55720.946914] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55720.966753] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55720.991155] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55721.015590] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55721.040002] Stack: [55721.047471] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [55721.072794] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [55721.098102] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [55721.123405] Call Trace: [55721.132387] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55721.154506] [] lnet_finalize+0xf6/0x690 [lnet] [55721.175476] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55721.198203] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55721.221767] [] ? dequeue_task_fair+0x41e/0x660 [55721.242759] [] ? wake_up_state+0x20/0x20 [55721.262049] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55721.284997] [] kthread+0xcf/0xe0 [55721.301965] [] ? kthread_create_on_node+0x140/0x140 [55721.324401] [] ret_from_fork+0x58/0x90 [55721.343126] [] ? kthread_create_on_node+0x140/0x140 [55721.365530] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55727.072951] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [55727.072953] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55727.072975] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55727.072981] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55727.072982] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55727.072983] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55727.072984] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55727.072985] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55727.072986] RSP: 0018:ffff8808262efa98 EFLAGS: 00000212 [55727.072986] RAX: 000000000000737f RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55727.072987] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55727.072988] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55727.072988] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55727.072989] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55727.072990] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55727.072990] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55727.072991] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55727.072992] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55727.072992] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55727.072992] Stack: [55727.072993] ffff8808262efac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55727.072994] ffff88083fb5e000 ffff8808262efb80 ffffffffa092c9b5 00000000000001a8 [55727.072995] ffffc9000e331730 ffff8808262efb68 ffffffffa0ebd587 ffffffffffffffff [55727.072995] Call Trace: [55727.073001] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55727.073006] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55727.073012] [] lnet_select_pathway+0x235/0x1140 [lnet] [55727.073016] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55727.073022] [] lnet_send+0x51/0x180 [lnet] [55727.073027] [] lnet_finalize+0x3f1/0x690 [lnet] [55727.073031] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55727.073033] [] ? __slab_free+0x81/0x2f0 [55727.073038] [] lnet_ni_recv+0xc3/0x320 [lnet] [55727.073044] [] lnet_parse+0x945/0xf90 [lnet] [55727.073047] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55727.073050] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55727.073053] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55727.073055] [] ? dequeue_task_fair+0x41e/0x660 [55727.073056] [] ? wake_up_state+0x20/0x20 [55727.073060] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55727.073061] [] kthread+0xcf/0xe0 [55727.073063] [] ? kthread_create_on_node+0x140/0x140 [55727.073065] [] ret_from_fork+0x58/0x90 [55727.073066] [] ? kthread_create_on_node+0x140/0x140 [55727.073076] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55728.410203] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55728.650368] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55728.721526] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55728.757000] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55728.794781] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55728.820262] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55728.848087] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55728.866436] RAX: 00000000000005fb RBX: 0000000000016c40 RCX: 0000000000000a1e [55728.890806] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55728.915201] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55728.939597] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55728.963990] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55728.988352] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55729.015911] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55729.035721] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55729.060113] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55729.084501] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55729.108890] Stack: [55729.116377] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55729.141625] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55729.166903] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55729.192224] Call Trace: [55729.201201] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55729.223354] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55729.244633] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55729.268177] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55729.294046] [] ? lock_timer_base.isra.33+0x2b/0x50 [55729.316162] [] ? try_to_del_timer_sync+0x5e/0x90 [55729.337735] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55729.361589] [] ? schedule_timeout+0x17d/0x2d0 [55729.382327] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55729.404448] [] ? wake_up_state+0x20/0x20 [55729.423732] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55729.448176] [] kthread+0xcf/0xe0 [55729.465178] [] ? kthread_create_on_node+0x140/0x140 [55729.487599] [] ret_from_fork+0x58/0x90 [55729.506330] [] ? kthread_create_on_node+0x140/0x140 [55729.528775] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55731.072539] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [55731.100474] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55731.341676] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55731.413282] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55731.450065] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55731.487993] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55731.513618] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55731.541533] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000202 [55731.559964] RAX: 0000000000004b25 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55731.584428] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55731.608904] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55731.633377] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55731.657810] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55731.682241] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55731.709828] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55731.729681] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55731.754148] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55731.778615] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55731.803082] Stack: [55731.810667] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55731.836042] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [55731.861405] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [55731.886739] Call Trace: [55731.895750] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55731.917907] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55731.940943] [] lnet_select_pathway+0x235/0x1140 [lnet] [55731.964267] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55731.986735] [] lnet_send+0x51/0x180 [lnet] [55732.006622] [] lnet_finalize+0x3f1/0x690 [lnet] [55732.027939] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55732.049820] [] ? __slab_free+0x81/0x2f0 [55732.068846] [] lnet_ni_recv+0xc3/0x320 [lnet] [55732.089564] [] lnet_parse+0x945/0xf90 [lnet] [55732.109993] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55732.133007] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55732.156290] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55732.177600] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55732.201190] [] ? dequeue_task_fair+0x41e/0x660 [55732.222187] [] ? wake_up_state+0x20/0x20 [55732.241455] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55732.264402] [] kthread+0xcf/0xe0 [55732.281352] [] ? kthread_create_on_node+0x140/0x140 [55732.303734] [] ret_from_fork+0x58/0x90 [55732.322370] [] ? kthread_create_on_node+0x140/0x140 [55732.344698] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55735.072127] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [lctl:36467] [55735.072129] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [kworker/3:2:27192] [55735.072148] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55735.072153] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55735.072154] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55735.072155] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55735.072157] Workqueue: ib_cm cm_work_handler [ib_cm] [55735.072157] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55735.072159] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55735.072159] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000216 [55735.072160] RAX: 000000000000091f RBX: 0000000000000000 RCX: 0000000000000a1e [55735.072160] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55735.072161] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55735.072161] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55735.072162] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55735.072162] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55735.072163] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55735.072164] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55735.072164] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55735.072165] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55735.072165] Stack: [55735.072166] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [55735.072167] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [55735.072168] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [55735.072168] Call Trace: [55735.072173] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55735.072178] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55735.072182] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55735.072184] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55735.072188] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55735.072190] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55735.072192] [] cma_req_handler+0x227/0x570 [rdma_cm] [55735.072194] [] cm_process_work+0x25/0x140 [ib_cm] [55735.072195] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55735.072197] [] cm_work_handler+0x185/0x1484 [ib_cm] [55735.072198] [] process_one_work+0x17b/0x470 [55735.072199] [] worker_thread+0x126/0x410 [55735.072200] [] ? rescuer_thread+0x460/0x460 [55735.072202] [] kthread+0xcf/0xe0 [55735.072203] [] ? kthread_create_on_node+0x140/0x140 [55735.072205] [] ret_from_fork+0x58/0x90 [55735.072206] [] ? kthread_create_on_node+0x140/0x140 [55735.072216] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55736.363803] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55736.604368] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55736.675676] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55736.709174] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55736.746993] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55736.772505] RIP: 0010:[] [] _raw_spin_lock+0x30/0x50 [55736.800320] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [55736.818680] RAX: 0000000000000242 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55736.843059] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55736.867437] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55736.891843] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55736.916220] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55736.940611] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55736.968126] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55736.987916] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55737.012299] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55737.036715] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55737.061129] Stack: [55737.068654] ffff8807f639fd88 ffffffffa08aa831 0000000200000002 ffff8807f63a2980 [55737.093974] 0000000000000000 ffff8807f639fdc0 ffffffffa0942e30 00000000c0086540 [55737.119312] ffff8807f63a2900 0000000000000002 0000000000650aa0 0000000000000000 [55737.144642] Call Trace: [55737.153595] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55737.175742] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55737.197277] [] LNetCtl+0x116/0x800 [lnet] [55737.216802] [] lnet_ioctl+0x60/0x250 [lnet] [55737.236885] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55737.258445] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55737.281117] [] do_vfs_ioctl+0x2d5/0x4b0 [55737.300084] [] ? __fd_install+0x47/0x60 [55737.319031] [] SyS_ioctl+0xa1/0xc0 [55737.336567] [] system_call_fastpath+0x16/0x1b [55737.357257] Code: b8 00 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 90 83 e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 [55739.071715] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [kiblnd_sd_01_02:2902] [55739.071716] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [kiblnd_sd_01_00:2900] [55739.071734] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55739.071739] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55739.071740] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55739.071741] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55739.071741] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55739.071743] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55739.071743] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55739.071744] RAX: 0000000000003e4b RBX: 00000201e39c5101 RCX: 0000000000000a1e [55739.071744] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55739.071745] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55739.071745] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55739.071746] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55739.071747] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55739.071747] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55739.071748] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55739.071749] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55739.071749] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55739.071749] Stack: [55739.071750] ffff8808217bbcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba3200 [55739.071751] 0000000000000001 ffff8808217bbd08 ffffffffa09217bb ffff8804c1ba3200 [55739.071752] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff8808217bbd70 [55739.071752] Call Trace: [55739.071758] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55739.071762] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55739.071768] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55739.071773] [] lnet_finalize+0x1e9/0x690 [lnet] [55739.071777] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55739.071780] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55739.071781] [] ? dequeue_task_fair+0x41e/0x660 [55739.071783] [] ? wake_up_state+0x20/0x20 [55739.071787] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55739.071788] [] kthread+0xcf/0xe0 [55739.071790] [] ? kthread_create_on_node+0x140/0x140 [55739.071791] [] ret_from_fork+0x58/0x90 [55739.071793] [] ? kthread_create_on_node+0x140/0x140 [55739.071803] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55740.262854] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55740.503862] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55740.575389] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55740.612144] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55740.650034] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55740.675626] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55740.703493] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000206 [55740.721893] RAX: 0000000000001fc5 RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55740.746275] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55740.770647] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55740.795019] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55740.819424] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55740.843829] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55740.871352] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55740.891196] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55740.915583] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55740.939962] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55740.964330] Stack: [55740.971818] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [55740.997145] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [55741.022440] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [55741.047770] Call Trace: [55741.056720] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55741.078853] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55741.101866] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55741.124258] [] lnet_finalize+0x1e9/0x690 [lnet] [55741.145515] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55741.168238] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55741.191774] [] ? dequeue_task_fair+0x41e/0x660 [55741.212751] [] ? wake_up_state+0x20/0x20 [55741.232023] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55741.255000] [] kthread+0xcf/0xe0 [55741.271959] [] ? kthread_create_on_node+0x140/0x140 [55741.294358] [] ret_from_fork+0x58/0x90 [55741.313037] [] ? kthread_create_on_node+0x140/0x140 [55741.335464] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55743.071304] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55743.071305] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [55743.071323] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55743.071328] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55743.071329] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55743.071329] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55743.071330] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55743.071332] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55743.071332] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [55743.071333] RAX: 0000000000001b96 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55743.071333] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55743.071334] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55743.071334] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55743.071335] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55743.071336] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55743.071336] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55743.071337] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55743.071337] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55743.071338] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55743.071338] Stack: [55743.071339] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [55743.071340] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [55743.071341] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [55743.071341] Call Trace: [55743.071347] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55743.071351] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55743.071358] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55743.071363] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55743.071368] [] lnet_finalize+0x1e9/0x690 [lnet] [55743.071372] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55743.071376] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55743.071377] [] ? dequeue_task_fair+0x41e/0x660 [55743.071378] [] ? wake_up_state+0x20/0x20 [55743.071382] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55743.071384] [] kthread+0xcf/0xe0 [55743.071385] [] ? kthread_create_on_node+0x140/0x140 [55743.071387] [] ret_from_fork+0x58/0x90 [55743.071388] [] ? kthread_create_on_node+0x140/0x140 [55743.071398] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55744.286870] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55744.527839] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55744.599424] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55744.635869] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55744.673753] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55744.699338] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55744.727184] RSP: 0018:ffff880820b03c00 EFLAGS: 00000206 [55744.745565] RAX: 00000000000073cc RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55744.769946] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55744.794332] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55744.818741] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55744.843126] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55744.867509] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55744.895042] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55744.914852] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55744.939235] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55744.963651] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55744.988027] Stack: [55744.995549] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [55745.020850] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [55745.046154] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [55745.071449] Call Trace: [55745.080439] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55745.102583] [] lnet_post_send_locked+0x446/0x880 [lnet] [55745.126165] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55745.152040] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55745.174183] [] lnet_finalize+0x1e9/0x690 [lnet] [55745.195441] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55745.218128] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55745.241682] [] ? dequeue_task_fair+0x41e/0x660 [55745.262681] [] ? sched_clock_cpu+0x85/0xc0 [55745.282525] [] ? wake_up_state+0x20/0x20 [55745.301819] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55745.324798] [] kthread+0xcf/0xe0 [55745.341790] [] ? kthread_create_on_node+0x140/0x140 [55745.364221] [] ret_from_fork+0x58/0x90 [55745.382906] [] ? kthread_create_on_node+0x140/0x140 [55745.405339] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55747.070891] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55747.070892] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55747.070910] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55747.070916] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55747.070917] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55747.070917] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55747.070918] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55747.070919] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55747.070920] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [55747.070920] RAX: 00000000000068dd RBX: 0000000100000000 RCX: 0000000000000a1e [55747.070921] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55747.070921] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55747.070922] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55747.070922] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55747.070923] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55747.070924] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55747.070925] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55747.070925] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55747.070926] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55747.070926] Stack: [55747.070927] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [55747.070928] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [55747.070929] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [55747.070929] Call Trace: [55747.070934] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55747.070940] [] lnet_finalize+0xf6/0x690 [lnet] [55747.070944] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55747.070945] [] ? enqueue_task_fair+0x208/0x6c0 [55747.070947] [] ? sched_clock_cpu+0x85/0xc0 [55747.070952] [] lnet_ni_recv+0xc3/0x320 [lnet] [55747.070958] [] lnet_parse+0x945/0xf90 [lnet] [55747.070962] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55747.070963] [] ? __wake_up+0x44/0x50 [55747.070966] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55747.070967] [] ? dequeue_task_fair+0x41e/0x660 [55747.070969] [] ? wake_up_state+0x20/0x20 [55747.070972] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55747.070974] [] kthread+0xcf/0xe0 [55747.070975] [] ? kthread_create_on_node+0x140/0x140 [55747.070977] [] ret_from_fork+0x58/0x90 [55747.070978] [] ? kthread_create_on_node+0x140/0x140 [55747.070988] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55748.336765] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55748.577356] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55748.648656] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55748.685013] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55748.722838] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55748.748364] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55748.776195] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55748.794541] RAX: 0000000000000bdf RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55748.818903] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55748.843271] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55748.867669] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55748.892083] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55748.916473] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55748.944045] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55748.963847] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55748.988247] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55749.012684] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55749.037105] Stack: [55749.044614] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [55749.069943] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [55749.095272] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [55749.120602] Call Trace: [55749.129543] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55749.151659] [] lnet_finalize+0xf6/0x690 [lnet] [55749.172660] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55749.195367] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55749.218952] [] ? dequeue_task_fair+0x41e/0x660 [55749.239923] [] ? wake_up_state+0x20/0x20 [55749.259193] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55749.282194] [] kthread+0xcf/0xe0 [55749.299191] [] ? kthread_create_on_node+0x140/0x140 [55749.321607] [] ret_from_fork+0x58/0x90 [55749.340297] [] ? kthread_create_on_node+0x140/0x140 [55749.362743] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55751.967388] INFO: rcu_sched self-detected stall on CPU [55751.968387] INFO: rcu_sched self-detected stall on CPU [55751.968389] INFO: rcu_sched self-detected stall on CPU [55751.968390] INFO: rcu_sched self-detected stall on CPU [55751.968391] INFO: rcu_sched self-detected stall on CPU [55751.968392] INFO: rcu_sched self-detected stall on CPU [55751.968393] INFO: rcu_sched self-detected stall on CPU [55751.968394] INFO: rcu_sched self-detected stall on CPU [55751.968395] { [55751.968396] { [55751.968396] { [55751.968398] { [55751.968398] { [55751.968399] { [55751.968400] { [55751.968401] 5 [55751.968402] 1 [55751.968403] 6 [55751.968404] 10 [55751.968405] 7 [55751.968406] 12 [55751.968407] 15 [55751.968408] } [55751.968409] } [55751.968409] } [55751.968410] } [55751.968411] } [55751.968412] } [55751.968412] } [55751.968420] (t=960015 jiffies g=99188 c=99187 q=24139) [55751.968421] (t=960015 jiffies g=99188 c=99187 q=24139) [55751.968423] (t=960015 jiffies g=99188 c=99187 q=24139) [55751.968424] (t=960015 jiffies g=99188 c=99187 q=24139) [55751.968425] (t=960015 jiffies g=99188 c=99187 q=24139) [55751.968426] (t=960015 jiffies g=99188 c=99187 q=24139) [55751.968427] (t=960015 jiffies g=99188 c=99187 q=24139) [55751.968428] Task dump for CPU 1: [55751.968429] kiblnd_sd_00_00 R [55751.968429] running task [55751.968430] 0 2896 2 0x00000088 [55751.968430] ffff880820b03d30 [55751.968431] 000000003f673d40 [55751.968431] ffffc90003ec02f0 [55751.968431] ffff88041d63d600 [55751.968431] [55751.968432] 0000000000000000 [55751.968432] 0000000000000000 [55751.968432] ffff88042aa84a28 [55751.968432] ffff880820b03da8 [55751.968432] [55751.968433] ffffffffa0eba166 [55751.968433] ffff880181447180 [55751.968433] ffff88042aa84a00 [55751.968433] 0000000000000002 [55751.968434] [55751.968434] Call Trace: [55751.968434] [55751.968438] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968438] [55751.968442] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968442] [55751.968443] [] ? dequeue_task_fair+0x41e/0x660 [55751.968443] [55751.968444] [] ? sched_clock_cpu+0x85/0xc0 [55751.968445] [55751.968446] [] ? wake_up_state+0x20/0x20 [55751.968446] [55751.968449] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968450] [55751.968451] [] ? kthread+0xcf/0xe0 [55751.968451] [55751.968452] [] ? kthread_create_on_node+0x140/0x140 [55751.968453] [55751.968454] [] ? ret_from_fork+0x58/0x90 [55751.968454] [55751.968455] [] ? kthread_create_on_node+0x140/0x140 [55751.968456] Task dump for CPU 5: [55751.968456] kiblnd_sd_00_02 R [55751.968457] running task [55751.968457] 0 2898 2 0x00000088 [55751.968457] ffff8804c1676dd0 [55751.968458] 000000000f26e3e5 [55751.968458] ffff88042e143db0 [55751.968458] ffffffff810c42f8 [55751.968458] [55751.968458] 0000000000000005 [55751.968459] ffffffff81a1a680 [55751.968459] ffff88042e143dc8 [55751.968459] ffffffff810c7b89 [55751.968459] [55751.968459] 0000000000000006 [55751.968460] ffff88042e143df8 [55751.968460] ffffffff811373f0 [55751.968460] ffff88042e1501c0 [55751.968460] [55751.968461] Call Trace: [55751.968461] [55751.968461] [55751.968462] [] sched_show_task+0xa8/0x110 [55751.968463] [55751.968464] [] dump_cpu_task+0x39/0x70 [55751.968464] [55751.968465] [] rcu_dump_cpu_stacks+0x90/0xd0 [55751.968465] [55751.968467] [] rcu_check_callbacks+0x442/0x720 [55751.968467] [55751.968468] [] ? tick_sched_handle.isra.13+0x60/0x60 [55751.968468] [55751.968470] [] update_process_times+0x47/0x80 [55751.968470] [55751.968471] [] tick_sched_handle.isra.13+0x25/0x60 [55751.968471] [55751.968472] [] tick_sched_timer+0x41/0x70 [55751.968472] [55751.968473] [] __hrtimer_run_queues+0xd2/0x260 [55751.968474] [55751.968475] [] hrtimer_interrupt+0xb0/0x1e0 [55751.968475] [55751.968476] [] ? call_softirq+0x1c/0x30 [55751.968476] [55751.968478] [] local_apic_timer_interrupt+0x37/0x60 [55751.968478] [55751.968479] [] smp_apic_timer_interrupt+0x3f/0x60 [55751.968479] [55751.968480] [] apic_timer_interrupt+0x6d/0x80 [55751.968480] [55751.968481] [55751.968482] [] ? _raw_spin_lock+0x37/0x50 [55751.968482] [55751.968487] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55751.968487] [55751.968493] [] lnet_finalize+0xf6/0x690 [lnet] [55751.968493] [55751.968496] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55751.968496] [55751.968498] [] ? enqueue_task_fair+0x208/0x6c0 [55751.968498] [55751.968499] [] ? sched_clock_cpu+0x85/0xc0 [55751.968500] [55751.968505] [] lnet_ni_recv+0xc3/0x320 [lnet] [55751.968505] [55751.968511] [] lnet_parse+0x945/0xf90 [lnet] [55751.968511] [55751.968514] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.968514] [55751.968515] [] ? __wake_up+0x44/0x50 [55751.968515] [55751.968518] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.968519] [55751.968520] [] ? dequeue_task_fair+0x41e/0x660 [55751.968520] [55751.968521] [] ? wake_up_state+0x20/0x20 [55751.968522] [55751.968525] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968525] [55751.968526] [] kthread+0xcf/0xe0 [55751.968526] [55751.968528] [] ? kthread_create_on_node+0x140/0x140 [55751.968528] [55751.968529] [] ret_from_fork+0x58/0x90 [55751.968529] [55751.968531] [] ? kthread_create_on_node+0x140/0x140 [55751.968531] Task dump for CPU 6: [55751.968532] kiblnd_sd_00_01 R [55751.968532] running task [55751.968532] 0 2897 2 0x00000088 [55751.968533] ffff88081f64fd30 [55751.968533] 00000000f1dbfe3f [55751.968533] ffffc90003ebfde0 [55751.968533] ffff88042dcd1e00 [55751.968533] [55751.968534] 0000000000000000 [55751.968534] 0000000000000000 [55751.968534] ffff88042aa84a28 [55751.968534] ffff88081f64fda8 [55751.968534] [55751.968535] ffffffffa0eba166 [55751.968535] ffff880181447180 [55751.968535] ffff88042aa84a00 [55751.968535] 0000000000000012 [55751.968535] [55751.968536] Call Trace: [55751.968536] [55751.968539] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968539] [55751.968542] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968543] [55751.968544] [] ? dequeue_task_fair+0x41e/0x660 [55751.968544] [55751.968545] [] ? wake_up_state+0x20/0x20 [55751.968545] [55751.968548] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968549] [55751.968550] [] ? kthread+0xcf/0xe0 [55751.968550] [55751.968551] [] ? kthread_create_on_node+0x140/0x140 [55751.968551] [55751.968553] [] ? ret_from_fork+0x58/0x90 [55751.968553] [55751.968554] [] ? kthread_create_on_node+0x140/0x140 [55751.968555] Task dump for CPU 7: [55751.968555] kiblnd_sd_00_03 R [55751.968555] running task [55751.968556] 0 2899 2 0x00000088 [55751.968556] ffff88082626bd30 [55751.968556] 00000000aaf7c743 [55751.968556] ffffc90003ebe8c8 [55751.968557] ffff8800b484ea00 [55751.968557] [55751.968557] 0000000000000000 [55751.968557] 0000000000000000 [55751.968558] ffff88042aa84a28 [55751.968558] ffff88082626bda8 [55751.968558] [55751.968558] ffffffffa0eba166 [55751.968558] ffff880181447180 [55751.968559] ffff88042aa84a00 [55751.968559] 0000000000000001 [55751.968559] [55751.968559] Call Trace: [55751.968559] [55751.968562] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968563] [55751.968566] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968566] [55751.968567] [] ? dequeue_task_fair+0x41e/0x660 [55751.968567] [55751.968568] [] ? wake_up_state+0x20/0x20 [55751.968569] [55751.968572] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968572] [55751.968573] [] ? kthread+0xcf/0xe0 [55751.968574] [55751.968575] [] ? kthread_create_on_node+0x140/0x140 [55751.968575] [55751.968576] [] ? ret_from_fork+0x58/0x90 [55751.968576] [55751.968578] [] ? kthread_create_on_node+0x140/0x140 [55751.968578] Task dump for CPU 10: [55751.968579] kiblnd_sd_01_01 R [55751.968579] running task [55751.968579] 0 2901 2 0x00000088 [55751.968580] ffff8807f886f000 [55751.968580] ffffc9001327c860 [55751.968580] ffff8808266e7a00 [55751.968580] 0000000000000000 [55751.968580] [55751.968581] ffff8808262efda8 [55751.968581] ffffffffa0ebedab [55751.968581] ffff8808262efe90 [55751.968581] ffff8808262efda8 [55751.968582] [55751.968582] ffffffffa03e7d2e [55751.968582] ffff88082b2fc3c0 [55751.968582] ffff88082bda6c00 [55751.968582] 0000000000000017 [55751.968583] [55751.968583] Call Trace: [55751.968583] [55751.968586] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.968587] [55751.968590] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.968590] [55751.968593] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.968593] [55751.968594] [] ? dequeue_task_fair+0x41e/0x660 [55751.968595] [55751.968596] [] ? wake_up_state+0x20/0x20 [55751.968596] [55751.968599] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968599] [55751.968601] [] ? kthread+0xcf/0xe0 [55751.968601] [55751.968602] [] ? kthread_create_on_node+0x140/0x140 [55751.968602] [55751.968604] [] ? ret_from_fork+0x58/0x90 [55751.968604] [55751.968605] [] ? kthread_create_on_node+0x140/0x140 [55751.968606] Task dump for CPU 11: [55751.968606] kiblnd_sd_01_02 R [55751.968606] running task [55751.968607] 0 2902 2 0x00000088 [55751.968607] ffff88082363bd30 [55751.968607] 000000005cf0ae83 [55751.968607] ffffc900061c2218 [55751.968608] ffff8804c1ba6800 [55751.968608] [55751.968608] 0000000000000000 [55751.968608] 0000000000000000 [55751.968608] ffff88081f6b6a28 [55751.968609] ffff88082363bda8 [55751.968609] [55751.968609] ffffffffa0eba166 [55751.968609] ffff88082b2fc3c0 [55751.968610] ffff88081f6b6a00 [55751.968610] 000000000000000b [55751.968610] [55751.968610] Call Trace: [55751.968610] [55751.968613] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968614] [55751.968617] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968617] [55751.968618] [] ? dequeue_task_fair+0x41e/0x660 [55751.968618] [55751.968619] [] ? wake_up_state+0x20/0x20 [55751.968620] [55751.968623] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968623] [55751.968624] [] ? kthread+0xcf/0xe0 [55751.968625] [55751.968626] [] ? kthread_create_on_node+0x140/0x140 [55751.968626] [55751.968627] [] ? ret_from_fork+0x58/0x90 [55751.968628] [55751.968629] [] ? kthread_create_on_node+0x140/0x140 [55751.968629] Task dump for CPU 12: [55751.968630] kiblnd_sd_01_03 R [55751.968630] running task [55751.968630] 0 2903 2 0x00000088 [55751.968631] ffff880820a21000 [55751.968631] ffffc90011f075b0 [55751.968631] ffff8808266e7a00 [55751.968631] 0000000000000000 [55751.968632] [55751.968632] ffff8808213b7da8 [55751.968632] ffffffffa0ebedab [55751.968632] ffff8808213b7e90 [55751.968632] ffff8808213b7da8 [55751.968633] [55751.968633] ffffffffa03e7d2e [55751.968633] ffff88082b2fc3c0 [55751.968633] ffff88082c5f3600 [55751.968634] 0000000000000011 [55751.968634] [55751.968634] Call Trace: [55751.968634] [55751.968637] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.968638] [55751.968640] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.968640] [55751.968643] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.968643] [55751.968644] [] ? dequeue_task_fair+0x41e/0x660 [55751.968645] [55751.968646] [] ? wake_up_state+0x20/0x20 [55751.968646] [55751.968649] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968649] [55751.968651] [] ? kthread+0xcf/0xe0 [55751.968651] [55751.968652] [] ? kthread_create_on_node+0x140/0x140 [55751.968652] [55751.968654] [] ? ret_from_fork+0x58/0x90 [55751.968654] [55751.968655] [] ? kthread_create_on_node+0x140/0x140 [55751.968656] Task dump for CPU 15: [55751.968656] kiblnd_sd_01_00 R [55751.968656] running task [55751.968657] 0 2900 2 0x00000088 [55751.968657] ffff8808217bbd30 [55751.968657] 00000000937dfd11 [55751.968657] ffffc9000e335900 [55751.968658] ffff8804c1ba3200 [55751.968658] [55751.968658] 0000000000000000 [55751.968658] 0000000000000000 [55751.968658] ffff88082bda6c28 [55751.968659] ffff8808217bbda8 [55751.968659] [55751.968659] ffffffffa0eba166 [55751.968659] ffff88082b2fc3c0 [55751.968659] ffff88082bda6c00 [55751.968660] 0000000000000014 [55751.968660] [55751.968660] Call Trace: [55751.968660] [55751.968663] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968663] [55751.968667] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968667] [55751.968668] [] ? dequeue_task_fair+0x41e/0x660 [55751.968668] [55751.968669] [] ? wake_up_state+0x20/0x20 [55751.968670] [55751.968673] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968673] [55751.968674] [] ? kthread+0xcf/0xe0 [55751.968674] [55751.968675] [] ? kthread_create_on_node+0x140/0x140 [55751.968676] [55751.968677] [] ? ret_from_fork+0x58/0x90 [55751.968677] [55751.968678] [] ? kthread_create_on_node+0x140/0x140 [55751.968679] Task dump for CPU 1: [55751.968680] kiblnd_sd_00_00 R [55751.968681] running task [55751.968681] 0 2896 2 0x00000088 [55751.968682] ffff880820b03d30 [55751.968682] 000000003f673d40 [55751.968683] ffffc90003ec02f0 [55751.968683] ffff88041d63d600 [55751.968683] [55751.968683] 0000000000000000 [55751.968684] 0000000000000000 [55751.968684] ffff88042aa84a28 [55751.968684] ffff880820b03da8 [55751.968685] [55751.968685] ffffffffa0eba166 [55751.968685] ffff880181447180 [55751.968685] ffff88042aa84a00 [55751.968686] 0000000000000002 [55751.968686] [55751.968686] Call Trace: [55751.968686] [55751.968690] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968691] [55751.968694] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968694] [55751.968695] [] ? dequeue_task_fair+0x41e/0x660 [55751.968696] [55751.968697] [] ? sched_clock_cpu+0x85/0xc0 [55751.968697] [55751.968699] [] ? wake_up_state+0x20/0x20 [55751.968699] [55751.968702] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968702] [55751.968704] [] ? kthread+0xcf/0xe0 [55751.968704] [55751.968705] [] ? kthread_create_on_node+0x140/0x140 [55751.968705] [55751.968707] [] ? ret_from_fork+0x58/0x90 [55751.968707] [55751.968708] [] ? kthread_create_on_node+0x140/0x140 [55751.968709] Task dump for CPU 5: [55751.968709] kiblnd_sd_00_02 R [55751.968710] running task [55751.968710] 0 2898 2 0x00000088 [55751.968711] ffff880415b13000 [55751.968711] ffff8800abaa43a8 [55751.968711] ffff88083fb5e000 [55751.968711] 0000000000000000 [55751.968711] [55751.968712] ffff88082aae7da8 [55751.968712] ffffffffa0ebedab [55751.968712] ffff880181447188 [55751.968712] ffff88082aae7da8 [55751.968713] [55751.968713] ffffffff810bc064 [55751.968713] ffff880181447180 [55751.968713] ffff88042aa84a00 [55751.968713] 0000000000000001 [55751.968714] [55751.968714] Call Trace: [55751.968714] [55751.968717] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.968717] [55751.968718] [] ? __wake_up+0x44/0x50 [55751.968719] [55751.968722] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.968722] [55751.968723] [] ? dequeue_task_fair+0x41e/0x660 [55751.968723] [55751.968724] [] ? wake_up_state+0x20/0x20 [55751.968725] [55751.968728] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968728] [55751.968729] [] ? kthread+0xcf/0xe0 [55751.968729] [55751.968731] [] ? kthread_create_on_node+0x140/0x140 [55751.968731] [55751.968732] [] ? ret_from_fork+0x58/0x90 [55751.968732] [55751.968733] [] ? kthread_create_on_node+0x140/0x140 [55751.968734] Task dump for CPU 6: [55751.968735] kiblnd_sd_00_01 R [55751.968735] running task [55751.968735] 0 2897 2 0x00000088 [55751.968736] ffff88081f64fd30 [55751.968736] 00000000f1dbfe3f [55751.968736] ffffc90003ebfde0 [55751.968736] ffff88042dcd1e00 [55751.968736] [55751.968737] 0000000000000000 [55751.968737] 0000000000000000 [55751.968737] ffff88042aa84a28 [55751.968737] ffff88081f64fda8 [55751.968737] [55751.968738] ffffffffa0eba166 [55751.968738] ffff880181447180 [55751.968738] ffff88042aa84a00 [55751.968738] 0000000000000012 [55751.968739] [55751.968739] Call Trace: [55751.968739] [55751.968742] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968742] [55751.968745] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968745] [55751.968746] [] ? dequeue_task_fair+0x41e/0x660 [55751.968747] [55751.968748] [] ? wake_up_state+0x20/0x20 [55751.968748] [55751.968751] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968752] [55751.968753] [] ? kthread+0xcf/0xe0 [55751.968753] [55751.968754] [] ? kthread_create_on_node+0x140/0x140 [55751.968754] [55751.968756] [] ? ret_from_fork+0x58/0x90 [55751.968756] [55751.968757] [] ? kthread_create_on_node+0x140/0x140 [55751.968758] Task dump for CPU 7: [55751.968758] kiblnd_sd_00_03 R [55751.968758] running task [55751.968759] 0 2899 2 0x00000088 [55751.968759] ffff88082626bd30 [55751.968759] 00000000aaf7c743 [55751.968759] ffffc90003ebe8c8 [55751.968760] ffff8800b484ea00 [55751.968760] [55751.968760] 0000000000000000 [55751.968760] 0000000000000000 [55751.968761] ffff88042aa84a28 [55751.968761] ffff88082626bda8 [55751.968761] [55751.968761] ffffffffa0eba166 [55751.968761] ffff880181447180 [55751.968762] ffff88042aa84a00 [55751.968762] 0000000000000001 [55751.968762] [55751.968762] Call Trace: [55751.968762] [55751.968765] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968766] [55751.968769] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968769] [55751.968770] [] ? dequeue_task_fair+0x41e/0x660 [55751.968770] [55751.968771] [] ? wake_up_state+0x20/0x20 [55751.968772] [55751.968775] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968775] [55751.968776] [] ? kthread+0xcf/0xe0 [55751.968777] [55751.968778] [] ? kthread_create_on_node+0x140/0x140 [55751.968778] [55751.968779] [] ? ret_from_fork+0x58/0x90 [55751.968779] [55751.968781] [] ? kthread_create_on_node+0x140/0x140 [55751.968781] Task dump for CPU 10: [55751.968781] kiblnd_sd_01_01 R [55751.968782] running task [55751.968782] 0 2901 2 0x00000088 [55751.968782] ffff88082aa3de20 [55751.968783] 00000000b1c67729 [55751.968783] ffff88082d883db0 [55751.968783] ffffffff810c42f8 [55751.968783] [55751.968783] 000000000000000a [55751.968784] ffffffff81a1a680 [55751.968784] ffff88082d883dc8 [55751.968784] ffffffff810c7b89 [55751.968784] [55751.968784] 000000000000000b [55751.968785] ffff88082d883df8 [55751.968785] ffffffff811373f0 [55751.968785] ffff88082d8901c0 [55751.968785] [55751.968785] Call Trace: [55751.968786] [55751.968786] [55751.968787] [] sched_show_task+0xa8/0x110 [55751.968788] [55751.968789] [] dump_cpu_task+0x39/0x70 [55751.968789] [55751.968790] [] rcu_dump_cpu_stacks+0x90/0xd0 [55751.968791] [55751.968792] [] rcu_check_callbacks+0x442/0x720 [55751.968792] [55751.968794] [] ? tick_sched_handle.isra.13+0x60/0x60 [55751.968794] [55751.968796] [] update_process_times+0x47/0x80 [55751.968796] [55751.968797] [] tick_sched_handle.isra.13+0x25/0x60 [55751.968797] [55751.968798] [] tick_sched_timer+0x41/0x70 [55751.968799] [55751.968800] [] __hrtimer_run_queues+0xd2/0x260 [55751.968800] [55751.968801] [] hrtimer_interrupt+0xb0/0x1e0 [55751.968801] [55751.968802] [] ? call_softirq+0x1c/0x30 [55751.968802] [55751.968805] [] local_apic_timer_interrupt+0x37/0x60 [55751.968805] [55751.968806] [] smp_apic_timer_interrupt+0x3f/0x60 [55751.968807] [55751.968807] [] apic_timer_interrupt+0x6d/0x80 [55751.968808] [55751.968808] [55751.968809] [] ? _raw_spin_lock+0x32/0x50 [55751.968809] [55751.968815] [] ? lnet_find_peer_ni_locked+0x18/0x130 [lnet] [55751.968816] [55751.968821] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55751.968821] [55751.968826] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55751.968826] [55751.968831] [] lnet_select_pathway+0x235/0x1140 [lnet] [55751.968832] [55751.968835] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55751.968835] [55751.968841] [] lnet_send+0x51/0x180 [lnet] [55751.968841] [55751.968846] [] lnet_finalize+0x3f1/0x690 [lnet] [55751.968846] [55751.968850] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55751.968850] [55751.968851] [] ? __slab_free+0x81/0x2f0 [55751.968852] [55751.968857] [] lnet_ni_recv+0xc3/0x320 [lnet] [55751.968857] [55751.968862] [] lnet_parse+0x945/0xf90 [lnet] [55751.968863] [55751.968866] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.968867] [55751.968869] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.968869] [55751.968872] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.968873] [55751.968874] [] ? dequeue_task_fair+0x41e/0x660 [55751.968874] [55751.968875] [] ? wake_up_state+0x20/0x20 [55751.968876] [55751.968879] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968879] [55751.968881] [] kthread+0xcf/0xe0 [55751.968881] [55751.968882] [] ? kthread_create_on_node+0x140/0x140 [55751.968883] [55751.968884] [] ret_from_fork+0x58/0x90 [55751.968884] [55751.968886] [] ? kthread_create_on_node+0x140/0x140 [55751.968886] Task dump for CPU 11: [55751.968887] kiblnd_sd_01_02 R [55751.968887] running task [55751.968888] 0 2902 2 0x00000088 [55751.968888] ffff88082363bd30 [55751.968888] 000000005cf0ae83 [55751.968888] ffffc900061c2218 [55751.968889] ffff8804c1ba6800 [55751.968889] [55751.968889] 0000000000000000 [55751.968889] 0000000000000000 [55751.968889] ffff88081f6b6a28 [55751.968890] ffff88082363bda8 [55751.968890] [55751.968890] ffffffffa0eba166 [55751.968890] ffff88082b2fc3c0 [55751.968891] ffff88081f6b6a00 [55751.968891] 000000000000000b [55751.968891] [55751.968891] Call Trace: [55751.968891] [55751.968894] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968895] [55751.968898] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968898] [55751.968899] [] ? dequeue_task_fair+0x41e/0x660 [55751.968899] [55751.968901] [] ? wake_up_state+0x20/0x20 [55751.968901] [55751.968904] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968904] [55751.968906] [] ? kthread+0xcf/0xe0 [55751.968906] [55751.968907] [] ? kthread_create_on_node+0x140/0x140 [55751.968907] [55751.968909] [] ? ret_from_fork+0x58/0x90 [55751.968909] [55751.968910] [] ? kthread_create_on_node+0x140/0x140 [55751.968911] Task dump for CPU 12: [55751.968911] kiblnd_sd_01_03 R [55751.968911] running task [55751.968911] 0 2903 2 0x00000088 [55751.968912] ffff880820a21000 [55751.968912] ffffc90011f075b0 [55751.968912] ffff8808266e7a00 [55751.968912] 0000000000000000 [55751.968913] [55751.968913] ffff8808213b7da8 [55751.968913] ffffffffa0ebedab [55751.968913] ffff8808213b7e90 [55751.968914] ffff8808213b7da8 [55751.968914] [55751.968914] ffffffffa03e7d2e [55751.968914] ffff88082b2fc3c0 [55751.968914] ffff88082c5f3600 [55751.968915] 0000000000000011 [55751.968915] [55751.968915] Call Trace: [55751.968915] [55751.968918] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.968919] [55751.968921] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.968921] [55751.968924] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.968924] [55751.968925] [] ? dequeue_task_fair+0x41e/0x660 [55751.968926] [55751.968927] [] ? wake_up_state+0x20/0x20 [55751.968927] [55751.968930] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968931] [55751.968932] [] ? kthread+0xcf/0xe0 [55751.968932] [55751.968933] [] ? kthread_create_on_node+0x140/0x140 [55751.968933] [55751.968935] [] ? ret_from_fork+0x58/0x90 [55751.968935] [55751.968936] [] ? kthread_create_on_node+0x140/0x140 [55751.968937] Task dump for CPU 15: [55751.968937] kiblnd_sd_01_00 R [55751.968937] running task [55751.968938] 0 2900 2 0x00000088 [55751.968938] ffff8808217bbd30 [55751.968938] 00000000937dfd11 [55751.968938] ffffc9000e335900 [55751.968939] ffff8804c1ba3200 [55751.968939] [55751.968939] 0000000000000000 [55751.968939] 0000000000000000 [55751.968939] ffff88082bda6c28 [55751.968940] ffff8808217bbda8 [55751.968940] [55751.968940] ffffffffa0eba166 [55751.968940] ffff88082b2fc3c0 [55751.968940] ffff88082bda6c00 [55751.968941] 0000000000000014 [55751.968941] [55751.968941] Call Trace: [55751.968941] [55751.968944] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.968944] [55751.968948] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.968948] [55751.968949] [] ? dequeue_task_fair+0x41e/0x660 [55751.968949] [55751.968950] [] ? wake_up_state+0x20/0x20 [55751.968951] [55751.968954] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.968954] [55751.968955] [] ? kthread+0xcf/0xe0 [55751.968955] [55751.968957] [] ? kthread_create_on_node+0x140/0x140 [55751.968957] [55751.968958] [] ? ret_from_fork+0x58/0x90 [55751.968958] [55751.968960] [] ? kthread_create_on_node+0x140/0x140 [55751.968961] Task dump for CPU 1: [55751.968961] kiblnd_sd_00_00 R [55751.968962] running task [55751.968963] 0 2896 2 0x00000088 [55751.968963] ffff8804c1673ec0 [55751.968963] 000000003f673d40 [55751.968964] ffff88042e043db0 [55751.968964] ffffffff810c42f8 [55751.968964] [55751.968965] 0000000000000001 [55751.968965] ffffffff81a1a680 [55751.968965] ffff88042e043dc8 [55751.968965] ffffffff810c7b89 [55751.968965] [55751.968966] 0000000000000002 [55751.968966] ffff88042e043df8 [55751.968966] ffffffff811373f0 [55751.968966] ffff88042e0501c0 [55751.968967] [55751.968967] Call Trace: [55751.968967] [55751.968968] [55751.968969] [] sched_show_task+0xa8/0x110 [55751.968970] [55751.968971] [] dump_cpu_task+0x39/0x70 [55751.968971] [55751.968972] [] rcu_dump_cpu_stacks+0x90/0xd0 [55751.968972] [55751.968973] [] rcu_check_callbacks+0x442/0x720 [55751.968974] [55751.968975] [] ? tick_sched_handle.isra.13+0x60/0x60 [55751.968975] [55751.968977] [] update_process_times+0x47/0x80 [55751.968977] [55751.968978] [] tick_sched_handle.isra.13+0x25/0x60 [55751.968978] [55751.968979] [] tick_sched_timer+0x41/0x70 [55751.968979] [55751.968981] [] __hrtimer_run_queues+0xd2/0x260 [55751.968981] [55751.968982] [] hrtimer_interrupt+0xb0/0x1e0 [55751.968982] [55751.968983] [] ? call_softirq+0x1c/0x30 [55751.968983] [55751.968985] [] local_apic_timer_interrupt+0x37/0x60 [55751.968985] [55751.968986] [] smp_apic_timer_interrupt+0x3f/0x60 [55751.968986] [55751.968987] [] apic_timer_interrupt+0x6d/0x80 [55751.968987] [55751.968988] [55751.968989] [] ? _raw_spin_lock+0x32/0x50 [55751.968989] [55751.968995] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55751.968995] [55751.969001] [] lnet_post_send_locked+0x446/0x880 [lnet] [55751.969001] [55751.969006] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55751.969007] [55751.969012] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55751.969012] [55751.969017] [] lnet_finalize+0x1e9/0x690 [lnet] [55751.969017] [55751.969021] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969021] [55751.969024] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969025] [55751.969026] [] ? dequeue_task_fair+0x41e/0x660 [55751.969026] [55751.969027] [] ? sched_clock_cpu+0x85/0xc0 [55751.969028] [55751.969029] [] ? wake_up_state+0x20/0x20 [55751.969029] [55751.969032] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969033] [55751.969034] [] kthread+0xcf/0xe0 [55751.969034] [55751.969035] [] ? kthread_create_on_node+0x140/0x140 [55751.969036] [55751.969037] [] ret_from_fork+0x58/0x90 [55751.969037] [55751.969039] [] ? kthread_create_on_node+0x140/0x140 [55751.969039] Task dump for CPU 5: [55751.969040] kiblnd_sd_00_02 R [55751.969040] running task [55751.969041] 0 2898 2 0x00000088 [55751.969041] ffff880415b13000 [55751.969041] ffff8800abaa43a8 [55751.969041] ffff88083fb5e000 [55751.969042] 0000000000000000 [55751.969042] [55751.969042] ffff88082aae7da8 [55751.969042] ffffffffa0ebedab [55751.969043] ffff880181447188 [55751.969043] ffff88082aae7da8 [55751.969043] [55751.969043] ffffffff810bc064 [55751.969043] ffff880181447180 [55751.969044] ffff88042aa84a00 [55751.969044] 0000000000000001 [55751.969044] [55751.969044] Call Trace: [55751.969045] [55751.969048] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969048] [55751.969049] [] ? __wake_up+0x44/0x50 [55751.969049] [55751.969052] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969052] [55751.969053] [] ? dequeue_task_fair+0x41e/0x660 [55751.969054] [55751.969055] [] ? wake_up_state+0x20/0x20 [55751.969055] [55751.969058] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969058] [55751.969060] [] ? kthread+0xcf/0xe0 [55751.969060] [55751.969061] [] ? kthread_create_on_node+0x140/0x140 [55751.969061] [55751.969063] [] ? ret_from_fork+0x58/0x90 [55751.969063] [55751.969064] [] ? kthread_create_on_node+0x140/0x140 [55751.969065] Task dump for CPU 6: [55751.969065] kiblnd_sd_00_01 R [55751.969065] running task [55751.969066] 0 2897 2 0x00000088 [55751.969066] ffff88081f64fd30 [55751.969066] 00000000f1dbfe3f [55751.969066] ffffc90003ebfde0 [55751.969067] ffff88042dcd1e00 [55751.969067] [55751.969067] 0000000000000000 [55751.969067] 0000000000000000 [55751.969068] ffff88042aa84a28 [55751.969068] ffff88081f64fda8 [55751.969068] [55751.969068] ffffffffa0eba166 [55751.969068] ffff880181447180 [55751.969069] ffff88042aa84a00 [55751.969069] 0000000000000012 [55751.969069] [55751.969069] Call Trace: [55751.969070] [55751.969073] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969073] [55751.969076] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969076] [55751.969077] [] ? dequeue_task_fair+0x41e/0x660 [55751.969077] [55751.969079] [] ? wake_up_state+0x20/0x20 [55751.969079] [55751.969082] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969082] [55751.969083] [] ? kthread+0xcf/0xe0 [55751.969084] [55751.969085] [] ? kthread_create_on_node+0x140/0x140 [55751.969085] [55751.969086] [] ? ret_from_fork+0x58/0x90 [55751.969087] [55751.969088] [] ? kthread_create_on_node+0x140/0x140 [55751.969088] Task dump for CPU 7: [55751.969089] kiblnd_sd_00_03 R [55751.969089] running task [55751.969089] 0 2899 2 0x00000088 [55751.969090] ffff88082626bd30 [55751.969090] 00000000aaf7c743 [55751.969090] ffffc90003ebe8c8 [55751.969090] ffff8800b484ea00 [55751.969091] [55751.969091] 0000000000000000 [55751.969091] 0000000000000000 [55751.969091] ffff88042aa84a28 [55751.969091] ffff88082626bda8 [55751.969092] [55751.969092] ffffffffa0eba166 [55751.969092] ffff880181447180 [55751.969092] ffff88042aa84a00 [55751.969092] 0000000000000001 [55751.969093] [55751.969093] Call Trace: [55751.969093] [55751.969096] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969096] [55751.969099] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969099] [55751.969101] [] ? dequeue_task_fair+0x41e/0x660 [55751.969101] [55751.969102] [] ? wake_up_state+0x20/0x20 [55751.969102] [55751.969105] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969106] [55751.969107] [] ? kthread+0xcf/0xe0 [55751.969107] [55751.969108] [] ? kthread_create_on_node+0x140/0x140 [55751.969108] [55751.969110] [] ? ret_from_fork+0x58/0x90 [55751.969110] [55751.969111] [] ? kthread_create_on_node+0x140/0x140 [55751.969112] Task dump for CPU 10: [55751.969112] kiblnd_sd_01_01 R [55751.969112] running task [55751.969113] 0 2901 2 0x00000088 [55751.969113] ffff8807f886f000 [55751.969113] ffffc9001327c860 [55751.969114] ffff8808266e7a00 [55751.969114] 0000000000000000 [55751.969114] [55751.969114] ffff8808262efda8 [55751.969114] ffffffffa0ebedab [55751.969115] ffff8808262efe90 [55751.969115] ffff8808262efda8 [55751.969115] [55751.969115] ffffffffa03e7d2e [55751.969116] ffff88082b2fc3c0 [55751.969116] ffff88082bda6c00 [55751.969116] 0000000000000017 [55751.969116] [55751.969116] Call Trace: [55751.969117] [55751.969120] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969121] [55751.969123] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.969124] [55751.969127] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969127] [55751.969128] [] ? dequeue_task_fair+0x41e/0x660 [55751.969128] [55751.969129] [] ? wake_up_state+0x20/0x20 [55751.969130] [55751.969133] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969133] [55751.969134] [] ? kthread+0xcf/0xe0 [55751.969135] [55751.969136] [] ? kthread_create_on_node+0x140/0x140 [55751.969136] [55751.969137] [] ? ret_from_fork+0x58/0x90 [55751.969137] [55751.969139] [] ? kthread_create_on_node+0x140/0x140 [55751.969139] Task dump for CPU 11: [55751.969139] kiblnd_sd_01_02 R [55751.969140] running task [55751.969140] 0 2902 2 0x00000088 [55751.969140] ffff88082363bd30 [55751.969141] 000000005cf0ae83 [55751.969141] ffffc900061c2218 [55751.969141] ffff8804c1ba6800 [55751.969141] [55751.969141] 0000000000000000 [55751.969142] 0000000000000000 [55751.969142] ffff88081f6b6a28 [55751.969142] ffff88082363bda8 [55751.969142] [55751.969142] ffffffffa0eba166 [55751.969143] ffff88082b2fc3c0 [55751.969143] ffff88081f6b6a00 [55751.969143] 000000000000000b [55751.969143] [55751.969143] Call Trace: [55751.969144] [55751.969147] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969147] [55751.969150] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969150] [55751.969151] [] ? dequeue_task_fair+0x41e/0x660 [55751.969151] [55751.969153] [] ? wake_up_state+0x20/0x20 [55751.969153] [55751.969156] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969156] [55751.969158] [] ? kthread+0xcf/0xe0 [55751.969158] [55751.969159] [] ? kthread_create_on_node+0x140/0x140 [55751.969159] [55751.969161] [] ? ret_from_fork+0x58/0x90 [55751.969161] [55751.969162] [] ? kthread_create_on_node+0x140/0x140 [55751.969162] Task dump for CPU 12: [55751.969163] kiblnd_sd_01_03 R [55751.969163] running task [55751.969163] 0 2903 2 0x00000088 [55751.969164] ffff880820a21000 [55751.969164] ffffc90011f075b0 [55751.969164] ffff8808266e7a00 [55751.969164] 0000000000000000 [55751.969164] [55751.969165] ffff8808213b7da8 [55751.969165] ffffffffa0ebedab [55751.969165] ffff8808213b7e90 [55751.969165] ffff8808213b7da8 [55751.969165] [55751.969166] ffffffffa03e7d2e [55751.969166] ffff88082b2fc3c0 [55751.969166] ffff88082c5f3600 [55751.969166] 0000000000000011 [55751.969167] [55751.969167] Call Trace: [55751.969167] [55751.969170] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969171] [55751.969172] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.969173] [55751.969176] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969176] [55751.969177] [] ? dequeue_task_fair+0x41e/0x660 [55751.969177] [55751.969179] [] ? wake_up_state+0x20/0x20 [55751.969179] [55751.969182] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969182] [55751.969184] [] ? kthread+0xcf/0xe0 [55751.969184] [55751.969185] [] ? kthread_create_on_node+0x140/0x140 [55751.969185] [55751.969187] [] ? ret_from_fork+0x58/0x90 [55751.969187] [55751.969188] [] ? kthread_create_on_node+0x140/0x140 [55751.969189] Task dump for CPU 15: [55751.969189] kiblnd_sd_01_00 R [55751.969189] running task [55751.969190] 0 2900 2 0x00000088 [55751.969190] ffff8808217bbd30 [55751.969190] 00000000937dfd11 [55751.969190] ffffc9000e335900 [55751.969191] ffff8804c1ba3200 [55751.969191] [55751.969191] 0000000000000000 [55751.969191] 0000000000000000 [55751.969191] ffff88082bda6c28 [55751.969192] ffff8808217bbda8 [55751.969192] [55751.969192] ffffffffa0eba166 [55751.969192] ffff88082b2fc3c0 [55751.969192] ffff88082bda6c00 [55751.969193] 0000000000000014 [55751.969193] [55751.969193] Call Trace: [55751.969193] [55751.969196] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969197] [55751.969200] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969200] [55751.969201] [] ? dequeue_task_fair+0x41e/0x660 [55751.969201] [55751.969202] [] ? wake_up_state+0x20/0x20 [55751.969203] [55751.969206] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969206] [55751.969207] [] ? kthread+0xcf/0xe0 [55751.969207] [55751.969209] [] ? kthread_create_on_node+0x140/0x140 [55751.969209] [55751.969210] [] ? ret_from_fork+0x58/0x90 [55751.969210] [55751.969212] [] ? kthread_create_on_node+0x140/0x140 [55751.969212] Task dump for CPU 1: [55751.969213] kiblnd_sd_00_00 R [55751.969213] running task [55751.969214] 0 2896 2 0x00000088 [55751.969215] ffff880820b03d30 [55751.969215] 000000003f673d40 [55751.969215] ffffc90003ec02f0 [55751.969215] ffff88041d63d600 [55751.969215] [55751.969216] 0000000000000000 [55751.969216] 0000000000000000 [55751.969216] ffff88042aa84a28 [55751.969216] ffff880820b03da8 [55751.969217] [55751.969217] ffffffffa0eba166 [55751.969217] ffff880181447180 [55751.969217] ffff88042aa84a00 [55751.969217] 0000000000000002 [55751.969218] [55751.969218] Call Trace: [55751.969218] [55751.969222] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969222] [55751.969225] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969226] [55751.969227] [] ? dequeue_task_fair+0x41e/0x660 [55751.969227] [55751.969228] [] ? sched_clock_cpu+0x85/0xc0 [55751.969229] [55751.969230] [] ? wake_up_state+0x20/0x20 [55751.969230] [55751.969233] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969234] [55751.969235] [] ? kthread+0xcf/0xe0 [55751.969235] [55751.969236] [] ? kthread_create_on_node+0x140/0x140 [55751.969237] [55751.969238] [] ? ret_from_fork+0x58/0x90 [55751.969238] [55751.969239] [] ? kthread_create_on_node+0x140/0x140 [55751.969240] Task dump for CPU 5: [55751.969240] kiblnd_sd_00_02 R [55751.969241] running task [55751.969241] 0 2898 2 0x00000088 [55751.969241] ffff880415b13000 [55751.969242] ffff8800abaa43a8 [55751.969242] ffff88083fb5e000 [55751.969242] 0000000000000000 [55751.969242] [55751.969242] ffff88082aae7da8 [55751.969243] ffffffffa0ebedab [55751.969243] ffff880181447188 [55751.969243] ffff88082aae7da8 [55751.969243] [55751.969243] ffffffff810bc064 [55751.969244] ffff880181447180 [55751.969244] ffff88042aa84a00 [55751.969244] 0000000000000001 [55751.969244] [55751.969245] Call Trace: [55751.969245] [55751.969248] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969248] [55751.969249] [] ? __wake_up+0x44/0x50 [55751.969249] [55751.969252] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969253] [55751.969254] [] ? dequeue_task_fair+0x41e/0x660 [55751.969254] [55751.969255] [] ? wake_up_state+0x20/0x20 [55751.969255] [55751.969259] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969259] [55751.969260] [] ? kthread+0xcf/0xe0 [55751.969260] [55751.969261] [] ? kthread_create_on_node+0x140/0x140 [55751.969262] [55751.969263] [] ? ret_from_fork+0x58/0x90 [55751.969263] [55751.969264] [] ? kthread_create_on_node+0x140/0x140 [55751.969265] Task dump for CPU 6: [55751.969265] kiblnd_sd_00_01 R [55751.969265] running task [55751.969266] 0 2897 2 0x00000088 [55751.969266] ffff8804c1675e20 [55751.969266] 00000000f1dbfe3f [55751.969267] ffff88042e183db0 [55751.969267] ffffffff810c42f8 [55751.969267] [55751.969267] 0000000000000006 [55751.969267] ffffffff81a1a680 [55751.969268] ffff88042e183dc8 [55751.969268] ffffffff810c7b89 [55751.969268] [55751.969268] 0000000000000007 [55751.969268] ffff88042e183df8 [55751.969269] ffffffff811373f0 [55751.969269] ffff88042e1901c0 [55751.969269] [55751.969269] Call Trace: [55751.969270] [55751.969270] [55751.969271] [] sched_show_task+0xa8/0x110 [55751.969271] [55751.969273] [] dump_cpu_task+0x39/0x70 [55751.969273] [55751.969274] [] rcu_dump_cpu_stacks+0x90/0xd0 [55751.969274] [55751.969275] [] rcu_check_callbacks+0x442/0x720 [55751.969275] [55751.969277] [] ? tick_sched_handle.isra.13+0x60/0x60 [55751.969277] [55751.969278] [] update_process_times+0x47/0x80 [55751.969278] [55751.969280] [] tick_sched_handle.isra.13+0x25/0x60 [55751.969280] [55751.969281] [] tick_sched_timer+0x41/0x70 [55751.969281] [55751.969282] [] __hrtimer_run_queues+0xd2/0x260 [55751.969282] [55751.969283] [] hrtimer_interrupt+0xb0/0x1e0 [55751.969283] [55751.969284] [] ? call_softirq+0x1c/0x30 [55751.969285] [55751.969286] [] local_apic_timer_interrupt+0x37/0x60 [55751.969286] [55751.969287] [] smp_apic_timer_interrupt+0x3f/0x60 [55751.969288] [55751.969288] [] apic_timer_interrupt+0x6d/0x80 [55751.969289] [55751.969289] [55751.969290] [] ? _raw_spin_lock+0x32/0x50 [55751.969290] [55751.969295] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55751.969295] [55751.969300] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55751.969300] [55751.969306] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55751.969306] [55751.969311] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55751.969312] [55751.969317] [] lnet_finalize+0x1e9/0x690 [lnet] [55751.969317] [55751.969320] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969320] [55751.969323] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969324] [55751.969325] [] ? dequeue_task_fair+0x41e/0x660 [55751.969325] [55751.969326] [] ? wake_up_state+0x20/0x20 [55751.969326] [55751.969330] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969330] [55751.969331] [] kthread+0xcf/0xe0 [55751.969331] [55751.969333] [] ? kthread_create_on_node+0x140/0x140 [55751.969333] [55751.969334] [] ret_from_fork+0x58/0x90 [55751.969334] [55751.969336] [] ? kthread_create_on_node+0x140/0x140 [55751.969336] Task dump for CPU 7: [55751.969336] kiblnd_sd_00_03 R [55751.969337] running task [55751.969337] 0 2899 2 0x00000088 [55751.969337] ffff88082626bd30 [55751.969338] 00000000aaf7c743 [55751.969338] ffffc90003ebe8c8 [55751.969338] ffff8800b484ea00 [55751.969338] [55751.969338] 0000000000000000 [55751.969339] 0000000000000000 [55751.969339] ffff88042aa84a28 [55751.969339] ffff88082626bda8 [55751.969339] [55751.969339] ffffffffa0eba166 [55751.969340] ffff880181447180 [55751.969340] ffff88042aa84a00 [55751.969340] 0000000000000001 [55751.969340] [55751.969341] Call Trace: [55751.969341] [55751.969344] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969344] [55751.969347] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969347] [55751.969348] [] ? dequeue_task_fair+0x41e/0x660 [55751.969349] [55751.969350] [] ? wake_up_state+0x20/0x20 [55751.969350] [55751.969353] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969353] [55751.969355] [] ? kthread+0xcf/0xe0 [55751.969355] [55751.969356] [] ? kthread_create_on_node+0x140/0x140 [55751.969356] [55751.969358] [] ? ret_from_fork+0x58/0x90 [55751.969358] [55751.969359] [] ? kthread_create_on_node+0x140/0x140 [55751.969360] Task dump for CPU 10: [55751.969360] kiblnd_sd_01_01 R [55751.969360] running task [55751.969360] 0 2901 2 0x00000088 [55751.969361] ffff8807f886f000 [55751.969361] ffffc9001327c860 [55751.969361] ffff8808266e7a00 [55751.969361] 0000000000000000 [55751.969362] [55751.969362] ffff8808262efda8 [55751.969362] ffffffffa0ebedab [55751.969362] ffff8808262efe90 [55751.969362] ffff8808262efda8 [55751.969363] [55751.969363] ffffffffa03e7d2e [55751.969363] ffff88082b2fc3c0 [55751.969363] ffff88082bda6c00 [55751.969364] 0000000000000017 [55751.969364] [55751.969364] Call Trace: [55751.969364] [55751.969367] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969369] [55751.969371] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.969371] [55751.969374] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969374] [55751.969375] [] ? dequeue_task_fair+0x41e/0x660 [55751.969376] [55751.969377] [] ? wake_up_state+0x20/0x20 [55751.969377] [55751.969380] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969380] [55751.969381] [] ? kthread+0xcf/0xe0 [55751.969382] [55751.969383] [] ? kthread_create_on_node+0x140/0x140 [55751.969383] [55751.969384] [] ? ret_from_fork+0x58/0x90 [55751.969385] [55751.969386] [] ? kthread_create_on_node+0x140/0x140 [55751.969386] Task dump for CPU 11: [55751.969387] kiblnd_sd_01_02 R [55751.969387] running task [55751.969387] 0 2902 2 0x00000088 [55751.969388] ffff88082363bd30 [55751.969388] 000000005cf0ae83 [55751.969388] ffffc900061c2218 [55751.969388] ffff8804c1ba6800 [55751.969388] [55751.969389] 0000000000000000 [55751.969389] 0000000000000000 [55751.969389] ffff88081f6b6a28 [55751.969389] ffff88082363bda8 [55751.969389] [55751.969390] ffffffffa0eba166 [55751.969390] ffff88082b2fc3c0 [55751.969390] ffff88081f6b6a00 [55751.969390] 000000000000000b [55751.969390] [55751.969391] Call Trace: [55751.969391] [55751.969394] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969394] [55751.969397] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969398] [55751.969399] [] ? dequeue_task_fair+0x41e/0x660 [55751.969399] [55751.969400] [] ? wake_up_state+0x20/0x20 [55751.969400] [55751.969404] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969404] [55751.969405] [] ? kthread+0xcf/0xe0 [55751.969405] [55751.969407] [] ? kthread_create_on_node+0x140/0x140 [55751.969407] [55751.969408] [] ? ret_from_fork+0x58/0x90 [55751.969408] [55751.969410] [] ? kthread_create_on_node+0x140/0x140 [55751.969410] Task dump for CPU 12: [55751.969411] kiblnd_sd_01_03 R [55751.969411] running task [55751.969411] 0 2903 2 0x00000088 [55751.969411] ffff880820a21000 [55751.969412] ffffc90011f075b0 [55751.969412] ffff8808266e7a00 [55751.969412] 0000000000000000 [55751.969412] [55751.969412] ffff8808213b7da8 [55751.969413] ffffffffa0ebedab [55751.969413] ffff8808213b7e90 [55751.969413] ffff8808213b7da8 [55751.969413] [55751.969414] ffffffffa03e7d2e [55751.969414] ffff88082b2fc3c0 [55751.969414] ffff88082c5f3600 [55751.969414] 0000000000000011 [55751.969414] [55751.969415] Call Trace: [55751.969415] [55751.969418] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969419] [55751.969420] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.969421] [55751.969424] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969424] [55751.969425] [] ? dequeue_task_fair+0x41e/0x660 [55751.969425] [55751.969426] [] ? wake_up_state+0x20/0x20 [55751.969427] [55751.969430] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969430] [55751.969431] [] ? kthread+0xcf/0xe0 [55751.969431] [55751.969433] [] ? kthread_create_on_node+0x140/0x140 [55751.969433] [55751.969434] [] ? ret_from_fork+0x58/0x90 [55751.969434] [55751.969435] [] ? kthread_create_on_node+0x140/0x140 [55751.969436] Task dump for CPU 15: [55751.969436] kiblnd_sd_01_00 R [55751.969437] running task [55751.969437] 0 2900 2 0x00000088 [55751.969437] ffff8808217bbd30 [55751.969437] 00000000937dfd11 [55751.969438] ffffc9000e335900 [55751.969438] ffff8804c1ba3200 [55751.969438] [55751.969438] 0000000000000000 [55751.969438] 0000000000000000 [55751.969439] ffff88082bda6c28 [55751.969439] ffff8808217bbda8 [55751.969439] [55751.969439] ffffffffa0eba166 [55751.969439] ffff88082b2fc3c0 [55751.969440] ffff88082bda6c00 [55751.969440] 0000000000000014 [55751.969440] [55751.969440] Call Trace: [55751.969441] [55751.969443] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969444] [55751.969447] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969447] [55751.969448] [] ? dequeue_task_fair+0x41e/0x660 [55751.969448] [55751.969449] [] ? wake_up_state+0x20/0x20 [55751.969450] [55751.969453] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969453] [55751.969454] [] ? kthread+0xcf/0xe0 [55751.969454] [55751.969456] [] ? kthread_create_on_node+0x140/0x140 [55751.969456] [55751.969457] [] ? ret_from_fork+0x58/0x90 [55751.969457] [55751.969459] [] ? kthread_create_on_node+0x140/0x140 [55751.969459] Task dump for CPU 1: [55751.969460] kiblnd_sd_00_00 R [55751.969460] running task [55751.969461] 0 2896 2 0x00000088 [55751.969461] ffff880820b03d30 [55751.969462] 000000003f673d40 [55751.969462] ffffc90003ec02f0 [55751.969462] ffff88041d63d600 [55751.969462] [55751.969463] 0000000000000000 [55751.969463] 0000000000000000 [55751.969463] ffff88042aa84a28 [55751.969463] ffff880820b03da8 [55751.969464] [55751.969464] ffffffffa0eba166 [55751.969464] ffff880181447180 [55751.969464] ffff88042aa84a00 [55751.969465] 0000000000000002 [55751.969465] [55751.969465] Call Trace: [55751.969465] [55751.969469] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969469] [55751.969473] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969473] [55751.969474] [] ? dequeue_task_fair+0x41e/0x660 [55751.969474] [55751.969476] [] ? sched_clock_cpu+0x85/0xc0 [55751.969476] [55751.969477] [] ? wake_up_state+0x20/0x20 [55751.969478] [55751.969481] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969481] [55751.969482] [] ? kthread+0xcf/0xe0 [55751.969483] [55751.969484] [] ? kthread_create_on_node+0x140/0x140 [55751.969484] [55751.969486] [] ? ret_from_fork+0x58/0x90 [55751.969486] [55751.969487] [] ? kthread_create_on_node+0x140/0x140 [55751.969488] Task dump for CPU 5: [55751.969488] kiblnd_sd_00_02 R [55751.969488] running task [55751.969489] 0 2898 2 0x00000088 [55751.969489] ffff880415b13000 [55751.969489] ffff8800abaa43a8 [55751.969489] ffff88083fb5e000 [55751.969490] 0000000000000000 [55751.969490] [55751.969490] ffff88082aae7da8 [55751.969490] ffffffffa0ebedab [55751.969490] ffff880181447188 [55751.969491] ffff88082aae7da8 [55751.969491] [55751.969491] ffffffff810bc064 [55751.969491] ffff880181447180 [55751.969491] ffff88042aa84a00 [55751.969492] 0000000000000001 [55751.969492] [55751.969492] Call Trace: [55751.969492] [55751.969495] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969496] [55751.969496] [] ? __wake_up+0x44/0x50 [55751.969497] [55751.969500] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969500] [55751.969501] [] ? dequeue_task_fair+0x41e/0x660 [55751.969501] [55751.969503] [] ? wake_up_state+0x20/0x20 [55751.969503] [55751.969506] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969506] [55751.969507] [] ? kthread+0xcf/0xe0 [55751.969508] [55751.969509] [] ? kthread_create_on_node+0x140/0x140 [55751.969509] [55751.969510] [] ? ret_from_fork+0x58/0x90 [55751.969511] [55751.969512] [] ? kthread_create_on_node+0x140/0x140 [55751.969512] Task dump for CPU 6: [55751.969513] kiblnd_sd_00_01 R [55751.969513] running task [55751.969513] 0 2897 2 0x00000088 [55751.969514] ffff88081f64fd30 [55751.969514] 00000000f1dbfe3f [55751.969514] ffffc90003ebfde0 [55751.969514] ffff88042dcd1e00 [55751.969515] [55751.969515] 0000000000000000 [55751.969515] 0000000000000000 [55751.969515] ffff88042aa84a28 [55751.969515] ffff88081f64fda8 [55751.969516] [55751.969516] ffffffffa0eba166 [55751.969516] ffff880181447180 [55751.969516] ffff88042aa84a00 [55751.969516] 0000000000000012 [55751.969517] [55751.969517] Call Trace: [55751.969517] [55751.969520] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969520] [55751.969523] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969524] [55751.969525] [] ? dequeue_task_fair+0x41e/0x660 [55751.969525] [55751.969526] [] ? wake_up_state+0x20/0x20 [55751.969526] [55751.969529] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969530] [55751.969531] [] ? kthread+0xcf/0xe0 [55751.969531] [55751.969532] [] ? kthread_create_on_node+0x140/0x140 [55751.969532] [55751.969534] [] ? ret_from_fork+0x58/0x90 [55751.969534] [55751.969535] [] ? kthread_create_on_node+0x140/0x140 [55751.969536] Task dump for CPU 7: [55751.969536] kiblnd_sd_00_03 R [55751.969536] running task [55751.969537] 0 2899 2 0x00000088 [55751.969537] ffff88082af10fb0 [55751.969537] 00000000aaf7c743 [55751.969537] ffff88042e1c3db0 [55751.969538] ffffffff810c42f8 [55751.969538] [55751.969538] 0000000000000007 [55751.969538] ffffffff81a1a680 [55751.969538] ffff88042e1c3dc8 [55751.969539] ffffffff810c7b89 [55751.969539] [55751.969539] 0000000000000008 [55751.969539] ffff88042e1c3df8 [55751.969539] ffffffff811373f0 [55751.969540] ffff88042e1d01c0 [55751.969540] [55751.969540] Call Trace: [55751.969541] [55751.969541] [55751.969542] [] sched_show_task+0xa8/0x110 [55751.969542] [55751.969544] [] dump_cpu_task+0x39/0x70 [55751.969544] [55751.969545] [] rcu_dump_cpu_stacks+0x90/0xd0 [55751.969545] [55751.969546] [] rcu_check_callbacks+0x442/0x720 [55751.969546] [55751.969548] [] ? tick_sched_handle.isra.13+0x60/0x60 [55751.969548] [55751.969549] [] update_process_times+0x47/0x80 [55751.969549] [55751.969551] [] tick_sched_handle.isra.13+0x25/0x60 [55751.969551] [55751.969552] [] tick_sched_timer+0x41/0x70 [55751.969552] [55751.969553] [] __hrtimer_run_queues+0xd2/0x260 [55751.969553] [55751.969554] [] hrtimer_interrupt+0xb0/0x1e0 [55751.969554] [55751.969555] [] ? call_softirq+0x1c/0x30 [55751.969555] [55751.969557] [] local_apic_timer_interrupt+0x37/0x60 [55751.969557] [55751.969558] [] smp_apic_timer_interrupt+0x3f/0x60 [55751.969559] [55751.969559] [] apic_timer_interrupt+0x6d/0x80 [55751.969560] [55751.969560] [55751.969561] [] ? _raw_spin_lock+0x32/0x50 [55751.969561] [55751.969566] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55751.969567] [55751.969572] [] lnet_finalize+0xf6/0x690 [lnet] [55751.969573] [55751.969576] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969576] [55751.969579] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969579] [55751.969580] [] ? dequeue_task_fair+0x41e/0x660 [55751.969581] [55751.969582] [] ? wake_up_state+0x20/0x20 [55751.969582] [55751.969585] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969585] [55751.969587] [] kthread+0xcf/0xe0 [55751.969587] [55751.969588] [] ? kthread_create_on_node+0x140/0x140 [55751.969588] [55751.969590] [] ret_from_fork+0x58/0x90 [55751.969590] [55751.969591] [] ? kthread_create_on_node+0x140/0x140 [55751.969592] Task dump for CPU 10: [55751.969592] kiblnd_sd_01_01 R [55751.969592] running task [55751.969593] 0 2901 2 0x00000088 [55751.969593] ffff8807f886f000 [55751.969593] ffffc9001327c860 [55751.969594] ffff8808266e7a00 [55751.969594] 0000000000000000 [55751.969594] [55751.969594] ffff8808262efda8 [55751.969594] ffffffffa0ebedab [55751.969595] ffff8808262efe90 [55751.969595] ffff8808262efda8 [55751.969595] [55751.969595] ffffffffa03e7d2e [55751.969596] ffff88082b2fc3c0 [55751.969596] ffff88082bda6c00 [55751.969596] 0000000000000017 [55751.969596] [55751.969596] Call Trace: [55751.969597] [55751.969600] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969601] [55751.969603] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.969604] [55751.969607] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969607] [55751.969608] [] ? dequeue_task_fair+0x41e/0x660 [55751.969608] [55751.969610] [] ? wake_up_state+0x20/0x20 [55751.969610] [55751.969613] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969613] [55751.969614] [] ? kthread+0xcf/0xe0 [55751.969615] [55751.969616] [] ? kthread_create_on_node+0x140/0x140 [55751.969616] [55751.969618] [] ? ret_from_fork+0x58/0x90 [55751.969618] [55751.969619] [] ? kthread_create_on_node+0x140/0x140 [55751.969620] Task dump for CPU 11: [55751.969620] kiblnd_sd_01_02 R [55751.969620] running task [55751.969621] 0 2902 2 0x00000088 [55751.969621] ffff88082363bd30 [55751.969621] 000000005cf0ae83 [55751.969621] ffffc900061c2218 [55751.969621] ffff8804c1ba6800 [55751.969622] [55751.969622] 0000000000000000 [55751.969622] 0000000000000000 [55751.969622] ffff88081f6b6a28 [55751.969623] ffff88082363bda8 [55751.969623] [55751.969623] ffffffffa0eba166 [55751.969623] ffff88082b2fc3c0 [55751.969623] ffff88081f6b6a00 [55751.969624] 000000000000000b [55751.969624] [55751.969624] Call Trace: [55751.969624] [55751.969627] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969627] [55751.969630] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969631] [55751.969632] [] ? dequeue_task_fair+0x41e/0x660 [55751.969632] [55751.969633] [] ? wake_up_state+0x20/0x20 [55751.969633] [55751.969637] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969637] [55751.969638] [] ? kthread+0xcf/0xe0 [55751.969638] [55751.969639] [] ? kthread_create_on_node+0x140/0x140 [55751.969640] [55751.969641] [] ? ret_from_fork+0x58/0x90 [55751.969641] [55751.969642] [] ? kthread_create_on_node+0x140/0x140 [55751.969643] Task dump for CPU 12: [55751.969643] kiblnd_sd_01_03 R [55751.969643] running task [55751.969644] 0 2903 2 0x00000088 [55751.969644] ffff880820a21000 [55751.969644] ffffc90011f075b0 [55751.969644] ffff8808266e7a00 [55751.969645] 0000000000000000 [55751.969645] [55751.969645] ffff8808213b7da8 [55751.969645] ffffffffa0ebedab [55751.969645] ffff8808213b7e90 [55751.969646] ffff8808213b7da8 [55751.969646] [55751.969646] ffffffffa03e7d2e [55751.969646] ffff88082b2fc3c0 [55751.969646] ffff88082c5f3600 [55751.969647] 0000000000000011 [55751.969647] [55751.969647] Call Trace: [55751.969647] [55751.969650] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969651] [55751.969653] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.969653] [55751.969656] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969656] [55751.969657] [] ? dequeue_task_fair+0x41e/0x660 [55751.969658] [55751.969659] [] ? wake_up_state+0x20/0x20 [55751.969659] [55751.969662] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969662] [55751.969664] [] ? kthread+0xcf/0xe0 [55751.969664] [55751.969665] [] ? kthread_create_on_node+0x140/0x140 [55751.969665] [55751.969667] [] ? ret_from_fork+0x58/0x90 [55751.969667] [55751.969668] [] ? kthread_create_on_node+0x140/0x140 [55751.969669] Task dump for CPU 15: [55751.969669] kiblnd_sd_01_00 R [55751.969669] running task [55751.969669] 0 2900 2 0x00000088 [55751.969670] ffff8808217bbd30 [55751.969670] 00000000937dfd11 [55751.969670] ffffc9000e335900 [55751.969670] ffff8804c1ba3200 [55751.969670] [55751.969671] 0000000000000000 [55751.969671] 0000000000000000 [55751.969671] ffff88082bda6c28 [55751.969671] ffff8808217bbda8 [55751.969672] [55751.969672] ffffffffa0eba166 [55751.969672] ffff88082b2fc3c0 [55751.969672] ffff88082bda6c00 [55751.969672] 0000000000000014 [55751.969673] [55751.969673] Call Trace: [55751.969673] [55751.969676] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969676] [55751.969679] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969679] [55751.969680] [] ? dequeue_task_fair+0x41e/0x660 [55751.969681] [55751.969682] [] ? wake_up_state+0x20/0x20 [55751.969682] [55751.969685] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969685] [55751.969687] [] ? kthread+0xcf/0xe0 [55751.969687] [55751.969688] [] ? kthread_create_on_node+0x140/0x140 [55751.969688] [55751.969690] [] ? ret_from_fork+0x58/0x90 [55751.969690] [55751.969691] [] ? kthread_create_on_node+0x140/0x140 [55751.969692] Task dump for CPU 1: [55751.969693] kiblnd_sd_00_00 R [55751.969694] running task [55751.969694] 0 2896 2 0x00000088 [55751.969695] ffff880820b03d30 [55751.969695] 000000003f673d40 [55751.969696] ffffc90003ec02f0 [55751.969696] ffff88041d63d600 [55751.969696] [55751.969696] 0000000000000000 [55751.969697] 0000000000000000 [55751.969697] ffff88042aa84a28 [55751.969697] ffff880820b03da8 [55751.969697] [55751.969698] ffffffffa0eba166 [55751.969698] ffff880181447180 [55751.969698] ffff88042aa84a00 [55751.969698] 0000000000000002 [55751.969698] [55751.969699] Call Trace: [55751.969699] [55751.969703] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969703] [55751.969707] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969707] [55751.969708] [] ? dequeue_task_fair+0x41e/0x660 [55751.969708] [55751.969710] [] ? sched_clock_cpu+0x85/0xc0 [55751.969710] [55751.969711] [] ? wake_up_state+0x20/0x20 [55751.969712] [55751.969715] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969715] [55751.969716] [] ? kthread+0xcf/0xe0 [55751.969717] [55751.969718] [] ? kthread_create_on_node+0x140/0x140 [55751.969718] [55751.969720] [] ? ret_from_fork+0x58/0x90 [55751.969720] [55751.969721] [] ? kthread_create_on_node+0x140/0x140 [55751.969722] Task dump for CPU 5: [55751.969722] kiblnd_sd_00_02 R [55751.969722] running task [55751.969723] 0 2898 2 0x00000088 [55751.969723] ffff880415b13000 [55751.969723] ffff8800abaa43a8 [55751.969724] ffff88083fb5e000 [55751.969724] 0000000000000000 [55751.969724] [55751.969724] ffff88082aae7da8 [55751.969724] ffffffffa0ebedab [55751.969725] ffff880181447188 [55751.969725] ffff88082aae7da8 [55751.969725] [55751.969725] ffffffff810bc064 [55751.969725] ffff880181447180 [55751.969726] ffff88042aa84a00 [55751.969726] 0000000000000001 [55751.969726] [55751.969726] Call Trace: [55751.969727] [55751.969730] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969730] [55751.969731] [] ? __wake_up+0x44/0x50 [55751.969731] [55751.969734] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969734] [55751.969735] [] ? dequeue_task_fair+0x41e/0x660 [55751.969736] [55751.969737] [] ? wake_up_state+0x20/0x20 [55751.969737] [55751.969740] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969740] [55751.969742] [] ? kthread+0xcf/0xe0 [55751.969742] [55751.969743] [] ? kthread_create_on_node+0x140/0x140 [55751.969743] [55751.969745] [] ? ret_from_fork+0x58/0x90 [55751.969745] [55751.969746] [] ? kthread_create_on_node+0x140/0x140 [55751.969747] Task dump for CPU 6: [55751.969747] kiblnd_sd_00_01 R [55751.969747] running task [55751.969748] 0 2897 2 0x00000088 [55751.969748] ffff88081f64fd30 [55751.969748] 00000000f1dbfe3f [55751.969748] ffffc90003ebfde0 [55751.969749] ffff88042dcd1e00 [55751.969749] [55751.969749] 0000000000000000 [55751.969749] 0000000000000000 [55751.969749] ffff88042aa84a28 [55751.969750] ffff88081f64fda8 [55751.969750] [55751.969750] ffffffffa0eba166 [55751.969750] ffff880181447180 [55751.969751] ffff88042aa84a00 [55751.969751] 0000000000000012 [55751.969751] [55751.969751] Call Trace: [55751.969751] [55751.969754] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969755] [55751.969758] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969758] [55751.969759] [] ? dequeue_task_fair+0x41e/0x660 [55751.969759] [55751.969760] [] ? wake_up_state+0x20/0x20 [55751.969761] [55751.969764] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969764] [55751.969765] [] ? kthread+0xcf/0xe0 [55751.969765] [55751.969767] [] ? kthread_create_on_node+0x140/0x140 [55751.969767] [55751.969768] [] ? ret_from_fork+0x58/0x90 [55751.969768] [55751.969769] [] ? kthread_create_on_node+0x140/0x140 [55751.969770] Task dump for CPU 7: [55751.969770] kiblnd_sd_00_03 R [55751.969771] running task [55751.969771] 0 2899 2 0x00000088 [55751.969771] ffff88082626bd30 [55751.969772] 00000000aaf7c743 [55751.969772] ffffc90003ebe8c8 [55751.969772] ffff8800b484ea00 [55751.969772] [55751.969772] 0000000000000000 [55751.969773] 0000000000000000 [55751.969773] ffff88042aa84a28 [55751.969773] ffff88082626bda8 [55751.969773] [55751.969774] ffffffffa0eba166 [55751.969774] ffff880181447180 [55751.969774] ffff88042aa84a00 [55751.969774] 0000000000000001 [55751.969774] [55751.969775] Call Trace: [55751.969775] [55751.969778] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969778] [55751.969781] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969781] [55751.969782] [] ? dequeue_task_fair+0x41e/0x660 [55751.969783] [55751.969784] [] ? wake_up_state+0x20/0x20 [55751.969784] [55751.969787] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969787] [55751.969788] [] ? kthread+0xcf/0xe0 [55751.969789] [55751.969790] [] ? kthread_create_on_node+0x140/0x140 [55751.969790] [55751.969792] [] ? ret_from_fork+0x58/0x90 [55751.969792] [55751.969793] [] ? kthread_create_on_node+0x140/0x140 [55751.969794] Task dump for CPU 10: [55751.969794] kiblnd_sd_01_01 R [55751.969794] running task [55751.969795] 0 2901 2 0x00000088 [55751.969795] ffff8807f886f000 [55751.969795] ffffc9001327c860 [55751.969795] ffff8808266e7a00 [55751.969795] 0000000000000000 [55751.969796] [55751.969796] ffff8808262efda8 [55751.969796] ffffffffa0ebedab [55751.969796] ffff8808262efe90 [55751.969797] ffff8808262efda8 [55751.969797] [55751.969797] ffffffffa03e7d2e [55751.969797] ffff88082b2fc3c0 [55751.969797] ffff88082bda6c00 [55751.969798] 0000000000000017 [55751.969798] [55751.969798] Call Trace: [55751.969798] [55751.969801] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969803] [55751.969805] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.969805] [55751.969808] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969808] [55751.969809] [] ? dequeue_task_fair+0x41e/0x660 [55751.969810] [55751.969811] [] ? wake_up_state+0x20/0x20 [55751.969811] [55751.969814] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969815] [55751.969816] [] ? kthread+0xcf/0xe0 [55751.969816] [55751.969817] [] ? kthread_create_on_node+0x140/0x140 [55751.969817] [55751.969819] [] ? ret_from_fork+0x58/0x90 [55751.969819] [55751.969820] [] ? kthread_create_on_node+0x140/0x140 [55751.969821] Task dump for CPU 11: [55751.969821] kiblnd_sd_01_02 R [55751.969821] running task [55751.969822] 0 2902 2 0x00000088 [55751.969822] ffff88082363bd30 [55751.969822] 000000005cf0ae83 [55751.969822] ffffc900061c2218 [55751.969823] ffff8804c1ba6800 [55751.969823] [55751.969823] 0000000000000000 [55751.969823] 0000000000000000 [55751.969824] ffff88081f6b6a28 [55751.969824] ffff88082363bda8 [55751.969824] [55751.969824] ffffffffa0eba166 [55751.969824] ffff88082b2fc3c0 [55751.969825] ffff88081f6b6a00 [55751.969825] 000000000000000b [55751.969825] [55751.969825] Call Trace: [55751.969825] [55751.969828] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969829] [55751.969832] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969832] [55751.969833] [] ? dequeue_task_fair+0x41e/0x660 [55751.969833] [55751.969834] [] ? wake_up_state+0x20/0x20 [55751.969835] [55751.969838] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969838] [55751.969839] [] ? kthread+0xcf/0xe0 [55751.969839] [55751.969841] [] ? kthread_create_on_node+0x140/0x140 [55751.969841] [55751.969842] [] ? ret_from_fork+0x58/0x90 [55751.969842] [55751.969843] [] ? kthread_create_on_node+0x140/0x140 [55751.969844] Task dump for CPU 12: [55751.969844] kiblnd_sd_01_03 R [55751.969844] running task [55751.969845] 0 2903 2 0x00000088 [55751.969845] ffff880826310fb0 [55751.969845] 000000007d9940fa [55751.969846] ffff88082d903db0 [55751.969846] ffffffff810c42f8 [55751.969846] [55751.969846] 000000000000000c [55751.969847] ffffffff81a1a680 [55751.969847] ffff88082d903dc8 [55751.969847] ffffffff810c7b89 [55751.969847] [55751.969847] 000000000000000d [55751.969848] ffff88082d903df8 [55751.969848] ffffffff811373f0 [55751.969848] ffff88082d9101c0 [55751.969848] [55751.969848] Call Trace: [55751.969849] [55751.969849] [55751.969850] [] sched_show_task+0xa8/0x110 [55751.969851] [55751.969852] [] dump_cpu_task+0x39/0x70 [55751.969852] [55751.969853] [] rcu_dump_cpu_stacks+0x90/0xd0 [55751.969853] [55751.969854] [] rcu_check_callbacks+0x442/0x720 [55751.969855] [55751.969856] [] ? tick_sched_handle.isra.13+0x60/0x60 [55751.969856] [55751.969858] [] update_process_times+0x47/0x80 [55751.969858] [55751.969859] [] tick_sched_handle.isra.13+0x25/0x60 [55751.969859] [55751.969860] [] tick_sched_timer+0x41/0x70 [55751.969860] [55751.969861] [] __hrtimer_run_queues+0xd2/0x260 [55751.969862] [55751.969863] [] hrtimer_interrupt+0xb0/0x1e0 [55751.969863] [55751.969864] [] ? call_softirq+0x1c/0x30 [55751.969864] [55751.969866] [] local_apic_timer_interrupt+0x37/0x60 [55751.969866] [55751.969867] [] smp_apic_timer_interrupt+0x3f/0x60 [55751.969867] [55751.969868] [] apic_timer_interrupt+0x6d/0x80 [55751.969868] [55751.969869] [55751.969869] [] ? _raw_spin_lock+0x32/0x50 [55751.969870] [55751.969876] [] ? lnet_find_peer_ni_locked+0x18/0x130 [lnet] [55751.969876] [55751.969881] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55751.969882] [55751.969886] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55751.969886] [55751.969892] [] lnet_select_pathway+0x235/0x1140 [lnet] [55751.969892] [55751.969895] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55751.969896] [55751.969901] [] lnet_send+0x51/0x180 [lnet] [55751.969901] [55751.969907] [] lnet_finalize+0x3f1/0x690 [lnet] [55751.969907] [55751.969910] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55751.969910] [55751.969912] [] ? __slab_free+0x81/0x2f0 [55751.969912] [55751.969918] [] lnet_ni_recv+0xc3/0x320 [lnet] [55751.969918] [55751.969923] [] lnet_parse+0x945/0xf90 [lnet] [55751.969923] [55751.969928] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55751.969928] [55751.969931] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.969932] [55751.969934] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.969934] [55751.969937] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.969938] [55751.969939] [] ? dequeue_task_fair+0x41e/0x660 [55751.969939] [55751.969940] [] ? wake_up_state+0x20/0x20 [55751.969941] [55751.969944] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969944] [55751.969945] [] kthread+0xcf/0xe0 [55751.969946] [55751.969947] [] ? kthread_create_on_node+0x140/0x140 [55751.969947] [55751.969948] [] ret_from_fork+0x58/0x90 [55751.969949] [55751.969950] [] ? kthread_create_on_node+0x140/0x140 [55751.969950] Task dump for CPU 15: [55751.969951] kiblnd_sd_01_00 R [55751.969951] running task [55751.969951] 0 2900 2 0x00000088 [55751.969952] ffff8808217bbd30 [55751.969952] 00000000937dfd11 [55751.969952] ffffc9000e335900 [55751.969952] ffff8804c1ba3200 [55751.969953] [55751.969953] 0000000000000000 [55751.969953] 0000000000000000 [55751.969953] ffff88082bda6c28 [55751.969953] ffff8808217bbda8 [55751.969954] [55751.969954] ffffffffa0eba166 [55751.969954] ffff88082b2fc3c0 [55751.969954] ffff88082bda6c00 [55751.969955] 0000000000000014 [55751.969955] [55751.969955] Call Trace: [55751.969955] [55751.969958] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969958] [55751.969961] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969962] [55751.969963] [] ? dequeue_task_fair+0x41e/0x660 [55751.969963] [55751.969964] [] ? wake_up_state+0x20/0x20 [55751.969965] [55751.969968] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969968] [55751.969969] [] ? kthread+0xcf/0xe0 [55751.969970] [55751.969971] [] ? kthread_create_on_node+0x140/0x140 [55751.969971] [55751.969972] [] ? ret_from_fork+0x58/0x90 [55751.969973] [55751.969974] [] ? kthread_create_on_node+0x140/0x140 [55751.969975] Task dump for CPU 1: [55751.969975] kiblnd_sd_00_00 R [55751.969975] running task [55751.969976] 0 2896 2 0x00000088 [55751.969976] ffff880820b03d30 [55751.969977] 000000003f673d40 [55751.969977] ffffc90003ec02f0 [55751.969977] ffff88041d63d600 [55751.969977] [55751.969978] 0000000000000000 [55751.969978] 0000000000000000 [55751.969978] ffff88042aa84a28 [55751.969978] ffff880820b03da8 [55751.969979] [55751.969979] ffffffffa0eba166 [55751.969979] ffff880181447180 [55751.969979] ffff88042aa84a00 [55751.969979] 0000000000000002 [55751.969980] [55751.969980] Call Trace: [55751.969980] [55751.969984] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.969984] [55751.969987] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.969987] [55751.969989] [] ? dequeue_task_fair+0x41e/0x660 [55751.969989] [55751.969990] [] ? sched_clock_cpu+0x85/0xc0 [55751.969991] [55751.969992] [] ? wake_up_state+0x20/0x20 [55751.969992] [55751.969995] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.969995] [55751.969997] [] ? kthread+0xcf/0xe0 [55751.969997] [55751.969998] [] ? kthread_create_on_node+0x140/0x140 [55751.969998] [55751.970000] [] ? ret_from_fork+0x58/0x90 [55751.970000] [55751.970001] [] ? kthread_create_on_node+0x140/0x140 [55751.970002] Task dump for CPU 5: [55751.970002] kiblnd_sd_00_02 R [55751.970003] running task [55751.970003] 0 2898 2 0x00000088 [55751.970003] ffff880415b13000 [55751.970004] ffff8800abaa43a8 [55751.970004] ffff88083fb5e000 [55751.970004] 0000000000000000 [55751.970004] [55751.970004] ffff88082aae7da8 [55751.970005] ffffffffa0ebedab [55751.970005] ffff880181447188 [55751.970005] ffff88082aae7da8 [55751.970005] [55751.970006] ffffffff810bc064 [55751.970006] ffff880181447180 [55751.970006] ffff88042aa84a00 [55751.970006] 0000000000000001 [55751.970006] [55751.970007] Call Trace: [55751.970007] [55751.970010] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.970010] [55751.970011] [] ? __wake_up+0x44/0x50 [55751.970011] [55751.970014] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.970015] [55751.970016] [] ? dequeue_task_fair+0x41e/0x660 [55751.970016] [55751.970017] [] ? wake_up_state+0x20/0x20 [55751.970018] [55751.970021] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.970021] [55751.970022] [] ? kthread+0xcf/0xe0 [55751.970022] [55751.970023] [] ? kthread_create_on_node+0x140/0x140 [55751.970024] [55751.970025] [] ? ret_from_fork+0x58/0x90 [55751.970025] [55751.970026] [] ? kthread_create_on_node+0x140/0x140 [55751.970027] Task dump for CPU 6: [55751.970027] kiblnd_sd_00_01 R [55751.970028] running task [55751.970028] 0 2897 2 0x00000088 [55751.970028] ffff88081f64fd30 [55751.970029] 00000000f1dbfe3f [55751.970029] ffffc90003ebfde0 [55751.970029] ffff88042dcd1e00 [55751.970029] [55751.970029] 0000000000000000 [55751.970030] 0000000000000000 [55751.970030] ffff88042aa84a28 [55751.970030] ffff88081f64fda8 [55751.970030] [55751.970030] ffffffffa0eba166 [55751.970031] ffff880181447180 [55751.970031] ffff88042aa84a00 [55751.970031] 0000000000000012 [55751.970031] [55751.970031] Call Trace: [55751.970032] [55751.970035] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.970035] [55751.970038] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.970038] [55751.970039] [] ? dequeue_task_fair+0x41e/0x660 [55751.970040] [55751.970041] [] ? wake_up_state+0x20/0x20 [55751.970041] [55751.970044] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.970044] [55751.970046] [] ? kthread+0xcf/0xe0 [55751.970046] [55751.970047] [] ? kthread_create_on_node+0x140/0x140 [55751.970047] [55751.970049] [] ? ret_from_fork+0x58/0x90 [55751.970049] [55751.970050] [] ? kthread_create_on_node+0x140/0x140 [55751.970050] Task dump for CPU 7: [55751.970051] kiblnd_sd_00_03 R [55751.970051] running task [55751.970051] 0 2899 2 0x00000088 [55751.970052] ffff88082626bd30 [55751.970052] 00000000aaf7c743 [55751.970052] ffffc90003ebe8c8 [55751.970052] ffff8800b484ea00 [55751.970052] [55751.970053] 0000000000000000 [55751.970053] 0000000000000000 [55751.970053] ffff88042aa84a28 [55751.970053] ffff88082626bda8 [55751.970054] [55751.970054] ffffffffa0eba166 [55751.970054] ffff880181447180 [55751.970054] ffff88042aa84a00 [55751.970054] 0000000000000001 [55751.970055] [55751.970055] Call Trace: [55751.970055] [55751.970058] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.970058] [55751.970061] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.970061] [55751.970062] [] ? dequeue_task_fair+0x41e/0x660 [55751.970063] [55751.970064] [] ? wake_up_state+0x20/0x20 [55751.970064] [55751.970067] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.970068] [55751.970069] [] ? kthread+0xcf/0xe0 [55751.970069] [55751.970070] [] ? kthread_create_on_node+0x140/0x140 [55751.970071] [55751.970072] [] ? ret_from_fork+0x58/0x90 [55751.970072] [55751.970073] [] ? kthread_create_on_node+0x140/0x140 [55751.970074] Task dump for CPU 10: [55751.970074] kiblnd_sd_01_01 R [55751.970074] running task [55751.970075] 0 2901 2 0x00000088 [55751.970075] ffff8807f886f000 [55751.970075] ffffc9001327c860 [55751.970075] ffff8808266e7a00 [55751.970076] 0000000000000000 [55751.970076] [55751.970076] ffff8808262efda8 [55751.970076] ffffffffa0ebedab [55751.970077] ffff8808262efe90 [55751.970077] ffff8808262efda8 [55751.970077] [55751.970077] ffffffffa03e7d2e [55751.970077] ffff88082b2fc3c0 [55751.970078] ffff88082bda6c00 [55751.970078] 0000000000000017 [55751.970078] [55751.970078] Call Trace: [55751.970078] [55751.970082] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.970083] [55751.970085] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.970085] [55751.970088] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.970088] [55751.970089] [] ? dequeue_task_fair+0x41e/0x660 [55751.970090] [55751.970091] [] ? wake_up_state+0x20/0x20 [55751.970091] [55751.970094] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.970094] [55751.970096] [] ? kthread+0xcf/0xe0 [55751.970096] [55751.970097] [] ? kthread_create_on_node+0x140/0x140 [55751.970097] [55751.970099] [] ? ret_from_fork+0x58/0x90 [55751.970099] [55751.970100] [] ? kthread_create_on_node+0x140/0x140 [55751.970101] Task dump for CPU 11: [55751.970101] kiblnd_sd_01_02 R [55751.970101] running task [55751.970101] 0 2902 2 0x00000088 [55751.970102] ffff88082363bd30 [55751.970102] 000000005cf0ae83 [55751.970102] ffffc900061c2218 [55751.970102] ffff8804c1ba6800 [55751.970103] [55751.970103] 0000000000000000 [55751.970103] 0000000000000000 [55751.970103] ffff88081f6b6a28 [55751.970103] ffff88082363bda8 [55751.970104] [55751.970104] ffffffffa0eba166 [55751.970104] ffff88082b2fc3c0 [55751.970104] ffff88081f6b6a00 [55751.970104] 000000000000000b [55751.970105] [55751.970105] Call Trace: [55751.970105] [55751.970108] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.970108] [55751.970111] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.970112] [55751.970113] [] ? dequeue_task_fair+0x41e/0x660 [55751.970113] [55751.970115] [] ? wake_up_state+0x20/0x20 [55751.970115] [55751.970118] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.970118] [55751.970119] [] ? kthread+0xcf/0xe0 [55751.970120] [55751.970121] [] ? kthread_create_on_node+0x140/0x140 [55751.970121] [55751.970122] [] ? ret_from_fork+0x58/0x90 [55751.970123] [55751.970124] [] ? kthread_create_on_node+0x140/0x140 [55751.970124] Task dump for CPU 12: [55751.970125] kiblnd_sd_01_03 R [55751.970125] running task [55751.970125] 0 2903 2 0x00000088 [55751.970126] ffff880820a21000 [55751.970126] ffffc90011f075b0 [55751.970126] ffff8808266e7a00 [55751.970126] 0000000000000000 [55751.970126] [55751.970127] ffff8808213b7da8 [55751.970127] ffffffffa0ebedab [55751.970127] ffff8808213b7e90 [55751.970127] ffff8808213b7da8 [55751.970128] [55751.970128] ffffffffa03e7d2e [55751.970128] ffff88082b2fc3c0 [55751.970128] ffff88082c5f3600 [55751.970129] 0000000000000011 [55751.970129] [55751.970129] Call Trace: [55751.970129] [55751.970132] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55751.970133] [55751.970135] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55751.970135] [55751.970138] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55751.970139] [55751.970140] [] ? dequeue_task_fair+0x41e/0x660 [55751.970140] [55751.970141] [] ? wake_up_state+0x20/0x20 [55751.970142] [55751.970145] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.970145] [55751.970146] [] ? kthread+0xcf/0xe0 [55751.970146] [55751.970148] [] ? kthread_create_on_node+0x140/0x140 [55751.970148] [55751.970149] [] ? ret_from_fork+0x58/0x90 [55751.970149] [55751.970150] [] ? kthread_create_on_node+0x140/0x140 [55751.970151] Task dump for CPU 15: [55751.970151] kiblnd_sd_01_00 R [55751.970151] running task [55751.970152] 0 2900 2 0x00000088 [55751.970152] ffff88082aa3edd0 [55751.970152] 00000000937dfd11 [55751.970153] ffff88082d9c3db0 [55751.970153] ffffffff810c42f8 [55751.970153] [55751.970153] 000000000000000f [55751.970154] ffffffff81a1a680 [55751.970154] ffff88082d9c3dc8 [55751.970154] ffffffff810c7b89 [55751.970154] [55751.970155] 0000000000000010 [55751.970155] ffff88082d9c3df8 [55751.970155] ffffffff811373f0 [55751.970155] ffff88082d9d01c0 [55751.970155] [55751.970156] Call Trace: [55751.970156] [55751.970156] [55751.970158] [] sched_show_task+0xa8/0x110 [55751.970158] [55751.970159] [] dump_cpu_task+0x39/0x70 [55751.970159] [55751.970160] [] rcu_dump_cpu_stacks+0x90/0xd0 [55751.970161] [55751.970162] [] rcu_check_callbacks+0x442/0x720 [55751.970162] [55751.970163] [] ? tick_sched_handle.isra.13+0x60/0x60 [55751.970163] [55751.970165] [] update_process_times+0x47/0x80 [55751.970165] [55751.970166] [] tick_sched_handle.isra.13+0x25/0x60 [55751.970166] [55751.970167] [] tick_sched_timer+0x41/0x70 [55751.970168] [55751.970168] [] __hrtimer_run_queues+0xd2/0x260 [55751.970169] [55751.970170] [] hrtimer_interrupt+0xb0/0x1e0 [55751.970170] [55751.970171] [] ? call_softirq+0x1c/0x30 [55751.970171] [55751.970173] [] local_apic_timer_interrupt+0x37/0x60 [55751.970173] [55751.970174] [] smp_apic_timer_interrupt+0x3f/0x60 [55751.970174] [55751.970175] [] apic_timer_interrupt+0x6d/0x80 [55751.970175] [55751.970176] [55751.970176] [] ? _raw_spin_lock+0x37/0x50 [55751.970177] [55751.970181] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55751.970182] [55751.970186] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55751.970187] [55751.970192] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55751.970192] [55751.970197] [] lnet_finalize+0x1e9/0x690 [lnet] [55751.970198] [55751.970201] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55751.970201] [55751.970204] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55751.970204] [55751.970205] [] ? dequeue_task_fair+0x41e/0x660 [55751.970206] [55751.970207] [] ? wake_up_state+0x20/0x20 [55751.970207] [55751.970210] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55751.970210] [55751.970212] [] kthread+0xcf/0xe0 [55751.970212] [55751.970213] [] ? kthread_create_on_node+0x140/0x140 [55751.970213] [55751.970215] [] ret_from_fork+0x58/0x90 [55751.970215] [55751.970216] [] ? kthread_create_on_node+0x140/0x140 [55755.070073] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [55755.070074] Modules linked in: [55755.070074] ko2iblnd(OE) [55755.070074] lustre(OE) [55755.070075] lmv(OE) [55755.070075] fld(OE) [55755.070075] mdc(OE) [55755.070075] fid(OE) [55755.070076] lov(OE) [55755.070076] ksocklnd(OE) [55755.070076] ptlrpc(OE) [55755.070077] obdclass(OE) [55755.070077] lnet(OE) [55755.070077] libcfs(OE) [55755.070077] rpcrdma [55755.070078] ib_isert [55755.070078] iscsi_target_mod [55755.070078] ib_iser [55755.070079] libiscsi [55755.070079] scsi_transport_iscsi [55755.070079] ib_srpt [55755.070079] target_core_mod [55755.070080] ib_srp [55755.070080] scsi_transport_srp [55755.070080] scsi_tgt [55755.070080] ib_ipoib [55755.070081] rdma_ucm [55755.070081] ib_ucm [55755.070081] ib_uverbs [55755.070081] ib_umad [55755.070082] rdma_cm [55755.070082] ib_cm [55755.070082] iw_cm [55755.070082] mlx4_ib [55755.070083] intel_powerclamp [55755.070083] coretemp [55755.070083] intel_rapl [55755.070084] iosf_mbi [55755.070084] kvm [55755.070084] irqbypass [55755.070084] crc32_pclmul [55755.070085] ghash_clmulni_intel [55755.070085] aesni_intel [55755.070085] ipmi_ssif [55755.070085] lrw [55755.070086] gf128mul [55755.070086] glue_helper [55755.070086] hfi1 [55755.070087] ipmi_devintf [55755.070087] rdmavt [55755.070087] ablk_helper [55755.070087] cryptd [55755.070088] iTCO_wdt [55755.070088] ipmi_si [55755.070088] sb_edac [55755.070088] sg [55755.070089] iTCO_vendor_support [55755.070089] ib_core [55755.070089] ioatdma [55755.070089] i2c_i801 [55755.070090] mei_me [55755.070090] mei [55755.070090] edac_core [55755.070090] ipmi_msghandler [55755.070091] lpc_ich [55755.070091] ntb [55755.070091] wmi [55755.070091] shpchp [55755.070092] pcspkr [55755.070092] nfsd [55755.070092] nfs_acl [55755.070093] lockd [55755.070093] grace [55755.070093] auth_rpcgss [55755.070093] sunrpc [55755.070094] ip_tables [55755.070094] ext4 [55755.070094] mbcache [55755.070094] jbd2 [55755.070095] sd_mod [55755.070095] crc_t10dif [55755.070095] crct10dif_generic [55755.070095] mlx4_en [55755.070096] mgag200 [55755.070096] drm_kms_helper [55755.070096] syscopyarea [55755.070096] sysfillrect [55755.070097] sysimgblt [55755.070097] fb_sys_fops [55755.070097] isci [55755.070097] ttm [55755.070098] igb [55755.070098] crct10dif_pclmul [55755.070098] crct10dif_common [55755.070098] ahci [55755.070099] ptp [55755.070099] crc32c_intel [55755.070099] libsas [55755.070099] libahci [55755.070100] pps_core [55755.070100] drm [55755.070100] mlx4_core [55755.070100] mpt2sas [55755.070101] dca [55755.070101] libata [55755.070101] raid_class [55755.070101] i2c_algo_bit [55755.070102] scsi_transport_sas [55755.070102] devlink [55755.070102] i2c_core [55755.070102] fjes [55755.070103] [55755.070104] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55755.070105] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55755.070106] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55755.070106] RIP: 0010:[] [55755.070108] [] _raw_spin_lock+0x37/0x50 [55755.070108] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000202 [55755.070109] RAX: 00000000000050cd RBX: 0000000000016c40 RCX: 0000000000000a1e [55755.070109] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55755.070110] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55755.070110] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55755.070111] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55755.070112] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55755.070113] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55755.070113] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55755.070114] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55755.070114] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55755.070115] Stack: [55755.070115] ffff88082aa33ca0 [55755.070115] ffffffffa08aa7c8 [55755.070115] ffff88082aa33ca8 [55755.070116] 0000000000000000 [55755.070116] [55755.070116] 00050000c0a80184 [55755.070116] ffff88082aa33cc8 [55755.070117] ffffffffa0916cd0 [55755.070117] ffff8800366a4530 [55755.070117] [55755.070117] ffff880821b758e8 [55755.070117] ffff88083fb5e000 [55755.070118] ffff88082aa33ce0 [55755.070118] ffffffffa0eb8460 [55755.070118] [55755.070118] Call Trace: [55755.070119] [55755.070124] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55755.070124] [55755.070129] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55755.070130] [55755.070133] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55755.070133] [55755.070136] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55755.070136] [55755.070138] [] ? lock_timer_base.isra.33+0x2b/0x50 [55755.070138] [55755.070139] [] ? try_to_del_timer_sync+0x5e/0x90 [55755.070139] [55755.070143] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55755.070143] [55755.070144] [] ? schedule_timeout+0x17d/0x2d0 [55755.070145] [55755.070148] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55755.070148] [55755.070150] [] ? wake_up_state+0x20/0x20 [55755.070150] [55755.070153] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55755.070153] [55755.070155] [] kthread+0xcf/0xe0 [55755.070155] [55755.070156] [] ? kthread_create_on_node+0x140/0x140 [55755.070156] [55755.070158] [] ret_from_fork+0x58/0x90 [55755.070158] [55755.070159] [] ? kthread_create_on_node+0x140/0x140 [55755.070160] Code: [55755.070160] c1 [55755.070160] 07 [55755.070160] 89 [55755.070161] c2 [55755.070161] c1 [55755.070161] ea [55755.070161] 10 [55755.070161] 66 [55755.070162] 39 [55755.070162] c2 [55755.070162] 75 [55755.070162] 01 [55755.070162] c3 [55755.070163] 55 [55755.070163] 83 [55755.070163] e2 [55755.070163] fe [55755.070163] 0f [55755.070164] b7 [55755.070164] f2 [55755.070164] 48 [55755.070164] 89 [55755.070164] e5 [55755.070165] b8 [55755.070165] 00 [55755.070165] 80 [55755.070165] 00 [55755.070165] 00 [55755.070165] eb [55755.070166] 0d [55755.070166] 66 [55755.070166] 0f [55755.070166] 1f [55755.070166] 44 [55755.070167] 00 [55755.070167] 00 [55755.070167] f3 [55755.070167] 90 [55755.070167] 83 [55755.070168] e8 [55755.070168] 01 [55755.070168] 74 [55755.070168] 0a [55755.070169] <0f> [55755.070169] b7 [55755.070169] 0f [55755.070169] 66 [55755.070169] 39 [55755.070170] ca [55755.070170] 75 [55755.070170] f1 [55755.070170] 5d [55755.070170] c3 [55755.070170] 66 [55755.070171] 66 [55755.070171] 66 [55755.070171] 90 [55755.070171] 66 [55755.070172] 66 [55755.070172] 90 [55755.070172] eb [55755.070172] d9 [55755.070172] 66 [55755.070173] 0f [55755.070173] [55763.069243] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55763.069245] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [55763.069250] Modules linked in: [55763.069251] Modules linked in: [55763.069252] ko2iblnd(OE) [55763.069253] ko2iblnd(OE) [55763.069254] lustre(OE) [55763.069254] lmv(OE) [55763.069255] fld(OE) [55763.069255] mdc(OE) [55763.069255] fid(OE) [55763.069255] lov(OE) [55763.069256] ksocklnd(OE) [55763.069256] ptlrpc(OE) [55763.069256] obdclass(OE) [55763.069256] lnet(OE) [55763.069257] libcfs(OE) [55763.069257] rpcrdma [55763.069257] ib_isert [55763.069257] iscsi_target_mod [55763.069258] ib_iser [55763.069258] libiscsi [55763.069258] scsi_transport_iscsi [55763.069258] ib_srpt [55763.069259] target_core_mod [55763.069260] lustre(OE) [55763.069261] ib_srp [55763.069262] lmv(OE) [55763.069263] scsi_transport_srp [55763.069264] fld(OE) [55763.069264] scsi_tgt [55763.069265] mdc(OE) [55763.069266] ib_ipoib [55763.069267] fid(OE) [55763.069268] rdma_ucm [55763.069269] lov(OE) [55763.069270] ib_ucm [55763.069270] ksocklnd(OE) [55763.069271] ib_uverbs [55763.069272] ptlrpc(OE) [55763.069273] ib_umad [55763.069274] obdclass(OE) [55763.069275] rdma_cm [55763.069276] lnet(OE) [55763.069277] ib_cm [55763.069277] libcfs(OE) [55763.069278] iw_cm [55763.069279] rpcrdma [55763.069280] mlx4_ib [55763.069281] ib_isert [55763.069282] intel_powerclamp [55763.069283] iscsi_target_mod [55763.069284] coretemp [55763.069285] ib_iser [55763.069286] intel_rapl [55763.069287] libiscsi [55763.069288] iosf_mbi [55763.069289] scsi_transport_iscsi [55763.069290] kvm [55763.069291] ib_srpt [55763.069291] irqbypass [55763.069292] target_core_mod [55763.069293] crc32_pclmul [55763.069294] ib_srp [55763.069295] ghash_clmulni_intel [55763.069296] scsi_transport_srp [55763.069297] aesni_intel [55763.069297] scsi_tgt [55763.069299] ipmi_ssif [55763.069299] ib_ipoib [55763.069301] lrw [55763.069301] rdma_ucm [55763.069302] gf128mul [55763.069303] ib_ucm [55763.069304] glue_helper [55763.069305] ib_uverbs [55763.069306] hfi1 [55763.069307] ib_umad [55763.069308] ipmi_devintf [55763.069308] rdma_cm [55763.069310] rdmavt [55763.069310] ib_cm [55763.069311] ablk_helper [55763.069312] iw_cm [55763.069313] cryptd [55763.069314] mlx4_ib [55763.069315] iTCO_wdt [55763.069316] intel_powerclamp [55763.069317] ipmi_si [55763.069317] coretemp [55763.069319] sb_edac [55763.069319] intel_rapl [55763.069320] sg [55763.069321] iosf_mbi [55763.069322] iTCO_vendor_support [55763.069323] kvm [55763.069324] ib_core [55763.069325] irqbypass [55763.069326] ioatdma [55763.069327] crc32_pclmul [55763.069328] i2c_i801 [55763.069329] ghash_clmulni_intel [55763.069330] mei_me [55763.069330] aesni_intel [55763.069331] mei [55763.069332] ipmi_ssif [55763.069333] edac_core [55763.069334] lrw [55763.069335] ipmi_msghandler [55763.069336] gf128mul [55763.069337] lpc_ich [55763.069338] glue_helper [55763.069339] ntb [55763.069339] hfi1 [55763.069340] wmi [55763.069341] ipmi_devintf [55763.069342] shpchp [55763.069343] rdmavt [55763.069344] pcspkr [55763.069345] ablk_helper [55763.069346] nfsd [55763.069346] cryptd [55763.069347] nfs_acl [55763.069348] iTCO_wdt [55763.069349] lockd [55763.069350] ipmi_si [55763.069351] grace [55763.069352] sb_edac [55763.069353] auth_rpcgss [55763.069354] sg [55763.069355] sunrpc [55763.069356] iTCO_vendor_support [55763.069356] ip_tables [55763.069357] ib_core [55763.069359] ext4 [55763.069360] ioatdma [55763.069361] mbcache [55763.069361] i2c_i801 [55763.069362] jbd2 [55763.069363] mei_me [55763.069364] sd_mod [55763.069365] mei [55763.069366] crc_t10dif [55763.069367] edac_core [55763.069368] crct10dif_generic [55763.069369] ipmi_msghandler [55763.069370] mlx4_en [55763.069370] lpc_ich [55763.069371] mgag200 [55763.069372] ntb [55763.069373] drm_kms_helper [55763.069374] wmi [55763.069375] syscopyarea [55763.069376] shpchp [55763.069377] sysfillrect [55763.069378] pcspkr [55763.069379] sysimgblt [55763.069380] nfsd [55763.069380] fb_sys_fops [55763.069381] nfs_acl [55763.069382] isci [55763.069383] lockd [55763.069384] ttm [55763.069385] grace [55763.069386] igb [55763.069387] auth_rpcgss [55763.069388] crct10dif_pclmul [55763.069389] sunrpc [55763.069390] crct10dif_common [55763.069390] ip_tables [55763.069392] ahci [55763.069392] ext4 [55763.069393] ptp [55763.069394] mbcache [55763.069395] crc32c_intel [55763.069396] jbd2 [55763.069397] libsas [55763.069398] sd_mod [55763.069399] libahci [55763.069400] crc_t10dif [55763.069401] pps_core [55763.069402] crct10dif_generic [55763.069403] drm [55763.069404] mlx4_en [55763.069405] mlx4_core [55763.069406] mgag200 [55763.069407] mpt2sas [55763.069408] drm_kms_helper [55763.069409] dca [55763.069410] syscopyarea [55763.069411] libata [55763.069412] sysfillrect [55763.069413] raid_class [55763.069414] sysimgblt [55763.069415] i2c_algo_bit [55763.069415] fb_sys_fops [55763.069416] scsi_transport_sas [55763.069417] isci [55763.069418] devlink [55763.069419] ttm [55763.069420] i2c_core [55763.069421] igb [55763.069422] fjes [55763.069422] crct10dif_pclmul [55763.069423] [55763.069424] crct10dif_common [55763.069426] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55763.069427] ahci [55763.069428] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55763.069429] ptp [55763.069432] Workqueue: ib_cm cm_work_handler [ib_cm] [55763.069433] crc32c_intel [55763.069434] [55763.069434] libsas [55763.069436] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55763.069437] libahci [55763.069438] RIP: 0010:[] [55763.069439] pps_core [55763.069441] [] _raw_spin_lock+0x32/0x50 [55763.069442] drm [55763.069443] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000216 [55763.069444] mlx4_core [55763.069445] RAX: 00000000000033ce RBX: 0000000000000000 RCX: 0000000000000a1e [55763.069446] mpt2sas [55763.069447] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55763.069448] dca [55763.069449] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55763.069450] libata [55763.069451] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55763.069452] raid_class [55763.069454] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55763.069455] i2c_algo_bit [55763.069456] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55763.069457] scsi_transport_sas [55763.069459] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55763.069460] devlink [55763.069461] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55763.069462] i2c_core [55763.069463] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55763.069464] fjes [55763.069465] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55763.069466] [55763.069467] Stack: [55763.069469] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55763.069471] ffff8803fea77b00 [55763.069472] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55763.069473] ffffffffa08aa7c8 [55763.069474] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55763.069475] ffff8803fea77bd0 [55763.069476] RIP: 0010:[] [55763.069477] 00050000c0a80173 [55763.069479] [] _raw_spin_lock+0x32/0x50 [55763.069480] [55763.069481] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [55763.069482] ffff88083fb5de00 [55763.069483] RAX: 0000000000006f4e RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55763.069484] ffff8803fea77b18 [55763.069485] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55763.069487] ffffffffa0916e4b [55763.069488] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55763.069489] ffff8803fea77c68 [55763.069490] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55763.069491] [55763.069492] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55763.069494] ffff8803fea77bc8 [55763.069495] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55763.069496] ffffffffa0ebfc24 [55763.069498] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55763.069498] ffff88041aa8a880 [55763.069500] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55763.069501] ffff8803fea77bd0 [55763.069502] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55763.069503] [55763.069505] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55763.069506] Call Trace: [55763.069506] Stack: [55763.069508] [55763.069508] ffff8807f639fd88 [55763.069514] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55763.069515] ffffffffa08aa831 [55763.069516] [55763.069517] 0000000200000002 [55763.069523] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55763.069524] ffff8807f63a2980 [55763.069525] [55763.069526] [55763.069530] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55763.069531] 0000000000000000 [55763.069532] [55763.069533] ffff8807f639fdc0 [55763.069535] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55763.069536] ffffffffa0942e30 [55763.069537] [55763.069538] 00000000c0086540 [55763.069542] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55763.069543] [55763.069544] [55763.069545] ffff8807f63a2900 [55763.069547] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55763.069548] 0000000000000002 [55763.069549] [55763.069550] 0000000000650aa0 [55763.069552] [] cma_req_handler+0x227/0x570 [rdma_cm] [55763.069553] 0000000000000000 [55763.069554] [55763.069555] [55763.069557] [] cm_process_work+0x25/0x140 [ib_cm] [55763.069557] Call Trace: [55763.069559] [55763.069560] [55763.069562] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55763.069568] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55763.069569] [55763.069570] [55763.069572] [] cm_work_handler+0x185/0x1484 [ib_cm] [55763.069579] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55763.069580] [55763.069581] [55763.069582] [] process_one_work+0x17b/0x470 [55763.069588] [] LNetCtl+0x116/0x800 [lnet] [55763.069589] [55763.069590] [55763.069591] [] worker_thread+0x126/0x410 [55763.069597] [] lnet_ioctl+0x60/0x250 [lnet] [55763.069598] [55763.069599] [55763.069600] [] ? rescuer_thread+0x460/0x460 [55763.069605] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55763.069606] [55763.069607] [55763.069609] [] kthread+0xcf/0xe0 [55763.069614] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55763.069615] [55763.069616] [55763.069618] [] ? kthread_create_on_node+0x140/0x140 [55763.069620] [] do_vfs_ioctl+0x2d5/0x4b0 [55763.069621] [55763.069622] [55763.069624] [] ret_from_fork+0x58/0x90 [55763.069626] [] ? __fd_install+0x47/0x60 [55763.069627] [55763.069627] [55763.069629] [] ? kthread_create_on_node+0x140/0x140 [55763.069631] [] SyS_ioctl+0xa1/0xc0 [55763.069632] Code: [55763.069633] [55763.069634] 00 [55763.069636] [] system_call_fastpath+0x16/0x1b [55763.069637] 02 [55763.069638] Code: [55763.069639] 00 [55763.069640] 00 [55763.069641] f0 [55763.069641] 02 [55763.069642] 0f [55763.069643] 00 [55763.069644] c1 [55763.069645] f0 [55763.069646] 07 [55763.069646] 0f [55763.069647] 89 [55763.069648] c1 [55763.069649] c2 [55763.069650] 07 [55763.069651] c1 [55763.069652] 89 [55763.069653] ea [55763.069653] c2 [55763.069654] 10 [55763.069655] c1 [55763.069656] 66 [55763.069657] ea [55763.069658] 39 [55763.069659] 10 [55763.069659] c2 [55763.069660] 66 [55763.069661] 75 [55763.069662] 39 [55763.069663] 01 [55763.069663] c2 [55763.069664] c3 [55763.069665] 75 [55763.069666] 55 [55763.069667] 01 [55763.069668] 83 [55763.069668] c3 [55763.069669] e2 [55763.069670] 55 [55763.069671] fe [55763.069672] 83 [55763.069673] 0f [55763.069673] e2 [55763.069674] b7 [55763.069675] fe [55763.069676] f2 [55763.069677] 0f [55763.069678] 48 [55763.069679] b7 [55763.069680] 89 [55763.069680] f2 [55763.069681] e5 [55763.069682] 48 [55763.069683] b8 [55763.069684] 89 [55763.069685] 00 [55763.069685] e5 [55763.069686] 80 [55763.069687] b8 [55763.069688] 00 [55763.069689] 00 [55763.069690] 00 [55763.069691] 80 [55763.069691] eb [55763.069692] 00 [55763.069693] 0d [55763.069694] 00 [55763.069695] 66 [55763.069696] eb [55763.069696] 0f [55763.069697] 0d [55763.069698] 1f [55763.069699] 66 [55763.069700] 44 [55763.069701] 0f [55763.069701] 00 [55763.069702] 1f [55763.069703] 00 [55763.069704] 44 [55763.069705] f3 [55763.069706] 00 [55763.069706] 90 [55763.069707] 00 [55763.069708] <83> [55763.069709] f3 [55763.069710] e8 [55763.069711] 90 [55763.069712] 01 [55763.069713] <83> [55763.069714] 74 [55763.069714] e8 [55763.069715] 0a [55763.069716] 01 [55763.069717] 0f [55763.069718] 74 [55763.069719] b7 [55763.069719] 0a [55763.069720] 0f [55763.069721] 0f [55763.069722] 66 [55763.069723] b7 [55763.069724] 39 [55763.069724] 0f [55763.069725] ca [55763.069726] 66 [55763.069727] 75 [55763.069728] 39 [55763.069729] f1 [55763.069729] ca [55763.069730] 5d [55763.069731] 75 [55763.069732] c3 [55763.069733] f1 [55763.069734] 66 [55763.069734] 5d [55763.069735] 66 [55763.069736] c3 [55763.069737] 66 [55763.069738] 66 [55763.069739] 90 [55763.069739] 66 [55763.069740] 66 [55763.069741] 66 [55763.069742] 66 [55763.069742] 90 [55763.069743] [55763.069744] 66 [55763.069744] 66 [55763.069745] [55776.883591] { [55776.889858] 11} (t=960015 jiffies g=99188 c=99187 q=24357) [55776.905032] Task dump for CPU 1: [55776.916430] kiblnd_sd_00_00 R running task 0 2896 2 0x00000088 [55776.940448] ffff880820b03d30 000000003f673d40 ffffc90003ec02f0 ffff88041d63d600 [55776.965630] 0000000000000000 0000000000000000 ffff88042aa84a28 ffff880820b03da8 [55776.990792] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000002 [55777.015957] Call Trace: [55777.024796] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55777.047884] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55777.071868] [] ? dequeue_task_fair+0x41e/0x660 [55777.092691] [] ? sched_clock_cpu+0x85/0xc0 [55777.112356] [] ? wake_up_state+0x20/0x20 [55777.131456] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55777.154272] [] ? kthread+0xcf/0xe0 [55777.171651] [] ? kthread_create_on_node+0x140/0x140 [55777.193874] [] ? ret_from_fork+0x58/0x90 [55777.212946] [] ? kthread_create_on_node+0x140/0x140 [55777.235153] Task dump for CPU 5: [55777.246470] kiblnd_sd_00_02 R running task 0 2898 2 0x00000088 [55777.270390] ffff880415b13000 ffff8800abaa43a8 ffff88083fb5e000 0000000000000000 [55777.295463] ffff88082aae7da8 ffffffffa0ebedab ffff880181447188 ffff88082aae7da8 [55777.320561] ffffffff810bc064 ffff880181447180 ffff88042aa84a00 0000000000000001 [55777.345658] Call Trace: [55777.354387] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55777.378003] [] ? __wake_up+0x44/0x50 [55777.395932] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55777.419842] [] ? dequeue_task_fair+0x41e/0x660 [55777.440597] [] ? wake_up_state+0x20/0x20 [55777.459636] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55777.482430] [] ? kthread+0xcf/0xe0 [55777.499784] [] ? kthread_create_on_node+0x140/0x140 [55777.522005] [] ? ret_from_fork+0x58/0x90 [55777.541058] [] ? kthread_create_on_node+0x140/0x140 [55777.563283] Task dump for CPU 6: [55777.574605] kiblnd_sd_00_01 R running task 0 2897 2 0x00000088 [55777.598564] ffff88081f64fd30 00000000f1dbfe3f ffffc90003ebfde0 ffff88042dcd1e00 [55777.623708] 0000000000000000 0000000000000000 ffff88042aa84a28 ffff88081f64fda8 [55777.648845] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000012 [55777.673995] Call Trace: [55777.682801] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55777.705920] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55777.729897] [] ? dequeue_task_fair+0x41e/0x660 [55777.750707] [] ? wake_up_state+0x20/0x20 [55777.769752] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55777.792515] [] ? kthread+0xcf/0xe0 [55777.809810] [] ? kthread_create_on_node+0x140/0x140 [55777.831993] [] ? ret_from_fork+0x58/0x90 [55777.851040] [] ? kthread_create_on_node+0x140/0x140 [55777.873210] Task dump for CPU 7: [55777.884521] kiblnd_sd_00_03 R running task 0 2899 2 0x00000088 [55777.908451] ffff88082626bd30 00000000aaf7c743 ffffc90003ebe8c8 ffff8800b484ea00 [55777.933546] 0000000000000000 0000000000000000 ffff88042aa84a28 ffff88082626bda8 [55777.958642] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000001 [55777.983713] Call Trace: [55777.992441] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55778.015493] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55778.039412] [] ? dequeue_task_fair+0x41e/0x660 [55778.060162] [] ? wake_up_state+0x20/0x20 [55778.079208] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55778.101978] [] ? kthread+0xcf/0xe0 [55778.119314] [] ? kthread_create_on_node+0x140/0x140 [55778.141504] [] ? ret_from_fork+0x58/0x90 [55778.160539] [] ? kthread_create_on_node+0x140/0x140 [55778.182735] Task dump for CPU 10: [55778.194343] kiblnd_sd_01_01 R running task 0 2901 2 0x00000088 [55778.218271] ffff8807f886f000 ffffc9001327c860 ffff8808266e7a00 0000000000000000 [55778.243375] ffff8808262efda8 ffffffffa0ebedab ffff8808262efe90 ffff8808262efda8 [55778.268419] ffffffffa03e7d2e ffff88082b2fc3c0 ffff88082bda6c00 0000000000000017 [55778.293516] Call Trace: [55778.302252] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55778.325857] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55778.346879] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55778.370782] [] ? dequeue_task_fair+0x41e/0x660 [55778.391546] [] ? wake_up_state+0x20/0x20 [55778.410591] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55778.433349] [] ? kthread+0xcf/0xe0 [55778.450666] [] ? kthread_create_on_node+0x140/0x140 [55778.472834] [] ? ret_from_fork+0x58/0x90 [55778.491848] [] ? kthread_create_on_node+0x140/0x140 [55778.514040] Task dump for CPU 11: [55778.525636] kiblnd_sd_01_02 R running task 0 2902 2 0x00000088 [55778.549560] ffff880826310000 000000005cf0ae83 ffff88082d8c3db0 ffffffff810c42f8 [55778.574666] 000000000000000b ffffffff81a1a680 ffff88082d8c3dc8 ffffffff810c7b89 [55778.599736] 000000000000000c ffff88082d8c3df8 ffffffff811373f0 ffff88082d8d01c0 [55778.624816] Call Trace: [55778.633535] [] sched_show_task+0xa8/0x110 [55778.654870] [] dump_cpu_task+0x39/0x70 [55778.673309] [] rcu_dump_cpu_stacks+0x90/0xd0 [55778.693478] [] rcu_check_callbacks+0x442/0x720 [55778.714215] [] ? tick_sched_handle.isra.13+0x60/0x60 [55778.736659] [] update_process_times+0x47/0x80 [55778.757096] [] tick_sched_handle.isra.13+0x25/0x60 [55778.778994] [] tick_sched_timer+0x41/0x70 [55778.798319] [] __hrtimer_run_queues+0xd2/0x260 [55778.819054] [] hrtimer_interrupt+0xb0/0x1e0 [55778.838931] [] ? call_softirq+0x1c/0x30 [55778.857662] [] local_apic_timer_interrupt+0x37/0x60 [55778.879830] [] smp_apic_timer_interrupt+0x3f/0x60 [55778.901421] [] apic_timer_interrupt+0x6d/0x80 [55778.921857] [] ? _raw_spin_lock+0x32/0x50 [55778.943157] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55778.965025] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55778.987757] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55779.009919] [] lnet_finalize+0x1e9/0x690 [lnet] [55779.030931] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55779.053355] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55779.067594] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [kiblnd_sd_01_01:2901] [55779.067601] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [kiblnd_sd_01_00:2900] [55779.067603] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [kiblnd_sd_00_03:2899] [55779.067604] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [kiblnd_sd_01_03:2903] [55779.067606] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [kiblnd_sd_00_02:2898] [55779.067607] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [kiblnd_sd_00_01:2897] [55779.067609] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [kiblnd_sd_00_00:2896] [55779.067611] Modules linked in: [55779.067612] Modules linked in: [55779.067613] Modules linked in: [55779.067615] Modules linked in: [55779.067616] Modules linked in: [55779.067617] Modules linked in: [55779.067618] Modules linked in: [55779.067619] ko2iblnd(OE) [55779.067620] ko2iblnd(OE) [55779.067621] ko2iblnd(OE) [55779.067622] ko2iblnd(OE) [55779.067623] ko2iblnd(OE) [55779.067624] ko2iblnd(OE) [55779.067625] ko2iblnd(OE) [55779.067626] lustre(OE) [55779.067627] lustre(OE) [55779.067628] lustre(OE) [55779.067630] lustre(OE) [55779.067631] lustre(OE) [55779.067631] lustre(OE) [55779.067632] lustre(OE) [55779.067633] lmv(OE) [55779.067634] lmv(OE) [55779.067635] lmv(OE) [55779.067637] lmv(OE) [55779.067637] lmv(OE) [55779.067638] lmv(OE) [55779.067639] lmv(OE) [55779.067640] fld(OE) [55779.067641] fld(OE) [55779.067642] fld(OE) [55779.067643] fld(OE) [55779.067644] fld(OE) [55779.067645] fld(OE) [55779.067646] fld(OE) [55779.067647] mdc(OE) [55779.067648] mdc(OE) [55779.067649] mdc(OE) [55779.067650] mdc(OE) [55779.067651] mdc(OE) [55779.067652] mdc(OE) [55779.067653] mdc(OE) [55779.067654] fid(OE) [55779.067655] fid(OE) [55779.067656] fid(OE) [55779.067657] fid(OE) [55779.067658] fid(OE) [55779.067659] fid(OE) [55779.067660] fid(OE) [55779.067660] lov(OE) [55779.067662] lov(OE) [55779.067663] lov(OE) [55779.067664] lov(OE) [55779.067664] lov(OE) [55779.067665] lov(OE) [55779.067666] lov(OE) [55779.067667] ksocklnd(OE) [55779.067668] ksocklnd(OE) [55779.067670] ksocklnd(OE) [55779.067671] ksocklnd(OE) [55779.067672] ksocklnd(OE) [55779.067673] ksocklnd(OE) [55779.067674] ksocklnd(OE) [55779.067675] ptlrpc(OE) [55779.067676] ptlrpc(OE) [55779.067677] ptlrpc(OE) [55779.067678] ptlrpc(OE) [55779.067679] ptlrpc(OE) [55779.067679] ptlrpc(OE) [55779.067680] ptlrpc(OE) [55779.067681] obdclass(OE) [55779.067683] obdclass(OE) [55779.067684] obdclass(OE) [55779.067685] obdclass(OE) [55779.067686] obdclass(OE) [55779.067687] obdclass(OE) [55779.067688] obdclass(OE) [55779.067689] lnet(OE) [55779.067690] lnet(OE) [55779.067691] lnet(OE) [55779.067692] lnet(OE) [55779.067693] lnet(OE) [55779.067694] lnet(OE) [55779.067695] lnet(OE) [55779.067696] libcfs(OE) [55779.067697] libcfs(OE) [55779.067698] libcfs(OE) [55779.067699] libcfs(OE) [55779.067700] libcfs(OE) [55779.067701] libcfs(OE) [55779.067702] libcfs(OE) [55779.067703] rpcrdma [55779.067704] rpcrdma [55779.067705] rpcrdma [55779.067706] rpcrdma [55779.067707] rpcrdma [55779.067708] rpcrdma [55779.067709] rpcrdma [55779.067710] ib_isert [55779.067711] ib_isert [55779.067712] ib_isert [55779.067714] ib_isert [55779.067714] ib_isert [55779.067715] ib_isert [55779.067716] ib_isert [55779.067717] iscsi_target_mod [55779.067718] iscsi_target_mod [55779.067719] iscsi_target_mod [55779.067721] iscsi_target_mod [55779.067722] iscsi_target_mod [55779.067723] iscsi_target_mod [55779.067724] iscsi_target_mod [55779.067725] ib_iser [55779.067727] ib_iser [55779.067728] ib_iser [55779.067729] ib_iser [55779.067730] ib_iser [55779.067731] ib_iser [55779.067732] ib_iser [55779.067732] libiscsi [55779.067734] libiscsi [55779.067735] libiscsi [55779.067736] libiscsi [55779.067736] libiscsi [55779.067737] libiscsi [55779.067738] libiscsi [55779.067739] scsi_transport_iscsi [55779.067740] scsi_transport_iscsi [55779.067741] scsi_transport_iscsi [55779.067742] scsi_transport_iscsi [55779.067743] scsi_transport_iscsi [55779.067744] scsi_transport_iscsi [55779.067745] scsi_transport_iscsi [55779.067746] ib_srpt [55779.067747] ib_srpt [55779.067749] ib_srpt [55779.067750] ib_srpt [55779.067750] ib_srpt [55779.067751] ib_srpt [55779.067752] ib_srpt [55779.067753] target_core_mod [55779.067755] target_core_mod [55779.067756] target_core_mod [55779.067757] target_core_mod [55779.067758] target_core_mod [55779.067760] target_core_mod [55779.067761] target_core_mod [55779.067762] ib_srp [55779.067763] ib_srp [55779.067764] ib_srp [55779.067765] ib_srp [55779.067766] ib_srp [55779.067767] ib_srp [55779.067768] ib_srp [55779.067769] scsi_transport_srp [55779.067770] scsi_transport_srp [55779.067771] scsi_transport_srp [55779.067772] scsi_transport_srp [55779.067773] scsi_transport_srp [55779.067774] scsi_transport_srp [55779.067775] scsi_transport_srp [55779.067776] scsi_tgt [55779.067777] scsi_tgt [55779.067778] scsi_tgt [55779.067779] scsi_tgt [55779.067780] scsi_tgt [55779.067781] scsi_tgt [55779.067782] scsi_tgt [55779.067783] ib_ipoib [55779.067784] ib_ipoib [55779.067785] ib_ipoib [55779.067786] ib_ipoib [55779.067787] ib_ipoib [55779.067788] ib_ipoib [55779.067789] ib_ipoib [55779.067790] rdma_ucm [55779.067792] rdma_ucm [55779.067793] rdma_ucm [55779.067794] rdma_ucm [55779.067795] rdma_ucm [55779.067796] rdma_ucm [55779.067797] rdma_ucm [55779.067798] ib_ucm [55779.067799] ib_ucm [55779.067800] ib_ucm [55779.067801] ib_ucm [55779.067802] ib_ucm [55779.067803] ib_ucm [55779.067804] ib_ucm [55779.067805] ib_uverbs [55779.067806] ib_uverbs [55779.067807] ib_uverbs [55779.067808] ib_uverbs [55779.067808] ib_uverbs [55779.067809] ib_uverbs [55779.067810] ib_uverbs [55779.067811] ib_umad [55779.067812] ib_umad [55779.067813] ib_umad [55779.067814] ib_umad [55779.067815] ib_umad [55779.067816] ib_umad [55779.067817] ib_umad [55779.067818] rdma_cm [55779.067819] rdma_cm [55779.067820] rdma_cm [55779.067821] rdma_cm [55779.067822] rdma_cm [55779.067823] rdma_cm [55779.067824] rdma_cm [55779.067825] ib_cm [55779.067826] ib_cm [55779.067827] ib_cm [55779.067828] ib_cm [55779.067829] ib_cm [55779.067830] ib_cm [55779.067831] ib_cm [55779.067832] iw_cm [55779.067833] iw_cm [55779.067834] iw_cm [55779.067835] iw_cm [55779.067836] iw_cm [55779.067837] iw_cm [55779.067838] iw_cm [55779.067839] mlx4_ib [55779.067840] mlx4_ib [55779.067842] mlx4_ib [55779.067843] mlx4_ib [55779.067843] mlx4_ib [55779.067844] mlx4_ib [55779.067845] mlx4_ib [55779.067846] intel_powerclamp [55779.067848] intel_powerclamp [55779.067849] intel_powerclamp [55779.067850] intel_powerclamp [55779.067851] intel_powerclamp [55779.067851] intel_powerclamp [55779.067853] intel_powerclamp [55779.067853] coretemp [55779.067854] coretemp [55779.067855] coretemp [55779.067856] coretemp [55779.067857] coretemp [55779.067858] coretemp [55779.067859] coretemp [55779.067860] intel_rapl [55779.067861] intel_rapl [55779.067862] intel_rapl [55779.067863] intel_rapl [55779.067863] intel_rapl [55779.067864] intel_rapl [55779.067865] intel_rapl [55779.067866] iosf_mbi [55779.067867] iosf_mbi [55779.067868] iosf_mbi [55779.067869] iosf_mbi [55779.067870] iosf_mbi [55779.067871] iosf_mbi [55779.067872] iosf_mbi [55779.067873] kvm [55779.067874] kvm [55779.067875] kvm [55779.067876] kvm [55779.067877] kvm [55779.067878] kvm [55779.067879] kvm [55779.067880] irqbypass [55779.067881] irqbypass [55779.067882] irqbypass [55779.067883] irqbypass [55779.067884] irqbypass [55779.067885] irqbypass [55779.067886] irqbypass [55779.067887] crc32_pclmul [55779.067889] crc32_pclmul [55779.067890] crc32_pclmul [55779.067891] crc32_pclmul [55779.067892] crc32_pclmul [55779.067893] crc32_pclmul [55779.067894] crc32_pclmul [55779.067895] ghash_clmulni_intel [55779.067896] ghash_clmulni_intel [55779.067897] ghash_clmulni_intel [55779.067898] ghash_clmulni_intel [55779.067899] ghash_clmulni_intel [55779.067900] ghash_clmulni_intel [55779.067901] ghash_clmulni_intel [55779.067902] aesni_intel [55779.067903] aesni_intel [55779.067904] aesni_intel [55779.067905] aesni_intel [55779.067906] aesni_intel [55779.067907] aesni_intel [55779.067908] aesni_intel [55779.067909] ipmi_ssif [55779.067910] ipmi_ssif [55779.067911] ipmi_ssif [55779.067912] ipmi_ssif [55779.067913] ipmi_ssif [55779.067914] ipmi_ssif [55779.067915] ipmi_ssif [55779.067916] lrw [55779.067917] lrw [55779.067918] lrw [55779.067919] lrw [55779.067920] lrw [55779.067921] lrw [55779.067922] lrw [55779.067922] gf128mul [55779.067924] gf128mul [55779.067925] gf128mul [55779.067926] gf128mul [55779.067927] gf128mul [55779.067928] gf128mul [55779.067929] gf128mul [55779.067930] glue_helper [55779.067931] glue_helper [55779.067932] glue_helper [55779.067933] glue_helper [55779.067934] glue_helper [55779.067935] glue_helper [55779.067937] glue_helper [55779.067937] hfi1 [55779.067938] hfi1 [55779.067940] hfi1 [55779.067941] hfi1 [55779.067942] hfi1 [55779.067942] hfi1 [55779.067944] hfi1 [55779.067944] ipmi_devintf [55779.067945] ipmi_devintf [55779.067947] ipmi_devintf [55779.067948] ipmi_devintf [55779.067949] ipmi_devintf [55779.067950] ipmi_devintf [55779.067951] ipmi_devintf [55779.067952] rdmavt [55779.067953] rdmavt [55779.067954] rdmavt [55779.067955] rdmavt [55779.067956] rdmavt [55779.067957] rdmavt [55779.067958] rdmavt [55779.067959] ablk_helper [55779.067960] ablk_helper [55779.067962] ablk_helper [55779.067962] ablk_helper [55779.067963] ablk_helper [55779.067964] ablk_helper [55779.067965] ablk_helper [55779.067966] cryptd [55779.067967] cryptd [55779.067969] cryptd [55779.067970] cryptd [55779.067971] cryptd [55779.067972] cryptd [55779.067973] cryptd [55779.067974] iTCO_wdt [55779.067975] iTCO_wdt [55779.067976] iTCO_wdt [55779.067977] iTCO_wdt [55779.067978] iTCO_wdt [55779.067979] iTCO_wdt [55779.067980] iTCO_wdt [55779.067980] ipmi_si [55779.067981] ipmi_si [55779.067983] ipmi_si [55779.067984] ipmi_si [55779.067984] ipmi_si [55779.067985] ipmi_si [55779.067986] ipmi_si [55779.067987] sb_edac [55779.067988] sb_edac [55779.067989] sb_edac [55779.067990] sb_edac [55779.067991] sb_edac [55779.067992] sb_edac [55779.067993] sb_edac [55779.067994] sg [55779.067995] sg [55779.067996] sg [55779.067997] sg [55779.067998] sg [55779.067999] sg [55779.068000] sg [55779.068001] iTCO_vendor_support [55779.068002] iTCO_vendor_support [55779.068003] iTCO_vendor_support [55779.068004] iTCO_vendor_support [55779.068006] iTCO_vendor_support [55779.068006] iTCO_vendor_support [55779.068008] iTCO_vendor_support [55779.068008] ib_core [55779.068010] ib_core [55779.068011] ib_core [55779.068012] ib_core [55779.068012] ib_core [55779.068014] ib_core [55779.068015] ib_core [55779.068016] ioatdma [55779.068017] ioatdma [55779.068018] ioatdma [55779.068019] ioatdma [55779.068020] ioatdma [55779.068021] ioatdma [55779.068022] ioatdma [55779.068023] i2c_i801 [55779.068024] i2c_i801 [55779.068025] i2c_i801 [55779.068026] i2c_i801 [55779.068027] i2c_i801 [55779.068028] i2c_i801 [55779.068029] i2c_i801 [55779.068029] mei_me [55779.068031] mei_me [55779.068032] mei_me [55779.068033] mei_me [55779.068034] mei_me [55779.068035] mei_me [55779.068036] mei_me [55779.068037] mei [55779.068038] mei [55779.068039] mei [55779.068041] mei [55779.068041] mei [55779.068042] mei [55779.068043] mei [55779.068044] edac_core [55779.068045] edac_core [55779.068046] edac_core [55779.068047] edac_core [55779.068048] edac_core [55779.068049] edac_core [55779.068050] edac_core [55779.068051] ipmi_msghandler [55779.068052] ipmi_msghandler [55779.068053] ipmi_msghandler [55779.068054] ipmi_msghandler [55779.068055] ipmi_msghandler [55779.068056] ipmi_msghandler [55779.068057] ipmi_msghandler [55779.068058] lpc_ich [55779.068060] lpc_ich [55779.068061] lpc_ich [55779.068062] lpc_ich [55779.068062] lpc_ich [55779.068063] lpc_ich [55779.068064] lpc_ich [55779.068065] ntb [55779.068067] ntb [55779.068068] ntb [55779.068069] ntb [55779.068070] ntb [55779.068070] ntb [55779.068071] ntb [55779.068072] wmi [55779.068073] wmi [55779.068075] wmi [55779.068076] wmi [55779.068077] wmi [55779.068077] wmi [55779.068079] wmi [55779.068080] shpchp [55779.068081] shpchp [55779.068082] shpchp [55779.068083] shpchp [55779.068084] shpchp [55779.068085] shpchp [55779.068086] shpchp [55779.068087] pcspkr [55779.068088] pcspkr [55779.068089] pcspkr [55779.068090] pcspkr [55779.068091] pcspkr [55779.068092] pcspkr [55779.068094] pcspkr [55779.068094] nfsd [55779.068096] nfsd [55779.068097] nfsd [55779.068098] nfsd [55779.068099] nfsd [55779.068099] nfsd [55779.068101] nfsd [55779.068101] nfs_acl [55779.068102] nfs_acl [55779.068103] nfs_acl [55779.068105] nfs_acl [55779.068105] nfs_acl [55779.068106] nfs_acl [55779.068107] nfs_acl [55779.068108] lockd [55779.068109] lockd [55779.068111] lockd [55779.068112] lockd [55779.068113] lockd [55779.068114] lockd [55779.068115] lockd [55779.068116] grace [55779.068117] grace [55779.068118] grace [55779.068119] grace [55779.068120] grace [55779.068121] grace [55779.068122] grace [55779.068123] auth_rpcgss [55779.068124] auth_rpcgss [55779.068126] auth_rpcgss [55779.068127] auth_rpcgss [55779.068128] auth_rpcgss [55779.068128] auth_rpcgss [55779.068129] auth_rpcgss [55779.068130] sunrpc [55779.068131] sunrpc [55779.068133] sunrpc [55779.068134] sunrpc [55779.068135] sunrpc [55779.068136] sunrpc [55779.068137] sunrpc [55779.068138] ip_tables [55779.068139] ip_tables [55779.068140] ip_tables [55779.068141] ip_tables [55779.068142] ip_tables [55779.068143] ip_tables [55779.068144] ip_tables [55779.068145] ext4 [55779.068146] ext4 [55779.068148] ext4 [55779.068149] ext4 [55779.068150] ext4 [55779.068150] ext4 [55779.068152] ext4 [55779.068153] mbcache [55779.068154] mbcache [55779.068155] mbcache [55779.068156] mbcache [55779.068157] mbcache [55779.068158] mbcache [55779.068159] mbcache [55779.068160] jbd2 [55779.068161] jbd2 [55779.068162] jbd2 [55779.068163] jbd2 [55779.068164] jbd2 [55779.068165] jbd2 [55779.068166] jbd2 [55779.068167] sd_mod [55779.068168] sd_mod [55779.068169] sd_mod [55779.068171] sd_mod [55779.068171] sd_mod [55779.068172] sd_mod [55779.068173] sd_mod [55779.068174] crc_t10dif [55779.068176] crc_t10dif [55779.068177] crc_t10dif [55779.068178] crc_t10dif [55779.068178] crc_t10dif [55779.068179] crc_t10dif [55779.068180] crc_t10dif [55779.068181] crct10dif_generic [55779.068182] crct10dif_generic [55779.068184] crct10dif_generic [55779.068185] crct10dif_generic [55779.068186] crct10dif_generic [55779.068187] crct10dif_generic [55779.068188] crct10dif_generic [55779.068189] mlx4_en [55779.068190] mlx4_en [55779.068191] mlx4_en [55779.068192] mlx4_en [55779.068193] mlx4_en [55779.068194] mlx4_en [55779.068195] mlx4_en [55779.068196] mgag200 [55779.068198] mgag200 [55779.068198] mgag200 [55779.068200] mgag200 [55779.068200] mgag200 [55779.068201] mgag200 [55779.068202] mgag200 [55779.068203] drm_kms_helper [55779.068204] drm_kms_helper [55779.068205] drm_kms_helper [55779.068207] drm_kms_helper [55779.068207] drm_kms_helper [55779.068208] drm_kms_helper [55779.068209] drm_kms_helper [55779.068210] syscopyarea [55779.068211] syscopyarea [55779.068213] syscopyarea [55779.068214] syscopyarea [55779.068215] syscopyarea [55779.068216] syscopyarea [55779.068217] syscopyarea [55779.068218] sysfillrect [55779.068219] sysfillrect [55779.068220] sysfillrect [55779.068221] sysfillrect [55779.068222] sysfillrect [55779.068223] sysfillrect [55779.068224] sysfillrect [55779.068225] sysimgblt [55779.068226] sysimgblt [55779.068227] sysimgblt [55779.068228] sysimgblt [55779.068228] sysimgblt [55779.068229] sysimgblt [55779.068230] sysimgblt [55779.068231] fb_sys_fops [55779.068233] fb_sys_fops [55779.068234] fb_sys_fops [55779.068235] fb_sys_fops [55779.068236] fb_sys_fops [55779.068237] fb_sys_fops [55779.068238] fb_sys_fops [55779.068239] isci [55779.068240] isci [55779.068241] isci [55779.068242] isci [55779.068243] isci [55779.068244] isci [55779.068245] isci [55779.068246] ttm [55779.068247] ttm [55779.068249] ttm [55779.068250] ttm [55779.068251] ttm [55779.068252] ttm [55779.068253] ttm [55779.068254] igb [55779.068255] igb [55779.068257] igb [55779.068258] igb [55779.068259] igb [55779.068260] igb [55779.068261] igb [55779.068262] crct10dif_pclmul [55779.068263] crct10dif_pclmul [55779.068264] crct10dif_pclmul [55779.068265] crct10dif_pclmul [55779.068266] crct10dif_pclmul [55779.068267] crct10dif_pclmul [55779.068268] crct10dif_pclmul [55779.068269] crct10dif_common [55779.068271] crct10dif_common [55779.068272] crct10dif_common [55779.068273] crct10dif_common [55779.068273] crct10dif_common [55779.068274] crct10dif_common [55779.068275] crct10dif_common [55779.068276] ahci [55779.068277] ahci [55779.068279] ahci [55779.068280] ahci [55779.068281] ahci [55779.068282] ahci [55779.068283] ahci [55779.068284] ptp [55779.068285] ptp [55779.068286] ptp [55779.068288] ptp [55779.068289] ptp [55779.068289] ptp [55779.068291] ptp [55779.068291] crc32c_intel [55779.068293] crc32c_intel [55779.068294] crc32c_intel [55779.068295] crc32c_intel [55779.068296] crc32c_intel [55779.068296] crc32c_intel [55779.068297] crc32c_intel [55779.068298] libsas [55779.068300] libsas [55779.068301] libsas [55779.068302] libsas [55779.068303] libsas [55779.068304] libsas [55779.068305] libsas [55779.068305] libahci [55779.068307] libahci [55779.068308] libahci [55779.068309] libahci [55779.068310] libahci [55779.068311] libahci [55779.068312] libahci [55779.068313] pps_core [55779.068314] pps_core [55779.068315] pps_core [55779.068316] pps_core [55779.068317] pps_core [55779.068318] pps_core [55779.068319] pps_core [55779.068320] drm [55779.068321] drm [55779.068322] drm [55779.068323] drm [55779.068324] drm [55779.068325] drm [55779.068326] drm [55779.068327] mlx4_core [55779.068328] mlx4_core [55779.068330] mlx4_core [55779.068331] mlx4_core [55779.068332] mlx4_core [55779.068332] mlx4_core [55779.068334] mlx4_core [55779.068335] mpt2sas [55779.068336] mpt2sas [55779.068337] mpt2sas [55779.068338] mpt2sas [55779.068339] mpt2sas [55779.068340] mpt2sas [55779.068341] mpt2sas [55779.068342] dca [55779.068344] dca [55779.068345] dca [55779.068346] dca [55779.068347] dca [55779.068347] dca [55779.068349] dca [55779.068350] libata [55779.068351] libata [55779.068352] libata [55779.068353] libata [55779.068354] libata [55779.068355] libata [55779.068356] libata [55779.068357] raid_class [55779.068358] raid_class [55779.068359] raid_class [55779.068360] raid_class [55779.068361] raid_class [55779.068362] raid_class [55779.068363] raid_class [55779.068364] i2c_algo_bit [55779.068365] i2c_algo_bit [55779.068366] i2c_algo_bit [55779.068367] i2c_algo_bit [55779.068368] i2c_algo_bit [55779.068369] i2c_algo_bit [55779.068371] i2c_algo_bit [55779.068372] scsi_transport_sas [55779.068373] scsi_transport_sas [55779.068375] scsi_transport_sas [55779.068376] scsi_transport_sas [55779.068377] scsi_transport_sas [55779.068377] scsi_transport_sas [55779.068379] scsi_transport_sas [55779.068379] devlink [55779.068381] devlink [55779.068382] devlink [55779.068383] devlink [55779.068384] devlink [55779.068385] devlink [55779.068386] devlink [55779.068387] i2c_core [55779.068388] i2c_core [55779.068389] i2c_core [55779.068390] i2c_core [55779.068391] i2c_core [55779.068392] i2c_core [55779.068393] i2c_core [55779.068394] fjes [55779.068395] fjes [55779.068396] fjes [55779.068397] fjes [55779.068398] fjes [55779.068399] fjes [55779.068400] [55779.068401] [55779.068402] [55779.068403] [55779.068404] [55779.068405] [55779.068406] fjes [55779.068408] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55779.068410] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55779.068412] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55779.068414] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55779.068416] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55779.068417] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55779.068418] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55779.068420] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55779.068421] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55779.068422] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55779.068423] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55779.068425] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55779.068426] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55779.068426] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55779.068428] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55779.068429] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55779.068430] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55779.068432] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55779.068433] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55779.068434] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55779.068435] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55779.068438] RIP: 0010:[] [55779.068439] RIP: 0010:[] [55779.068441] RIP: 0010:[] [55779.068443] RIP: 0010:[] [55779.068444] RIP: 0010:[] [55779.068445] RIP: 0010:[] [55779.068447] [] _raw_spin_lock+0x32/0x50 [55779.068448] [] _raw_spin_lock+0x32/0x50 [55779.068450] [] _raw_spin_lock+0x32/0x50 [55779.068452] [] _raw_spin_lock+0x32/0x50 [55779.068454] [] _raw_spin_lock+0x32/0x50 [55779.068455] [] _raw_spin_lock+0x30/0x50 [55779.068457] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55779.068458] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [55779.068459] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55779.068460] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55779.068462] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [55779.068463] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [55779.068464] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [55779.068465] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [55779.068467] RAX: 0000000000005fef RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55779.068468] RAX: 00000000000022e7 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55779.068469] RAX: 0000000000006fc5 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55779.068470] RAX: 00000000000070ef RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55779.068472] RAX: 00000000000078ae RBX: 0000000100000000 RCX: 0000000000000a1e [55779.068472] RAX: 00000000000076f6 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55779.068473] RAX: 000000000000414b RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55779.068475] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55779.068476] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55779.068477] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55779.068478] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55779.068480] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55779.068480] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55779.068481] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55779.068483] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55779.068484] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55779.068485] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55779.068486] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55779.068488] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55779.068488] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55779.068489] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55779.068490] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55779.068491] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55779.068493] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55779.068494] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55779.068495] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55779.068497] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55779.068498] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55779.068499] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55779.068500] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55779.068501] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55779.068503] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55779.068504] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55779.068505] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55779.068506] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55779.068507] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55779.068508] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55779.068510] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55779.068511] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55779.068512] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55779.068514] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55779.068515] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55779.068516] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55779.068517] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55779.068519] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55779.068520] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55779.068521] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55779.068522] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55779.068524] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55779.068525] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55779.068526] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55779.068527] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55779.068528] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55779.068530] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55779.068531] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55779.068532] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55779.068533] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55779.068534] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55779.068535] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55779.068537] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55779.068538] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55779.068539] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55779.068540] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55779.068541] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55779.068542] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55779.068543] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55779.068544] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55779.068546] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55779.068547] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55779.068548] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55779.068549] Stack: [55779.068550] Stack: [55779.068550] Stack: [55779.068551] Stack: [55779.068552] Stack: [55779.068553] Stack: [55779.068553] Stack: [55779.068555] ffff8808262efac0 [55779.068557] ffff8808217bbcd8 [55779.068558] ffff88082626bd08 [55779.068559] ffff8808213b7ac0 [55779.068560] ffff88082aae7ba0 [55779.068561] ffff88081f64fc98 [55779.068562] ffff880820b03c28 [55779.068563] ffffffffa08aa7c8 [55779.068564] ffffffffa08aa7c8 [55779.068565] ffffffffa08aa7c8 [55779.068566] ffffffffa08aa7c8 [55779.068567] ffffffffa08aa7c8 [55779.068568] ffffffffa08aa7c8 [55779.068569] ffffffffa08aa7c8 [55779.068570] ffffffffa08aa8b6 [55779.068571] ffffffffa08aa8b6 [55779.068572] ffff88082af11648 [55779.068573] ffffffffa08aa8b6 [55779.068574] 000000000000763d [55779.068575] ffffffffa08aa8b6 [55779.068576] ffff880820b03c28 [55779.068577] 0000000000000000 [55779.068578] ffff8804c1ba3200 [55779.068579] ffff8800b484ea00 [55779.068580] 0000000000000000 [55779.068581] ffff8803fe4d0400 [55779.068581] ffff88042dcd1e00 [55779.068582] [55779.068583] [55779.068584] [55779.068585] [55779.068586] [55779.068587] [55779.068587] 0000000000000000 [55779.068589] ffff88083fb5e000 [55779.068590] 0000000000000001 [55779.068591] 0000000000000000 [55779.068593] ffff88083fb5e000 [55779.068593] 0000000000000000 [55779.068594] ffff8800b4864200 [55779.068596] ffff88083fb5e000 [55779.068597] ffff8808262efb80 [55779.068598] ffff8808217bbd08 [55779.068599] ffff88082626bd70 [55779.068600] ffff8808213b7b80 [55779.068601] ffff88082aae7c08 [55779.068602] ffff88081f64fcd8 [55779.068603] ffff880820b03c98 [55779.068604] ffffffffa092c9b5 [55779.068605] ffffffffa09217bb [55779.068606] ffffffffa0922006 [55779.068608] ffffffffa092c9b5 [55779.068608] ffffffffa0922006 [55779.068609] ffffffffa092e857 [55779.068610] ffffffffa092c346 [55779.068611] 00000000000001a8 [55779.068612] ffff8804c1ba3200 [55779.068613] ffff88082af10fb0 [55779.068615] 00000000000000c0 [55779.068615] ffff8804c1676dd0 [55779.068616] ffff8800b484d600 [55779.068617] [55779.068618] [55779.068619] [55779.068620] [55779.068621] [55779.068621] [55779.068622] ffff880820b03c60 [55779.068624] ffffc9000e331730 [55779.068625] 0000000000000001 [55779.068626] ffff88082af10fb0 [55779.068627] ffffc9000e335900 [55779.068627] ffff8804c1676dd0 [55779.068628] ffff88042dcd1e00 [55779.068629] 00000000000000b4 [55779.068630] ffff8808262efb68 [55779.068631] ffff88082abba340 [55779.068632] ffff88082af10fb0 [55779.068633] ffff8808213b7b68 [55779.068634] ffff8804c1676dd0 [55779.068635] 0000000000000000 [55779.068636] 00000001033f89f4 [55779.068637] ffffffffa0ebd587 [55779.068638] ffff88082abba348 [55779.068639] ffff88082626bd30 [55779.068640] ffffffffa0ebd587 [55779.068641] 0000000000000078 [55779.068642] ffff88041d604a00 [55779.068643] ffff8800b4864274 [55779.068644] ffffffffffffffff [55779.068645] ffff8808217bbd70 [55779.068646] ffff88082626bd30 [55779.068647] ffffffffffffffff [55779.068647] ffff88082aae7c48 [55779.068648] 0000000000000000 [55779.068649] [55779.068650] [55779.068651] [55779.068652] [55779.068653] [55779.068653] [55779.068654] 0000000000000000 [55779.068655] Call Trace: [55779.068656] Call Trace: [55779.068657] Call Trace: [55779.068658] Call Trace: [55779.068660] Call Trace: [55779.068661] Call Trace: [55779.068662] Call Trace: [55779.068672] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55779.068677] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55779.068682] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55779.068688] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55779.068694] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55779.068699] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55779.068705] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55779.068714] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55779.068719] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55779.068725] [] lnet_finalize+0xf6/0x690 [lnet] [55779.068730] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55779.068736] [] lnet_finalize+0xf6/0x690 [lnet] [55779.068741] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55779.068747] [] lnet_post_send_locked+0x446/0x880 [lnet] [55779.068758] [] lnet_select_pathway+0x235/0x1140 [lnet] [55779.068763] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55779.068767] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55779.068774] [] lnet_select_pathway+0x235/0x1140 [lnet] [55779.068778] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55779.068784] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55779.068789] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55779.068798] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55779.068803] [] lnet_finalize+0x1e9/0x690 [lnet] [55779.068807] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55779.068811] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55779.068813] [] ? enqueue_task_fair+0x208/0x6c0 [55779.068818] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55779.068824] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55779.068834] [] lnet_send+0x51/0x180 [lnet] [55779.068838] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55779.068840] [] ? dequeue_task_fair+0x41e/0x660 [55779.068846] [] lnet_send+0x51/0x180 [lnet] [55779.068848] [] ? sched_clock_cpu+0x85/0xc0 [55779.068853] [] lnet_finalize+0x1e9/0x690 [lnet] [55779.068859] [] lnet_finalize+0x1e9/0x690 [lnet] [55779.068869] [] lnet_finalize+0x3f1/0x690 [lnet] [55779.068873] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55779.068875] [] ? wake_up_state+0x20/0x20 [55779.068881] [] lnet_finalize+0x3f1/0x690 [lnet] [55779.068887] [] lnet_ni_recv+0xc3/0x320 [lnet] [55779.068891] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55779.068894] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55779.068902] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55779.068904] [] ? dequeue_task_fair+0x41e/0x660 [55779.068908] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55779.068912] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55779.068918] [] lnet_parse+0x945/0xf90 [lnet] [55779.068922] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55779.068925] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55779.068932] [] ? __slab_free+0x81/0x2f0 [55779.068933] [] ? wake_up_state+0x20/0x20 [55779.068935] [] kthread+0xcf/0xe0 [55779.068938] [] ? __slab_free+0x81/0x2f0 [55779.068942] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55779.068943] [] ? dequeue_task_fair+0x41e/0x660 [55779.068945] [] ? dequeue_task_fair+0x41e/0x660 [55779.068955] [] lnet_ni_recv+0xc3/0x320 [lnet] [55779.068959] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55779.068961] [] ? kthread_create_on_node+0x140/0x140 [55779.068967] [] lnet_ni_recv+0xc3/0x320 [lnet] [55779.068968] [] ? __wake_up+0x44/0x50 [55779.068970] [] ? wake_up_state+0x20/0x20 [55779.068972] [] ? sched_clock_cpu+0x85/0xc0 [55779.068982] [] lnet_parse+0x945/0xf90 [lnet] [55779.068983] [] kthread+0xcf/0xe0 [55779.068986] [] ret_from_fork+0x58/0x90 [55779.068991] [] lnet_parse+0x945/0xf90 [lnet] [55779.068995] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55779.068999] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55779.069000] [] ? wake_up_state+0x20/0x20 [55779.069009] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55779.069011] [] ? kthread_create_on_node+0x140/0x140 [55779.069012] [] ? kthread_create_on_node+0x140/0x140 [55779.069018] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55779.069019] [] ? dequeue_task_fair+0x41e/0x660 [55779.069021] [] kthread+0xcf/0xe0 [55779.069025] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55779.069028] Code: [55779.069032] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55779.069034] [] ret_from_fork+0x58/0x90 [55779.069039] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55779.069041] [] ? wake_up_state+0x20/0x20 [55779.069043] [] ? kthread_create_on_node+0x140/0x140 [55779.069044] [] kthread+0xcf/0xe0 [55779.069045] 00 [55779.069048] 02 [55779.069053] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55779.069055] [] ? kthread_create_on_node+0x140/0x140 [55779.069058] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55779.069062] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55779.069065] [] ret_from_fork+0x58/0x90 [55779.069066] [] ? kthread_create_on_node+0x140/0x140 [55779.069067] 00 [55779.069069] Code: [55779.069070] f0 [55779.069073] [] ? dequeue_task_fair+0x41e/0x660 [55779.069075] 00 [55779.069079] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55779.069081] [] kthread+0xcf/0xe0 [55779.069083] [] ? kthread_create_on_node+0x140/0x140 [55779.069085] [] ret_from_fork+0x58/0x90 [55779.069085] 0f [55779.069087] 02 [55779.069088] c1 [55779.069090] Code: [55779.069093] [] ? wake_up_state+0x20/0x20 [55779.069095] 00 [55779.069097] [] ? dequeue_task_fair+0x41e/0x660 [55779.069099] [] ? kthread_create_on_node+0x140/0x140 [55779.069099] 07 [55779.069101] [] ? kthread_create_on_node+0x140/0x140 [55779.069102] b8 [55779.069104] f0 [55779.069105] 89 [55779.069107] 00 [55779.069111] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55779.069112] Code: [55779.069113] 0f [55779.069115] [] ? wake_up_state+0x20/0x20 [55779.069117] [] ret_from_fork+0x58/0x90 [55779.069118] c2 [55779.069119] 00 [55779.069119] 00 [55779.069121] c1 [55779.069122] c1 [55779.069124] 02 [55779.069126] [] kthread+0xcf/0xe0 [55779.069127] 02 [55779.069128] 07 [55779.069133] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55779.069135] [] ? kthread_create_on_node+0x140/0x140 [55779.069135] ea [55779.069137] 00 [55779.069138] 00 [55779.069139] 89 [55779.069140] 10 [55779.069142] Code: [55779.069143] f0 [55779.069145] [] ? kthread_create_on_node+0x140/0x140 [55779.069146] f0 [55779.069147] c2 [55779.069149] [] kthread+0xcf/0xe0 [55779.069150] 66 [55779.069151] 00 [55779.069151] 0f [55779.069152] 0f [55779.069154] c1 [55779.069155] 39 [55779.069157] 02 [55779.069158] c1 [55779.069160] [] ret_from_fork+0x58/0x90 [55779.069161] c1 [55779.069162] ea [55779.069164] [] ? kthread_create_on_node+0x140/0x140 [55779.069165] c2 [55779.069166] 00 [55779.069166] 07 [55779.069167] 07 [55779.069169] 10 [55779.069170] 75 [55779.069172] f0 [55779.069172] 89 [55779.069174] [] ? kthread_create_on_node+0x140/0x140 [55779.069175] 89 [55779.069176] 66 [55779.069179] [] ret_from_fork+0x58/0x90 [55779.069180] 01 [55779.069180] 0f [55779.069181] c2 [55779.069182] c2 [55779.069183] Code: [55779.069184] 39 [55779.069185] c3 [55779.069187] c1 [55779.069188] c1 [55779.069189] c1 [55779.069190] 00 [55779.069191] c2 [55779.069193] [] ? kthread_create_on_node+0x140/0x140 [55779.069194] 55 [55779.069195] 07 [55779.069196] ea [55779.069197] ea [55779.069198] 02 [55779.069199] 75 [55779.069200] 83 [55779.069201] Code: [55779.069202] 89 [55779.069203] 10 [55779.069204] 10 [55779.069205] 00 [55779.069206] 01 [55779.069207] e2 [55779.069208] 00 [55779.069209] c2 [55779.069210] 66 [55779.069211] 66 [55779.069212] f0 [55779.069213] c3 [55779.069214] fe [55779.069216] 02 [55779.069216] c1 [55779.069217] 39 [55779.069218] 39 [55779.069219] 0f [55779.069221] 55 [55779.069222] 0f [55779.069223] 00 [55779.069223] ea [55779.069224] c2 [55779.069225] c2 [55779.069226] c1 [55779.069228] 83 [55779.069229] b7 [55779.069230] f0 [55779.069230] 10 [55779.069231] 75 [55779.069232] 75 [55779.069233] 07 [55779.069234] e2 [55779.069235] f2 [55779.069236] 0f [55779.069237] 66 [55779.069238] 01 [55779.069239] 01 [55779.069240] 89 [55779.069241] fe [55779.069242] 48 [55779.069243] c1 [55779.069243] 39 [55779.069245] c3 [55779.069246] c3 [55779.069247] c2 [55779.069248] 0f [55779.069249] 89 [55779.069250] 07 [55779.069250] c2 [55779.069251] 55 [55779.069252] 55 [55779.069253] c1 [55779.069254] b7 [55779.069255] e5 [55779.069256] 89 [55779.069257] 75 [55779.069257] 83 [55779.069258] 83 [55779.069259] ea [55779.069260] f2 [55779.069261] b8 [55779.069262] c2 [55779.069263] 01 [55779.069264] e2 [55779.069265] e2 [55779.069266] 10 [55779.069267] 48 [55779.069268] 00 [55779.069269] c1 [55779.069269] c3 [55779.069270] fe [55779.069271] fe [55779.069272] 66 [55779.069273] 89 [55779.069274] 80 [55779.069275] ea [55779.069276] 55 [55779.069277] 0f [55779.069278] 0f [55779.069279] 39 [55779.069280] e5 [55779.069281] 00 [55779.069282] 10 [55779.069283] 83 [55779.069284] b7 [55779.069285] b7 [55779.069286] c2 [55779.069287] b8 [55779.069288] 00 [55779.069290] 66 [55779.069290] e2 [55779.069291] f2 [55779.069292] f2 [55779.069293] 75 [55779.069294] 00 [55779.069295] eb [55779.069297] 39 [55779.069297] fe [55779.069298] 48 [55779.069300] 48 [55779.069300] 01 [55779.069301] 80 [55779.069302] 0d [55779.069304] c2 [55779.069304] 0f [55779.069305] 89 [55779.069306] 89 [55779.069307] c3 [55779.069308] 00 [55779.069309] 66 [55779.069311] 75 [55779.069311] b7 [55779.069312] e5 [55779.069313] e5 [55779.069314] 55 [55779.069315] 00 [55779.069316] 0f [55779.069317] 01 [55779.069318] f2 [55779.069319] b8 [55779.069320] b8 [55779.069321] 83 [55779.069322] eb [55779.069323] 1f [55779.069324] c3 [55779.069325] 48 [55779.069326] 00 [55779.069327] 00 [55779.069328] e2 [55779.069329] 0d [55779.069330] 44 [55779.069331] 55 [55779.069332] 89 [55779.069333] 80 [55779.069334] 80 [55779.069335] fe [55779.069336] 66 [55779.069337] 00 [55779.069338] 83 [55779.069339] e5 [55779.069339] 00 [55779.069340] 00 [55779.069341] 0f [55779.069342] 0f [55779.069343] 00 [55779.069344] e2 [55779.069345] b8 [55779.069346] 00 [55779.069347] 00 [55779.069348] b7 [55779.069349] 1f [55779.069350] f3 [55779.069351] fe [55779.069352] 00 [55779.069352] eb [55779.069354] eb [55779.069354] f2 [55779.069355] 44 [55779.069357] 90 [55779.069358] 0f [55779.069358] 80 [55779.069359] 0d [55779.069360] 0d [55779.069361] 48 [55779.069362] 00 [55779.069364] <83> [55779.069365] b7 [55779.069366] 00 [55779.069366] 66 [55779.069367] 66 [55779.069368] 89 [55779.069369] 00 [55779.069370] e8 [55779.069372] f2 [55779.069372] 00 [55779.069373] 0f [55779.069374] 0f [55779.069375] e5 [55779.069376] f3 [55779.069377] 01 [55779.069378] 48 [55779.069379] eb [55779.069380] 1f [55779.069381] 1f [55779.069382] b8 [55779.069383] 90 [55779.069384] 74 [55779.069385] 89 [55779.069385] 0d [55779.069386] 44 [55779.069387] 44 [55779.069388] 00 [55779.069389] <83> [55779.069390] 0a [55779.069391] e5 [55779.069392] 66 [55779.069392] 00 [55779.069394] 00 [55779.069395] 80 [55779.069396] e8 [55779.069397] 0f [55779.069398] b8 [55779.069399] 0f [55779.069400] 00 [55779.069401] 00 [55779.069402] 00 [55779.069403] 01 [55779.069404] b7 [55779.069405] 00 [55779.069406] 1f [55779.069407] [55779.069408] f3 [55779.069408] 00 [55779.069410] 74 [55779.069411] 0f [55779.069412] 80 [55779.069413] 44 [55779.069414] 90 [55779.069415] 90 [55779.069416] eb [55779.069417] 0a [55779.069418] 66 [55779.069419] 00 [55779.069420] 00 [55779.069421] 83 [55779.069422] <83> [55779.069423] 0d [55779.069424] 0f [55779.069425] 39 [55779.069426] 00 [55779.069427] 00 [55779.069428] e8 [55779.069429] e8 [55779.069430] 66 [55779.069431] b7 [55779.069432] ca [55779.069433] eb [55779.069434] f3 [55779.069435] 01 [55779.069436] 01 [55779.069437] 0f [55779.069437] 0f [55779.069439] 75 [55779.069440] 0d [55779.069441] 90 [55779.069441] 74 [55779.069443] 74 [55779.069444] 1f [55779.069444] 66 [55779.069446] f1 [55779.069447] 66 [55779.069447] <83> [55779.069448] 0a [55779.069450] 0a [55779.069450] 44 [55779.069452] 39 [55779.069453] 5d [55779.069454] 0f [55779.069455] e8 [55779.069456] 0f [55779.069457] 0f [55779.069458] 00 [55779.069459] ca [55779.069460] c3 [55779.069461] 1f [55779.069462] 01 [55779.069463] b7 [55779.069464] b7 [55779.069465] 00 [55779.069467] 75 [55779.069468] 66 [55779.069469] 44 [55779.069470] 74 [55779.069471] 0f [55779.069472] 0f [55779.069473] f3 [55779.069474] f1 [55779.069475] 66 [55779.069477] 00 [55779.069477] 0a [55779.069478] 66 [55779.069479] 66 [55779.069480] 90 [55779.069481] 5d [55779.069483] 66 [55779.069484] 00 [55779.069485] 0f [55779.069486] 39 [55779.069487] 39 [55779.069488] <83> [55779.069489] c3 [55779.069490] 90 [55779.069491] f3 [55779.069492] b7 [55779.069492] ca [55779.069494] ca [55779.069495] e8 [55779.069495] 66 [55779.069496] 66 [55779.069498] 90 [55779.069498] 0f [55779.069499] 75 [55779.069500] 75 [55779.069501] 01 [55779.069502] 66 [55779.069503] 66 [55779.069504] <83> [55779.069505] 66 [55779.069506] f1 [55779.069507] f1 [55779.069508] 74 [55779.069509] [55779.069510] 66 [55779.069511] e8 [55779.069512] 39 [55779.069512] 5d [55779.069513] 5d [55779.069515] 0a [55779.069515] 90 [55779.069516] 01 [55779.069517] ca [55779.069518] c3 [55779.069519] c3 [55779.069520] 0f [55779.069520] 66 [55779.069521] 74 [55779.069522] 75 [55779.069523] 66 [55779.069524] 66 [55779.069525] b7 [55779.069526] 66 [55779.069527] 0a [55779.069527] f1 [55779.069528] 66 [55779.069529] 66 [55779.069530] [55779.069531] 0f [55779.069532] 0f [55779.069533] 5d [55779.069534] 66 [55779.069535] 66 [55779.069536] 66 [55779.069537] b7 [55779.069537] c3 [55779.069539] 90 [55779.069540] 90 [55779.069541] 39 [55779.069542] 0f [55779.069543] [55779.069543] 66 [55779.069544] 66 [55779.069545] ca [55779.069546] 66 [55779.069547] 66 [55779.069548] 66 [55779.069549] 75 [55779.069550] 39 [55779.069550] [55779.069551] 66 [55779.069552] f1 [55779.069553] ca [55779.069554] 90 [55779.069555] 5d [55779.069556] 75 [55779.069557] 66 [55779.069558] c3 [55779.069559] f1 [55779.069560] 66 [55779.069561] 66 [55779.069562] [55779.069563] 5d [55779.069564] 66 [55779.069564] c3 [55779.069565] 66 [55779.069565] 66 [55779.069566] 90 [55779.069566] 66 [55779.069566] 66 [55779.069567] 66 [55779.069567] 66 [55779.069568] [55779.069568] 90 66 66 [55783.067182] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [kiblnd_connd:2895] [55783.067202] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55783.067207] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55783.067208] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55783.067209] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55783.067209] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55783.067211] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55783.067211] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000206 [55783.067212] RAX: 000000000000297e RBX: 0000000000016c40 RCX: 0000000000000a1e [55783.067213] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55783.067213] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55783.067214] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55783.067214] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55783.067215] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55783.067216] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55783.067216] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55783.067217] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55783.067217] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55783.067217] Stack: [55783.067218] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55783.067219] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55783.067220] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55783.067221] Call Trace: [55783.067226] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55783.067231] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55783.067234] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55783.067237] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55783.067239] [] ? lock_timer_base.isra.33+0x2b/0x50 [55783.067240] [] ? try_to_del_timer_sync+0x5e/0x90 [55783.067243] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55783.067245] [] ? schedule_timeout+0x17d/0x2d0 [55783.067248] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55783.067250] [] ? wake_up_state+0x20/0x20 [55783.067253] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55783.067255] [] kthread+0xcf/0xe0 [55783.067256] [] ? kthread_create_on_node+0x140/0x140 [55783.067258] [] ret_from_fork+0x58/0x90 [55783.067259] [] ? kthread_create_on_node+0x140/0x140 [55783.067269] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55791.066358] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55791.066365] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [55791.066367] Modules linked in: [55791.066368] Modules linked in: [55791.066370] ko2iblnd(OE) [55791.066371] ko2iblnd(OE) [55791.066372] lustre(OE) [55791.066373] lustre(OE) [55791.066374] lmv(OE) [55791.066375] lmv(OE) [55791.066376] fld(OE) [55791.066377] fld(OE) [55791.066378] mdc(OE) [55791.066379] mdc(OE) [55791.066380] fid(OE) [55791.066381] fid(OE) [55791.066382] lov(OE) [55791.066383] lov(OE) [55791.066384] ksocklnd(OE) [55791.066385] ksocklnd(OE) [55791.066386] ptlrpc(OE) [55791.066387] ptlrpc(OE) [55791.066388] obdclass(OE) [55791.066390] obdclass(OE) [55791.066391] lnet(OE) [55791.066392] lnet(OE) [55791.066393] libcfs(OE) [55791.066394] libcfs(OE) [55791.066394] rpcrdma [55791.066395] rpcrdma [55791.066396] ib_isert [55791.066397] ib_isert [55791.066398] iscsi_target_mod [55791.066399] iscsi_target_mod [55791.066400] ib_iser [55791.066401] ib_iser [55791.066402] libiscsi [55791.066403] libiscsi [55791.066404] scsi_transport_iscsi [55791.066405] scsi_transport_iscsi [55791.066406] ib_srpt [55791.066408] ib_srpt [55791.066408] target_core_mod [55791.066409] target_core_mod [55791.066411] ib_srp [55791.066412] ib_srp [55791.066413] scsi_transport_srp [55791.066414] scsi_transport_srp [55791.066414] scsi_tgt [55791.066416] scsi_tgt [55791.066417] ib_ipoib [55791.066418] ib_ipoib [55791.066419] rdma_ucm [55791.066420] rdma_ucm [55791.066421] ib_ucm [55791.066422] ib_ucm [55791.066423] ib_uverbs [55791.066424] ib_uverbs [55791.066425] ib_umad [55791.066426] ib_umad [55791.066427] rdma_cm [55791.066428] rdma_cm [55791.066429] ib_cm [55791.066430] ib_cm [55791.066431] iw_cm [55791.066432] iw_cm [55791.066433] mlx4_ib [55791.066434] mlx4_ib [55791.066435] intel_powerclamp [55791.066436] intel_powerclamp [55791.066437] coretemp [55791.066438] coretemp [55791.066439] intel_rapl [55791.066441] intel_rapl [55791.066442] iosf_mbi [55791.066443] iosf_mbi [55791.066444] kvm [55791.066445] kvm [55791.066446] irqbypass [55791.066448] irqbypass [55791.066449] crc32_pclmul [55791.066450] crc32_pclmul [55791.066451] ghash_clmulni_intel [55791.066452] ghash_clmulni_intel [55791.066454] aesni_intel [55791.066455] aesni_intel [55791.066456] ipmi_ssif [55791.066457] ipmi_ssif [55791.066458] lrw [55791.066459] lrw [55791.066460] gf128mul [55791.066461] gf128mul [55791.066462] glue_helper [55791.066463] glue_helper [55791.066464] hfi1 [55791.066465] hfi1 [55791.066466] ipmi_devintf [55791.066467] ipmi_devintf [55791.066468] rdmavt [55791.066469] rdmavt [55791.066470] ablk_helper [55791.066471] ablk_helper [55791.066472] cryptd [55791.066473] cryptd [55791.066474] iTCO_wdt [55791.066475] iTCO_wdt [55791.066476] ipmi_si [55791.066477] ipmi_si [55791.066478] sb_edac [55791.066479] sb_edac [55791.066480] sg [55791.066481] sg [55791.066483] iTCO_vendor_support [55791.066484] iTCO_vendor_support [55791.066485] ib_core [55791.066486] ib_core [55791.066487] ioatdma [55791.066488] ioatdma [55791.066489] i2c_i801 [55791.066490] i2c_i801 [55791.066491] mei_me [55791.066492] mei_me [55791.066493] mei [55791.066494] mei [55791.066495] edac_core [55791.066496] edac_core [55791.066497] ipmi_msghandler [55791.066498] ipmi_msghandler [55791.066499] lpc_ich [55791.066500] lpc_ich [55791.066501] ntb [55791.066502] ntb [55791.066503] wmi [55791.066504] wmi [55791.066505] shpchp [55791.066507] shpchp [55791.066508] pcspkr [55791.066509] pcspkr [55791.066510] nfsd [55791.066511] nfsd [55791.066512] nfs_acl [55791.066513] nfs_acl [55791.066514] lockd [55791.066515] lockd [55791.066516] grace [55791.066517] grace [55791.066518] auth_rpcgss [55791.066519] auth_rpcgss [55791.066520] sunrpc [55791.066521] sunrpc [55791.066522] ip_tables [55791.066523] ip_tables [55791.066524] ext4 [55791.066525] ext4 [55791.066526] mbcache [55791.066527] mbcache [55791.066528] jbd2 [55791.066529] jbd2 [55791.066530] sd_mod [55791.066531] sd_mod [55791.066532] crc_t10dif [55791.066533] crc_t10dif [55791.066534] crct10dif_generic [55791.066535] crct10dif_generic [55791.066536] mlx4_en [55791.066537] mlx4_en [55791.066538] mgag200 [55791.066540] mgag200 [55791.066541] drm_kms_helper [55791.066542] drm_kms_helper [55791.066543] syscopyarea [55791.066544] syscopyarea [55791.066545] sysfillrect [55791.066546] sysfillrect [55791.066547] sysimgblt [55791.066548] sysimgblt [55791.066549] fb_sys_fops [55791.066550] fb_sys_fops [55791.066552] isci [55791.066553] isci [55791.066554] ttm [55791.066555] ttm [55791.066556] igb [55791.066557] igb [55791.066558] crct10dif_pclmul [55791.066559] crct10dif_pclmul [55791.066560] crct10dif_common [55791.066562] crct10dif_common [55791.066563] ahci [55791.066564] ahci [55791.066565] ptp [55791.066566] ptp [55791.066567] crc32c_intel [55791.066568] crc32c_intel [55791.066569] libsas [55791.066570] libsas [55791.066571] libahci [55791.066573] libahci [55791.066573] pps_core [55791.066574] pps_core [55791.066575] drm [55791.066577] drm [55791.066578] mlx4_core [55791.066579] mlx4_core [55791.066580] mpt2sas [55791.066581] mpt2sas [55791.066582] dca [55791.066583] dca [55791.066584] libata [55791.066585] libata [55791.066586] raid_class [55791.066587] raid_class [55791.066588] i2c_algo_bit [55791.066590] i2c_algo_bit [55791.066590] scsi_transport_sas [55791.066591] scsi_transport_sas [55791.066593] devlink [55791.066594] devlink [55791.066595] i2c_core [55791.066596] i2c_core [55791.066597] fjes [55791.066598] [55791.066599] fjes [55791.066601] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55791.066603] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55791.066604] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55791.066605] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55791.066609] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55791.066610] Workqueue: ib_cm cm_work_handler [ib_cm] [55791.066612] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55791.066614] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55791.066616] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000202 [55791.066617] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55791.066619] RAX: 0000000000007902 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55791.066620] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000216 [55791.066621] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55791.066623] RAX: 0000000000002045 RBX: 0000000000000000 RCX: 0000000000000a1e [55791.066624] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55791.066625] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55791.066626] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55791.066627] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55791.066629] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55791.066630] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55791.066631] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55791.066632] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55791.066634] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55791.066635] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55791.066636] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55791.066637] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55791.066638] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55791.066640] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55791.066641] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55791.066642] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55791.066643] Stack: [55791.066644] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55791.066646] Stack: [55791.066648] ffff8807f639fd88 ffffffffa08aa831 [55791.066649] ffff8803fea77b00 [55791.066650] 0000000200000002 [55791.066651] ffffffffa08aa7c8 [55791.066652] ffff8807f63a2980 [55791.066653] [55791.066655] ffff8803fea77bd0 00050000c0a80173 [55791.066656] [55791.066657] 0000000000000000 ffff8807f639fdc0 [55791.066658] ffff88083fb5de00 [55791.066659] ffffffffa0942e30 [55791.066660] ffff8803fea77b18 [55791.066662] 00000000c0086540 [55791.066663] [55791.066664] ffffffffa0916e4b ffff8803fea77c68 [55791.066665] [55791.066667] ffff8807f63a2900 0000000000000002 [55791.066668] ffff8803fea77bc8 [55791.066669] 0000000000650aa0 [55791.066670] ffffffffa0ebfc24 [55791.066671] 0000000000000000 [55791.066672] [55791.066673] Call Trace: [55791.066675] ffff88041aa8a880 ffff8803fea77bd0 [55791.066676] Call Trace: [55791.066682] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55791.066689] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55791.066695] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55791.066702] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55791.066707] [] LNetCtl+0x116/0x800 [lnet] [55791.066713] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55791.066719] [] lnet_ioctl+0x60/0x250 [lnet] [55791.066723] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55791.066728] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55791.066733] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55791.066738] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55791.066742] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55791.066743] [] do_vfs_ioctl+0x2d5/0x4b0 [55791.066747] [] cma_req_handler+0x227/0x570 [rdma_cm] [55791.066749] [] ? __fd_install+0x47/0x60 [55791.066752] [] cm_process_work+0x25/0x140 [ib_cm] [55791.066754] [] SyS_ioctl+0xa1/0xc0 [55791.066758] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55791.066760] [] system_call_fastpath+0x16/0x1b [55791.066764] [] cm_work_handler+0x185/0x1484 [ib_cm] [55791.066766] Code: c2 [55791.066768] [] process_one_work+0x17b/0x470 [55791.066770] c1 ea [55791.066772] [] worker_thread+0x126/0x410 [55791.066773] 10 66 [55791.066776] [] ? rescuer_thread+0x460/0x460 [55791.066777] 39 c2 [55791.066780] [] kthread+0xcf/0xe0 [55791.066781] 75 01 [55791.066784] [] ? kthread_create_on_node+0x140/0x140 [55791.066786] c3 55 [55791.066789] [] ret_from_fork+0x58/0x90 [55791.066790] 83 e2 [55791.066793] [] ? kthread_create_on_node+0x140/0x140 [55791.066794] fe 0f [55791.066795] Code: [55791.066796] b7 [55791.066797] 00 [55791.066798] f2 [55791.066799] 02 [55791.066800] 48 [55791.066801] 00 [55791.066802] 89 [55791.066803] f0 [55791.066804] e5 [55791.066805] 0f [55791.066806] b8 [55791.066807] c1 [55791.066808] 00 [55791.066809] 07 [55791.066810] 80 [55791.066811] 89 [55791.066811] 00 [55791.066812] c2 [55791.066813] 00 [55791.066814] c1 [55791.066815] eb [55791.066816] ea [55791.066817] 0d [55791.066818] 10 [55791.066819] 66 [55791.066819] 66 [55791.066821] 0f [55791.066821] 39 [55791.066822] 1f [55791.066823] c2 [55791.066824] 44 [55791.066825] 75 [55791.066826] 00 [55791.066827] 01 [55791.066828] 00 [55791.066829] c3 [55791.066830] f3 [55791.066831] 55 [55791.066832] 90 [55791.066833] 83 [55791.066834] 83 [55791.066835] e2 [55791.066836] e8 [55791.066837] fe [55791.066838] 01 [55791.066839] 0f [55791.066840] 74 [55791.066841] b7 [55791.066842] 0a [55791.066843] f2 [55791.066844] 0f [55791.066845] 48 [55791.066846] b7 [55791.066847] 89 [55791.066848] 0f [55791.066849] e5 [55791.066850] <66> [55791.066851] b8 [55791.066852] 39 [55791.066853] 00 [55791.066854] ca [55791.066855] 80 [55791.066856] 75 [55791.066857] 00 [55791.066858] f1 [55791.066859] 00 [55791.066860] 5d [55791.066861] eb [55791.066862] c3 [55791.066862] 0d [55791.066864] 66 [55791.066864] 66 [55791.066866] 66 [55791.066866] 0f [55791.066867] 66 [55791.066868] 1f [55791.066869] 90 [55791.066870] 44 [55791.066871] 66 [55791.066872] 00 [55791.066873] 66 [55791.066874] 00 [55791.066875] 90 [55791.066876] f3 [55791.066877] eb [55791.066877] 90 [55791.066879] d9 [55791.066880] <83> [55791.066880] 66 [55791.066881] e8 [55791.066882] 0f [55791.066883] 01 [55791.066884] 1f [55791.066885] 74 [55791.066886] 44 [55791.066887] 0a [55791.066888] 00 [55791.066889] [55791.066892] 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55799.161966] [] ? dequeue_task_fair+0x41e/0x660 [55799.182129] [] ? wake_up_state+0x20/0x20 [55799.200582] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55799.222761] [] kthread+0xcf/0xe0 [55799.238942] [] ? kthread_create_on_node+0x140/0x140 [55799.260578] [] ret_from_fork+0x58/0x90 [55799.278497] [] ? kthread_create_on_node+0x140/0x140 [55799.300149] Task dump for CPU 12: [55799.311227] kiblnd_sd_01_03 R running task 0 2903 2 0x00000088 [55799.334646] ffff880820a21000 ffffc90011f075b0 ffff8808266e7a00 0000000000000000 [55799.359255] ffff8808213b7da8 ffffffffa0ebedab ffff8808213b7e90 ffff8808213b7da8 [55799.383872] ffffffffa03e7d2e ffff88082b2fc3c0 ffff88082c5f3600 0000000000000011 [55799.408501] Call Trace: [55799.416797] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55799.439995] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55799.460621] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55799.484131] [] ? dequeue_task_fair+0x41e/0x660 [55799.504494] [] ? wake_up_state+0x20/0x20 [55799.523162] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55799.545548] [] ? kthread+0xcf/0xe0 [55799.562510] [] ? kthread_create_on_node+0x140/0x140 [55799.584346] [] ? ret_from_fork+0x58/0x90 [55799.603057] [] ? kthread_create_on_node+0x140/0x140 [55799.624915] Task dump for CPU 15: [55799.636197] kiblnd_sd_01_00 R running task 0 2900 2 0x00000088 [55799.659822] ffff8808217bbd30 00000000937dfd11 ffffc9000e335900 ffff8804c1ba3200 [55799.684652] 0000000000000000 0000000000000000 ffff88082bda6c28 ffff8808217bbda8 [55799.709473] ffffffffa0eba166 ffff88082b2fc3c0 ffff88082bda6c00 0000000000000014 [55799.734308] Call Trace: [55799.742815] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55799.765657] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55799.789349] [] ? dequeue_task_fair+0x41e/0x660 [55799.809911] [] ? wake_up_state+0x20/0x20 [55799.828769] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55799.851365] [] ? kthread+0xcf/0xe0 [55799.868520] [] ? kthread_create_on_node+0x140/0x140 [55799.890563] [] ? ret_from_fork+0x58/0x90 [55799.909455] [] ? kthread_create_on_node+0x140/0x140 [55807.064711] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55807.064713] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55807.064715] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [55807.064716] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55807.064718] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55807.064719] Modules linked in: [55807.064720] Modules linked in: [55807.064722] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [kiblnd_sd_01_00:2900] [55807.064724] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [55807.064725] Modules linked in: [55807.064727] Modules linked in: [55807.064728] ko2iblnd(OE) [55807.064729] ko2iblnd(OE) [55807.064730] ko2iblnd(OE) [55807.064731] Modules linked in: [55807.064733] Modules linked in: [55807.064734] ko2iblnd(OE) [55807.064734] lustre(OE) [55807.064735] lustre(OE) [55807.064736] lustre(OE) [55807.064737] ko2iblnd(OE) [55807.064738] ko2iblnd(OE) [55807.064739] lustre(OE) [55807.064740] lmv(OE) [55807.064741] lmv(OE) [55807.064742] lmv(OE) [55807.064743] lustre(OE) [55807.064745] lustre(OE) [55807.064746] lmv(OE) [55807.064746] fld(OE) [55807.064747] fld(OE) [55807.064748] fld(OE) [55807.064749] lmv(OE) [55807.064750] lmv(OE) [55807.064751] fld(OE) [55807.064751] mdc(OE) [55807.064752] mdc(OE) [55807.064753] mdc(OE) [55807.064754] fld(OE) [55807.064755] fld(OE) [55807.064756] mdc(OE) [55807.064757] fid(OE) [55807.064758] fid(OE) [55807.064759] fid(OE) [55807.064759] mdc(OE) [55807.064760] mdc(OE) [55807.064761] fid(OE) [55807.064763] lov(OE) [55807.064763] lov(OE) [55807.064764] lov(OE) [55807.064765] fid(OE) [55807.064766] fid(OE) [55807.064767] lov(OE) [55807.064767] ksocklnd(OE) [55807.064768] ksocklnd(OE) [55807.064769] ksocklnd(OE) [55807.064770] lov(OE) [55807.064772] lov(OE) [55807.064772] ksocklnd(OE) [55807.064774] ptlrpc(OE) [55807.064774] ptlrpc(OE) [55807.064775] ptlrpc(OE) [55807.064776] ksocklnd(OE) [55807.064778] ksocklnd(OE) [55807.064779] ptlrpc(OE) [55807.064779] obdclass(OE) [55807.064780] obdclass(OE) [55807.064781] obdclass(OE) [55807.064782] ptlrpc(OE) [55807.064783] ptlrpc(OE) [55807.064784] obdclass(OE) [55807.064785] lnet(OE) [55807.064786] lnet(OE) [55807.064787] lnet(OE) [55807.064788] obdclass(OE) [55807.064789] obdclass(OE) [55807.064790] lnet(OE) [55807.064791] libcfs(OE) [55807.064791] libcfs(OE) [55807.064793] libcfs(OE) [55807.064793] lnet(OE) [55807.064795] lnet(OE) [55807.064796] libcfs(OE) [55807.064796] rpcrdma [55807.064797] rpcrdma [55807.064798] rpcrdma [55807.064799] libcfs(OE) [55807.064801] libcfs(OE) [55807.064801] rpcrdma [55807.064802] ib_isert [55807.064803] ib_isert [55807.064804] ib_isert [55807.064805] rpcrdma [55807.064807] rpcrdma [55807.064808] ib_isert [55807.064809] iscsi_target_mod [55807.064810] iscsi_target_mod [55807.064811] iscsi_target_mod [55807.064812] ib_isert [55807.064813] ib_isert [55807.064814] iscsi_target_mod [55807.064814] ib_iser [55807.064816] ib_iser [55807.064817] ib_iser [55807.064818] iscsi_target_mod [55807.064819] iscsi_target_mod [55807.064820] ib_iser [55807.064821] libiscsi [55807.064822] libiscsi [55807.064823] libiscsi [55807.064824] ib_iser [55807.064825] ib_iser [55807.064826] libiscsi [55807.064827] scsi_transport_iscsi [55807.064828] scsi_transport_iscsi [55807.064829] scsi_transport_iscsi [55807.064830] libiscsi [55807.064831] libiscsi [55807.064832] scsi_transport_iscsi [55807.064833] ib_srpt [55807.064834] ib_srpt [55807.064835] ib_srpt [55807.064836] scsi_transport_iscsi [55807.064837] scsi_transport_iscsi [55807.064837] ib_srpt [55807.064839] target_core_mod [55807.064840] target_core_mod [55807.064841] target_core_mod [55807.064842] ib_srpt [55807.064843] ib_srpt [55807.064844] target_core_mod [55807.064845] ib_srp [55807.064845] ib_srp [55807.064847] ib_srp [55807.064847] target_core_mod [55807.064849] target_core_mod [55807.064849] ib_srp [55807.064850] scsi_transport_srp [55807.064851] scsi_transport_srp [55807.064852] scsi_transport_srp [55807.064853] ib_srp [55807.064855] ib_srp [55807.064856] scsi_transport_srp [55807.064857] scsi_tgt [55807.064858] scsi_tgt [55807.064859] scsi_tgt [55807.064859] scsi_transport_srp [55807.064860] scsi_transport_srp [55807.064861] scsi_tgt [55807.064862] ib_ipoib [55807.064863] ib_ipoib [55807.064864] ib_ipoib [55807.064865] scsi_tgt [55807.064866] scsi_tgt [55807.064867] ib_ipoib [55807.064868] rdma_ucm [55807.064869] rdma_ucm [55807.064870] rdma_ucm [55807.064871] ib_ipoib [55807.064872] ib_ipoib [55807.064873] rdma_ucm [55807.064874] ib_ucm [55807.064874] ib_ucm [55807.064876] ib_ucm [55807.064877] rdma_ucm [55807.064878] rdma_ucm [55807.064879] ib_ucm [55807.064880] ib_uverbs [55807.064880] ib_uverbs [55807.064882] ib_uverbs [55807.064882] ib_ucm [55807.064884] ib_ucm [55807.064884] ib_uverbs [55807.064886] ib_umad [55807.064886] ib_umad [55807.064887] ib_umad [55807.064888] ib_uverbs [55807.064890] ib_uverbs [55807.064891] ib_umad [55807.064892] rdma_cm [55807.064892] rdma_cm [55807.064893] rdma_cm [55807.064894] ib_umad [55807.064895] ib_umad [55807.064896] rdma_cm [55807.064897] ib_cm [55807.064898] ib_cm [55807.064899] ib_cm [55807.064899] rdma_cm [55807.064900] rdma_cm [55807.064901] ib_cm [55807.064902] iw_cm [55807.064903] iw_cm [55807.064904] iw_cm [55807.064904] ib_cm [55807.064906] ib_cm [55807.064907] iw_cm [55807.064907] mlx4_ib [55807.064908] mlx4_ib [55807.064909] mlx4_ib [55807.064910] iw_cm [55807.064911] iw_cm [55807.064912] mlx4_ib [55807.064913] intel_powerclamp [55807.064913] intel_powerclamp [55807.064915] intel_powerclamp [55807.064916] mlx4_ib [55807.064917] mlx4_ib [55807.064918] intel_powerclamp [55807.064918] coretemp [55807.064919] coretemp [55807.064920] coretemp [55807.064921] intel_powerclamp [55807.064923] intel_powerclamp [55807.064924] coretemp [55807.064925] intel_rapl [55807.064926] intel_rapl [55807.064927] intel_rapl [55807.064928] coretemp [55807.064929] coretemp [55807.064930] intel_rapl [55807.064931] iosf_mbi [55807.064932] iosf_mbi [55807.064933] iosf_mbi [55807.064934] intel_rapl [55807.064936] intel_rapl [55807.064937] iosf_mbi [55807.064937] kvm [55807.064938] kvm [55807.064939] kvm [55807.064940] iosf_mbi [55807.064942] iosf_mbi [55807.064943] kvm [55807.064943] irqbypass [55807.064944] irqbypass [55807.064945] irqbypass [55807.064946] kvm [55807.064947] kvm [55807.064948] irqbypass [55807.064948] crc32_pclmul [55807.064950] crc32_pclmul [55807.064951] crc32_pclmul [55807.064952] irqbypass [55807.064953] irqbypass [55807.064954] crc32_pclmul [55807.064955] ghash_clmulni_intel [55807.064956] ghash_clmulni_intel [55807.064957] ghash_clmulni_intel [55807.064958] crc32_pclmul [55807.064959] crc32_pclmul [55807.064960] ghash_clmulni_intel [55807.064961] aesni_intel [55807.064962] aesni_intel [55807.064963] aesni_intel [55807.064963] ghash_clmulni_intel [55807.064965] ghash_clmulni_intel [55807.064966] aesni_intel [55807.064967] ipmi_ssif [55807.064968] ipmi_ssif [55807.064969] ipmi_ssif [55807.064970] aesni_intel [55807.064971] aesni_intel [55807.064972] ipmi_ssif [55807.064973] lrw [55807.064974] lrw [55807.064975] lrw [55807.064976] ipmi_ssif [55807.064978] ipmi_ssif [55807.064979] lrw [55807.064979] gf128mul [55807.064980] gf128mul [55807.064981] gf128mul [55807.064982] lrw [55807.064984] lrw [55807.064984] gf128mul [55807.064985] glue_helper [55807.064986] glue_helper [55807.064987] glue_helper [55807.064988] gf128mul [55807.064989] gf128mul [55807.064990] glue_helper [55807.064991] hfi1 [55807.064992] hfi1 [55807.064993] hfi1 [55807.064994] glue_helper [55807.064995] glue_helper [55807.064996] hfi1 [55807.064997] ipmi_devintf [55807.064998] ipmi_devintf [55807.064999] ipmi_devintf [55807.065000] hfi1 [55807.065001] hfi1 [55807.065002] ipmi_devintf [55807.065004] rdmavt [55807.065004] rdmavt [55807.065006] rdmavt [55807.065007] ipmi_devintf [55807.065008] ipmi_devintf [55807.065009] rdmavt [55807.065009] ablk_helper [55807.065010] ablk_helper [55807.065012] ablk_helper [55807.065013] rdmavt [55807.065014] rdmavt [55807.065014] ablk_helper [55807.065015] cryptd [55807.065016] cryptd [55807.065018] cryptd [55807.065019] ablk_helper [55807.065020] ablk_helper [55807.065021] cryptd [55807.065022] iTCO_wdt [55807.065023] iTCO_wdt [55807.065024] iTCO_wdt [55807.065025] cryptd [55807.065026] cryptd [55807.065027] iTCO_wdt [55807.065028] ipmi_si [55807.065029] ipmi_si [55807.065030] ipmi_si [55807.065030] iTCO_wdt [55807.065032] iTCO_wdt [55807.065032] ipmi_si [55807.065033] sb_edac [55807.065034] sb_edac [55807.065035] sb_edac [55807.065036] ipmi_si [55807.065038] ipmi_si [55807.065038] sb_edac [55807.065039] sg [55807.065040] sg [55807.065041] sg [55807.065042] sb_edac [55807.065043] sb_edac [55807.065044] sg [55807.065045] iTCO_vendor_support [55807.065046] iTCO_vendor_support [55807.065047] iTCO_vendor_support [55807.065048] sg [55807.065049] sg [55807.065050] iTCO_vendor_support [55807.065051] ib_core [55807.065052] ib_core [55807.065053] ib_core [55807.065054] iTCO_vendor_support [55807.065055] iTCO_vendor_support [55807.065056] ib_core [55807.065057] ioatdma [55807.065057] ioatdma [55807.065059] ioatdma [55807.065059] ib_core [55807.065060] ib_core [55807.065062] ioatdma [55807.065063] i2c_i801 [55807.065063] i2c_i801 [55807.065065] i2c_i801 [55807.065066] ioatdma [55807.065067] ioatdma [55807.065068] i2c_i801 [55807.065069] mei_me [55807.065070] mei_me [55807.065071] mei_me [55807.065072] i2c_i801 [55807.065073] i2c_i801 [55807.065074] mei_me [55807.065075] mei [55807.065076] mei [55807.065077] mei [55807.065078] mei_me [55807.065079] mei_me [55807.065080] mei [55807.065081] edac_core [55807.065082] edac_core [55807.065083] edac_core [55807.065084] mei [55807.065086] mei [55807.065087] edac_core [55807.065088] ipmi_msghandler [55807.065089] ipmi_msghandler [55807.065090] ipmi_msghandler [55807.065090] edac_core [55807.065092] edac_core [55807.065092] ipmi_msghandler [55807.065093] lpc_ich [55807.065094] lpc_ich [55807.065095] lpc_ich [55807.065096] ipmi_msghandler [55807.065097] ipmi_msghandler [55807.065099] lpc_ich [55807.065099] ntb [55807.065100] ntb [55807.065102] ntb [55807.065102] lpc_ich [55807.065104] lpc_ich [55807.065105] ntb [55807.065106] wmi [55807.065107] wmi [55807.065108] wmi [55807.065109] ntb [55807.065110] ntb [55807.065111] wmi [55807.065112] shpchp [55807.065113] shpchp [55807.065114] shpchp [55807.065114] wmi [55807.065116] wmi [55807.065117] shpchp [55807.065118] pcspkr [55807.065118] pcspkr [55807.065120] pcspkr [55807.065121] shpchp [55807.065122] shpchp [55807.065123] pcspkr [55807.065123] nfsd [55807.065124] nfsd [55807.065125] nfsd [55807.065126] pcspkr [55807.065128] pcspkr [55807.065129] nfsd [55807.065129] nfs_acl [55807.065130] nfs_acl [55807.065132] nfs_acl [55807.065132] nfsd [55807.065133] nfsd [55807.065134] nfs_acl [55807.065135] lockd [55807.065136] lockd [55807.065137] lockd [55807.065138] nfs_acl [55807.065139] nfs_acl [55807.065140] lockd [55807.065141] grace [55807.065142] grace [55807.065143] grace [55807.065144] lockd [55807.065145] lockd [55807.065146] grace [55807.065147] auth_rpcgss [55807.065148] auth_rpcgss [55807.065149] auth_rpcgss [55807.065150] grace [55807.065151] grace [55807.065152] auth_rpcgss [55807.065153] sunrpc [55807.065153] sunrpc [55807.065154] sunrpc [55807.065155] auth_rpcgss [55807.065156] auth_rpcgss [55807.065157] sunrpc [55807.065158] ip_tables [55807.065159] ip_tables [55807.065160] ip_tables [55807.065161] sunrpc [55807.065162] sunrpc [55807.065163] ip_tables [55807.065164] ext4 [55807.065165] ext4 [55807.065166] ext4 [55807.065167] ip_tables [55807.065168] ip_tables [55807.065169] ext4 [55807.065170] mbcache [55807.065171] mbcache [55807.065172] mbcache [55807.065173] ext4 [55807.065174] ext4 [55807.065175] mbcache [55807.065176] jbd2 [55807.065178] jbd2 [55807.065179] jbd2 [55807.065179] mbcache [55807.065181] mbcache [55807.065182] jbd2 [55807.065183] sd_mod [55807.065184] sd_mod [55807.065185] sd_mod [55807.065186] jbd2 [55807.065187] jbd2 [55807.065188] sd_mod [55807.065189] crc_t10dif [55807.065190] crc_t10dif [55807.065191] crc_t10dif [55807.065192] sd_mod [55807.065194] sd_mod [55807.065194] crc_t10dif [55807.065195] crct10dif_generic [55807.065196] crct10dif_generic [55807.065198] crct10dif_generic [55807.065199] crc_t10dif [55807.065200] crc_t10dif [55807.065201] crct10dif_generic [55807.065202] mlx4_en [55807.065202] mlx4_en [55807.065203] mlx4_en [55807.065204] crct10dif_generic [55807.065205] crct10dif_generic [55807.065206] mlx4_en [55807.065207] mgag200 [55807.065208] mgag200 [55807.065209] mgag200 [55807.065210] mlx4_en [55807.065211] mlx4_en [55807.065212] mgag200 [55807.065213] drm_kms_helper [55807.065214] drm_kms_helper [55807.065215] drm_kms_helper [55807.065216] mgag200 [55807.065217] mgag200 [55807.065218] drm_kms_helper [55807.065219] syscopyarea [55807.065220] syscopyarea [55807.065221] syscopyarea [55807.065222] drm_kms_helper [55807.065223] drm_kms_helper [55807.065224] syscopyarea [55807.065225] sysfillrect [55807.065226] sysfillrect [55807.065227] sysfillrect [55807.065228] syscopyarea [55807.065229] syscopyarea [55807.065230] sysfillrect [55807.065231] sysimgblt [55807.065232] sysimgblt [55807.065233] sysimgblt [55807.065234] sysfillrect [55807.065235] sysfillrect [55807.065236] sysimgblt [55807.065237] fb_sys_fops [55807.065238] fb_sys_fops [55807.065239] fb_sys_fops [55807.065240] sysimgblt [55807.065241] sysimgblt [55807.065242] fb_sys_fops [55807.065243] isci [55807.065244] isci [55807.065245] isci [55807.065246] fb_sys_fops [55807.065247] fb_sys_fops [55807.065248] isci [55807.065249] ttm [55807.065250] ttm [55807.065251] ttm [55807.065252] isci [55807.065253] isci [55807.065254] ttm [55807.065255] igb [55807.065255] igb [55807.065256] igb [55807.065257] ttm [55807.065258] ttm [55807.065260] igb [55807.065260] crct10dif_pclmul [55807.065262] crct10dif_pclmul [55807.065263] crct10dif_pclmul [55807.065264] igb [55807.065265] igb [55807.065266] crct10dif_pclmul [55807.065267] crct10dif_common [55807.065267] crct10dif_common [55807.065269] crct10dif_common [55807.065270] crct10dif_pclmul [55807.065271] crct10dif_pclmul [55807.065272] crct10dif_common [55807.065272] ahci [55807.065273] ahci [55807.065274] ahci [55807.065275] crct10dif_common [55807.065277] crct10dif_common [55807.065277] ahci [55807.065278] ptp [55807.065279] ptp [55807.065280] ptp [55807.065281] ahci [55807.065282] ahci [55807.065283] ptp [55807.065284] crc32c_intel [55807.065285] crc32c_intel [55807.065286] crc32c_intel [55807.065287] ptp [55807.065288] ptp [55807.065289] crc32c_intel [55807.065290] libsas [55807.065291] libsas [55807.065292] libsas [55807.065293] crc32c_intel [55807.065295] crc32c_intel [55807.065295] libsas [55807.065296] libahci [55807.065297] libahci [55807.065298] libahci [55807.065299] libsas [55807.065300] libsas [55807.065301] libahci [55807.065302] pps_core [55807.065303] pps_core [55807.065304] pps_core [55807.065305] libahci [55807.065306] libahci [55807.065307] pps_core [55807.065308] drm [55807.065309] drm [55807.065310] drm [55807.065311] pps_core [55807.065312] pps_core [55807.065313] drm [55807.065314] mlx4_core [55807.065315] mlx4_core [55807.065316] mlx4_core [55807.065317] drm [55807.065318] drm [55807.065319] mlx4_core [55807.065320] mpt2sas [55807.065320] mpt2sas [55807.065322] mpt2sas [55807.065322] mlx4_core [55807.065324] mlx4_core [55807.065324] mpt2sas [55807.065325] dca [55807.065326] dca [55807.065327] dca [55807.065328] mpt2sas [55807.065329] mpt2sas [55807.065330] dca [55807.065331] libata [55807.065332] libata [55807.065333] libata [55807.065334] dca [55807.065335] dca [55807.065336] libata [55807.065337] raid_class [55807.065338] raid_class [55807.065339] raid_class [55807.065339] libata [55807.065340] libata [55807.065342] raid_class [55807.065342] i2c_algo_bit [55807.065343] i2c_algo_bit [55807.065344] i2c_algo_bit [55807.065345] raid_class [55807.065347] raid_class [55807.065348] i2c_algo_bit [55807.065348] scsi_transport_sas [55807.065349] scsi_transport_sas [55807.065350] scsi_transport_sas [55807.065351] i2c_algo_bit [55807.065352] i2c_algo_bit [55807.065353] scsi_transport_sas [55807.065354] devlink [55807.065355] devlink [55807.065356] devlink [55807.065357] scsi_transport_sas [55807.065358] scsi_transport_sas [55807.065359] devlink [55807.065360] i2c_core [55807.065361] i2c_core [55807.065362] i2c_core [55807.065363] devlink [55807.065364] devlink [55807.065365] i2c_core [55807.065366] fjes [55807.065366] fjes [55807.065368] fjes [55807.065369] i2c_core [55807.065370] i2c_core [55807.065371] [55807.065371] [55807.065372] [55807.065373] fjes [55807.065374] fjes [55807.065375] [55807.065376] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55807.065378] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55807.065379] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55807.065380] [55807.065381] fjes [55807.065383] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55807.065384] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55807.065384] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55807.065386] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55807.065387] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55807.065389] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55807.065390] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55807.065391] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55807.065392] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55807.065393] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55807.065394] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55807.065395] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55807.065397] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55807.065399] RIP: 0010:[] [55807.065400] RIP: 0010:[] [55807.065401] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55807.065403] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55807.065404] RIP: 0010:[] [55807.065406] [] _raw_spin_lock+0x3a/0x50 [55807.065407] [] _raw_spin_lock+0x32/0x50 [55807.065409] [] _raw_spin_lock+0x32/0x50 [55807.065410] RIP: 0010:[] [55807.065411] RIP: 0010:[] [55807.065413] [] _raw_spin_lock+0x32/0x50 [55807.065414] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000206 [55807.065415] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [55807.065416] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [55807.065419] [] _raw_spin_lock+0x32/0x50 [55807.065420] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55807.065422] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55807.065423] RAX: 0000000000007f0a RBX: 0000000100000000 RCX: 0000000000000a1e [55807.065423] RAX: 0000000000000422 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55807.065424] RAX: 0000000000001652 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55807.065425] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55807.065426] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [55807.065428] RAX: 0000000000006758 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55807.065429] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55807.065430] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55807.065430] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55807.065432] RAX: 0000000000002956 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55807.065433] RAX: 00000000000062b5 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55807.065434] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55807.065435] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55807.065436] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55807.065437] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55807.065438] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55807.065439] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55807.065441] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55807.065442] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55807.065443] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55807.065443] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55807.065445] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55807.065446] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55807.065447] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55807.065448] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55807.065449] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55807.065450] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55807.065451] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55807.065452] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55807.065454] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55807.065455] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55807.065456] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55807.065457] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55807.065458] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55807.065459] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55807.065461] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55807.065462] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55807.065463] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55807.065464] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55807.065465] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55807.065466] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55807.065468] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55807.065469] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55807.065470] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55807.065471] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55807.065472] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55807.065473] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55807.065475] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55807.065476] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55807.065477] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55807.065477] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55807.065479] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55807.065479] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55807.065481] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55807.065482] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55807.065483] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55807.065483] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55807.065485] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55807.065485] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55807.065487] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55807.065487] Stack: [55807.065488] Stack: [55807.065488] Stack: [55807.065490] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55807.065491] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55807.065492] Stack: [55807.065493] ffff88082aae7ba0 [55807.065494] ffff88081f64fc98 [55807.065495] Stack: [55807.065496] Stack: [55807.065497] ffff880820b03c28 [55807.065498] ffff88082626bd08 [55807.065499] ffffffffa08aa7c8 [55807.065500] ffffffffa08aa7c8 [55807.065501] ffffffffa08aa7c8 [55807.065502] ffff8808217bbcd8 [55807.065503] ffff8808213b7ac0 [55807.065504] ffffffffa08aa7c8 [55807.065505] 000000000000763d [55807.065505] ffffffffa08aa8b6 [55807.065506] ffff880820b03c28 [55807.065508] ffffffffa08aa7c8 [55807.065508] ffffffffa08aa7c8 [55807.065509] ffff88082af11648 [55807.065510] ffff8803fe4d0400 [55807.065511] ffff88042dcd1e00 [55807.065512] 0000000000000000 [55807.065513] ffffffffa08aa8b6 [55807.065514] ffffffffa08aa8b6 [55807.065515] [55807.065516] [55807.065517] [55807.065518] ffff8800b484ea00 [55807.065519] ffff8804c1ba3200 [55807.065520] [55807.065520] 0000000000000000 [55807.065521] 0000000000000000 [55807.065522] ffff8800b4864200 [55807.065523] [55807.065524] [55807.065525] ffff88083fb5e000 [55807.065526] 0000000000000000 [55807.065527] ffff88082aae7c08 [55807.065528] ffff88081f64fcd8 [55807.065529] ffff880820b03c98 [55807.065530] 0000000000000001 [55807.065531] ffff88083fb5e000 [55807.065531] ffff88082626bd70 [55807.065532] ffffffffa0922006 [55807.065533] ffffffffa092e857 [55807.065534] ffffffffa092c346 [55807.065535] ffff8808217bbd08 [55807.065536] ffff8808213b7b80 [55807.065537] ffffffffa0922006 [55807.065538] ffff8804c1676dd0 [55807.065538] ffff8800b484d600 [55807.065540] ffff880820b03c60 [55807.065540] ffffffffa09217bb [55807.065541] ffffffffa092c9b5 [55807.065542] [55807.065543] [55807.065544] [55807.065545] ffff88082af10fb0 [55807.065545] ffff8804c1ba3200 [55807.065547] [55807.065547] 00000000000000c0 [55807.065548] ffff8804c1676dd0 [55807.065549] ffff88042dcd1e00 [55807.065550] [55807.065551] [55807.065552] 00000000000000b4 [55807.065553] ffff88082af10fb0 [55807.065554] ffff8804c1676dd0 [55807.065555] 0000000000000000 [55807.065556] 00000001033f89f4 [55807.065557] 0000000000000001 [55807.065558] ffffc9000e335900 [55807.065558] ffff88082af10fb0 [55807.065559] 0000000000000078 [55807.065560] ffff88041d604a00 [55807.065562] ffff8800b4864274 [55807.065562] ffff88082abba340 [55807.065563] ffff8808213b7b68 [55807.065564] ffff88082626bd30 [55807.065565] ffff88082aae7c48 [55807.065566] 0000000000000000 [55807.065567] 0000000000000000 [55807.065568] ffff88082abba348 [55807.065569] ffffffffa0ebd587 [55807.065570] [55807.065571] [55807.065571] [55807.065572] ffff88082626bd30 [55807.065573] ffff8808217bbd70 [55807.065574] [55807.065575] Call Trace: [55807.065575] Call Trace: [55807.065576] Call Trace: [55807.065577] [55807.065578] ffffffffffffffff [55807.065579] Call Trace: [55807.065582] Call Trace: [55807.065582] Call Trace: [55807.065589] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55807.065595] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55807.065600] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55807.065607] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55807.065616] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55807.065622] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55807.065628] [] lnet_finalize+0xf6/0x690 [lnet] [55807.065633] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55807.065639] [] lnet_post_send_locked+0x446/0x880 [lnet] [55807.065647] [] lnet_finalize+0xf6/0x690 [lnet] [55807.065653] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55807.065658] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55807.065662] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55807.065669] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55807.065674] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55807.065680] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55807.065690] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55807.065698] [] lnet_select_pathway+0x235/0x1140 [lnet] [55807.065700] [] ? enqueue_task_fair+0x208/0x6c0 [55807.065706] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55807.065712] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55807.065718] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55807.065724] [] lnet_finalize+0x1e9/0x690 [lnet] [55807.065731] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55807.065733] [] ? sched_clock_cpu+0x85/0xc0 [55807.065739] [] lnet_finalize+0x1e9/0x690 [lnet] [55807.065745] [] lnet_finalize+0x1e9/0x690 [lnet] [55807.065748] [] ? dequeue_task_fair+0x41e/0x660 [55807.065753] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55807.065759] [] lnet_send+0x51/0x180 [lnet] [55807.065765] [] lnet_ni_recv+0xc3/0x320 [lnet] [55807.065770] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55807.065774] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55807.065777] [] ? wake_up_state+0x20/0x20 [55807.065783] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55807.065789] [] lnet_finalize+0x3f1/0x690 [lnet] [55807.065795] [] lnet_parse+0x945/0xf90 [lnet] [55807.065799] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55807.065802] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55807.065808] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55807.065811] [] ? dequeue_task_fair+0x41e/0x660 [55807.065815] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55807.065819] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55807.065821] [] ? dequeue_task_fair+0x41e/0x660 [55807.065823] [] ? dequeue_task_fair+0x41e/0x660 [55807.065826] [] kthread+0xcf/0xe0 [55807.065829] [] ? wake_up_state+0x20/0x20 [55807.065832] [] ? __slab_free+0x81/0x2f0 [55807.065834] [] ? __wake_up+0x44/0x50 [55807.065836] [] ? wake_up_state+0x20/0x20 [55807.065838] [] ? sched_clock_cpu+0x85/0xc0 [55807.065841] [] ? kthread_create_on_node+0x140/0x140 [55807.065846] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55807.065851] [] lnet_ni_recv+0xc3/0x320 [lnet] [55807.065855] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55807.065859] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55807.065861] [] ? wake_up_state+0x20/0x20 [55807.065865] [] ret_from_fork+0x58/0x90 [55807.065868] [] kthread+0xcf/0xe0 [55807.065874] [] lnet_parse+0x945/0xf90 [lnet] [55807.065876] [] ? dequeue_task_fair+0x41e/0x660 [55807.065878] [] kthread+0xcf/0xe0 [55807.065882] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55807.065885] [] ? kthread_create_on_node+0x140/0x140 [55807.065888] [] ? kthread_create_on_node+0x140/0x140 [55807.065893] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55807.065895] [] ? wake_up_state+0x20/0x20 [55807.065897] [] ? kthread_create_on_node+0x140/0x140 [55807.065899] [] kthread+0xcf/0xe0 [55807.065900] Code: [55807.065903] 00 [55807.065906] [] ret_from_fork+0x58/0x90 [55807.065909] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55807.065913] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55807.065915] [] ret_from_fork+0x58/0x90 [55807.065918] [] ? kthread_create_on_node+0x140/0x140 [55807.065919] 02 [55807.065921] 00 [55807.065924] [] ? kthread_create_on_node+0x140/0x140 [55807.065926] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55807.065928] [] kthread+0xcf/0xe0 [55807.065930] [] ? kthread_create_on_node+0x140/0x140 [55807.065932] [] ret_from_fork+0x58/0x90 [55807.065933] f0 [55807.065934] Code: [55807.065936] Code: [55807.065937] 0f [55807.065942] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55807.065944] [] ? kthread_create_on_node+0x140/0x140 [55807.065945] 00 [55807.065945] 00 [55807.065947] [] ? kthread_create_on_node+0x140/0x140 [55807.065948] c1 [55807.065949] 02 [55807.065951] 02 [55807.065952] 07 [55807.065953] Code: [55807.065955] [] ? dequeue_task_fair+0x41e/0x660 [55807.065957] [] ret_from_fork+0x58/0x90 [55807.065958] 00 [55807.065959] 00 [55807.065959] 89 [55807.065960] 00 [55807.065961] f0 [55807.065963] f0 [55807.065964] c2 [55807.065965] 02 [55807.065967] [] ? wake_up_state+0x20/0x20 [55807.065969] [] ? kthread_create_on_node+0x140/0x140 [55807.065970] 0f [55807.065971] 0f [55807.065972] c1 [55807.065973] 00 [55807.065974] c1 [55807.065975] Code: [55807.065976] c1 [55807.065977] ea [55807.065978] f0 [55807.065982] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55807.065983] 07 [55807.065984] c2 [55807.065985] 07 [55807.065986] 10 [55807.065987] 0f [55807.065988] 89 [55807.065989] c1 [55807.065990] 89 [55807.065991] 66 [55807.065992] c1 [55807.065994] [] kthread+0xcf/0xe0 [55807.065995] c2 [55807.065995] ea [55807.065996] c2 [55807.065997] 39 [55807.065998] 07 [55807.065999] c1 [55807.066000] 10 [55807.066001] c1 [55807.066002] c2 [55807.066003] 89 [55807.066005] [] ? kthread_create_on_node+0x140/0x140 [55807.066007] ea [55807.066007] 66 [55807.066008] ea [55807.066009] 75 [55807.066010] c2 [55807.066011] 10 [55807.066013] 39 [55807.066014] 10 [55807.066014] 01 [55807.066016] c1 [55807.066018] [] ret_from_fork+0x58/0x90 [55807.066019] 66 [55807.066019] c2 [55807.066021] 66 [55807.066021] c3 [55807.066023] ea [55807.066023] 39 [55807.066025] 75 [55807.066026] 39 [55807.066027] 55 [55807.066028] 10 [55807.066030] [] ? kthread_create_on_node+0x140/0x140 [55807.066031] c2 [55807.066032] 01 [55807.066032] c2 [55807.066033] 83 [55807.066035] 66 [55807.066035] 75 [55807.066036] Code: [55807.066037] c3 [55807.066038] 75 [55807.066039] e2 [55807.066040] 39 [55807.066041] 01 [55807.066042] 00 [55807.066042] 55 [55807.066043] 01 [55807.066044] fe [55807.066045] c2 [55807.066047] c3 [55807.066048] 02 [55807.066048] 83 [55807.066049] c3 [55807.066050] 0f [55807.066051] 75 [55807.066052] 55 [55807.066053] 00 [55807.066054] e2 [55807.066055] 55 [55807.066055] b7 [55807.066056] 01 [55807.066057] 83 [55807.066058] f0 [55807.066059] fe [55807.066060] 83 [55807.066061] f2 [55807.066062] c3 [55807.066063] e2 [55807.066064] 0f [55807.066064] 0f [55807.066066] e2 [55807.066066] 48 [55807.066068] 55 [55807.066068] fe [55807.066069] c1 [55807.066070] b7 [55807.066071] fe [55807.066072] 89 [55807.066073] 83 [55807.066074] 0f [55807.066075] 07 [55807.066076] f2 [55807.066077] 0f [55807.066078] e5 [55807.066079] e2 [55807.066080] b7 [55807.066081] 89 [55807.066081] 48 [55807.066082] b7 [55807.066083] b8 [55807.066084] fe [55807.066085] f2 [55807.066086] c2 [55807.066087] 89 [55807.066088] f2 [55807.066088] 00 [55807.066089] 0f [55807.066090] 48 [55807.066091] c1 [55807.066092] e5 [55807.066093] 48 [55807.066094] 80 [55807.066095] b7 [55807.066096] 89 [55807.066097] ea [55807.066098] b8 [55807.066099] 89 [55807.066100] 00 [55807.066101] f2 [55807.066102] e5 [55807.066103] 10 [55807.066103] 00 [55807.066104] e5 [55807.066105] 00 [55807.066106] 48 [55807.066107] b8 [55807.066108] 66 [55807.066109] 80 [55807.066110] b8 [55807.066110] eb [55807.066111] 89 [55807.066112] 00 [55807.066113] 39 [55807.066114] 00 [55807.066115] 00 [55807.066116] 0d [55807.066117] e5 [55807.066118] 80 [55807.066119] c2 [55807.066120] 00 [55807.066121] 80 [55807.066122] 66 [55807.066123] b8 [55807.066124] 00 [55807.066125] 75 [55807.066126] eb [55807.066127] 00 [55807.066128] 0f [55807.066129] 00 [55807.066130] 00 [55807.066131] 01 [55807.066132] 0d [55807.066133] 00 [55807.066133] 1f [55807.066135] 80 [55807.066136] eb [55807.066137] c3 [55807.066138] 66 [55807.066139] eb [55807.066140] 44 [55807.066141] 00 [55807.066142] 0d [55807.066143] 55 [55807.066143] 0f [55807.066144] 0d [55807.066145] 00 [55807.066146] 00 [55807.066147] 66 [55807.066148] 83 [55807.066148] 1f [55807.066149] 66 [55807.066150] 00 [55807.066151] eb [55807.066152] 0f [55807.066153] e2 [55807.066154] 44 [55807.066155] 0f [55807.066156] f3 [55807.066157] 0d [55807.066158] 1f [55807.066159] fe [55807.066160] 00 [55807.066161] 1f [55807.066161] 90 [55807.066163] 66 [55807.066163] 44 [55807.066164] 0f [55807.066165] 00 [55807.066166] 44 [55807.066167] <83> [55807.066168] 0f [55807.066169] 00 [55807.066170] b7 [55807.066170] f3 [55807.066171] 00 [55807.066172] e8 [55807.066173] 1f [55807.066174] 00 [55807.066175] f2 [55807.066176] 90 [55807.066176] 00 [55807.066177] 01 [55807.066178] 44 [55807.066179] f3 [55807.066180] 48 [55807.066181] 83 [55807.066181] f3 [55807.066182] 74 [55807.066183] 00 [55807.066184] 90 [55807.066185] 89 [55807.066186] e8 [55807.066187] 90 [55807.066187] 0a [55807.066188] 00 [55807.066189] <83> [55807.066190] e5 [55807.066191] 01 [55807.066192] <83> [55807.066193] 0f [55807.066194] f3 [55807.066195] e8 [55807.066196] b8 [55807.066196] 74 [55807.066197] e8 [55807.066198] b7 [55807.066199] 90 [55807.066200] 01 [55807.066201] 00 [55807.066202] 0a [55807.066203] 01 [55807.066204] 0f [55807.066205] <83> [55807.066206] 74 [55807.066208] 80 [55807.066208] 0f [55807.066210] 74 [55807.066210] 66 [55807.066211] e8 [55807.066212] 0a [55807.066213] 00 [55807.066214] b7 [55807.066215] 0a [55807.066216] 39 [55807.066217] 01 [55807.066218] 0f [55807.066219] 00 [55807.066220] 0f [55807.066220] 0f [55807.066221] ca [55807.066222] 74 [55807.066223] b7 [55807.066224] eb [55807.066225] <66> [55807.066226] b7 [55807.066227] 75 [55807.066228] 0a [55807.066229] 0f [55807.066230] 0d [55807.066230] 39 [55807.066231] 0f [55807.066232] f1 [55807.066233] 0f [55807.066234] 66 [55807.066235] 66 [55807.066235] ca [55807.066236] 66 [55807.066237] 5d [55807.066238] b7 [55807.066239] 39 [55807.066240] 0f [55807.066241] 75 [55807.066242] 39 [55807.066242] c3 [55807.066244] 0f [55807.066245] ca [55807.066246] 1f [55807.066246] f1 [55807.066247] ca [55807.066248] 66 [55807.066249] 66 [55807.066250] 75 [55807.066251] 44 [55807.066252] 5d [55807.066253] 75 [55807.066254] 66 [55807.066255] 39 [55807.066256] f1 [55807.066257] 00 [55807.066258] c3 [55807.066258] f1 [55807.066259] 66 [55807.066260] ca [55807.066261] 5d [55807.066262] 00 [55807.066263] 66 [55807.066264] 5d [55807.066264] 90 [55807.066266] 75 [55807.066266] c3 [55807.066267] f3 [55807.066268] 66 [55807.066269] c3 [55807.066270] 66 [55807.066271] f1 [55807.066272] 66 [55807.066273] 90 [55807.066273] 66 [55807.066274] 66 [55807.066275] 66 [55807.066276] 5d [55807.066278] 66 [55807.066279] <83> [55807.066279] 90 [55807.066280] [55807.066281] 66 [55807.066282] c3 [55807.066283] 66 [55807.066284] e8 [55807.066284] 66 [55807.066285] 66 [55807.066286] 66 [55807.066287] 90 [55807.066288] 01 [55807.066289] 66 [55807.066290] 90 [55807.066291] 66 [55807.066292] 66 [55807.066293] 74 [55807.066294] 90 [55807.066294] 66 [55807.066296] 66 [55807.066296] 66 [55807.066297] 0a [55807.066298] eb [55807.066299] 66 [55807.066300] [55807.066301] 90 [55807.066302] 0f [55807.066303] [55807.066303] d9 [55807.066304] 66 [55807.066305] b7 [55807.066306] 66 [55807.066307] 66 [55807.066309] 0f [55807.066310] [55807.066311] 0f [55807.066312] 66 [55807.066312] 1f [55807.066314] 39 [55807.066315] 44 [55807.066316] ca [55807.066317] 00 [55807.066318] [55807.066320] 75 f1 5d c3 66 66 66 90 66 66 [55811.064299] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [kiblnd_connd:2895] [55811.064318] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55811.064323] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55811.064324] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55811.064325] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55811.064326] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55811.064327] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55811.064328] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55811.064328] RAX: 0000000000002404 RBX: 0000000000016c40 RCX: 0000000000000a1e [55811.064329] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55811.064330] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55811.064330] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55811.064331] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55811.064331] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55811.064332] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55811.064332] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55811.064333] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55811.064334] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55811.064334] Stack: [55811.064335] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55811.064336] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55811.064337] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55811.064337] Call Trace: [55811.064343] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55811.064348] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55811.064351] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55811.064355] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55811.064356] [] ? lock_timer_base.isra.33+0x2b/0x50 [55811.064358] [] ? try_to_del_timer_sync+0x5e/0x90 [55811.064361] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55811.064363] [] ? schedule_timeout+0x17d/0x2d0 [55811.064366] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55811.064368] [] ? wake_up_state+0x20/0x20 [55811.064371] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55811.064373] [] kthread+0xcf/0xe0 [55811.064374] [] ? kthread_create_on_node+0x140/0x140 [55811.064376] [] ret_from_fork+0x58/0x90 [55811.064377] [] ? kthread_create_on_node+0x140/0x140 [55811.064387] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55819.063476] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55819.063478] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [55819.063480] Modules linked in: [55819.063481] Modules linked in: [55819.063482] ko2iblnd(OE) [55819.063483] ko2iblnd(OE) [55819.063484] lustre(OE) [55819.063485] lustre(OE) [55819.063486] lmv(OE) [55819.063487] lmv(OE) [55819.063488] fld(OE) [55819.063489] fld(OE) [55819.063491] mdc(OE) [55819.063492] mdc(OE) [55819.063493] fid(OE) [55819.063494] fid(OE) [55819.063495] lov(OE) [55819.063496] lov(OE) [55819.063497] ksocklnd(OE) [55819.063498] ksocklnd(OE) [55819.063499] ptlrpc(OE) [55819.063500] ptlrpc(OE) [55819.063501] obdclass(OE) [55819.063502] obdclass(OE) [55819.063503] lnet(OE) [55819.063504] lnet(OE) [55819.063505] libcfs(OE) [55819.063506] libcfs(OE) [55819.063507] rpcrdma [55819.063508] rpcrdma [55819.063509] ib_isert [55819.063510] ib_isert [55819.063511] iscsi_target_mod [55819.063512] iscsi_target_mod [55819.063513] ib_iser [55819.063514] ib_iser [55819.063515] libiscsi [55819.063516] libiscsi [55819.063517] scsi_transport_iscsi [55819.063518] scsi_transport_iscsi [55819.063519] ib_srpt [55819.063520] ib_srpt [55819.063521] target_core_mod [55819.063522] target_core_mod [55819.063523] ib_srp [55819.063525] ib_srp [55819.063525] scsi_transport_srp [55819.063527] scsi_transport_srp [55819.063527] scsi_tgt [55819.063528] scsi_tgt [55819.063529] ib_ipoib [55819.063530] ib_ipoib [55819.063531] rdma_ucm [55819.063532] rdma_ucm [55819.063534] ib_ucm [55819.063535] ib_ucm [55819.063536] ib_uverbs [55819.063537] ib_uverbs [55819.063538] ib_umad [55819.063539] ib_umad [55819.063540] rdma_cm [55819.063541] rdma_cm [55819.063542] ib_cm [55819.063543] ib_cm [55819.063544] iw_cm [55819.063545] iw_cm [55819.063546] mlx4_ib [55819.063547] mlx4_ib [55819.063548] intel_powerclamp [55819.063549] intel_powerclamp [55819.063550] coretemp [55819.063551] coretemp [55819.063552] intel_rapl [55819.063553] intel_rapl [55819.063554] iosf_mbi [55819.063555] iosf_mbi [55819.063556] kvm [55819.063558] kvm [55819.063559] irqbypass [55819.063560] irqbypass [55819.063561] crc32_pclmul [55819.063562] crc32_pclmul [55819.063563] ghash_clmulni_intel [55819.063564] ghash_clmulni_intel [55819.063565] aesni_intel [55819.063566] aesni_intel [55819.063567] ipmi_ssif [55819.063568] ipmi_ssif [55819.063569] lrw [55819.063571] lrw [55819.063571] gf128mul [55819.063572] gf128mul [55819.063574] glue_helper [55819.063575] glue_helper [55819.063576] hfi1 [55819.063577] hfi1 [55819.063578] ipmi_devintf [55819.063579] ipmi_devintf [55819.063581] rdmavt [55819.063582] rdmavt [55819.063583] ablk_helper [55819.063584] ablk_helper [55819.063585] cryptd [55819.063586] cryptd [55819.063587] iTCO_wdt [55819.063588] iTCO_wdt [55819.063589] ipmi_si [55819.063590] ipmi_si [55819.063591] sb_edac [55819.063592] sb_edac [55819.063593] sg [55819.063594] sg [55819.063595] iTCO_vendor_support [55819.063596] iTCO_vendor_support [55819.063598] ib_core [55819.063599] ib_core [55819.063600] ioatdma [55819.063601] ioatdma [55819.063602] i2c_i801 [55819.063603] i2c_i801 [55819.063604] mei_me [55819.063605] mei_me [55819.063606] mei [55819.063607] mei [55819.063608] edac_core [55819.063609] edac_core [55819.063610] ipmi_msghandler [55819.063611] ipmi_msghandler [55819.063612] lpc_ich [55819.063613] lpc_ich [55819.063614] ntb [55819.063615] ntb [55819.063616] wmi [55819.063617] wmi [55819.063619] shpchp [55819.063620] shpchp [55819.063621] pcspkr [55819.063622] pcspkr [55819.063623] nfsd [55819.063624] nfsd [55819.063625] nfs_acl [55819.063626] nfs_acl [55819.063627] lockd [55819.063629] lockd [55819.063630] grace [55819.063631] grace [55819.063632] auth_rpcgss [55819.063633] auth_rpcgss [55819.063634] sunrpc [55819.063635] sunrpc [55819.063636] ip_tables [55819.063637] ip_tables [55819.063639] ext4 [55819.063640] ext4 [55819.063641] mbcache [55819.063642] mbcache [55819.063643] jbd2 [55819.063644] jbd2 [55819.063645] sd_mod [55819.063646] sd_mod [55819.063647] crc_t10dif [55819.063648] crc_t10dif [55819.063649] crct10dif_generic [55819.063650] crct10dif_generic [55819.063651] mlx4_en [55819.063652] mlx4_en [55819.063653] mgag200 [55819.063654] mgag200 [55819.063656] drm_kms_helper [55819.063657] drm_kms_helper [55819.063658] syscopyarea [55819.063659] syscopyarea [55819.063660] sysfillrect [55819.063661] sysfillrect [55819.063662] sysimgblt [55819.063663] sysimgblt [55819.063664] fb_sys_fops [55819.063665] fb_sys_fops [55819.063666] isci [55819.063667] isci [55819.063668] ttm [55819.063670] ttm [55819.063671] igb [55819.063672] igb [55819.063673] crct10dif_pclmul [55819.063674] crct10dif_pclmul [55819.063675] crct10dif_common [55819.063676] crct10dif_common [55819.063677] ahci [55819.063678] ahci [55819.063679] ptp [55819.063680] ptp [55819.063681] crc32c_intel [55819.063682] crc32c_intel [55819.063683] libsas [55819.063684] libsas [55819.063685] libahci [55819.063686] libahci [55819.063687] pps_core [55819.063688] pps_core [55819.063690] drm [55819.063691] drm [55819.063692] mlx4_core [55819.063693] mlx4_core [55819.063694] mpt2sas [55819.063695] mpt2sas [55819.063696] dca [55819.063697] dca [55819.063698] libata [55819.063699] libata [55819.063700] raid_class [55819.063701] raid_class [55819.063702] i2c_algo_bit [55819.063703] i2c_algo_bit [55819.063704] scsi_transport_sas [55819.063705] scsi_transport_sas [55819.063706] devlink [55819.063707] devlink [55819.063708] i2c_core [55819.063709] i2c_core [55819.063710] fjes [55819.063711] [55819.063712] fjes [55819.063714] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55819.063716] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55819.063718] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55819.063719] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55819.063723] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55819.063723] Workqueue: ib_cm cm_work_handler [ib_cm] [55819.063726] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55819.063728] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55819.063730] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [55819.063732] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55819.063733] RAX: 0000000000000d59 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55819.063734] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000216 [55819.063736] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55819.063737] RAX: 0000000000000dc2 RBX: 0000000000000000 RCX: 0000000000000a1e [55819.063738] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55819.063739] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55819.063740] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55819.063742] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55819.063743] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55819.063744] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55819.063745] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55819.063746] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55819.063748] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55819.063749] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55819.063750] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55819.063752] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55819.063753] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55819.063754] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55819.063755] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55819.063757] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55819.063757] Stack: [55819.063759] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55819.063761] Stack: [55819.063762] ffff8807f639fd88 ffffffffa08aa831 [55819.063763] ffff8803fea77b00 [55819.063764] 0000000200000002 [55819.063765] ffffffffa08aa7c8 [55819.063766] ffff8807f63a2980 [55819.063767] [55819.063769] ffff8803fea77bd0 00050000c0a80173 [55819.063770] [55819.063772] 0000000000000000 ffff8807f639fdc0 [55819.063773] ffff88083fb5de00 [55819.063774] ffffffffa0942e30 [55819.063775] ffff8803fea77b18 [55819.063776] 00000000c0086540 [55819.063777] [55819.063779] ffffffffa0916e4b ffff8803fea77c68 [55819.063780] [55819.063782] ffff8807f63a2900 0000000000000002 [55819.063783] ffff8803fea77bc8 [55819.063784] 0000000000650aa0 [55819.063785] ffffffffa0ebfc24 [55819.063786] 0000000000000000 [55819.063787] [55819.063788] Call Trace: [55819.063789] ffff88041aa8a880 ffff8803fea77bd0 [55819.063791] Call Trace: [55819.063797] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55819.063804] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55819.063811] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55819.063818] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55819.063824] [] LNetCtl+0x116/0x800 [lnet] [55819.063830] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55819.063836] [] lnet_ioctl+0x60/0x250 [lnet] [55819.063840] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55819.063845] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55819.063850] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55819.063855] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55819.063859] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55819.063862] [] do_vfs_ioctl+0x2d5/0x4b0 [55819.063866] [] cma_req_handler+0x227/0x570 [rdma_cm] [55819.063868] [] ? __fd_install+0x47/0x60 [55819.063872] [] cm_process_work+0x25/0x140 [ib_cm] [55819.063873] [] SyS_ioctl+0xa1/0xc0 [55819.063877] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55819.063879] [] system_call_fastpath+0x16/0x1b [55819.063883] [] cm_work_handler+0x185/0x1484 [ib_cm] [55819.063885] Code: 00 [55819.063887] [] process_one_work+0x17b/0x470 [55819.063889] 02 00 [55819.063891] [] worker_thread+0x126/0x410 [55819.063892] f0 0f [55819.063895] [] ? rescuer_thread+0x460/0x460 [55819.063896] c1 07 [55819.063899] [] kthread+0xcf/0xe0 [55819.063900] 89 c2 [55819.063903] [] ? kthread_create_on_node+0x140/0x140 [55819.063904] c1 ea [55819.063907] [] ret_from_fork+0x58/0x90 [55819.063909] 10 66 [55819.063911] [] ? kthread_create_on_node+0x140/0x140 [55819.063913] 39 c2 [55819.063914] Code: [55819.063915] 75 [55819.063916] 00 [55819.063917] 01 [55819.063918] 02 [55819.063919] c3 [55819.063920] 00 [55819.063921] 55 [55819.063922] f0 [55819.063923] 83 [55819.063924] 0f [55819.063925] e2 [55819.063926] c1 [55819.063927] fe [55819.063928] 07 [55819.063929] 0f [55819.063929] 89 [55819.063930] b7 [55819.063931] c2 [55819.063932] f2 [55819.063933] c1 [55819.063934] 48 [55819.063935] ea [55819.063936] 89 [55819.063937] 10 [55819.063938] e5 [55819.063939] 66 [55819.063940] b8 [55819.063941] 39 [55819.063942] 00 [55819.063943] c2 [55819.063944] 80 [55819.063945] 75 [55819.063946] 00 [55819.063947] 01 [55819.063948] 00 [55819.063949] c3 [55819.063949] eb [55819.063950] 55 [55819.063951] 0d [55819.063952] 83 [55819.063953] 66 [55819.063954] e2 [55819.063955] 0f [55819.063956] fe [55819.063957] 1f [55819.063958] 0f [55819.063959] 44 [55819.063960] b7 [55819.063961] 00 [55819.063962] f2 [55819.063963] 00 [55819.063964] 48 [55819.063965] f3 [55819.063966] 89 [55819.063967] 90 [55819.063968] e5 [55819.063969] <83> [55819.063970] b8 [55819.063971] e8 [55819.063972] 00 [55819.063973] 01 [55819.063974] 80 [55819.063975] 74 [55819.063976] 00 [55819.063977] 0a [55819.063978] 00 [55819.063979] 0f [55819.063980] eb [55819.063981] b7 [55819.063982] 0d [55819.063983] 0f [55819.063983] 66 [55819.063985] 66 [55819.063986] 0f [55819.063987] 39 [55819.063987] 1f [55819.063989] ca [55819.063990] 44 [55819.063991] 75 [55819.063991] 00 [55819.063992] f1 [55819.063993] 00 [55819.063994] 5d [55819.063995] f3 [55819.063996] c3 [55819.063997] 90 [55819.063998] 66 [55819.063999] <83> [55819.064000] 66 [55819.064001] e8 [55819.064002] 66 [55819.064003] 01 [55819.064004] 90 [55819.064005] 74 [55819.064006] 66 [55819.064007] 0a [55819.064008] 66 [55819.064009] [55819.064012] 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55823.063072] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kiblnd_sd_01_02:2902] [55823.063093] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55823.063099] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55823.063100] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55823.063101] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55823.063101] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55823.063103] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55823.063104] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [55823.063104] RAX: 000000000000532d RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55823.063105] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55823.063105] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55823.063106] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55823.063106] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55823.063107] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55823.063108] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55823.063108] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55823.063109] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55823.063109] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55823.063110] Stack: [55823.063111] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [55823.063112] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [55823.063113] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [55823.063113] Call Trace: [55823.063118] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55823.063123] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55823.063129] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55823.063134] [] lnet_finalize+0x1e9/0x690 [lnet] [55823.063138] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55823.063142] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55823.063143] [] ? dequeue_task_fair+0x41e/0x660 [55823.063145] [] ? wake_up_state+0x20/0x20 [55823.063148] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55823.063150] [] kthread+0xcf/0xe0 [55823.063151] [] ? kthread_create_on_node+0x140/0x140 [55823.063153] [] ret_from_fork+0x58/0x90 [55823.063154] [] ? kthread_create_on_node+0x140/0x140 [55823.063164] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55826.138943] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55826.379987] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55826.451580] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55826.488343] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55826.526219] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55826.551777] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55826.579629] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [55826.598058] RAX: 0000000000005e2d RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55826.622444] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55826.646823] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55826.671198] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55826.695593] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55826.719991] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55826.747511] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55826.767326] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55826.791723] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55826.816141] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55826.840530] Stack: [55826.848008] ffff8808262efac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55826.873329] ffff88083fb5e000 ffff8808262efb80 ffffffffa092c9b5 00000000000001a8 [55826.898620] ffffc9000e331730 ffff8808262efb68 ffffffffa0ebd587 ffffffffffffffff [55826.923947] Call Trace: [55826.932885] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55826.954989] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55826.977978] [] lnet_select_pathway+0x235/0x1140 [lnet] [55827.001225] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55827.023612] [] lnet_send+0x51/0x180 [lnet] [55827.043435] [] lnet_finalize+0x3f1/0x690 [lnet] [55827.064685] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55827.086528] [] ? __slab_free+0x81/0x2f0 [55827.105506] [] lnet_ni_recv+0xc3/0x320 [lnet] [55827.126189] [] lnet_parse+0x945/0xf90 [lnet] [55827.146611] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55827.169864] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55827.191130] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55827.214685] [] ? dequeue_task_fair+0x41e/0x660 [55827.235654] [] ? wake_up_state+0x20/0x20 [55827.254925] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55827.277930] [] kthread+0xcf/0xe0 [55827.294892] [] ? kthread_create_on_node+0x140/0x140 [55827.317292] [] ret_from_fork+0x58/0x90 [55827.335923] [] ? kthread_create_on_node+0x140/0x140 [55827.358282] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55835.061829] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [kiblnd_sd_01_00:2900] [55835.061831] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [55835.061833] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [55835.061834] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55835.061836] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55835.061837] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55835.061839] Modules linked in: [55835.061841] Modules linked in: [55835.061842] Modules linked in: [55835.061844] Modules linked in: [55835.061845] Modules linked in: [55835.061846] ko2iblnd(OE) [55835.061847] ko2iblnd(OE) [55835.061848] ko2iblnd(OE) [55835.061849] ko2iblnd(OE) [55835.061850] ko2iblnd(OE) [55835.061851] lustre(OE) [55835.061852] lustre(OE) [55835.061853] lustre(OE) [55835.061854] lustre(OE) [55835.061855] lustre(OE) [55835.061857] lmv(OE) [55835.061858] lmv(OE) [55835.061859] lmv(OE) [55835.061860] lmv(OE) [55835.061860] lmv(OE) [55835.061862] fld(OE) [55835.061863] fld(OE) [55835.061863] fld(OE) [55835.061864] fld(OE) [55835.061865] fld(OE) [55835.061866] mdc(OE) [55835.061868] mdc(OE) [55835.061869] mdc(OE) [55835.061870] mdc(OE) [55835.061871] mdc(OE) [55835.061872] fid(OE) [55835.061873] fid(OE) [55835.061874] fid(OE) [55835.061875] fid(OE) [55835.061876] fid(OE) [55835.061877] lov(OE) [55835.061879] lov(OE) [55835.061879] lov(OE) [55835.061880] lov(OE) [55835.061881] lov(OE) [55835.061882] ksocklnd(OE) [55835.061883] ksocklnd(OE) [55835.061884] ksocklnd(OE) [55835.061885] ksocklnd(OE) [55835.061886] ksocklnd(OE) [55835.061888] ptlrpc(OE) [55835.061889] ptlrpc(OE) [55835.061889] ptlrpc(OE) [55835.061890] ptlrpc(OE) [55835.061891] ptlrpc(OE) [55835.061892] obdclass(OE) [55835.061894] obdclass(OE) [55835.061895] obdclass(OE) [55835.061895] obdclass(OE) [55835.061896] obdclass(OE) [55835.061897] lnet(OE) [55835.061899] lnet(OE) [55835.061899] lnet(OE) [55835.061900] lnet(OE) [55835.061901] lnet(OE) [55835.061902] libcfs(OE) [55835.061903] libcfs(OE) [55835.061904] libcfs(OE) [55835.061905] libcfs(OE) [55835.061906] libcfs(OE) [55835.061907] rpcrdma [55835.061908] rpcrdma [55835.061909] rpcrdma [55835.061910] rpcrdma [55835.061911] rpcrdma [55835.061912] ib_isert [55835.061913] ib_isert [55835.061914] ib_isert [55835.061915] ib_isert [55835.061916] ib_isert [55835.061917] iscsi_target_mod [55835.061918] iscsi_target_mod [55835.061919] iscsi_target_mod [55835.061920] iscsi_target_mod [55835.061921] iscsi_target_mod [55835.061922] ib_iser [55835.061924] ib_iser [55835.061924] ib_iser [55835.061925] ib_iser [55835.061926] ib_iser [55835.061927] libiscsi [55835.061928] libiscsi [55835.061929] libiscsi [55835.061929] libiscsi [55835.061930] libiscsi [55835.061931] scsi_transport_iscsi [55835.061932] scsi_transport_iscsi [55835.061933] scsi_transport_iscsi [55835.061934] scsi_transport_iscsi [55835.061935] scsi_transport_iscsi [55835.061936] ib_srpt [55835.061937] ib_srpt [55835.061938] ib_srpt [55835.061938] ib_srpt [55835.061939] ib_srpt [55835.061940] target_core_mod [55835.061942] target_core_mod [55835.061943] target_core_mod [55835.061944] target_core_mod [55835.061944] target_core_mod [55835.061946] ib_srp [55835.061947] ib_srp [55835.061948] ib_srp [55835.061949] ib_srp [55835.061949] ib_srp [55835.061950] scsi_transport_srp [55835.061952] scsi_transport_srp [55835.061952] scsi_transport_srp [55835.061954] scsi_transport_srp [55835.061954] scsi_transport_srp [55835.061956] scsi_tgt [55835.061957] scsi_tgt [55835.061957] scsi_tgt [55835.061958] scsi_tgt [55835.061959] scsi_tgt [55835.061960] ib_ipoib [55835.061961] ib_ipoib [55835.061962] ib_ipoib [55835.061963] ib_ipoib [55835.061964] ib_ipoib [55835.061965] rdma_ucm [55835.061966] rdma_ucm [55835.061967] rdma_ucm [55835.061968] rdma_ucm [55835.061969] rdma_ucm [55835.061970] ib_ucm [55835.061971] ib_ucm [55835.061972] ib_ucm [55835.061973] ib_ucm [55835.061974] ib_ucm [55835.061975] ib_uverbs [55835.061976] ib_uverbs [55835.061977] ib_uverbs [55835.061978] ib_uverbs [55835.061979] ib_uverbs [55835.061980] ib_umad [55835.061981] ib_umad [55835.061982] ib_umad [55835.061983] ib_umad [55835.061984] ib_umad [55835.061985] rdma_cm [55835.061986] rdma_cm [55835.061987] rdma_cm [55835.061987] rdma_cm [55835.061988] rdma_cm [55835.061989] ib_cm [55835.061991] ib_cm [55835.061991] ib_cm [55835.061992] ib_cm [55835.061993] ib_cm [55835.061994] iw_cm [55835.061995] iw_cm [55835.061996] iw_cm [55835.061997] iw_cm [55835.061998] iw_cm [55835.061999] mlx4_ib [55835.062000] mlx4_ib [55835.062001] mlx4_ib [55835.062002] mlx4_ib [55835.062003] mlx4_ib [55835.062004] intel_powerclamp [55835.062006] intel_powerclamp [55835.062007] intel_powerclamp [55835.062007] intel_powerclamp [55835.062009] intel_powerclamp [55835.062010] coretemp [55835.062011] coretemp [55835.062011] coretemp [55835.062013] coretemp [55835.062013] coretemp [55835.062014] intel_rapl [55835.062016] intel_rapl [55835.062017] intel_rapl [55835.062018] intel_rapl [55835.062018] intel_rapl [55835.062020] iosf_mbi [55835.062021] iosf_mbi [55835.062022] iosf_mbi [55835.062023] iosf_mbi [55835.062023] iosf_mbi [55835.062025] kvm [55835.062026] kvm [55835.062027] kvm [55835.062027] kvm [55835.062028] kvm [55835.062030] irqbypass [55835.062030] irqbypass [55835.062031] irqbypass [55835.062032] irqbypass [55835.062032] irqbypass [55835.062034] crc32_pclmul [55835.062035] crc32_pclmul [55835.062036] crc32_pclmul [55835.062036] crc32_pclmul [55835.062037] crc32_pclmul [55835.062038] ghash_clmulni_intel [55835.062040] ghash_clmulni_intel [55835.062041] ghash_clmulni_intel [55835.062042] ghash_clmulni_intel [55835.062043] ghash_clmulni_intel [55835.062044] aesni_intel [55835.062046] aesni_intel [55835.062046] aesni_intel [55835.062047] aesni_intel [55835.062048] aesni_intel [55835.062049] ipmi_ssif [55835.062050] ipmi_ssif [55835.062051] ipmi_ssif [55835.062052] ipmi_ssif [55835.062053] ipmi_ssif [55835.062054] lrw [55835.062055] lrw [55835.062056] lrw [55835.062057] lrw [55835.062058] lrw [55835.062059] gf128mul [55835.062060] gf128mul [55835.062061] gf128mul [55835.062062] gf128mul [55835.062063] gf128mul [55835.062064] glue_helper [55835.062065] glue_helper [55835.062066] glue_helper [55835.062067] glue_helper [55835.062068] glue_helper [55835.062069] hfi1 [55835.062070] hfi1 [55835.062071] hfi1 [55835.062072] hfi1 [55835.062073] hfi1 [55835.062074] ipmi_devintf [55835.062075] ipmi_devintf [55835.062076] ipmi_devintf [55835.062077] ipmi_devintf [55835.062078] ipmi_devintf [55835.062079] rdmavt [55835.062080] rdmavt [55835.062081] rdmavt [55835.062081] rdmavt [55835.062082] rdmavt [55835.062083] ablk_helper [55835.062084] ablk_helper [55835.062085] ablk_helper [55835.062087] ablk_helper [55835.062087] ablk_helper [55835.062089] cryptd [55835.062090] cryptd [55835.062091] cryptd [55835.062092] cryptd [55835.062093] cryptd [55835.062094] iTCO_wdt [55835.062095] iTCO_wdt [55835.062095] iTCO_wdt [55835.062096] iTCO_wdt [55835.062097] iTCO_wdt [55835.062098] ipmi_si [55835.062099] ipmi_si [55835.062100] ipmi_si [55835.062101] ipmi_si [55835.062102] ipmi_si [55835.062103] sb_edac [55835.062105] sb_edac [55835.062105] sb_edac [55835.062106] sb_edac [55835.062107] sb_edac [55835.062108] sg [55835.062109] sg [55835.062110] sg [55835.062111] sg [55835.062112] sg [55835.062113] iTCO_vendor_support [55835.062115] iTCO_vendor_support [55835.062116] iTCO_vendor_support [55835.062117] iTCO_vendor_support [55835.062117] iTCO_vendor_support [55835.062118] ib_core [55835.062119] ib_core [55835.062120] ib_core [55835.062121] ib_core [55835.062122] ib_core [55835.062123] ioatdma [55835.062124] ioatdma [55835.062125] ioatdma [55835.062126] ioatdma [55835.062127] ioatdma [55835.062128] i2c_i801 [55835.062129] i2c_i801 [55835.062130] i2c_i801 [55835.062131] i2c_i801 [55835.062132] i2c_i801 [55835.062133] mei_me [55835.062134] mei_me [55835.062135] mei_me [55835.062136] mei_me [55835.062137] mei_me [55835.062138] mei [55835.062139] mei [55835.062140] mei [55835.062141] mei [55835.062142] mei [55835.062143] edac_core [55835.062144] edac_core [55835.062145] edac_core [55835.062146] edac_core [55835.062147] edac_core [55835.062148] ipmi_msghandler [55835.062150] ipmi_msghandler [55835.062150] ipmi_msghandler [55835.062151] ipmi_msghandler [55835.062152] ipmi_msghandler [55835.062153] lpc_ich [55835.062155] lpc_ich [55835.062155] lpc_ich [55835.062156] lpc_ich [55835.062157] lpc_ich [55835.062158] ntb [55835.062160] ntb [55835.062161] ntb [55835.062161] ntb [55835.062162] ntb [55835.062163] wmi [55835.062164] wmi [55835.062165] wmi [55835.062166] wmi [55835.062167] wmi [55835.062168] shpchp [55835.062170] shpchp [55835.062171] shpchp [55835.062172] shpchp [55835.062173] shpchp [55835.062174] pcspkr [55835.062175] pcspkr [55835.062176] pcspkr [55835.062177] pcspkr [55835.062178] pcspkr [55835.062179] nfsd [55835.062180] nfsd [55835.062181] nfsd [55835.062182] nfsd [55835.062182] nfsd [55835.062184] nfs_acl [55835.062185] nfs_acl [55835.062185] nfs_acl [55835.062186] nfs_acl [55835.062187] nfs_acl [55835.062188] lockd [55835.062189] lockd [55835.062190] lockd [55835.062191] lockd [55835.062192] lockd [55835.062193] grace [55835.062194] grace [55835.062195] grace [55835.062196] grace [55835.062196] grace [55835.062197] auth_rpcgss [55835.062199] auth_rpcgss [55835.062200] auth_rpcgss [55835.062201] auth_rpcgss [55835.062201] auth_rpcgss [55835.062202] sunrpc [55835.062203] sunrpc [55835.062205] sunrpc [55835.062205] sunrpc [55835.062206] sunrpc [55835.062207] ip_tables [55835.062209] ip_tables [55835.062210] ip_tables [55835.062211] ip_tables [55835.062212] ip_tables [55835.062213] ext4 [55835.062214] ext4 [55835.062215] ext4 [55835.062216] ext4 [55835.062217] ext4 [55835.062218] mbcache [55835.062219] mbcache [55835.062220] mbcache [55835.062221] mbcache [55835.062222] mbcache [55835.062223] jbd2 [55835.062225] jbd2 [55835.062226] jbd2 [55835.062226] jbd2 [55835.062227] jbd2 [55835.062228] sd_mod [55835.062230] sd_mod [55835.062231] sd_mod [55835.062232] sd_mod [55835.062232] sd_mod [55835.062233] crc_t10dif [55835.062235] crc_t10dif [55835.062235] crc_t10dif [55835.062236] crc_t10dif [55835.062237] crc_t10dif [55835.062238] crct10dif_generic [55835.062239] crct10dif_generic [55835.062240] crct10dif_generic [55835.062241] crct10dif_generic [55835.062242] crct10dif_generic [55835.062243] mlx4_en [55835.062244] mlx4_en [55835.062245] mlx4_en [55835.062246] mlx4_en [55835.062246] mlx4_en [55835.062247] mgag200 [55835.062248] mgag200 [55835.062249] mgag200 [55835.062251] mgag200 [55835.062251] mgag200 [55835.062252] drm_kms_helper [55835.062253] drm_kms_helper [55835.062254] drm_kms_helper [55835.062256] drm_kms_helper [55835.062257] drm_kms_helper [55835.062258] syscopyarea [55835.062259] syscopyarea [55835.062260] syscopyarea [55835.062260] syscopyarea [55835.062261] syscopyarea [55835.062263] sysfillrect [55835.062264] sysfillrect [55835.062265] sysfillrect [55835.062266] sysfillrect [55835.062266] sysfillrect [55835.062267] sysimgblt [55835.062269] sysimgblt [55835.062270] sysimgblt [55835.062271] sysimgblt [55835.062273] sysimgblt [55835.062274] fb_sys_fops [55835.062275] fb_sys_fops [55835.062276] fb_sys_fops [55835.062277] fb_sys_fops [55835.062278] fb_sys_fops [55835.062279] isci [55835.062281] isci [55835.062281] isci [55835.062282] isci [55835.062283] isci [55835.062284] ttm [55835.062285] ttm [55835.062287] ttm [55835.062287] ttm [55835.062288] ttm [55835.062289] igb [55835.062291] igb [55835.062292] igb [55835.062293] igb [55835.062293] igb [55835.062295] crct10dif_pclmul [55835.062296] crct10dif_pclmul [55835.062297] crct10dif_pclmul [55835.062298] crct10dif_pclmul [55835.062299] crct10dif_pclmul [55835.062300] crct10dif_common [55835.062301] crct10dif_common [55835.062302] crct10dif_common [55835.062303] crct10dif_common [55835.062304] crct10dif_common [55835.062306] ahci [55835.062307] ahci [55835.062308] ahci [55835.062309] ahci [55835.062309] ahci [55835.062310] ptp [55835.062311] ptp [55835.062312] ptp [55835.062313] ptp [55835.062314] ptp [55835.062315] crc32c_intel [55835.062316] crc32c_intel [55835.062317] crc32c_intel [55835.062319] crc32c_intel [55835.062319] crc32c_intel [55835.062320] libsas [55835.062322] libsas [55835.062322] libsas [55835.062323] libsas [55835.062324] libsas [55835.062325] libahci [55835.062326] libahci [55835.062327] libahci [55835.062328] libahci [55835.062329] libahci [55835.062330] pps_core [55835.062332] pps_core [55835.062332] pps_core [55835.062333] pps_core [55835.062334] pps_core [55835.062335] drm [55835.062336] drm [55835.062337] drm [55835.062338] drm [55835.062338] drm [55835.062340] mlx4_core [55835.062341] mlx4_core [55835.062341] mlx4_core [55835.062342] mlx4_core [55835.062343] mlx4_core [55835.062345] mpt2sas [55835.062346] mpt2sas [55835.062346] mpt2sas [55835.062347] mpt2sas [55835.062348] mpt2sas [55835.062349] dca [55835.062350] dca [55835.062351] dca [55835.062352] dca [55835.062353] dca [55835.062354] libata [55835.062355] libata [55835.062356] libata [55835.062357] libata [55835.062358] libata [55835.062359] raid_class [55835.062360] raid_class [55835.062361] raid_class [55835.062361] raid_class [55835.062362] raid_class [55835.062364] i2c_algo_bit [55835.062364] i2c_algo_bit [55835.062365] i2c_algo_bit [55835.062366] i2c_algo_bit [55835.062367] i2c_algo_bit [55835.062369] scsi_transport_sas [55835.062370] scsi_transport_sas [55835.062371] scsi_transport_sas [55835.062372] scsi_transport_sas [55835.062373] scsi_transport_sas [55835.062374] devlink [55835.062375] devlink [55835.062376] devlink [55835.062377] devlink [55835.062378] devlink [55835.062380] i2c_core [55835.062381] i2c_core [55835.062382] i2c_core [55835.062383] i2c_core [55835.062384] i2c_core [55835.062385] fjes [55835.062386] fjes [55835.062387] fjes [55835.062388] fjes [55835.062388] [55835.062389] [55835.062391] [55835.062391] [55835.062392] fjes [55835.062394] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55835.062395] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55835.062397] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55835.062398] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55835.062400] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55835.062401] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55835.062402] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55835.062403] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55835.062404] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55835.062405] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55835.062406] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55835.062407] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55835.062408] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55835.062409] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55835.062410] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55835.062412] RIP: 0010:[] [55835.062413] RIP: 0010:[] [55835.062414] RIP: 0010:[] [55835.062416] RIP: 0010:[] [55835.062418] [] _raw_spin_lock+0x37/0x50 [55835.062419] [] _raw_spin_lock+0x37/0x50 [55835.062421] [] _raw_spin_lock+0x32/0x50 [55835.062423] [] _raw_spin_lock+0x32/0x50 [55835.062424] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55835.062425] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000202 [55835.062427] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000202 [55835.062428] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [55835.062429] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55835.062430] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [55835.062431] RAX: 0000000000000b92 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55835.062432] RAX: 0000000000001734 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55835.062433] RAX: 0000000000002742 RBX: 0000000100000000 RCX: 0000000000000a1e [55835.062434] RAX: 0000000000003db3 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55835.062435] RAX: 0000000000005170 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55835.062436] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55835.062437] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55835.062438] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55835.062439] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55835.062440] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55835.062441] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55835.062442] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55835.062443] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55835.062444] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55835.062445] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55835.062446] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55835.062448] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55835.062449] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55835.062450] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55835.062451] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55835.062452] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55835.062453] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55835.062455] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55835.062455] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55835.062456] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55835.062457] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55835.062459] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55835.062460] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55835.062461] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55835.062462] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55835.062463] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55835.062465] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55835.062466] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55835.062467] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55835.062468] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55835.062469] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55835.062470] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55835.062471] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55835.062472] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55835.062473] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55835.062474] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55835.062475] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55835.062476] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55835.062478] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55835.062478] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55835.062479] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55835.062480] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55835.062482] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55835.062483] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55835.062484] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55835.062484] Stack: [55835.062485] Stack: [55835.062486] Stack: [55835.062487] Stack: [55835.062487] Stack: [55835.062489] ffff88081f64fc98 [55835.062490] ffff8808213b7ac0 [55835.062491] ffff88082aae7ba0 [55835.062491] ffff88082626bd08 [55835.062492] ffff880820b03c28 [55835.062494] ffffffffa08aa7c8 [55835.062495] ffffffffa08aa7c8 [55835.062495] ffffffffa08aa7c8 [55835.062496] ffffffffa08aa7c8 [55835.062497] ffffffffa08aa7c8 [55835.062498] ffffffffa08aa8b6 [55835.062499] ffffffffa08aa8b6 [55835.062501] 000000000000763d [55835.062501] ffff88082af11648 [55835.062502] ffff880820b03c28 [55835.062504] ffff88042dcd1e00 [55835.062505] 0000000000000000 [55835.062506] ffff8803fe4d0400 [55835.062506] ffff8800b484ea00 [55835.062507] [55835.062508] [55835.062509] [55835.062510] [55835.062510] 0000000000000000 [55835.062512] ffff8800b4864200 [55835.062513] ffff88083fb5e000 [55835.062514] 0000000000000000 [55835.062515] 0000000000000000 [55835.062516] ffff88083fb5e000 [55835.062517] ffff88081f64fcd8 [55835.062518] ffff8808213b7b80 [55835.062519] ffff88082aae7c08 [55835.062520] ffff88082626bd70 [55835.062520] ffff880820b03c98 [55835.062521] ffffffffa092e857 [55835.062522] ffffffffa092c9b5 [55835.062523] ffffffffa0922006 [55835.062524] ffffffffa0922006 [55835.062525] ffffffffa092c346 [55835.062526] ffff8800b484d600 [55835.062527] 00000000000000c0 [55835.062528] ffff8804c1676dd0 [55835.062528] ffff88082af10fb0 [55835.062529] [55835.062530] [55835.062531] [55835.062531] [55835.062532] ffff880820b03c60 [55835.062534] ffff88042dcd1e00 [55835.062535] ffffc9000e335900 [55835.062535] ffff8804c1676dd0 [55835.062537] ffff88082af10fb0 [55835.062537] 00000000000000b4 [55835.062539] 0000000000000000 [55835.062540] ffff8808213b7b68 [55835.062540] ffff8804c1676dd0 [55835.062541] ffff88082af10fb0 [55835.062542] 00000001033f89f4 [55835.062543] ffff88041d604a00 [55835.062544] ffffffffa0ebd587 [55835.062545] 0000000000000078 [55835.062546] ffff88082626bd30 [55835.062547] ffff8800b4864274 [55835.062548] 0000000000000000 [55835.062550] ffffffffffffffff [55835.062551] ffff88082aae7c48 [55835.062552] ffff88082626bd30 [55835.062553] [55835.062554] [55835.062555] [55835.062555] [55835.062556] 0000000000000000 [55835.062557] Call Trace: [55835.062558] Call Trace: [55835.062559] Call Trace: [55835.062560] Call Trace: [55835.062561] Call Trace: [55835.062569] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55835.062575] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55835.062580] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55835.062586] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55835.062591] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55835.062599] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55835.062604] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55835.062610] [] lnet_finalize+0xf6/0x690 [lnet] [55835.062616] [] lnet_finalize+0xf6/0x690 [lnet] [55835.062622] [] lnet_post_send_locked+0x446/0x880 [lnet] [55835.062631] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55835.062637] [] lnet_select_pathway+0x235/0x1140 [lnet] [55835.062641] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55835.062645] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55835.062651] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55835.062659] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55835.062663] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55835.062665] [] ? enqueue_task_fair+0x208/0x6c0 [55835.062669] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55835.062675] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55835.062683] [] lnet_finalize+0x1e9/0x690 [lnet] [55835.062688] [] lnet_send+0x51/0x180 [lnet] [55835.062691] [] ? sched_clock_cpu+0x85/0xc0 [55835.062693] [] ? dequeue_task_fair+0x41e/0x660 [55835.062698] [] lnet_finalize+0x1e9/0x690 [lnet] [55835.062704] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55835.062710] [] lnet_finalize+0x3f1/0x690 [lnet] [55835.062716] [] lnet_ni_recv+0xc3/0x320 [lnet] [55835.062718] [] ? wake_up_state+0x20/0x20 [55835.062721] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55835.062727] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55835.062731] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55835.062737] [] lnet_parse+0x945/0xf90 [lnet] [55835.062741] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55835.062745] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55835.062749] [] ? dequeue_task_fair+0x41e/0x660 [55835.062751] [] ? __slab_free+0x81/0x2f0 [55835.062755] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55835.062757] [] kthread+0xcf/0xe0 [55835.062759] [] ? dequeue_task_fair+0x41e/0x660 [55835.062763] [] ? wake_up_state+0x20/0x20 [55835.062769] [] lnet_ni_recv+0xc3/0x320 [lnet] [55835.062771] [] ? __wake_up+0x44/0x50 [55835.062772] [] ? kthread_create_on_node+0x140/0x140 [55835.062774] [] ? sched_clock_cpu+0x85/0xc0 [55835.062780] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55835.062786] [] lnet_parse+0x945/0xf90 [lnet] [55835.062790] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55835.062792] [] ret_from_fork+0x58/0x90 [55835.062793] [] ? wake_up_state+0x20/0x20 [55835.062797] [] kthread+0xcf/0xe0 [55835.062803] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55835.062804] [] ? dequeue_task_fair+0x41e/0x660 [55835.062806] [] ? kthread_create_on_node+0x140/0x140 [55835.062810] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55835.062813] Code: [55835.062814] [] ? kthread_create_on_node+0x140/0x140 [55835.062818] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55835.062820] [] ? wake_up_state+0x20/0x20 [55835.062823] [] kthread+0xcf/0xe0 [55835.062824] 00 [55835.062826] 02 [55835.062829] [] ret_from_fork+0x58/0x90 [55835.062831] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55835.062835] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55835.062838] [] ? kthread_create_on_node+0x140/0x140 [55835.062839] 00 [55835.062842] f0 [55835.062843] [] ? kthread_create_on_node+0x140/0x140 [55835.062847] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55835.062850] [] kthread+0xcf/0xe0 [55835.062852] [] ret_from_fork+0x58/0x90 [55835.062853] 0f [55835.062854] Code: [55835.062856] c1 [55835.062859] [] ? dequeue_task_fair+0x41e/0x660 [55835.062861] [] ? kthread_create_on_node+0x140/0x140 [55835.062862] c1 [55835.062863] [] ? kthread_create_on_node+0x140/0x140 [55835.062864] 07 [55835.062865] 07 [55835.062867] 89 [55835.062868] Code: [55835.062871] [] ? wake_up_state+0x20/0x20 [55835.062873] [] ret_from_fork+0x58/0x90 [55835.062874] 89 [55835.062875] c2 [55835.062875] c2 [55835.062876] c2 [55835.062878] c1 [55835.062879] c1 [55835.062883] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55835.062885] [] ? kthread_create_on_node+0x140/0x140 [55835.062886] c1 [55835.062887] ea [55835.062888] ea [55835.062889] ea [55835.062891] Code: [55835.062892] 10 [55835.062892] 10 [55835.062895] [] kthread+0xcf/0xe0 [55835.062896] 10 [55835.062897] 00 [55835.062898] 66 [55835.062899] 66 [55835.062899] 66 [55835.062901] 02 [55835.062903] 39 [55835.062903] 39 [55835.062906] [] ? kthread_create_on_node+0x140/0x140 [55835.062907] 39 [55835.062908] 00 [55835.062909] c2 [55835.062910] c2 [55835.062911] c2 [55835.062912] f0 [55835.062913] 75 [55835.062914] 75 [55835.062916] [] ret_from_fork+0x58/0x90 [55835.062917] 75 [55835.062918] 0f [55835.062919] 01 [55835.062920] 01 [55835.062921] 01 [55835.062923] c1 [55835.062924] c3 [55835.062925] c3 [55835.062927] [] ? kthread_create_on_node+0x140/0x140 [55835.062928] c3 [55835.062929] 07 [55835.062930] 55 [55835.062931] 55 [55835.062932] 55 [55835.062933] Code: [55835.062933] 89 [55835.062934] 83 [55835.062935] 83 [55835.062936] 83 [55835.062938] c1 [55835.062938] c2 [55835.062939] e2 [55835.062940] e2 [55835.062941] e2 [55835.062942] 07 [55835.062943] c1 [55835.062944] fe [55835.062945] fe [55835.062946] fe [55835.062947] 89 [55835.062948] ea [55835.062949] 0f [55835.062949] 0f [55835.062950] 0f [55835.062952] c2 [55835.062953] 10 [55835.062954] b7 [55835.062955] b7 [55835.062956] b7 [55835.062957] c1 [55835.062958] 66 [55835.062959] f2 [55835.062960] f2 [55835.062961] f2 [55835.062962] ea [55835.062963] 39 [55835.062964] 48 [55835.062965] 48 [55835.062966] 48 [55835.062966] 10 [55835.062967] c2 [55835.062968] 89 [55835.062969] 89 [55835.062970] 89 [55835.062971] 66 [55835.062972] 75 [55835.062973] e5 [55835.062973] e5 [55835.062974] e5 [55835.062975] 39 [55835.062976] 01 [55835.062977] b8 [55835.062977] b8 [55835.062978] b8 [55835.062980] c2 [55835.062981] c3 [55835.062982] 00 [55835.062983] 00 [55835.062984] 00 [55835.062985] 75 [55835.062986] 55 [55835.062987] 80 [55835.062988] 80 [55835.062989] 80 [55835.062990] 01 [55835.062991] 83 [55835.062991] 00 [55835.062992] 00 [55835.062993] 00 [55835.062994] c3 [55835.062995] e2 [55835.062996] 00 [55835.062997] 00 [55835.062998] 00 [55835.062999] 55 [55835.063000] fe [55835.063001] eb [55835.063002] eb [55835.063003] eb [55835.063004] 83 [55835.063005] 0f [55835.063006] 0d [55835.063007] 0d [55835.063008] 0d [55835.063010] e2 [55835.063010] b7 [55835.063011] 66 [55835.063012] 66 [55835.063013] 66 [55835.063014] fe [55835.063015] f2 [55835.063016] 0f [55835.063017] 0f [55835.063018] 0f [55835.063019] 0f [55835.063020] 48 [55835.063021] 1f [55835.063021] 1f [55835.063022] 1f [55835.063024] b7 [55835.063024] 89 [55835.063025] 44 [55835.063026] 44 [55835.063027] 44 [55835.063028] f2 [55835.063029] e5 [55835.063030] 00 [55835.063031] 00 [55835.063032] 00 [55835.063033] 48 [55835.063034] b8 [55835.063035] 00 [55835.063035] 00 [55835.063036] 00 [55835.063038] 89 [55835.063038] 00 [55835.063039] f3 [55835.063040] f3 [55835.063041] f3 [55835.063042] e5 [55835.063043] 80 [55835.063044] 90 [55835.063045] 90 [55835.063046] 90 [55835.063047] b8 [55835.063048] 00 [55835.063049] <83> [55835.063050] 83 [55835.063051] 83 [55835.063052] 00 [55835.063052] 00 [55835.063053] e8 [55835.063054] e8 [55835.063055] e8 [55835.063056] 80 [55835.063058] eb [55835.063059] 01 [55835.063060] 01 [55835.063061] 01 [55835.063062] 00 [55835.063062] 0d [55835.063063] 74 [55835.063064] 74 [55835.063065] 74 [55835.063066] 00 [55835.063067] 66 [55835.063068] 0a [55835.063068] 0a [55835.063070] 0a [55835.063071] eb [55835.063072] 0f [55835.063072] 0f [55835.063073] 0f [55835.063074] <0f> [55835.063075] 0d [55835.063076] 1f [55835.063077] b7 [55835.063078] b7 [55835.063079] b7 [55835.063080] 66 [55835.063081] 44 [55835.063081] 0f [55835.063082] 0f [55835.063083] 0f [55835.063085] 0f [55835.063085] 00 [55835.063086] 66 [55835.063087] <66> [55835.063088] 66 [55835.063089] 1f [55835.063090] 00 [55835.063091] 39 [55835.063092] 39 [55835.063093] 39 [55835.063094] 44 [55835.063095] f3 [55835.063095] ca [55835.063096] ca [55835.063097] ca [55835.063099] 00 [55835.063099] 90 [55835.063100] 75 [55835.063101] 75 [55835.063103] 75 [55835.063104] 00 [55835.063105] <83> [55835.063106] f1 [55835.063107] f1 [55835.063108] f1 [55835.063109] f3 [55835.063110] e8 [55835.063111] 5d [55835.063112] 5d [55835.063113] 5d [55835.063114] 90 [55835.063114] 01 [55835.063115] c3 [55835.063116] c3 [55835.063117] c3 [55835.063118] 83 [55835.063119] 74 [55835.063120] 66 [55835.063120] 66 [55835.063122] 66 [55835.063123] e8 [55835.063123] 0a [55835.063124] 66 [55835.063125] 66 [55835.063126] 66 [55835.063127] 01 [55835.063128] 0f [55835.063129] 66 [55835.063130] 66 [55835.063131] 66 [55835.063132] 74 [55835.063133] b7 [55835.063133] 90 [55835.063134] 90 [55835.063135] 90 [55835.063137] 0a [55835.063137] 0f [55835.063138] 66 [55835.063139] 66 [55835.063140] 66 [55835.063141] <0f> [55835.063142] 66 [55835.063143] 66 [55835.063144] 66 [55835.063145] 66 [55835.063146] b7 [55835.063147] [55835.063147] 39 [55835.063148] 90 [55835.063149] 90 [55835.063151] 0f [55835.063151] ca [55835.063152] eb [55835.063153] eb [55835.063154] 66 [55835.063155] 75 [55835.063155] d9 [55835.063156] d9 [55835.063158] 39 [55835.063158] f1 [55835.063159] 66 [55835.063160] 66 [55835.063162] ca [55835.063162] 5d [55835.063163] 0f [55835.063164] 0f [55835.063165] 75 [55835.063165] c3 [55835.063166] [55835.063167] 1f [55835.063168] f1 [55835.063169] 66 [55835.063170] 44 [55835.063171] 5d [55835.063172] 66 [55835.063172] 00 [55835.063174] c3 [55835.063175] [55835.063176] 66 [55835.063177] 66 [55835.063178] 90 [55835.063179] 66 [55835.063180] 66 [55835.063181] 66 [55835.063182] 66 [55835.063183] [55835.063185] 90 66 66 90 eb d9 66 0f [55839.061418] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [kiblnd_connd:2895] [55839.061435] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55839.061440] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55839.061442] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55839.061442] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55839.061443] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55839.061445] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55839.061445] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55839.061446] RAX: 00000000000040b4 RBX: 0000000000016c40 RCX: 0000000000000a1e [55839.061446] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55839.061447] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55839.061447] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55839.061448] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55839.061448] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55839.061449] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55839.061449] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55839.061450] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55839.061450] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55839.061451] Stack: [55839.061452] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55839.061453] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55839.061454] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55839.061454] Call Trace: [55839.061460] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55839.061465] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55839.061468] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55839.061472] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55839.061473] [] ? lock_timer_base.isra.33+0x2b/0x50 [55839.061474] [] ? try_to_del_timer_sync+0x5e/0x90 [55839.061478] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55839.061479] [] ? schedule_timeout+0x17d/0x2d0 [55839.061483] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55839.061484] [] ? wake_up_state+0x20/0x20 [55839.061488] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55839.061489] [] kthread+0xcf/0xe0 [55839.061491] [] ? kthread_create_on_node+0x140/0x140 [55839.061493] [] ret_from_fork+0x58/0x90 [55839.061494] [] ? kthread_create_on_node+0x140/0x140 [55839.061504] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55847.060593] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [55847.060595] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55846.840380] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 [55847.060597] Modules linked in: [55847.060598] Modules linked in: [55847.060599] ko2iblnd(OE) [55847.060600] ko2iblnd(OE) [55847.060600] lustre(OE) [55847.060602] lustre(OE) [55847.060602] lmv(OE) [55847.060604] lmv(OE) [55847.060605] fld(OE) [55847.060606] fld(OE) [55847.060606] mdc(OE) [55847.060608] mdc(OE) [55847.060609] fid(OE) [55847.060609] fid(OE) [55847.060610] lov(OE) [55847.060611] lov(OE) [55847.060612] ksocklnd(OE) [55847.060613] ksocklnd(OE) [55847.060614] ptlrpc(OE) [55847.060615] ptlrpc(OE) [55847.060615] obdclass(OE) [55847.060616] obdclass(OE) [55847.060617] lnet(OE) [55847.060619] lnet(OE) [55847.060619] libcfs(OE) [55847.060620] libcfs(OE) [55847.060622] rpcrdma [55847.060623] rpcrdma [55847.060623] ib_isert [55847.060624] ib_isert [55847.060625] iscsi_target_mod [55847.060626] iscsi_target_mod [55847.060627] ib_iser [55847.060628] ib_iser [55847.060629] libiscsi [55847.060630] libiscsi [55847.060631] scsi_transport_iscsi [55847.060632] scsi_transport_iscsi [55847.060633] ib_srpt [55847.060634] ib_srpt [55847.060635] target_core_mod [55847.060636] target_core_mod [55847.060637] ib_srp [55847.060638] ib_srp [55847.060639] scsi_transport_srp [55847.060640] scsi_transport_srp [55847.060640] scsi_tgt [55847.060641] scsi_tgt [55847.060642] ib_ipoib [55847.060643] ib_ipoib [55847.060644] rdma_ucm [55847.060645] rdma_ucm [55847.060646] ib_ucm [55847.060647] ib_ucm [55847.060648] ib_uverbs [55847.060649] ib_uverbs [55847.060649] ib_umad [55847.060651] ib_umad [55847.060651] rdma_cm [55847.060652] rdma_cm [55847.060653] ib_cm [55847.060654] ib_cm [55847.060655] iw_cm [55847.060656] iw_cm [55847.060657] mlx4_ib [55847.060658] mlx4_ib [55847.060658] intel_powerclamp [55847.060660] intel_powerclamp [55847.060660] coretemp [55847.060662] coretemp [55847.060662] intel_rapl [55847.060663] intel_rapl [55847.060664] iosf_mbi [55847.060666] iosf_mbi [55847.060666] kvm [55847.060667] kvm [55847.060668] irqbypass [55847.060669] irqbypass [55847.060670] crc32_pclmul [55847.060671] crc32_pclmul [55847.060672] ghash_clmulni_intel [55847.060673] ghash_clmulni_intel [55847.060674] aesni_intel [55847.060675] aesni_intel [55847.060675] ipmi_ssif [55847.060677] ipmi_ssif [55847.060677] lrw [55847.060678] lrw [55847.060679] gf128mul [55847.060680] gf128mul [55847.060681] glue_helper [55847.060682] glue_helper [55847.060683] hfi1 [55847.060684] hfi1 [55847.060685] ipmi_devintf [55847.060686] ipmi_devintf [55847.060686] rdmavt [55847.060687] rdmavt [55847.060688] ablk_helper [55847.060689] ablk_helper [55847.060690] cryptd [55847.060691] cryptd [55847.060692] iTCO_wdt [55847.060693] iTCO_wdt [55847.060694] ipmi_si [55847.060695] ipmi_si [55847.060695] sb_edac [55847.060696] sb_edac [55847.060697] sg [55847.060699] sg [55847.060699] iTCO_vendor_support [55847.060700] iTCO_vendor_support [55847.060701] ib_core [55847.060702] ib_core [55847.060703] ioatdma [55847.060704] ioatdma [55847.060705] i2c_i801 [55847.060706] i2c_i801 [55847.060707] mei_me [55847.060708] mei_me [55847.060709] mei [55847.060710] mei [55847.060710] edac_core [55847.060711] edac_core [55847.060712] ipmi_msghandler [55847.060714] ipmi_msghandler [55847.060714] lpc_ich [55847.060715] lpc_ich [55847.060716] ntb [55847.060717] ntb [55847.060718] wmi [55847.060719] wmi [55847.060720] shpchp [55847.060720] shpchp [55847.060721] pcspkr [55847.060722] pcspkr [55847.060723] nfsd [55847.060724] nfsd [55847.060725] nfs_acl [55847.060726] nfs_acl [55847.060727] lockd [55847.060728] lockd [55847.060729] grace [55847.060730] grace [55847.060731] auth_rpcgss [55847.060732] auth_rpcgss [55847.060733] sunrpc [55847.060734] sunrpc [55847.060735] ip_tables [55847.060736] ip_tables [55847.060737] ext4 [55847.060738] ext4 [55847.060739] mbcache [55847.060739] mbcache [55847.060740] jbd2 [55847.060741] jbd2 [55847.060742] sd_mod [55847.060743] sd_mod [55847.060744] crc_t10dif [55847.060745] crc_t10dif [55847.060746] crct10dif_generic [55847.060747] crct10dif_generic [55847.060747] mlx4_en [55847.060748] mlx4_en [55847.060749] mgag200 [55847.060750] mgag200 [55847.060751] drm_kms_helper [55847.060752] drm_kms_helper [55847.060753] syscopyarea [55847.060754] syscopyarea [55847.060755] sysfillrect [55847.060756] sysfillrect [55847.060757] sysimgblt [55847.060758] sysimgblt [55847.060758] fb_sys_fops [55847.060760] fb_sys_fops [55847.060760] isci [55847.060761] isci [55847.060762] ttm [55847.060763] ttm [55847.060764] igb [55847.060765] igb [55847.060766] crct10dif_pclmul [55847.060767] crct10dif_pclmul [55847.060768] crct10dif_common [55847.060769] crct10dif_common [55847.060770] ahci [55847.060771] ahci [55847.060772] ptp [55847.060773] ptp [55847.060774] crc32c_intel [55847.060775] crc32c_intel [55847.060776] libsas [55847.060777] libsas [55847.060777] libahci [55847.060779] libahci [55847.060779] pps_core [55847.060780] pps_core [55847.060781] drm [55847.060782] drm [55847.060783] mlx4_core [55847.060784] mlx4_core [55847.060785] mpt2sas [55847.060786] mpt2sas [55847.060787] dca [55847.060788] dca [55847.060789] libata [55847.060789] libata [55847.060790] raid_class [55847.060791] raid_class [55847.060792] i2c_algo_bit [55847.060793] i2c_algo_bit [55847.060794] scsi_transport_sas [55847.060795] scsi_transport_sas [55847.060796] devlink [55847.060797] devlink [55847.060798] i2c_core [55847.060799] i2c_core [55847.060800] fjes [55847.060801] fjes [55847.060802] [55847.060803] [55847.060804] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55847.060806] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55847.060807] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55847.060809] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55847.060810] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55847.060813] Workqueue: ib_cm cm_work_handler [ib_cm] [55847.060814] RIP: 0010:[] [55847.060815] [55847.060816] [] _raw_spin_lock+0x32/0x50 [55847.060818] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55847.060819] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [55847.060820] RIP: 0010:[] [55847.060821] RAX: 000000000000751f RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55847.060823] [] _raw_spin_lock+0x37/0x50 [55847.060825] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55847.060826] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000206 [55847.060827] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55847.060829] RAX: 000000000000710a RBX: 0000000000000000 RCX: 0000000000000a1e [55847.060830] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55847.060831] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55847.060832] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55847.060834] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55847.060835] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55847.060836] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55847.060838] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55847.060839] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55847.060840] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55847.060841] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55847.060843] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55847.060844] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55847.060845] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55847.060846] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55847.060847] Stack: [55847.060849] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55847.060850] ffff8807f639fd88 [55847.060851] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55847.060852] ffffffffa08aa831 [55847.060853] Stack: [55847.060854] 0000000200000002 [55847.060855] ffff8803fea77b00 [55847.060855] ffff8807f63a2980 [55847.060856] ffffffffa08aa7c8 [55847.060857] [55847.060858] ffff8803fea77bd0 [55847.060859] 0000000000000000 [55847.060860] 00050000c0a80173 [55847.060860] ffff8807f639fdc0 [55847.060861] [55847.060862] ffffffffa0942e30 [55847.060863] ffff88083fb5de00 [55847.060864] 00000000c0086540 [55847.060865] ffff8803fea77b18 [55847.060866] [55847.060867] ffffffffa0916e4b [55847.060867] ffff8807f63a2900 [55847.060868] ffff8803fea77c68 [55847.060869] 0000000000000002 [55847.060870] [55847.060871] 0000000000650aa0 [55847.060872] ffff8803fea77bc8 [55847.060873] 0000000000000000 [55847.060874] ffffffffa0ebfc24 [55847.060874] [55847.060875] ffff88041aa8a880 [55847.060876] Call Trace: [55847.060877] ffff8803fea77bd0 [55847.060878] [55847.060879] [55847.060885] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55847.060886] Call Trace: [55847.060887] [55847.060888] [55847.060895] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55847.060900] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55847.060901] [55847.060902] [55847.060907] [] LNetCtl+0x116/0x800 [lnet] [55847.060913] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55847.060914] [55847.060915] [55847.060921] [] lnet_ioctl+0x60/0x250 [lnet] [55847.060925] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55847.060926] [55847.060927] [55847.060933] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55847.060935] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55847.060936] [55847.060937] [55847.060942] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55847.060946] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55847.060947] [55847.060948] [55847.060950] [] do_vfs_ioctl+0x2d5/0x4b0 [55847.060952] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55847.060953] [55847.060954] [55847.060956] [] ? __fd_install+0x47/0x60 [55847.060959] [] cma_req_handler+0x227/0x570 [rdma_cm] [55847.060960] [55847.060961] [55847.060963] [] SyS_ioctl+0xa1/0xc0 [55847.060965] [] cm_process_work+0x25/0x140 [ib_cm] [55847.060966] [55847.060967] [55847.060969] [] system_call_fastpath+0x16/0x1b [55847.060971] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55847.060972] Code: [55847.060973] [55847.060974] 00 [55847.060976] [] cm_work_handler+0x185/0x1484 [ib_cm] [55847.060978] 02 [55847.060979] [55847.060980] 00 [55847.060981] [] process_one_work+0x17b/0x470 [55847.060982] f0 [55847.060983] [55847.060984] 0f [55847.060985] [] worker_thread+0x126/0x410 [55847.060986] c1 [55847.060987] [55847.060987] 07 [55847.060989] [] ? rescuer_thread+0x460/0x460 [55847.060990] 89 [55847.060990] [55847.060991] c2 [55847.060993] [] kthread+0xcf/0xe0 [55847.060994] c1 [55847.060995] [55847.060996] ea [55847.060997] [] ? kthread_create_on_node+0x140/0x140 [55847.060998] 10 [55847.060999] [55847.061000] 66 [55847.061002] [] ret_from_fork+0x58/0x90 [55847.061003] 39 [55847.061004] [55847.061004] c2 [55847.061006] [] ? kthread_create_on_node+0x140/0x140 [55847.061007] 75 [55847.061008] Code: [55847.061009] 01 [55847.061010] c1 [55847.061011] c3 [55847.061012] 07 [55847.061013] 55 [55847.061014] 89 [55847.061014] 83 [55847.061015] c2 [55847.061016] e2 [55847.061017] c1 [55847.061018] fe [55847.061019] ea [55847.061020] 0f [55847.061020] 10 [55847.061021] b7 [55847.061022] 66 [55847.061023] f2 [55847.061024] 39 [55847.061024] 48 [55847.061025] c2 [55847.061026] 89 [55847.061027] 75 [55847.061027] e5 [55847.061028] 01 [55847.061029] b8 [55847.061030] c3 [55847.061031] 00 [55847.061032] 55 [55847.061032] 80 [55847.061034] 83 [55847.061034] 00 [55847.061035] e2 [55847.061036] 00 [55847.061037] fe [55847.061037] eb [55847.061038] 0f [55847.061039] 0d [55847.061040] b7 [55847.061041] 66 [55847.061042] f2 [55847.061042] 0f [55847.061043] 48 [55847.061044] 1f [55847.061045] 89 [55847.061046] 44 [55847.061047] e5 [55847.061047] 00 [55847.061048] b8 [55847.061049] 00 [55847.061050] 00 [55847.061051] f3 [55847.061052] 80 [55847.061052] 90 [55847.061053] 00 [55847.061054] <83> [55847.061055] 00 [55847.061056] e8 [55847.061057] eb [55847.061058] 01 [55847.061059] 0d [55847.061060] 74 [55847.061061] 66 [55847.061061] 0a [55847.061062] 0f [55847.061063] 0f [55847.061064] 1f [55847.061065] b7 [55847.061066] 44 [55847.061066] 0f [55847.061067] 00 [55847.061068] 66 [55847.061069] 00 [55847.061070] 39 [55847.061070] f3 [55847.061071] ca [55847.061072] 90 [55847.061073] 75 [55847.061074] 83 [55847.061075] f1 [55847.061076] e8 [55847.061076] 5d [55847.061077] 01 [55847.061078] c3 [55847.061079] 74 [55847.061080] 66 [55847.061081] 0a [55847.061081] 66 [55847.061082] <0f> [55847.061083] 66 [55847.061084] b7 [55847.061085] 90 [55847.061085] 0f [55847.061086] 66 [55847.061087] 66 [55847.061088] 66 [55847.061089] 39 [55847.061090] [55847.061091] ca [55847.061091] 75 [55847.061091] f1 [55847.061091] 5d [55847.061092] c3 [55847.061092] 66 [55847.061092] 66 [55847.061092] 66 [55847.061092] 90 [55847.061093] 66 [55847.061093] 66 [55847.061093] 90 [55847.061093] eb [55847.061093] d9 [55847.061094] 66 [55847.061094] 0f [55847.061094] [55849.439244] sd_mod [55849.446851] crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55849.535125] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55849.571712] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55849.609462] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55849.634928] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55849.662664] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55849.680966] RAX: 0000000000006245 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55849.705226] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55849.729514] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55849.753809] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55849.778093] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55849.802406] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55849.829844] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55849.849591] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55849.873917] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55849.898200] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55849.922488] Stack: [55849.929891] ffff8808217bbcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba3200 [55849.955128] 0000000000000001 ffff8808217bbd08 ffffffffa09217bb ffff8804c1ba3200 [55849.980323] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff8808217bbd70 [55850.005517] Call Trace: [55850.014366] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55850.036383] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55850.059296] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55850.081606] [] lnet_finalize+0x1e9/0x690 [lnet] [55850.102762] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55850.125345] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55850.148813] [] ? dequeue_task_fair+0x41e/0x660 [55850.169710] [] ? wake_up_state+0x20/0x20 [55850.188856] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55850.211732] [] kthread+0xcf/0xe0 [55850.228622] [] ? kthread_create_on_node+0x140/0x140 [55850.250955] [] ret_from_fork+0x58/0x90 [55850.269563] [] ? kthread_create_on_node+0x140/0x140 [55850.291850] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55851.060181] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kiblnd_sd_01_02:2902] [55851.060183] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55851.060202] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55851.060208] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55851.060209] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55851.060210] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55851.060210] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55851.060212] RIP: 0010:[] [] _raw_spin_lock+0x30/0x50 [55851.060213] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [55851.060213] RAX: 0000000000005a32 RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55851.060214] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55851.060215] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55851.060216] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55851.060216] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55851.060217] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55851.060218] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55851.060218] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55851.060219] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55851.060219] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55851.060220] Stack: [55851.060221] ffff8808262efac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55851.060222] ffff88083fb5e000 ffff8808262efb80 ffffffffa092c9b5 00000000000001a8 [55851.060223] ffffc9000e331730 ffff8808262efb68 ffffffffa0ebd587 ffffffffffffffff [55851.060223] Call Trace: [55851.060229] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55851.060233] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55851.060239] [] lnet_select_pathway+0x235/0x1140 [lnet] [55851.060243] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55851.060249] [] lnet_send+0x51/0x180 [lnet] [55851.060254] [] lnet_finalize+0x3f1/0x690 [lnet] [55851.060258] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55851.060260] [] ? __slab_free+0x81/0x2f0 [55851.060265] [] lnet_ni_recv+0xc3/0x320 [lnet] [55851.060271] [] lnet_parse+0x945/0xf90 [lnet] [55851.060274] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55851.060277] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55851.060281] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55851.060282] [] ? dequeue_task_fair+0x41e/0x660 [55851.060284] [] ? wake_up_state+0x20/0x20 [55851.060287] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55851.060289] [] kthread+0xcf/0xe0 [55851.060290] [] ? kthread_create_on_node+0x140/0x140 [55851.060292] [] ret_from_fork+0x58/0x90 [55851.060294] [] ? kthread_create_on_node+0x140/0x140 [55851.060304] Code: b8 00 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 90 83 e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 [55852.396678] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55852.636858] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55852.708036] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55852.744654] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55852.782429] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55852.807911] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55852.835745] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000202 [55852.854131] RAX: 00000000000027bc RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55852.878540] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55852.902935] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55852.927332] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55852.951716] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55852.976105] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55853.003632] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55853.023431] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55853.047844] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55853.072256] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55853.096597] Stack: [55853.104117] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [55853.129436] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [55853.154718] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [55853.180010] Call Trace: [55853.188941] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55853.211087] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55853.234046] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55853.256433] [] lnet_finalize+0x1e9/0x690 [lnet] [55853.277721] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55853.300413] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55853.323984] [] ? dequeue_task_fair+0x41e/0x660 [55853.344961] [] ? wake_up_state+0x20/0x20 [55853.364200] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55853.387186] [] kthread+0xcf/0xe0 [55853.404133] [] ? kthread_create_on_node+0x140/0x140 [55853.426535] [] ret_from_fork+0x58/0x90 [55853.445243] [] ? kthread_create_on_node+0x140/0x140 [55853.467634] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55863.058946] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [55863.058947] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [55863.058949] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55863.058950] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55863.058951] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55863.058953] Modules linked in: [55863.058954] Modules linked in: [55863.058955] Modules linked in: [55863.058957] Modules linked in: [55863.058957] ko2iblnd(OE) [55863.058958] ko2iblnd(OE) [55863.058958] ko2iblnd(OE) [55863.058959] ko2iblnd(OE) [55863.058959] lustre(OE) [55863.058960] lustre(OE) [55863.058960] lustre(OE) [55863.058961] lustre(OE) [55863.058961] lmv(OE) [55863.058962] lmv(OE) [55863.058962] lmv(OE) [55863.058963] lmv(OE) [55863.058963] fld(OE) [55863.058963] fld(OE) [55863.058964] fld(OE) [55863.058964] fld(OE) [55863.058965] mdc(OE) [55863.058965] mdc(OE) [55863.058966] mdc(OE) [55863.058966] mdc(OE) [55863.058966] fid(OE) [55863.058967] fid(OE) [55863.058967] fid(OE) [55863.058968] fid(OE) [55863.058968] lov(OE) [55863.058969] lov(OE) [55863.058969] lov(OE) [55863.058970] lov(OE) [55863.058970] ksocklnd(OE) [55863.058971] ksocklnd(OE) [55863.058971] ksocklnd(OE) [55863.058972] ksocklnd(OE) [55863.058972] ptlrpc(OE) [55863.058972] ptlrpc(OE) [55863.058973] ptlrpc(OE) [55863.058973] ptlrpc(OE) [55863.058974] obdclass(OE) [55863.058974] obdclass(OE) [55863.058975] obdclass(OE) [55863.058975] obdclass(OE) [55863.058976] lnet(OE) [55863.058976] lnet(OE) [55863.058977] lnet(OE) [55863.058977] lnet(OE) [55863.058978] libcfs(OE) [55863.058978] libcfs(OE) [55863.058978] libcfs(OE) [55863.058979] libcfs(OE) [55863.058979] rpcrdma [55863.058980] rpcrdma [55863.058980] rpcrdma [55863.058981] rpcrdma [55863.058981] ib_isert [55863.058982] ib_isert [55863.058982] ib_isert [55863.058983] ib_isert [55863.058983] iscsi_target_mod [55863.058983] iscsi_target_mod [55863.058984] iscsi_target_mod [55863.058984] iscsi_target_mod [55863.058985] ib_iser [55863.058985] ib_iser [55863.058986] ib_iser [55863.058986] ib_iser [55863.058987] libiscsi [55863.058987] libiscsi [55863.058988] libiscsi [55863.058988] libiscsi [55863.058989] scsi_transport_iscsi [55863.058989] scsi_transport_iscsi [55863.058990] scsi_transport_iscsi [55863.058990] scsi_transport_iscsi [55863.058990] ib_srpt [55863.058991] ib_srpt [55863.058991] ib_srpt [55863.058992] ib_srpt [55863.058992] target_core_mod [55863.058993] target_core_mod [55863.058993] target_core_mod [55863.058994] target_core_mod [55863.058994] ib_srp [55863.058995] ib_srp [55863.058995] ib_srp [55863.058996] ib_srp [55863.058996] scsi_transport_srp [55863.058997] scsi_transport_srp [55863.058997] scsi_transport_srp [55863.058998] scsi_transport_srp [55863.058998] scsi_tgt [55863.058998] scsi_tgt [55863.058999] scsi_tgt [55863.058999] scsi_tgt [55863.059000] ib_ipoib [55863.059000] ib_ipoib [55863.059001] ib_ipoib [55863.059001] ib_ipoib [55863.059002] rdma_ucm [55863.059002] rdma_ucm [55863.059002] rdma_ucm [55863.059003] rdma_ucm [55863.059003] ib_ucm [55863.059004] ib_ucm [55863.059004] ib_ucm [55863.059005] ib_ucm [55863.059005] ib_uverbs [55863.059006] ib_uverbs [55863.059006] ib_uverbs [55863.059007] ib_uverbs [55863.059007] ib_umad [55863.059008] ib_umad [55863.059008] ib_umad [55863.059008] ib_umad [55863.059009] rdma_cm [55863.059009] rdma_cm [55863.059010] rdma_cm [55863.059010] rdma_cm [55863.059011] ib_cm [55863.059011] ib_cm [55863.059012] ib_cm [55863.059012] ib_cm [55863.059013] iw_cm [55863.059013] iw_cm [55863.059014] iw_cm [55863.059014] iw_cm [55863.059014] mlx4_ib [55863.059015] mlx4_ib [55863.059015] mlx4_ib [55863.059016] mlx4_ib [55863.059016] intel_powerclamp [55863.059017] intel_powerclamp [55863.059017] intel_powerclamp [55863.059018] intel_powerclamp [55863.059018] coretemp [55863.059019] coretemp [55863.059019] coretemp [55863.059019] coretemp [55863.059020] intel_rapl [55863.059020] intel_rapl [55863.059021] intel_rapl [55863.059021] intel_rapl [55863.059022] iosf_mbi [55863.059022] iosf_mbi [55863.059023] iosf_mbi [55863.059023] iosf_mbi [55863.059024] kvm [55863.059024] kvm [55863.059025] kvm [55863.059025] kvm [55863.059025] irqbypass [55863.059026] irqbypass [55863.059026] irqbypass [55863.059027] irqbypass [55863.059027] crc32_pclmul [55863.059028] crc32_pclmul [55863.059028] crc32_pclmul [55863.059029] crc32_pclmul [55863.059029] ghash_clmulni_intel [55863.059030] ghash_clmulni_intel [55863.059030] ghash_clmulni_intel [55863.059031] ghash_clmulni_intel [55863.059031] aesni_intel [55863.059031] aesni_intel [55863.059032] aesni_intel [55863.059032] aesni_intel [55863.059033] ipmi_ssif [55863.059033] ipmi_ssif [55863.059034] ipmi_ssif [55863.059034] ipmi_ssif [55863.059035] lrw [55863.059035] lrw [55863.059036] lrw [55863.059036] lrw [55863.059036] gf128mul [55863.059037] gf128mul [55863.059037] gf128mul [55863.059038] gf128mul [55863.059038] glue_helper [55863.059039] glue_helper [55863.059039] glue_helper [55863.059040] glue_helper [55863.059040] hfi1 [55863.059041] hfi1 [55863.059041] hfi1 [55863.059042] hfi1 [55863.059042] ipmi_devintf [55863.059043] ipmi_devintf [55863.059043] ipmi_devintf [55863.059043] ipmi_devintf [55863.059044] rdmavt [55863.059044] rdmavt [55863.059045] rdmavt [55863.059045] rdmavt [55863.059046] ablk_helper [55863.059046] ablk_helper [55863.059047] ablk_helper [55863.059047] ablk_helper [55863.059048] cryptd [55863.059048] cryptd [55863.059049] cryptd [55863.059049] cryptd [55863.059050] iTCO_wdt [55863.059050] iTCO_wdt [55863.059050] iTCO_wdt [55863.059051] iTCO_wdt [55863.059051] ipmi_si [55863.059052] ipmi_si [55863.059052] ipmi_si [55863.059053] ipmi_si [55863.059053] sb_edac [55863.059053] sb_edac [55863.059054] sb_edac [55863.059054] sb_edac [55863.059055] sg [55863.059055] sg [55863.059056] sg [55863.059056] sg [55863.059057] iTCO_vendor_support [55863.059057] iTCO_vendor_support [55863.059058] iTCO_vendor_support [55863.059058] iTCO_vendor_support [55863.059059] ib_core [55863.059059] ib_core [55863.059059] ib_core [55863.059060] ib_core [55863.059060] ioatdma [55863.059061] ioatdma [55863.059061] ioatdma [55863.059062] ioatdma [55863.059062] i2c_i801 [55863.059063] i2c_i801 [55863.059063] i2c_i801 [55863.059063] i2c_i801 [55863.059064] mei_me [55863.059064] mei_me [55863.059065] mei_me [55863.059065] mei_me [55863.059066] mei [55863.059066] mei [55863.059067] mei [55863.059067] mei [55863.059068] edac_core [55863.059068] edac_core [55863.059069] edac_core [55863.059069] edac_core [55863.059069] ipmi_msghandler [55863.059070] ipmi_msghandler [55863.059070] ipmi_msghandler [55863.059071] ipmi_msghandler [55863.059071] lpc_ich [55863.059072] lpc_ich [55863.059072] lpc_ich [55863.059073] lpc_ich [55863.059073] ntb [55863.059074] ntb [55863.059074] ntb [55863.059074] ntb [55863.059075] wmi [55863.059075] wmi [55863.059076] wmi [55863.059076] wmi [55863.059077] shpchp [55863.059077] shpchp [55863.059078] shpchp [55863.059078] shpchp [55863.059079] pcspkr [55863.059079] pcspkr [55863.059079] pcspkr [55863.059080] pcspkr [55863.059080] nfsd [55863.059081] nfsd [55863.059081] nfsd [55863.059082] nfsd [55863.059082] nfs_acl [55863.059083] nfs_acl [55863.059083] nfs_acl [55863.059084] nfs_acl [55863.059084] lockd [55863.059084] lockd [55863.059085] lockd [55863.059085] lockd [55863.059086] grace [55863.059086] grace [55863.059087] grace [55863.059087] grace [55863.059088] auth_rpcgss [55863.059088] auth_rpcgss [55863.059089] auth_rpcgss [55863.059089] auth_rpcgss [55863.059090] sunrpc [55863.059090] sunrpc [55863.059090] sunrpc [55863.059091] sunrpc [55863.059091] ip_tables [55863.059092] ip_tables [55863.059092] ip_tables [55863.059093] ip_tables [55863.059093] ext4 [55863.059094] ext4 [55863.059094] ext4 [55863.059094] ext4 [55863.059095] mbcache [55863.059095] mbcache [55863.059096] mbcache [55863.059096] mbcache [55863.059097] jbd2 [55863.059097] jbd2 [55863.059098] jbd2 [55863.059098] jbd2 [55863.059098] sd_mod [55863.059099] sd_mod [55863.059099] sd_mod [55863.059100] sd_mod [55863.059100] crc_t10dif [55863.059101] crc_t10dif [55863.059101] crc_t10dif [55863.059102] crc_t10dif [55863.059102] crct10dif_generic [55863.059103] crct10dif_generic [55863.059103] crct10dif_generic [55863.059103] crct10dif_generic [55863.059104] mlx4_en [55863.059104] mlx4_en [55863.059105] mlx4_en [55863.059105] mlx4_en [55863.059106] mgag200 [55863.059106] mgag200 [55863.059107] mgag200 [55863.059107] mgag200 [55863.059108] drm_kms_helper [55863.059108] drm_kms_helper [55863.059108] drm_kms_helper [55863.059109] drm_kms_helper [55863.059109] syscopyarea [55863.059110] syscopyarea [55863.059110] syscopyarea [55863.059111] syscopyarea [55863.059111] sysfillrect [55863.059112] sysfillrect [55863.059112] sysfillrect [55863.059113] sysfillrect [55863.059113] sysimgblt [55863.059113] sysimgblt [55863.059114] sysimgblt [55863.059114] sysimgblt [55863.059115] fb_sys_fops [55863.059115] fb_sys_fops [55863.059116] fb_sys_fops [55863.059116] fb_sys_fops [55863.059117] isci [55863.059117] isci [55863.059118] isci [55863.059118] isci [55863.059119] ttm [55863.059119] ttm [55863.059119] ttm [55863.059120] ttm [55863.059120] igb [55863.059121] igb [55863.059121] igb [55863.059122] igb [55863.059122] crct10dif_pclmul [55863.059123] crct10dif_pclmul [55863.059123] crct10dif_pclmul [55863.059124] crct10dif_pclmul [55863.059124] crct10dif_common [55863.059125] crct10dif_common [55863.059125] crct10dif_common [55863.059125] crct10dif_common [55863.059126] ahci [55863.059126] ahci [55863.059127] ahci [55863.059127] ahci [55863.059128] ptp [55863.059128] ptp [55863.059129] ptp [55863.059129] ptp [55863.059130] crc32c_intel [55863.059130] crc32c_intel [55863.059131] crc32c_intel [55863.059131] crc32c_intel [55863.059131] libsas [55863.059132] libsas [55863.059132] libsas [55863.059133] libsas [55863.059133] libahci [55863.059134] libahci [55863.059134] libahci [55863.059135] libahci [55863.059135] pps_core [55863.059135] pps_core [55863.059136] pps_core [55863.059136] pps_core [55863.059137] drm [55863.059137] drm [55863.059138] drm [55863.059138] drm [55863.059139] mlx4_core [55863.059139] mlx4_core [55863.059140] mlx4_core [55863.059140] mlx4_core [55863.059140] mpt2sas [55863.059141] mpt2sas [55863.059141] mpt2sas [55863.059142] mpt2sas [55863.059142] dca [55863.059143] dca [55863.059143] dca [55863.059144] dca [55863.059144] libata [55863.059145] libata [55863.059145] libata [55863.059146] libata [55863.059146] raid_class [55863.059146] raid_class [55863.059147] raid_class [55863.059147] raid_class [55863.059148] i2c_algo_bit [55863.059148] i2c_algo_bit [55863.059149] i2c_algo_bit [55863.059149] i2c_algo_bit [55863.059150] scsi_transport_sas [55863.059150] scsi_transport_sas [55863.059151] scsi_transport_sas [55863.059151] scsi_transport_sas [55863.059152] devlink [55863.059152] devlink [55863.059152] devlink [55863.059153] devlink [55863.059153] i2c_core [55863.059154] i2c_core [55863.059154] i2c_core [55863.059155] i2c_core [55863.059155] fjes [55863.059156] fjes [55863.059156] fjes [55863.059157] [55863.059157] [55863.059157] [55863.059158] fjes [55863.059159] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55863.059160] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55863.059161] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55863.059162] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55863.059162] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55863.059163] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55863.059163] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55863.059164] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55863.059165] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55863.059165] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55863.059166] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55863.059167] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55863.059168] RIP: 0010:[] [55863.059169] RIP: 0010:[] [55863.059170] RIP: 0010:[] [55863.059171] [] _raw_spin_lock+0x32/0x50 [55863.059172] [] _raw_spin_lock+0x32/0x50 [55863.059173] [] _raw_spin_lock+0x3a/0x50 [55863.059175] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55863.059175] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [55863.059176] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55863.059176] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000206 [55863.059177] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [55863.059178] RAX: 0000000000005c03 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55863.059178] RAX: 0000000000003fa4 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55863.059179] RAX: 0000000000006e35 RBX: 0000000100000000 RCX: 0000000000000a1e [55863.059180] RAX: 0000000000006ca0 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55863.059180] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55863.059181] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55863.059182] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55863.059182] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55863.059183] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55863.059184] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55863.059184] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55863.059185] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55863.059186] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55863.059186] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55863.059187] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55863.059187] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55863.059188] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55863.059189] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55863.059189] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55863.059190] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55863.059191] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55863.059191] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55863.059192] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55863.059193] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55863.059194] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55863.059194] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55863.059195] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55863.059196] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55863.059196] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55863.059197] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55863.059198] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55863.059198] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55863.059199] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55863.059200] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55863.059200] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55863.059201] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55863.059201] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55863.059202] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55863.059203] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55863.059203] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55863.059204] Stack: [55863.059204] Stack: [55863.059204] Stack: [55863.059205] Stack: [55863.059205] ffff88081f64fc98 [55863.059206] ffff88082626bd08 [55863.059206] ffff88082aae7ba0 [55863.059207] ffff880820b03c28 [55863.059207] ffffffffa08aa7c8 [55863.059208] ffffffffa08aa7c8 [55863.059208] ffffffffa08aa7c8 [55863.059209] ffffffffa08aa7c8 [55863.059209] ffffffffa08aa8b6 [55863.059210] ffff88082af11648 [55863.059210] 000000000000763d [55863.059211] ffff880820b03c28 [55863.059211] ffff88042dcd1e00 [55863.059212] ffff8800b484ea00 [55863.059212] ffff8803fe4d0400 [55863.059212] [55863.059213] [55863.059213] [55863.059213] 0000000000000000 [55863.059214] ffff8800b4864200 [55863.059215] 0000000000000000 [55863.059215] 0000000000000000 [55863.059216] ffff88083fb5e000 [55863.059216] ffff88081f64fcd8 [55863.059216] ffff88082626bd70 [55863.059217] ffff88082aae7c08 [55863.059217] ffff880820b03c98 [55863.059218] ffffffffa092e857 [55863.059218] ffffffffa0922006 [55863.059219] ffffffffa0922006 [55863.059219] ffffffffa092c346 [55863.059220] ffff8800b484d600 [55863.059220] ffff88082af10fb0 [55863.059221] ffff8804c1676dd0 [55863.059221] [55863.059221] [55863.059222] [55863.059222] ffff880820b03c60 [55863.059223] ffff88042dcd1e00 [55863.059223] ffff88082af10fb0 [55863.059224] ffff8804c1676dd0 [55863.059224] 00000000000000b4 [55863.059225] 0000000000000000 [55863.059225] ffff88082af10fb0 [55863.059226] ffff8804c1676dd0 [55863.059226] 00000001033f89f4 [55863.059227] ffff88041d604a00 [55863.059227] ffff88082626bd30 [55863.059228] 0000000000000078 [55863.059228] ffff8800b4864274 [55863.059229] 0000000000000000 [55863.059229] ffff88082626bd30 [55863.059229] ffff88082aae7c48 [55863.059230] [55863.059230] [55863.059230] [55863.059231] 0000000000000000 [55863.059231] Call Trace: [55863.059232] Call Trace: [55863.059232] Call Trace: [55863.059232] Call Trace: [55863.059238] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55863.059244] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55863.059249] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55863.059254] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55863.059259] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55863.059265] [] lnet_finalize+0xf6/0x690 [lnet] [55863.059271] [] lnet_finalize+0xf6/0x690 [lnet] [55863.059277] [] lnet_post_send_locked+0x446/0x880 [lnet] [55863.059283] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55863.059287] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55863.059291] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55863.059297] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55863.059302] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55863.059305] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55863.059307] [] ? enqueue_task_fair+0x208/0x6c0 [55863.059313] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55863.059318] [] lnet_finalize+0x1e9/0x690 [lnet] [55863.059319] [] ? dequeue_task_fair+0x41e/0x660 [55863.059321] [] ? sched_clock_cpu+0x85/0xc0 [55863.059326] [] lnet_finalize+0x1e9/0x690 [lnet] [55863.059330] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55863.059331] [] ? wake_up_state+0x20/0x20 [55863.059337] [] lnet_ni_recv+0xc3/0x320 [lnet] [55863.059341] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55863.059345] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55863.059348] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55863.059354] [] lnet_parse+0x945/0xf90 [lnet] [55863.059358] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55863.059359] [] ? dequeue_task_fair+0x41e/0x660 [55863.059361] [] kthread+0xcf/0xe0 [55863.059364] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55863.059366] [] ? dequeue_task_fair+0x41e/0x660 [55863.059367] [] ? wake_up_state+0x20/0x20 [55863.059369] [] ? kthread_create_on_node+0x140/0x140 [55863.059371] [] ? __wake_up+0x44/0x50 [55863.059372] [] ? sched_clock_cpu+0x85/0xc0 [55863.059376] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55863.059378] [] ret_from_fork+0x58/0x90 [55863.059382] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55863.059383] [] ? wake_up_state+0x20/0x20 [55863.059385] [] kthread+0xcf/0xe0 [55863.059386] [] ? kthread_create_on_node+0x140/0x140 [55863.059388] [] ? dequeue_task_fair+0x41e/0x660 [55863.059391] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55863.059393] [] ? kthread_create_on_node+0x140/0x140 [55863.059393] Code: [55863.059394] 00 [55863.059396] [] ? wake_up_state+0x20/0x20 [55863.059397] [] kthread+0xcf/0xe0 [55863.059399] [] ret_from_fork+0x58/0x90 [55863.059399] 02 [55863.059400] 00 [55863.059404] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55863.059405] [] ? kthread_create_on_node+0x140/0x140 [55863.059407] [] ? kthread_create_on_node+0x140/0x140 [55863.059407] f0 [55863.059408] 0f [55863.059410] [] kthread+0xcf/0xe0 [55863.059411] [] ret_from_fork+0x58/0x90 [55863.059412] Code: [55863.059412] c1 [55863.059413] 00 [55863.059414] 07 [55863.059415] [] ? kthread_create_on_node+0x140/0x140 [55863.059417] [] ? kthread_create_on_node+0x140/0x140 [55863.059417] 02 [55863.059417] 89 [55863.059418] 00 [55863.059418] Code: [55863.059419] c2 [55863.059421] [] ret_from_fork+0x58/0x90 [55863.059421] f0 [55863.059422] 00 [55863.059422] c1 [55863.059422] 0f [55863.059423] 02 [55863.059423] ea [55863.059425] [] ? kthread_create_on_node+0x140/0x140 [55863.059425] c1 [55863.059426] 00 [55863.059426] 10 [55863.059427] 07 [55863.059427] Code: [55863.059427] f0 [55863.059428] 66 [55863.059428] 89 [55863.059429] c2 [55863.059429] 0f [55863.059430] 39 [55863.059430] c2 [55863.059430] c1 [55863.059431] c1 [55863.059431] c2 [55863.059432] c1 [55863.059432] ea [55863.059433] 07 [55863.059433] 75 [55863.059433] ea [55863.059434] 10 [55863.059434] 89 [55863.059435] 01 [55863.059435] 10 [55863.059436] 66 [55863.059436] c2 [55863.059436] c3 [55863.059437] 66 [55863.059437] 39 [55863.059438] c1 [55863.059438] 55 [55863.059439] 39 [55863.059439] c2 [55863.059439] ea [55863.059440] 83 [55863.059440] c2 [55863.059441] 75 [55863.059441] 10 [55863.059442] e2 [55863.059442] 75 [55863.059442] 01 [55863.059443] 66 [55863.059443] fe [55863.059444] 01 [55863.059444] c3 [55863.059445] 39 [55863.059445] 0f [55863.059445] c3 [55863.059446] 55 [55863.059446] c2 [55863.059447] b7 [55863.059447] 55 [55863.059448] 83 [55863.059448] 75 [55863.059448] f2 [55863.059449] 83 [55863.059449] e2 [55863.059450] 01 [55863.059450] 48 [55863.059451] e2 [55863.059451] fe [55863.059451] c3 [55863.059452] 89 [55863.059452] fe [55863.059453] 0f [55863.059453] 55 [55863.059454] e5 [55863.059454] 0f [55863.059455] b7 [55863.059455] 83 [55863.059455] b8 [55863.059456] b7 [55863.059456] f2 [55863.059457] e2 [55863.059457] 00 [55863.059458] f2 [55863.059458] 48 [55863.059458] fe [55863.059459] 80 [55863.059459] 48 [55863.059460] 89 [55863.059460] 0f [55863.059461] 00 [55863.059461] 89 [55863.059461] e5 [55863.059462] b7 [55863.059462] 00 [55863.059463] e5 [55863.059463] b8 [55863.059464] f2 [55863.059464] eb [55863.059464] b8 [55863.059465] 00 [55863.059465] 48 [55863.059466] 0d [55863.059466] 00 [55863.059467] 80 [55863.059467] 89 [55863.059467] 66 [55863.059468] 80 [55863.059468] 00 [55863.059469] e5 [55863.059469] 0f [55863.059470] 00 [55863.059470] 00 [55863.059470] b8 [55863.059471] 1f [55863.059471] 00 [55863.059472] eb [55863.059472] 00 [55863.059473] 44 [55863.059473] eb [55863.059473] 0d [55863.059474] 80 [55863.059474] 00 [55863.059475] 0d [55863.059475] 66 [55863.059476] 00 [55863.059476] 00 [55863.059476] 66 [55863.059477] 0f [55863.059477] 00 [55863.059478] f3 [55863.059478] 0f [55863.059479] 1f [55863.059479] eb [55863.059480] 90 [55863.059480] 1f [55863.059480] 44 [55863.059481] 0d [55863.059481] <83> [55863.059482] 44 [55863.059482] 00 [55863.059483] 66 [55863.059483] e8 [55863.059484] 00 [55863.059484] 00 [55863.059484] 0f [55863.059485] 01 [55863.059485] 00 [55863.059486] f3 [55863.059486] 1f [55863.059487] 74 [55863.059487] f3 [55863.059487] 90 [55863.059488] 44 [55863.059488] 0a [55863.059489] 90 [55863.059489] 83 [55863.059490] 00 [55863.059490] 0f [55863.059491] <83> [55863.059491] e8 [55863.059491] 00 [55863.059492] b7 [55863.059492] e8 [55863.059493] 01 [55863.059493] f3 [55863.059494] 0f [55863.059494] 01 [55863.059494] 74 [55863.059495] 90 [55863.059495] 66 [55863.059496] 74 [55863.059496] 0a [55863.059497] <83> [55863.059497] 39 [55863.059498] 0a [55863.059498] 0f [55863.059498] e8 [55863.059499] ca [55863.059499] 0f [55863.059500] b7 [55863.059500] 01 [55863.059501] 75 [55863.059501] b7 [55863.059501] 0f [55863.059502] 74 [55863.059502] f1 [55863.059503] 0f [55863.059503] <66> [55863.059504] 0a [55863.059504] 5d [55863.059505] 66 [55863.059505] 39 [55863.059505] 0f [55863.059506] c3 [55863.059506] 39 [55863.059507] ca [55863.059507] b7 [55863.059508] 66 [55863.059508] ca [55863.059508] 75 [55863.059509] 0f [55863.059509] 66 [55863.059510] 75 [55863.059510] f1 [55863.059511] 66 [55863.059511] 66 [55863.059511] f1 [55863.059512] 5d [55863.059512] 39 [55863.059513] 90 [55863.059513] 5d [55863.059514] c3 [55863.059514] ca [55863.059515] 66 [55863.059515] c3 [55863.059515] 66 [55863.059516] 75 [55863.059516] 66 [55863.059517] 66 [55863.059517] 66 [55863.059517] [55863.059518] f1 [55863.059518] 66 [55863.059519] 66 [55863.059519] 5d [55863.059520] 66 [55863.059520] 90 [55863.059520] c3 [55863.059521] 90 [55863.059521] 66 [55863.059522] 66 [55863.059522] 66 [55863.059522] 66 [55863.059523] 66 [55863.059523] 66 [55863.059524] 90 [55863.059524] [55863.059524] 66 [55863.059525] eb [55863.059525] 90 [55863.059526] d9 [55863.059526] 66 [55863.059526] 66 [55863.059527] 66 [55863.059527] [55863.059528] 0f 1f 44 00 [55867.058534] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [kiblnd_connd:2895] [55867.058551] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55867.058557] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55867.058558] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55867.058558] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55867.058559] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55867.058561] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55867.058561] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55867.058562] RAX: 0000000000004f3d RBX: 0000000000016c40 RCX: 0000000000000a1e [55867.058563] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55867.058563] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55867.058563] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55867.058564] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55867.058565] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55867.058565] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55867.058566] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55867.058566] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55867.058567] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55867.058567] Stack: [55867.058568] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55867.058569] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55867.058570] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55867.058570] Call Trace: [55867.058576] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55867.058581] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55867.058584] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55867.058588] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55867.058589] [] ? lock_timer_base.isra.33+0x2b/0x50 [55867.058591] [] ? try_to_del_timer_sync+0x5e/0x90 [55867.058594] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55867.058595] [] ? schedule_timeout+0x17d/0x2d0 [55867.058599] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55867.058601] [] ? wake_up_state+0x20/0x20 [55867.058604] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55867.058606] [] kthread+0xcf/0xe0 [55867.058607] [] ? kthread_create_on_node+0x140/0x140 [55867.058609] [] ret_from_fork+0x58/0x90 [55867.058610] [] ? kthread_create_on_node+0x140/0x140 [55867.058620] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55872.619901] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55872.860924] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55872.932479] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55872.969162] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55873.007019] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55873.032594] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55873.060481] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [55873.078895] RAX: 000000000000504f RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55873.103292] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55873.127685] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55873.152120] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55873.176523] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55873.200915] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55873.228513] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55873.248347] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55873.272754] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55873.297193] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55873.321590] Stack: [55873.329139] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55873.354452] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [55873.379796] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [55873.405096] Call Trace: [55873.414082] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55873.436206] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55873.459200] [] lnet_select_pathway+0x235/0x1140 [lnet] [55873.482504] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55873.504913] [] lnet_send+0x51/0x180 [lnet] [55873.524741] [] lnet_finalize+0x3f1/0x690 [lnet] [55873.546004] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55873.567851] [] ? __slab_free+0x81/0x2f0 [55873.586814] [] lnet_ni_recv+0xc3/0x320 [lnet] [55873.607509] [] lnet_parse+0x945/0xf90 [lnet] [55873.627939] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55873.650911] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55873.674189] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55873.695465] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55873.719031] [] ? dequeue_task_fair+0x41e/0x660 [55873.740005] [] ? wake_up_state+0x20/0x20 [55873.759282] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55873.782258] [] kthread+0xcf/0xe0 [55873.799183] [] ? kthread_create_on_node+0x140/0x140 [55873.821539] [] ret_from_fork+0x58/0x90 [55873.840173] [] ? kthread_create_on_node+0x140/0x140 [55873.862494] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55875.057710] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [kiblnd_sd_01_00:2900] [55875.057712] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55875.057714] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [55875.057716] Modules linked in: [55875.057718] Modules linked in: [55875.057719] ko2iblnd(OE) [55875.057720] ko2iblnd(OE) [55875.057721] lustre(OE) [55875.057722] lustre(OE) [55875.057723] lmv(OE) [55875.057724] lmv(OE) [55875.057725] fld(OE) [55875.057726] fld(OE) [55875.057727] mdc(OE) [55875.057728] mdc(OE) [55875.057728] fid(OE) [55875.057730] fid(OE) [55875.057731] lov(OE) [55875.057732] lov(OE) [55875.057733] ksocklnd(OE) [55875.057734] ksocklnd(OE) [55875.057735] ptlrpc(OE) [55875.057736] ptlrpc(OE) [55875.057737] obdclass(OE) [55875.057738] obdclass(OE) [55875.057739] lnet(OE) [55875.057740] lnet(OE) [55875.057741] libcfs(OE) [55875.057742] libcfs(OE) [55875.057743] rpcrdma [55875.057744] rpcrdma [55875.057745] ib_isert [55875.057746] ib_isert [55875.057747] iscsi_target_mod [55875.057748] iscsi_target_mod [55875.057749] ib_iser [55875.057751] ib_iser [55875.057751] libiscsi [55875.057753] libiscsi [55875.057754] scsi_transport_iscsi [55875.057755] scsi_transport_iscsi [55875.057756] ib_srpt [55875.057758] ib_srpt [55875.057758] target_core_mod [55875.057760] target_core_mod [55875.057761] ib_srp [55875.057762] ib_srp [55875.057763] scsi_transport_srp [55875.057764] scsi_transport_srp [55875.057764] scsi_tgt [55875.057766] scsi_tgt [55875.057766] ib_ipoib [55875.057768] ib_ipoib [55875.057769] rdma_ucm [55875.057770] rdma_ucm [55875.057771] ib_ucm [55875.057772] ib_ucm [55875.057773] ib_uverbs [55875.057774] ib_uverbs [55875.057775] ib_umad [55875.057776] ib_umad [55875.057777] rdma_cm [55875.057778] rdma_cm [55875.057780] ib_cm [55875.057781] ib_cm [55875.057782] iw_cm [55875.057783] iw_cm [55875.057784] mlx4_ib [55875.057785] mlx4_ib [55875.057786] intel_powerclamp [55875.057787] intel_powerclamp [55875.057788] coretemp [55875.057789] coretemp [55875.057790] intel_rapl [55875.057791] intel_rapl [55875.057792] iosf_mbi [55875.057793] iosf_mbi [55875.057795] kvm [55875.057796] kvm [55875.057797] irqbypass [55875.057798] irqbypass [55875.057799] crc32_pclmul [55875.057800] crc32_pclmul [55875.057801] ghash_clmulni_intel [55875.057802] ghash_clmulni_intel [55875.057803] aesni_intel [55875.057804] aesni_intel [55875.057805] ipmi_ssif [55875.057806] ipmi_ssif [55875.057808] lrw [55875.057809] lrw [55875.057810] gf128mul [55875.057811] gf128mul [55875.057812] glue_helper [55875.057813] glue_helper [55875.057814] hfi1 [55875.057815] hfi1 [55875.057816] ipmi_devintf [55875.057817] ipmi_devintf [55875.057818] rdmavt [55875.057819] rdmavt [55875.057821] ablk_helper [55875.057822] ablk_helper [55875.057823] cryptd [55875.057824] cryptd [55875.057825] iTCO_wdt [55875.057826] iTCO_wdt [55875.057827] ipmi_si [55875.057828] ipmi_si [55875.057829] sb_edac [55875.057830] sb_edac [55875.057831] sg [55875.057832] sg [55875.057833] iTCO_vendor_support [55875.057834] iTCO_vendor_support [55875.057835] ib_core [55875.057836] ib_core [55875.057837] ioatdma [55875.057838] ioatdma [55875.057839] i2c_i801 [55875.057840] i2c_i801 [55875.057841] mei_me [55875.057842] mei_me [55875.057844] mei [55875.057845] mei [55875.057846] edac_core [55875.057847] edac_core [55875.057848] ipmi_msghandler [55875.057849] ipmi_msghandler [55875.057850] lpc_ich [55875.057851] lpc_ich [55875.057852] ntb [55875.057853] ntb [55875.057854] wmi [55875.057855] wmi [55875.057856] shpchp [55875.057857] shpchp [55875.057858] pcspkr [55875.057860] pcspkr [55875.057861] nfsd [55875.057862] nfsd [55875.057863] nfs_acl [55875.057864] nfs_acl [55875.057865] lockd [55875.057866] lockd [55875.057867] grace [55875.057868] grace [55875.057870] auth_rpcgss [55875.057871] auth_rpcgss [55875.057872] sunrpc [55875.057873] sunrpc [55875.057874] ip_tables [55875.057875] ip_tables [55875.057876] ext4 [55875.057877] ext4 [55875.057878] mbcache [55875.057879] mbcache [55875.057880] jbd2 [55875.057881] jbd2 [55875.057883] sd_mod [55875.057884] sd_mod [55875.057885] crc_t10dif [55875.057886] crc_t10dif [55875.057887] crct10dif_generic [55875.057888] crct10dif_generic [55875.057889] mlx4_en [55875.057890] mlx4_en [55875.057891] mgag200 [55875.057892] mgag200 [55875.057893] drm_kms_helper [55875.057894] drm_kms_helper [55875.057895] syscopyarea [55875.057896] syscopyarea [55875.057897] sysfillrect [55875.057898] sysfillrect [55875.057899] sysimgblt [55875.057900] sysimgblt [55875.057902] fb_sys_fops [55875.057903] fb_sys_fops [55875.057904] isci [55875.057905] isci [55875.057906] ttm [55875.057907] ttm [55875.057908] igb [55875.057909] igb [55875.057910] crct10dif_pclmul [55875.057911] crct10dif_pclmul [55875.057912] crct10dif_common [55875.057913] crct10dif_common [55875.057915] ahci [55875.057916] ahci [55875.057917] ptp [55875.057918] ptp [55875.057919] crc32c_intel [55875.057920] crc32c_intel [55875.057921] libsas [55875.057922] libsas [55875.057923] libahci [55875.057924] libahci [55875.057925] pps_core [55875.057926] pps_core [55875.057927] drm [55875.057928] drm [55875.057929] mlx4_core [55875.057930] mlx4_core [55875.057931] mpt2sas [55875.057932] mpt2sas [55875.057933] dca [55875.057934] dca [55875.057935] libata [55875.057936] libata [55875.057937] raid_class [55875.057939] raid_class [55875.057940] i2c_algo_bit [55875.057941] i2c_algo_bit [55875.057941] scsi_transport_sas [55875.057943] scsi_transport_sas [55875.057943] devlink [55875.057945] devlink [55875.057946] i2c_core [55875.057947] i2c_core [55875.057948] fjes [55875.057949] [55875.057950] fjes [55875.057952] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55875.057954] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55875.057955] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55875.057956] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55875.057960] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55875.057961] Workqueue: ib_cm cm_work_handler [ib_cm] [55875.057964] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55875.057965] RIP: 0010:[] [] _raw_spin_lock+0x30/0x50 [55875.057968] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [55875.057970] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55875.057971] RAX: 000000000000330d RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55875.057972] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000206 [55875.057973] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55875.057975] RAX: 0000000000002812 RBX: 0000000000000000 RCX: 0000000000000a1e [55875.057976] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55875.057977] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55875.057978] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55875.057979] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55875.057980] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55875.057982] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55875.057983] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55875.057984] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55875.057985] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55875.057987] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55875.057988] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55875.057989] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55875.057990] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55875.057992] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55875.057993] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55875.057994] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55875.057995] Stack: [55875.057996] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55875.057998] Stack: [55875.058000] ffff8807f639fd88 ffffffffa08aa831 [55875.058001] ffff8803fea77b00 [55875.058002] 0000000200000002 [55875.058003] ffffffffa08aa7c8 [55875.058004] ffff8807f63a2980 [55875.058005] [55875.058007] ffff8803fea77bd0 00050000c0a80173 [55875.058008] [55875.058010] 0000000000000000 ffff8807f639fdc0 [55875.058011] ffff88083fb5de00 [55875.058013] ffffffffa0942e30 [55875.058013] ffff8803fea77b18 [55875.058015] 00000000c0086540 [55875.058015] [55875.058017] ffffffffa0916e4b ffff8803fea77c68 [55875.058018] [55875.058020] ffff8807f63a2900 0000000000000002 [55875.058021] ffff8803fea77bc8 [55875.058023] 0000000000650aa0 [55875.058024] ffffffffa0ebfc24 [55875.058025] 0000000000000000 [55875.058025] [55875.058027] Call Trace: [55875.058028] ffff88041aa8a880 ffff8803fea77bd0 [55875.058030] Call Trace: [55875.058036] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55875.058043] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55875.058050] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55875.058056] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55875.058062] [] LNetCtl+0x116/0x800 [lnet] [55875.058068] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55875.058074] [] lnet_ioctl+0x60/0x250 [lnet] [55875.058078] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55875.058083] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55875.058089] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55875.058094] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55875.058098] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55875.058099] [] do_vfs_ioctl+0x2d5/0x4b0 [55875.058103] [] cma_req_handler+0x227/0x570 [rdma_cm] [55875.058105] [] ? __fd_install+0x47/0x60 [55875.058109] [] cm_process_work+0x25/0x140 [ib_cm] [55875.058110] [] SyS_ioctl+0xa1/0xc0 [55875.058114] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55875.058116] [] system_call_fastpath+0x16/0x1b [55875.058120] [] cm_work_handler+0x185/0x1484 [ib_cm] [55875.058122] Code: b8 [55875.058124] [] process_one_work+0x17b/0x470 [55875.058126] 00 00 [55875.058128] [] worker_thread+0x126/0x410 [55875.058130] 02 00 [55875.058132] [] ? rescuer_thread+0x460/0x460 [55875.058133] f0 0f [55875.058136] [] kthread+0xcf/0xe0 [55875.058138] c1 07 [55875.058140] [] ? kthread_create_on_node+0x140/0x140 [55875.058142] 89 c2 [55875.058145] [] ret_from_fork+0x58/0x90 [55875.058146] c1 ea [55875.058149] [] ? kthread_create_on_node+0x140/0x140 [55875.058151] 10 66 [55875.058152] Code: [55875.058153] 39 [55875.058154] c2 [55875.058155] c2 [55875.058156] c1 [55875.058157] 75 [55875.058158] ea [55875.058159] 01 [55875.058160] 10 [55875.058161] c3 [55875.058162] 66 [55875.058163] 55 [55875.058164] 39 [55875.058165] 83 [55875.058166] c2 [55875.058167] e2 [55875.058168] 75 [55875.058169] fe [55875.058169] 01 [55875.058170] 0f [55875.058171] c3 [55875.058172] b7 [55875.058173] 55 [55875.058174] f2 [55875.058175] 83 [55875.058176] 48 [55875.058177] e2 [55875.058178] 89 [55875.058179] fe [55875.058180] e5 [55875.058181] 0f [55875.058182] b8 [55875.058183] b7 [55875.058184] 00 [55875.058184] f2 [55875.058186] 80 [55875.058186] 48 [55875.058187] 00 [55875.058188] 89 [55875.058189] 00 [55875.058190] e5 [55875.058191] eb [55875.058192] b8 [55875.058193] 0d [55875.058194] 00 [55875.058195] 66 [55875.058196] 80 [55875.058197] 0f [55875.058198] 00 [55875.058199] 1f [55875.058200] 00 [55875.058201] 44 [55875.058202] eb [55875.058203] 00 [55875.058204] 0d [55875.058205] 00 [55875.058206] 66 [55875.058207] [55875.058208] 0f [55875.058209] 90 [55875.058210] 1f [55875.058211] 83 [55875.058212] 44 [55875.058213] e8 [55875.058214] 00 [55875.058215] 01 [55875.058216] 00 [55875.058217] 74 [55875.058218] f3 [55875.058219] 0a [55875.058220] 90 [55875.058221] 0f [55875.058222] 83 [55875.058223] b7 [55875.058224] e8 [55875.058225] 0f [55875.058226] 01 [55875.058227] 66 [55875.058228] 74 [55875.058229] 39 [55875.058230] 0a [55875.058231] ca [55875.058232] 0f [55875.058233] 75 [55875.058234] b7 [55875.058235] f1 [55875.058236] 0f [55875.058237] 5d [55875.058238] <66> [55875.058239] c3 [55875.058240] 39 [55875.058241] 66 [55875.058242] ca [55875.058243] 66 [55875.058244] 75 [55875.058245] 66 [55875.058246] f1 [55875.058247] 90 [55875.058248] [55875.058251] 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55879.057298] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kiblnd_sd_01_02:2902] [55879.057299] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55879.057300] Modules linked in: [55879.057300] Modules linked in: [55879.057301] ko2iblnd(OE) [55879.057301] ko2iblnd(OE) [55879.057302] lustre(OE) [55879.057302] lustre(OE) [55879.057303] lmv(OE) [55879.057303] lmv(OE) [55879.057304] fld(OE) [55879.057304] fld(OE) [55879.057305] mdc(OE) [55879.057305] mdc(OE) [55879.057306] fid(OE) [55879.057306] fid(OE) [55879.057306] lov(OE) [55879.057307] lov(OE) [55879.057307] ksocklnd(OE) [55879.057308] ksocklnd(OE) [55879.057308] ptlrpc(OE) [55879.057309] ptlrpc(OE) [55879.057309] obdclass(OE) [55879.057310] obdclass(OE) [55879.057310] lnet(OE) [55879.057310] lnet(OE) [55879.057311] libcfs(OE) [55879.057311] libcfs(OE) [55879.057312] rpcrdma [55879.057312] rpcrdma [55879.057313] ib_isert [55879.057313] ib_isert [55879.057314] iscsi_target_mod [55879.057314] iscsi_target_mod [55879.057315] ib_iser [55879.057315] ib_iser [55879.057316] libiscsi [55879.057316] libiscsi [55879.057317] scsi_transport_iscsi [55879.057317] scsi_transport_iscsi [55879.057317] ib_srpt [55879.057318] ib_srpt [55879.057318] target_core_mod [55879.057319] target_core_mod [55879.057319] ib_srp [55879.057320] ib_srp [55879.057320] scsi_transport_srp [55879.057321] scsi_transport_srp [55879.057321] scsi_tgt [55879.057322] scsi_tgt [55879.057322] ib_ipoib [55879.057322] ib_ipoib [55879.057323] rdma_ucm [55879.057323] rdma_ucm [55879.057324] ib_ucm [55879.057324] ib_ucm [55879.057325] ib_uverbs [55879.057325] ib_uverbs [55879.057325] ib_umad [55879.057326] ib_umad [55879.057326] rdma_cm [55879.057327] rdma_cm [55879.057327] ib_cm [55879.057328] ib_cm [55879.057328] iw_cm [55879.057329] iw_cm [55879.057329] mlx4_ib [55879.057329] mlx4_ib [55879.057330] intel_powerclamp [55879.057330] intel_powerclamp [55879.057331] coretemp [55879.057331] coretemp [55879.057332] intel_rapl [55879.057332] intel_rapl [55879.057333] iosf_mbi [55879.057333] iosf_mbi [55879.057333] kvm [55879.057334] kvm [55879.057334] irqbypass [55879.057335] irqbypass [55879.057335] crc32_pclmul [55879.057336] crc32_pclmul [55879.057336] ghash_clmulni_intel [55879.057337] ghash_clmulni_intel [55879.057337] aesni_intel [55879.057337] aesni_intel [55879.057338] ipmi_ssif [55879.057338] ipmi_ssif [55879.057339] lrw [55879.057339] lrw [55879.057340] gf128mul [55879.057340] gf128mul [55879.057341] glue_helper [55879.057341] glue_helper [55879.057342] hfi1 [55879.057342] hfi1 [55879.057342] ipmi_devintf [55879.057343] ipmi_devintf [55879.057343] rdmavt [55879.057344] rdmavt [55879.057344] ablk_helper [55879.057345] ablk_helper [55879.057345] cryptd [55879.057346] cryptd [55879.057346] iTCO_wdt [55879.057347] iTCO_wdt [55879.057347] ipmi_si [55879.057347] ipmi_si [55879.057348] sb_edac [55879.057348] sb_edac [55879.057349] sg [55879.057349] sg [55879.057350] iTCO_vendor_support [55879.057350] iTCO_vendor_support [55879.057351] ib_core [55879.057351] ib_core [55879.057352] ioatdma [55879.057352] ioatdma [55879.057352] i2c_i801 [55879.057353] i2c_i801 [55879.057353] mei_me [55879.057354] mei_me [55879.057354] mei [55879.057355] mei [55879.057355] edac_core [55879.057356] edac_core [55879.057356] ipmi_msghandler [55879.057356] ipmi_msghandler [55879.057357] lpc_ich [55879.057357] lpc_ich [55879.057358] ntb [55879.057358] ntb [55879.057359] wmi [55879.057359] wmi [55879.057360] shpchp [55879.057360] shpchp [55879.057361] pcspkr [55879.057361] pcspkr [55879.057361] nfsd [55879.057362] nfsd [55879.057362] nfs_acl [55879.057363] nfs_acl [55879.057363] lockd [55879.057364] lockd [55879.057364] grace [55879.057365] grace [55879.057365] auth_rpcgss [55879.057366] auth_rpcgss [55879.057366] sunrpc [55879.057366] sunrpc [55879.057367] ip_tables [55879.057367] ip_tables [55879.057368] ext4 [55879.057368] ext4 [55879.057369] mbcache [55879.057369] mbcache [55879.057370] jbd2 [55879.057370] jbd2 [55879.057371] sd_mod [55879.057371] sd_mod [55879.057371] crc_t10dif [55879.057372] crc_t10dif [55879.057372] crct10dif_generic [55879.057373] crct10dif_generic [55879.057373] mlx4_en [55879.057374] mlx4_en [55879.057374] mgag200 [55879.057375] mgag200 [55879.057375] drm_kms_helper [55879.057376] drm_kms_helper [55879.057376] syscopyarea [55879.057376] syscopyarea [55879.057377] sysfillrect [55879.057377] sysfillrect [55879.057378] sysimgblt [55879.057378] sysimgblt [55879.057379] fb_sys_fops [55879.057379] fb_sys_fops [55879.057380] isci [55879.057380] isci [55879.057381] ttm [55879.057381] ttm [55879.057381] igb [55879.057382] igb [55879.057382] crct10dif_pclmul [55879.057383] crct10dif_pclmul [55879.057383] crct10dif_common [55879.057384] crct10dif_common [55879.057384] ahci [55879.057385] ahci [55879.057385] ptp [55879.057386] ptp [55879.057386] crc32c_intel [55879.057386] crc32c_intel [55879.057387] libsas [55879.057387] libsas [55879.057388] libahci [55879.057388] libahci [55879.057389] pps_core [55879.057389] pps_core [55879.057390] drm [55879.057390] drm [55879.057391] mlx4_core [55879.057391] mlx4_core [55879.057391] mpt2sas [55879.057392] mpt2sas [55879.057392] dca [55879.057393] dca [55879.057393] libata [55879.057394] libata [55879.057394] raid_class [55879.057395] raid_class [55879.057395] i2c_algo_bit [55879.057396] i2c_algo_bit [55879.057396] scsi_transport_sas [55879.057396] scsi_transport_sas [55879.057397] devlink [55879.057397] devlink [55879.057398] i2c_core [55879.057398] i2c_core [55879.057399] fjes [55879.057399] [55879.057399] fjes [55879.057401] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55879.057402] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55879.057402] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55879.057403] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55879.057404] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55879.057404] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55879.057406] RIP: 0010:[] [55879.057407] [] _raw_spin_lock+0x32/0x50 [55879.057409] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55879.057409] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [55879.057410] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [55879.057411] RAX: 0000000000003b4c RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55879.057411] RAX: 00000000000017e1 RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55879.057412] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55879.057413] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55879.057414] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55879.057414] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55879.057415] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55879.057415] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55879.057416] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55879.057417] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55879.057418] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55879.057418] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55879.057419] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55879.057420] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55879.057421] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55879.057421] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55879.057422] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55879.057422] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55879.057423] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55879.057424] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55879.057424] Stack: [55879.057424] Stack: [55879.057425] ffff88082363bcd8 [55879.057426] ffff8808262efac0 [55879.057426] ffffffffa08aa7c8 [55879.057427] ffffffffa08aa7c8 [55879.057427] ffffffffa08aa8b6 [55879.057428] ffffffffa08aa8b6 [55879.057428] ffff8804c1ba6800 [55879.057429] [55879.057429] 0000000000000000 [55879.057430] 0000000000000001 [55879.057430] ffff88083fb5e000 [55879.057431] ffff88082363bd08 [55879.057431] ffff8808262efb80 [55879.057431] ffffffffa09217bb [55879.057432] ffffffffa092c9b5 [55879.057432] ffff8804c1ba6800 [55879.057433] [55879.057433] 00000000000001a8 [55879.057434] 0000000000000001 [55879.057434] ffffc9000e331730 [55879.057435] ffff88082abba340 [55879.057435] ffff8808262efb68 [55879.057436] ffff88082abba348 [55879.057436] ffffffffa0ebd587 [55879.057437] ffff88082363bd70 [55879.057437] [55879.057437] ffffffffffffffff [55879.057438] Call Trace: [55879.057438] Call Trace: [55879.057444] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55879.057449] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55879.057454] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55879.057459] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55879.057465] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55879.057471] [] lnet_select_pathway+0x235/0x1140 [lnet] [55879.057476] [] lnet_finalize+0x1e9/0x690 [lnet] [55879.057480] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55879.057483] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55879.057489] [] lnet_send+0x51/0x180 [lnet] [55879.057493] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55879.057499] [] lnet_finalize+0x3f1/0x690 [lnet] [55879.057500] [] ? dequeue_task_fair+0x41e/0x660 [55879.057504] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55879.057505] [] ? wake_up_state+0x20/0x20 [55879.057508] [] ? __slab_free+0x81/0x2f0 [55879.057511] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55879.057517] [] lnet_ni_recv+0xc3/0x320 [lnet] [55879.057518] [] kthread+0xcf/0xe0 [55879.057524] [] lnet_parse+0x945/0xf90 [lnet] [55879.057525] [] ? kthread_create_on_node+0x140/0x140 [55879.057529] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55879.057531] [] ret_from_fork+0x58/0x90 [55879.057533] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55879.057534] [] ? kthread_create_on_node+0x140/0x140 [55879.057538] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55879.057539] Code: [55879.057541] [] ? dequeue_task_fair+0x41e/0x660 [55879.057541] 00 02 [55879.057543] [] ? wake_up_state+0x20/0x20 [55879.057544] 00 f0 [55879.057547] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55879.057548] 0f c1 [55879.057550] [] kthread+0xcf/0xe0 [55879.057550] 07 89 [55879.057552] [] ? kthread_create_on_node+0x140/0x140 [55879.057552] c2 c1 [55879.057554] [] ret_from_fork+0x58/0x90 [55879.057555] ea 10 [55879.057557] [] ? kthread_create_on_node+0x140/0x140 [55879.057557] 66 39 c2 [55879.057558] Code: [55879.057558] 75 [55879.057559] 00 [55879.057559] 01 [55879.057560] 02 [55879.057560] c3 [55879.057561] 00 [55879.057561] 55 [55879.057562] f0 [55879.057562] 83 [55879.057562] 0f [55879.057563] e2 [55879.057563] c1 [55879.057564] fe [55879.057564] 07 [55879.057565] 0f [55879.057565] 89 [55879.057566] b7 [55879.057566] c2 [55879.057566] f2 [55879.057567] c1 [55879.057567] 48 [55879.057568] ea [55879.057568] 89 [55879.057568] 10 [55879.057569] e5 [55879.057569] 66 [55879.057570] b8 [55879.057570] 39 [55879.057571] 00 [55879.057571] c2 [55879.057571] 80 [55879.057572] 75 [55879.057572] 00 [55879.057573] 01 [55879.057573] 00 [55879.057573] c3 [55879.057574] eb [55879.057574] 55 [55879.057575] 0d [55879.057575] 83 [55879.057576] 66 [55879.057576] e2 [55879.057576] 0f [55879.057577] fe [55879.057577] 1f [55879.057578] 0f [55879.057578] 44 [55879.057579] b7 [55879.057579] 00 [55879.057579] f2 [55879.057580] 00 [55879.057580] 48 [55879.057581] f3 [55879.057581] 89 [55879.057581] 90 [55879.057582] e5 [55879.057582] <83> [55879.057583] b8 [55879.057583] e8 [55879.057584] 00 [55879.057584] 01 [55879.057585] 80 [55879.057585] 74 [55879.057585] 00 [55879.057586] 0a [55879.057586] 00 [55879.057587] 0f [55879.057587] eb [55879.057588] b7 [55879.057588] 0d [55879.057588] 0f [55879.057589] 66 [55879.057589] 66 [55879.057590] 0f [55879.057590] 39 [55879.057591] 1f [55879.057591] ca [55879.057591] 44 [55879.057592] 75 [55879.057592] 00 [55879.057593] f1 [55879.057593] 00 [55879.057594] 5d [55879.057594] f3 [55879.057594] c3 [55879.057595] 90 [55879.057595] 66 [55879.057596] <83> [55879.057596] 66 [55879.057597] e8 [55879.057597] 66 [55879.057597] 01 [55879.057598] 90 [55879.057598] 74 [55879.057599] 66 [55879.057599] 0a [55879.057600] 66 [55879.057600] [55879.057603] 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55883.279639] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55883.516190] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55883.585932] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55883.622131] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55883.659495] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55883.684591] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55883.711991] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55883.729952] RAX: 0000000000001fb2 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55883.753945] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55883.777960] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55883.801971] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55883.825992] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55883.849996] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55883.877193] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55883.896704] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55883.920783] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55883.944876] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55883.968998] Stack: [55883.976235] ffff8808217bbcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba3200 [55884.001267] 0000000000000001 ffff8808217bbd08 ffffffffa09217bb ffff8804c1ba3200 [55884.026307] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff8808217bbd70 [55884.051356] Call Trace: [55884.060066] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55884.081977] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55884.104750] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55884.126972] [] lnet_finalize+0x1e9/0x690 [lnet] [55884.148051] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55884.170569] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55884.193969] [] ? dequeue_task_fair+0x41e/0x660 [55884.214786] [] ? wake_up_state+0x20/0x20 [55884.233894] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55884.256716] [] kthread+0xcf/0xe0 [55884.273581] [] ? kthread_create_on_node+0x140/0x140 [55884.295894] [] ret_from_fork+0x58/0x90 [55884.314474] [] ? kthread_create_on_node+0x140/0x140 [55884.336799] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55891.056063] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55891.056064] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [55891.056065] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55891.056066] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55891.056067] Modules linked in: [55891.056068] Modules linked in: [55891.056069] Modules linked in: [55891.056069] ko2iblnd(OE) [55891.056070] ko2iblnd(OE) [55891.056070] ko2iblnd(OE) [55891.056071] lustre(OE) [55891.056071] lustre(OE) [55891.056071] lustre(OE) [55891.056072] lmv(OE) [55891.056072] lmv(OE) [55891.056073] lmv(OE) [55891.056073] fld(OE) [55891.056074] fld(OE) [55891.056074] fld(OE) [55891.056075] mdc(OE) [55891.056075] mdc(OE) [55891.056075] mdc(OE) [55891.056076] fid(OE) [55891.056076] fid(OE) [55891.056077] fid(OE) [55891.056077] lov(OE) [55891.056078] lov(OE) [55891.056078] lov(OE) [55891.056079] ksocklnd(OE) [55891.056079] ksocklnd(OE) [55891.056080] ksocklnd(OE) [55891.056080] ptlrpc(OE) [55891.056080] ptlrpc(OE) [55891.056081] ptlrpc(OE) [55891.056081] obdclass(OE) [55891.056082] obdclass(OE) [55891.056082] obdclass(OE) [55891.056083] lnet(OE) [55891.056083] lnet(OE) [55891.056084] lnet(OE) [55891.056084] libcfs(OE) [55891.056084] libcfs(OE) [55891.056085] libcfs(OE) [55891.056085] rpcrdma [55891.056086] rpcrdma [55891.056086] rpcrdma [55891.056087] ib_isert [55891.056087] ib_isert [55891.056088] ib_isert [55891.056088] iscsi_target_mod [55891.056089] iscsi_target_mod [55891.056089] iscsi_target_mod [55891.056089] ib_iser [55891.056090] ib_iser [55891.056090] ib_iser [55891.056091] libiscsi [55891.056091] libiscsi [55891.056092] libiscsi [55891.056092] scsi_transport_iscsi [55891.056093] scsi_transport_iscsi [55891.056093] scsi_transport_iscsi [55891.056094] ib_srpt [55891.056094] ib_srpt [55891.056094] ib_srpt [55891.056095] target_core_mod [55891.056095] target_core_mod [55891.056096] target_core_mod [55891.056096] ib_srp [55891.056097] ib_srp [55891.056097] ib_srp [55891.056098] scsi_transport_srp [55891.056098] scsi_transport_srp [55891.056099] scsi_transport_srp [55891.056099] scsi_tgt [55891.056099] scsi_tgt [55891.056100] scsi_tgt [55891.056100] ib_ipoib [55891.056101] ib_ipoib [55891.056101] ib_ipoib [55891.056102] rdma_ucm [55891.056102] rdma_ucm [55891.056103] rdma_ucm [55891.056103] ib_ucm [55891.056103] ib_ucm [55891.056104] ib_ucm [55891.056104] ib_uverbs [55891.056105] ib_uverbs [55891.056105] ib_uverbs [55891.056106] ib_umad [55891.056106] ib_umad [55891.056107] ib_umad [55891.056107] rdma_cm [55891.056107] rdma_cm [55891.056108] rdma_cm [55891.056108] ib_cm [55891.056109] ib_cm [55891.056109] ib_cm [55891.056110] iw_cm [55891.056110] iw_cm [55891.056111] iw_cm [55891.056111] mlx4_ib [55891.056111] mlx4_ib [55891.056112] mlx4_ib [55891.056112] intel_powerclamp [55891.056113] intel_powerclamp [55891.056113] intel_powerclamp [55891.056114] coretemp [55891.056114] coretemp [55891.056115] coretemp [55891.056115] intel_rapl [55891.056115] intel_rapl [55891.056116] intel_rapl [55891.056116] iosf_mbi [55891.056117] iosf_mbi [55891.056117] iosf_mbi [55891.056118] kvm [55891.056118] kvm [55891.056119] kvm [55891.056119] irqbypass [55891.056120] irqbypass [55891.056120] irqbypass [55891.056120] crc32_pclmul [55891.056121] crc32_pclmul [55891.056121] crc32_pclmul [55891.056122] ghash_clmulni_intel [55891.056122] ghash_clmulni_intel [55891.056123] ghash_clmulni_intel [55891.056123] aesni_intel [55891.056124] aesni_intel [55891.056124] aesni_intel [55891.056125] ipmi_ssif [55891.056125] ipmi_ssif [55891.056125] ipmi_ssif [55891.056126] lrw [55891.056126] lrw [55891.056127] lrw [55891.056127] gf128mul [55891.056128] gf128mul [55891.056128] gf128mul [55891.056128] glue_helper [55891.056129] glue_helper [55891.056129] glue_helper [55891.056130] hfi1 [55891.056130] hfi1 [55891.056131] hfi1 [55891.056131] ipmi_devintf [55891.056132] ipmi_devintf [55891.056132] ipmi_devintf [55891.056133] rdmavt [55891.056133] rdmavt [55891.056134] rdmavt [55891.056134] ablk_helper [55891.056134] ablk_helper [55891.056135] ablk_helper [55891.056135] cryptd [55891.056136] cryptd [55891.056136] cryptd [55891.056137] iTCO_wdt [55891.056137] iTCO_wdt [55891.056137] iTCO_wdt [55891.056138] ipmi_si [55891.056138] ipmi_si [55891.056139] ipmi_si [55891.056139] sb_edac [55891.056140] sb_edac [55891.056140] sb_edac [55891.056140] sg [55891.056141] sg [55891.056141] sg [55891.056142] iTCO_vendor_support [55891.056142] iTCO_vendor_support [55891.056143] iTCO_vendor_support [55891.056143] ib_core [55891.056144] ib_core [55891.056144] ib_core [55891.056145] ioatdma [55891.056145] ioatdma [55891.056145] ioatdma [55891.056146] i2c_i801 [55891.056146] i2c_i801 [55891.056147] i2c_i801 [55891.056147] mei_me [55891.056148] mei_me [55891.056148] mei_me [55891.056149] mei [55891.056149] mei [55891.056149] mei [55891.056150] edac_core [55891.056150] edac_core [55891.056151] edac_core [55891.056151] ipmi_msghandler [55891.056152] ipmi_msghandler [55891.056152] ipmi_msghandler [55891.056153] lpc_ich [55891.056153] lpc_ich [55891.056154] lpc_ich [55891.056154] ntb [55891.056154] ntb [55891.056155] ntb [55891.056155] wmi [55891.056156] wmi [55891.056156] wmi [55891.056157] shpchp [55891.056157] shpchp [55891.056158] shpchp [55891.056158] pcspkr [55891.056159] pcspkr [55891.056159] pcspkr [55891.056159] nfsd [55891.056160] nfsd [55891.056160] nfsd [55891.056161] nfs_acl [55891.056161] nfs_acl [55891.056162] nfs_acl [55891.056162] lockd [55891.056163] lockd [55891.056163] lockd [55891.056164] grace [55891.056164] grace [55891.056164] grace [55891.056165] auth_rpcgss [55891.056165] auth_rpcgss [55891.056166] auth_rpcgss [55891.056166] sunrpc [55891.056167] sunrpc [55891.056167] sunrpc [55891.056168] ip_tables [55891.056168] ip_tables [55891.056168] ip_tables [55891.056169] ext4 [55891.056169] ext4 [55891.056170] ext4 [55891.056170] mbcache [55891.056171] mbcache [55891.056171] mbcache [55891.056172] jbd2 [55891.056172] jbd2 [55891.056173] jbd2 [55891.056173] sd_mod [55891.056173] sd_mod [55891.056174] sd_mod [55891.056174] crc_t10dif [55891.056175] crc_t10dif [55891.056175] crc_t10dif [55891.056176] crct10dif_generic [55891.056176] crct10dif_generic [55891.056177] crct10dif_generic [55891.056177] mlx4_en [55891.056178] mlx4_en [55891.056178] mlx4_en [55891.056178] mgag200 [55891.056179] mgag200 [55891.056179] mgag200 [55891.056180] drm_kms_helper [55891.056180] drm_kms_helper [55891.056181] drm_kms_helper [55891.056181] syscopyarea [55891.056182] syscopyarea [55891.056182] syscopyarea [55891.056183] sysfillrect [55891.056183] sysfillrect [55891.056183] sysfillrect [55891.056184] sysimgblt [55891.056184] sysimgblt [55891.056185] sysimgblt [55891.056185] fb_sys_fops [55891.056186] fb_sys_fops [55891.056186] fb_sys_fops [55891.056187] isci [55891.056187] isci [55891.056188] isci [55891.056188] ttm [55891.056188] ttm [55891.056189] ttm [55891.056189] igb [55891.056190] igb [55891.056190] igb [55891.056191] crct10dif_pclmul [55891.056191] crct10dif_pclmul [55891.056192] crct10dif_pclmul [55891.056192] crct10dif_common [55891.056193] crct10dif_common [55891.056193] crct10dif_common [55891.056193] ahci [55891.056194] ahci [55891.056194] ahci [55891.056195] ptp [55891.056195] ptp [55891.056196] ptp [55891.056196] crc32c_intel [55891.056197] crc32c_intel [55891.056197] crc32c_intel [55891.056197] libsas [55891.056198] libsas [55891.056198] libsas [55891.056199] libahci [55891.056199] libahci [55891.056200] libahci [55891.056200] pps_core [55891.056201] pps_core [55891.056201] pps_core [55891.056201] drm [55891.056202] drm [55891.056202] drm [55891.056203] mlx4_core [55891.056203] mlx4_core [55891.056204] mlx4_core [55891.056204] mpt2sas [55891.056204] mpt2sas [55891.056205] mpt2sas [55891.056205] dca [55891.056206] dca [55891.056206] dca [55891.056207] libata [55891.056207] libata [55891.056208] libata [55891.056208] raid_class [55891.056209] raid_class [55891.056209] raid_class [55891.056210] i2c_algo_bit [55891.056210] i2c_algo_bit [55891.056210] i2c_algo_bit [55891.056211] scsi_transport_sas [55891.056211] scsi_transport_sas [55891.056212] scsi_transport_sas [55891.056212] devlink [55891.056213] devlink [55891.056213] devlink [55891.056214] i2c_core [55891.056214] i2c_core [55891.056214] i2c_core [55891.056215] fjes [55891.056215] fjes [55891.056216] [55891.056216] [55891.056216] fjes [55891.056218] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55891.056219] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55891.056220] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55891.056220] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55891.056221] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55891.056221] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55891.056222] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55891.056223] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55891.056224] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55891.056225] RIP: 0010:[] [55891.056225] RIP: 0010:[] [55891.056227] [] _raw_spin_lock+0x37/0x50 [55891.056228] [] _raw_spin_lock+0x37/0x50 [55891.056229] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55891.056230] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000216 [55891.056230] RSP: 0018:ffff880820b03c00 EFLAGS: 00000206 [55891.056231] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55891.056232] RAX: 000000000000295f RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55891.056232] RAX: 000000000000526a RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55891.056233] RAX: 0000000000003948 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55891.056234] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55891.056234] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55891.056235] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55891.056235] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55891.056236] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55891.056237] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55891.056237] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55891.056238] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55891.056239] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55891.056239] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55891.056240] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55891.056240] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55891.056241] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55891.056242] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55891.056243] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55891.056243] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55891.056244] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55891.056245] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55891.056246] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55891.056246] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55891.056247] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55891.056247] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55891.056248] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55891.056249] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55891.056249] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55891.056250] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55891.056251] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55891.056251] Stack: [55891.056251] Stack: [55891.056252] Stack: [55891.056252] ffff88081f64fc98 [55891.056253] ffff880820b03c28 [55891.056253] ffff88082626bd08 [55891.056254] ffffffffa08aa7c8 [55891.056254] ffffffffa08aa7c8 [55891.056255] ffffffffa08aa7c8 [55891.056255] ffffffffa08aa8b6 [55891.056256] ffff880820b03c28 [55891.056256] ffff88082af11648 [55891.056256] ffff88042dcd1e00 [55891.056257] 0000000000000000 [55891.056257] [55891.056258] [55891.056258] ffff8800b484ea00 [55891.056259] ffff8800b4864200 [55891.056259] ffff88083fb5e000 [55891.056260] 0000000000000000 [55891.056260] ffff88081f64fcd8 [55891.056261] ffff880820b03c98 [55891.056261] ffff88082626bd70 [55891.056262] ffffffffa092e857 [55891.056262] ffffffffa092c346 [55891.056262] ffffffffa0922006 [55891.056263] ffff8800b484d600 [55891.056263] ffff880820b03c60 [55891.056264] [55891.056264] [55891.056264] ffff88082af10fb0 [55891.056265] ffff88042dcd1e00 [55891.056266] 00000000000000b4 [55891.056266] ffff88082af10fb0 [55891.056266] 0000000000000000 [55891.056267] 00000001033f89f4 [55891.056267] ffff88082af10fb0 [55891.056268] ffff88041d604a00 [55891.056268] ffff8800b4864274 [55891.056269] ffff88082626bd30 [55891.056269] 0000000000000000 [55891.056270] 0000000000000000 [55891.056270] [55891.056270] [55891.056271] ffff88082626bd30 [55891.056271] Call Trace: [55891.056271] Call Trace: [55891.056272] Call Trace: [55891.056277] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55891.056283] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55891.056288] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55891.056293] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55891.056299] [] lnet_post_send_locked+0x446/0x880 [lnet] [55891.056305] [] lnet_finalize+0xf6/0x690 [lnet] [55891.056311] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55891.056316] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55891.056320] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55891.056325] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55891.056331] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55891.056335] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55891.056340] [] lnet_finalize+0x1e9/0x690 [lnet] [55891.056345] [] lnet_finalize+0x1e9/0x690 [lnet] [55891.056347] [] ? dequeue_task_fair+0x41e/0x660 [55891.056350] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55891.056354] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55891.056356] [] ? wake_up_state+0x20/0x20 [55891.056359] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55891.056363] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55891.056367] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55891.056368] [] ? dequeue_task_fair+0x41e/0x660 [55891.056370] [] ? dequeue_task_fair+0x41e/0x660 [55891.056371] [] kthread+0xcf/0xe0 [55891.056373] [] ? wake_up_state+0x20/0x20 [55891.056375] [] ? sched_clock_cpu+0x85/0xc0 [55891.056376] [] ? kthread_create_on_node+0x140/0x140 [55891.056380] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55891.056382] [] ? wake_up_state+0x20/0x20 [55891.056383] [] ret_from_fork+0x58/0x90 [55891.056385] [] kthread+0xcf/0xe0 [55891.056389] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55891.056390] [] ? kthread_create_on_node+0x140/0x140 [55891.056392] [] ? kthread_create_on_node+0x140/0x140 [55891.056392] Code: [55891.056394] [] kthread+0xcf/0xe0 [55891.056396] [] ret_from_fork+0x58/0x90 [55891.056396] 00 [55891.056397] 02 [55891.056398] [] ? kthread_create_on_node+0x140/0x140 [55891.056400] [] ? kthread_create_on_node+0x140/0x140 [55891.056400] 00 [55891.056402] [] ret_from_fork+0x58/0x90 [55891.056403] f0 [55891.056403] Code: [55891.056404] 0f [55891.056404] c1 [55891.056406] [] ? kthread_create_on_node+0x140/0x140 [55891.056406] c1 [55891.056406] 07 [55891.056407] 07 [55891.056407] Code: [55891.056408] 89 [55891.056408] 89 [55891.056409] c1 [55891.056409] c2 [55891.056410] c2 [55891.056410] 07 [55891.056410] c1 [55891.056411] c1 [55891.056411] 89 [55891.056412] ea [55891.056412] ea [55891.056413] c2 [55891.056413] 10 [55891.056413] 10 [55891.056414] c1 [55891.056414] 66 [55891.056415] 66 [55891.056415] ea [55891.056415] 39 [55891.056416] 39 [55891.056416] 10 [55891.056417] c2 [55891.056417] c2 [55891.056418] 66 [55891.056418] 75 [55891.056418] 75 [55891.056419] 39 [55891.056419] 01 [55891.056420] 01 [55891.056420] c2 [55891.056421] c3 [55891.056421] c3 [55891.056421] 75 [55891.056422] 55 [55891.056422] 55 [55891.056423] 01 [55891.056423] 83 [55891.056424] 83 [55891.056424] c3 [55891.056424] e2 [55891.056425] e2 [55891.056425] 55 [55891.056426] fe [55891.056426] fe [55891.056426] 83 [55891.056427] 0f [55891.056427] 0f [55891.056428] e2 [55891.056428] b7 [55891.056429] b7 [55891.056429] fe [55891.056429] f2 [55891.056430] f2 [55891.056430] 0f [55891.056431] 48 [55891.056431] 48 [55891.056432] b7 [55891.056432] 89 [55891.056432] 89 [55891.056433] f2 [55891.056433] e5 [55891.056434] e5 [55891.056434] 48 [55891.056435] b8 [55891.056435] b8 [55891.056435] 89 [55891.056436] 00 [55891.056436] 00 [55891.056437] e5 [55891.056437] 80 [55891.056438] 80 [55891.056438] b8 [55891.056438] 00 [55891.056439] 00 [55891.056439] 00 [55891.056440] 00 [55891.056440] 00 [55891.056440] 80 [55891.056441] eb [55891.056441] eb [55891.056442] 00 [55891.056442] 0d [55891.056443] 0d [55891.056443] 00 [55891.056443] 66 [55891.056444] 66 [55891.056444] eb [55891.056445] 0f [55891.056445] 0f [55891.056446] 0d [55891.056446] 1f [55891.056446] 1f [55891.056447] 66 [55891.056447] 44 [55891.056448] 44 [55891.056448] 0f [55891.056449] 00 [55891.056449] 00 [55891.056449] 1f [55891.056450] 00 [55891.056450] 00 [55891.056451] 44 [55891.056451] f3 [55891.056451] f3 [55891.056452] 00 [55891.056452] 90 [55891.056453] 90 [55891.056453] 00 [55891.056454] 83 [55891.056454] <83> [55891.056455] f3 [55891.056455] e8 [55891.056455] e8 [55891.056456] 90 [55891.056456] 01 [55891.056457] 01 [55891.056457] 83 [55891.056458] 74 [55891.056458] 74 [55891.056458] e8 [55891.056459] 0a [55891.056459] 0a [55891.056460] 01 [55891.056460] <0f> [55891.056461] 0f [55891.056461] 74 [55891.056461] b7 [55891.056462] b7 [55891.056462] 0a [55891.056463] 0f [55891.056463] 0f [55891.056464] <0f> [55891.056464] 66 [55891.056464] 66 [55891.056465] b7 [55891.056465] 39 [55891.056466] 39 [55891.056466] 0f [55891.056467] ca [55891.056467] ca [55891.056467] 66 [55891.056468] 75 [55891.056468] 75 [55891.056469] 39 [55891.056469] f1 [55891.056470] f1 [55891.056470] ca [55891.056470] 5d [55891.056471] 5d [55891.056471] 75 [55891.056472] c3 [55891.056472] c3 [55891.056472] f1 [55891.056473] 66 [55891.056473] 66 [55891.056474] 5d [55891.056474] 66 [55891.056475] 66 [55891.056475] c3 [55891.056475] 66 [55891.056476] 66 [55891.056476] 66 [55891.056477] 90 [55891.056477] 90 [55891.056478] 66 [55891.056478] 66 [55891.056479] 66 [55891.056479] 66 [55891.056479] 66 [55891.056480] 66 [55891.056480] 90 [55891.056481] [55891.056481] 90 [55891.056481] 66 [55891.056482] eb [55891.056482] 66 [55891.056483] d9 [55891.056483] 90 [55891.056483] 66 [55891.056484] eb [55891.056484] 0f [55891.056485] [55891.056485] d9 66 0f [55895.055650] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [kiblnd_connd:2895] [55895.055668] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55895.055673] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55895.055674] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55895.055675] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55895.055675] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55895.055677] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55895.055677] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55895.055678] RAX: 0000000000000efc RBX: 0000000000016c40 RCX: 0000000000000a1e [55895.055678] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55895.055679] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55895.055679] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55895.055680] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55895.055681] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55895.055681] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55895.055682] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55895.055682] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55895.055683] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55895.055683] Stack: [55895.055684] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55895.055685] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55895.055686] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55895.055686] Call Trace: [55895.055692] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55895.055697] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55895.055700] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55895.055704] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55895.055705] [] ? lock_timer_base.isra.33+0x2b/0x50 [55895.055707] [] ? try_to_del_timer_sync+0x5e/0x90 [55895.055710] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55895.055712] [] ? schedule_timeout+0x17d/0x2d0 [55895.055715] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55895.055717] [] ? wake_up_state+0x20/0x20 [55895.055720] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55895.055722] [] kthread+0xcf/0xe0 [55895.055723] [] ? kthread_create_on_node+0x140/0x140 [55895.055725] [] ret_from_fork+0x58/0x90 [55895.055726] [] ? kthread_create_on_node+0x140/0x140 [55895.055736] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55898.480576] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55898.721489] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55898.793007] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55898.829465] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55898.867358] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55898.892925] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55898.920777] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000202 [55898.939204] RAX: 0000000000002be0 RBX: 0000000100000000 RCX: 0000000000000a1e [55898.963637] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55898.988023] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55899.012396] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55899.036780] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55899.055239] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 24s! [kiblnd_sd_01_03:2903] [55899.055258] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55899.055264] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55899.055266] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55899.055266] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55899.055267] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55899.055268] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55899.055269] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [55899.055270] RAX: 0000000000002067 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55899.055270] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55899.055271] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55899.055271] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55899.055272] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55899.055273] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55899.055274] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55899.055274] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55899.055275] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55899.055275] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55899.055276] Stack: [55899.055277] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55899.055278] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [55899.055278] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [55899.055279] Call Trace: [55899.055284] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55899.055289] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55899.055295] [] lnet_select_pathway+0x235/0x1140 [lnet] [55899.055299] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55899.055305] [] lnet_send+0x51/0x180 [lnet] [55899.055310] [] lnet_finalize+0x3f1/0x690 [lnet] [55899.055314] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55899.055316] [] ? __slab_free+0x81/0x2f0 [55899.055321] [] lnet_ni_recv+0xc3/0x320 [lnet] [55899.055326] [] lnet_parse+0x945/0xf90 [lnet] [55899.055331] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55899.055335] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55899.055337] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55899.055341] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55899.055342] [] ? dequeue_task_fair+0x41e/0x660 [55899.055344] [] ? wake_up_state+0x20/0x20 [55899.055347] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55899.055349] [] kthread+0xcf/0xe0 [55899.055351] [] ? kthread_create_on_node+0x140/0x140 [55899.055352] [] ret_from_fork+0x58/0x90 [55899.055354] [] ? kthread_create_on_node+0x140/0x140 [55899.055364] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55900.398453] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55900.425873] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55900.445528] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55900.469718] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55900.493938] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55900.518126] Stack: [55900.525412] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [55900.550487] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [55900.575545] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [55900.600587] Call Trace: [55900.609302] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55900.631197] [] lnet_finalize+0xf6/0x690 [lnet] [55900.651927] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55900.673524] [] ? enqueue_task_fair+0x208/0x6c0 [55900.694252] [] ? sched_clock_cpu+0x85/0xc0 [55900.713825] [] lnet_ni_recv+0xc3/0x320 [lnet] [55900.734278] [] lnet_parse+0x945/0xf90 [lnet] [55900.754411] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55900.777438] [] ? __wake_up+0x44/0x50 [55900.795320] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55900.818631] [] ? dequeue_task_fair+0x41e/0x660 [55900.839362] [] ? wake_up_state+0x20/0x20 [55900.858362] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55900.881097] [] kthread+0xcf/0xe0 [55900.897800] [] ? kthread_create_on_node+0x140/0x140 [55900.919965] [] ret_from_fork+0x58/0x90 [55900.938412] [] ? kthread_create_on_node+0x140/0x140 [55900.960568] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55903.054827] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [55903.054828] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55903.054848] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55903.054853] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55903.054854] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55903.054855] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55903.054857] Workqueue: ib_cm cm_work_handler [ib_cm] [55903.054857] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55903.054859] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55903.054859] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000202 [55903.054860] RAX: 00000000000079d0 RBX: 0000000000000000 RCX: 0000000000000a1e [55903.054860] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55903.054861] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55903.054861] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55903.054862] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55903.054863] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55903.054863] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55903.054864] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55903.054864] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55903.054865] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55903.054865] Stack: [55903.054866] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [55903.054867] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [55903.054868] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [55903.054869] Call Trace: [55903.054874] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55903.054879] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55903.054883] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55903.054885] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55903.054888] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55903.054890] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55903.054892] [] cma_req_handler+0x227/0x570 [rdma_cm] [55903.054894] [] cm_process_work+0x25/0x140 [ib_cm] [55903.054896] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55903.054898] [] cm_work_handler+0x185/0x1484 [ib_cm] [55903.054899] [] process_one_work+0x17b/0x470 [55903.054900] [] worker_thread+0x126/0x410 [55903.054901] [] ? rescuer_thread+0x460/0x460 [55903.054902] [] kthread+0xcf/0xe0 [55903.054904] [] ? kthread_create_on_node+0x140/0x140 [55903.054905] [] ret_from_fork+0x58/0x90 [55903.054907] [] ? kthread_create_on_node+0x140/0x140 [55903.054917] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55904.346780] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55904.587329] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55904.658635] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55904.692131] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55904.729965] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55904.755467] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55904.783256] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000206 [55904.801649] RAX: 0000000000004b71 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55904.826023] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55904.850399] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55904.874774] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55904.899184] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55904.923579] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55904.951125] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55904.970914] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55904.995294] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55905.019678] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55905.044083] Stack: [55905.051569] ffff8807f639fd88 ffffffffa08aa831 0000000200000002 ffff8807f63a2980 [55905.076854] 0000000000000000 ffff8807f639fdc0 ffffffffa0942e30 00000000c0086540 [55905.102151] ffff8807f63a2900 0000000000000002 0000000000650aa0 0000000000000000 [55905.127436] Call Trace: [55905.136380] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55905.158490] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55905.180040] [] LNetCtl+0x116/0x800 [lnet] [55905.199603] [] lnet_ioctl+0x60/0x250 [lnet] [55905.219704] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55905.241245] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55905.263922] [] do_vfs_ioctl+0x2d5/0x4b0 [55905.282909] [] ? __fd_install+0x47/0x60 [55905.301880] [] SyS_ioctl+0xa1/0xc0 [55905.319428] [] system_call_fastpath+0x16/0x1b [55905.340113] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55907.054415] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 21s! [kiblnd_sd_01_00:2900] [55907.054416] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kiblnd_sd_01_02:2902] [55907.054417] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55907.054418] Modules linked in: [55907.054419] Modules linked in: [55907.054419] ko2iblnd(OE) [55907.054420] ko2iblnd(OE) [55907.054420] lustre(OE) [55907.054420] lustre(OE) [55907.054421] lmv(OE) [55907.054421] lmv(OE) [55907.054422] fld(OE) [55907.054422] fld(OE) [55907.054423] mdc(OE) [55907.054423] mdc(OE) [55907.054424] fid(OE) [55907.054424] fid(OE) [55907.054424] lov(OE) [55907.054425] lov(OE) [55907.054425] ksocklnd(OE) [55907.054426] ksocklnd(OE) [55907.054426] ptlrpc(OE) [55907.054427] ptlrpc(OE) [55907.054427] obdclass(OE) [55907.054428] obdclass(OE) [55907.054428] lnet(OE) [55907.054429] lnet(OE) [55907.054429] libcfs(OE) [55907.054429] libcfs(OE) [55907.054430] rpcrdma [55907.054430] rpcrdma [55907.054431] ib_isert [55907.054431] ib_isert [55907.054432] iscsi_target_mod [55907.054432] iscsi_target_mod [55907.054433] ib_iser [55907.054433] ib_iser [55907.054433] libiscsi [55907.054434] libiscsi [55907.054434] scsi_transport_iscsi [55907.054435] scsi_transport_iscsi [55907.054435] ib_srpt [55907.054436] ib_srpt [55907.054436] target_core_mod [55907.054437] target_core_mod [55907.054437] ib_srp [55907.054438] ib_srp [55907.054438] scsi_transport_srp [55907.054439] scsi_transport_srp [55907.054439] scsi_tgt [55907.054440] scsi_tgt [55907.054440] ib_ipoib [55907.054440] ib_ipoib [55907.054441] rdma_ucm [55907.054441] rdma_ucm [55907.054442] ib_ucm [55907.054442] ib_ucm [55907.054443] ib_uverbs [55907.054443] ib_uverbs [55907.054444] ib_umad [55907.054444] ib_umad [55907.054445] rdma_cm [55907.054445] rdma_cm [55907.054445] ib_cm [55907.054446] ib_cm [55907.054446] iw_cm [55907.054447] iw_cm [55907.054447] mlx4_ib [55907.054448] mlx4_ib [55907.054448] intel_powerclamp [55907.054449] intel_powerclamp [55907.054449] coretemp [55907.054449] coretemp [55907.054450] intel_rapl [55907.054450] intel_rapl [55907.054451] iosf_mbi [55907.054451] iosf_mbi [55907.054452] kvm [55907.054452] kvm [55907.054453] irqbypass [55907.054453] irqbypass [55907.054454] crc32_pclmul [55907.054454] crc32_pclmul [55907.054455] ghash_clmulni_intel [55907.054455] ghash_clmulni_intel [55907.054455] aesni_intel [55907.054456] aesni_intel [55907.054456] ipmi_ssif [55907.054457] ipmi_ssif [55907.054457] lrw [55907.054458] lrw [55907.054458] gf128mul [55907.054459] gf128mul [55907.054459] glue_helper [55907.054460] glue_helper [55907.054460] hfi1 [55907.054460] hfi1 [55907.054461] ipmi_devintf [55907.054461] ipmi_devintf [55907.054462] rdmavt [55907.054462] rdmavt [55907.054463] ablk_helper [55907.054463] ablk_helper [55907.054464] cryptd [55907.054464] cryptd [55907.054465] iTCO_wdt [55907.054465] iTCO_wdt [55907.054465] ipmi_si [55907.054466] ipmi_si [55907.054466] sb_edac [55907.054467] sb_edac [55907.054467] sg [55907.054468] sg [55907.054468] iTCO_vendor_support [55907.054469] iTCO_vendor_support [55907.054469] ib_core [55907.054470] ib_core [55907.054470] ioatdma [55907.054471] ioatdma [55907.054471] i2c_i801 [55907.054471] i2c_i801 [55907.054472] mei_me [55907.054472] mei_me [55907.054473] mei [55907.054473] mei [55907.054474] edac_core [55907.054474] edac_core [55907.054475] ipmi_msghandler [55907.054475] ipmi_msghandler [55907.054476] lpc_ich [55907.054476] lpc_ich [55907.054476] ntb [55907.054477] ntb [55907.054477] wmi [55907.054478] wmi [55907.054478] shpchp [55907.054479] shpchp [55907.054479] pcspkr [55907.054480] pcspkr [55907.054480] nfsd [55907.054480] nfsd [55907.054481] nfs_acl [55907.054481] nfs_acl [55907.054482] lockd [55907.054482] lockd [55907.054483] grace [55907.054483] grace [55907.054484] auth_rpcgss [55907.054484] auth_rpcgss [55907.054485] sunrpc [55907.054485] sunrpc [55907.054485] ip_tables [55907.054486] ip_tables [55907.054486] ext4 [55907.054487] ext4 [55907.054487] mbcache [55907.054488] mbcache [55907.054488] jbd2 [55907.054489] jbd2 [55907.054489] sd_mod [55907.054490] sd_mod [55907.054490] crc_t10dif [55907.054490] crc_t10dif [55907.054491] crct10dif_generic [55907.054491] crct10dif_generic [55907.054492] mlx4_en [55907.054492] mlx4_en [55907.054493] mgag200 [55907.054493] mgag200 [55907.054494] drm_kms_helper [55907.054494] drm_kms_helper [55907.054495] syscopyarea [55907.054495] syscopyarea [55907.054495] sysfillrect [55907.054496] sysfillrect [55907.054496] sysimgblt [55907.054497] sysimgblt [55907.054497] fb_sys_fops [55907.054498] fb_sys_fops [55907.054498] isci [55907.054499] isci [55907.054499] ttm [55907.054500] ttm [55907.054500] igb [55907.054500] igb [55907.054501] crct10dif_pclmul [55907.054501] crct10dif_pclmul [55907.054502] crct10dif_common [55907.054502] crct10dif_common [55907.054503] ahci [55907.054503] ahci [55907.054504] ptp [55907.054504] ptp [55907.054505] crc32c_intel [55907.054505] crc32c_intel [55907.054506] libsas [55907.054506] libsas [55907.054506] libahci [55907.054507] libahci [55907.054507] pps_core [55907.054508] pps_core [55907.054508] drm [55907.054509] drm [55907.054509] mlx4_core [55907.054510] mlx4_core [55907.054510] mpt2sas [55907.054510] mpt2sas [55907.054511] dca [55907.054511] dca [55907.054512] libata [55907.054512] libata [55907.054513] raid_class [55907.054513] raid_class [55907.054514] i2c_algo_bit [55907.054514] i2c_algo_bit [55907.054515] scsi_transport_sas [55907.054515] scsi_transport_sas [55907.054516] devlink [55907.054516] devlink [55907.054516] i2c_core [55907.054517] i2c_core [55907.054517] fjes [55907.054518] [55907.054518] fjes [55907.054519] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55907.054520] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55907.054521] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55907.054521] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55907.054522] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55907.054523] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55907.054524] RIP: 0010:[] [55907.054525] [] _raw_spin_lock+0x32/0x50 [55907.054526] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55907.054527] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [55907.054528] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [55907.054528] RAX: 00000000000063aa RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55907.054529] RAX: 0000000000001d39 RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55907.054530] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55907.054530] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55907.054531] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55907.054532] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55907.054532] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55907.054533] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55907.054534] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55907.054534] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55907.054535] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55907.054536] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55907.054537] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55907.054537] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55907.054538] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55907.054539] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55907.054539] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55907.054540] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55907.054541] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55907.054541] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55907.054542] Stack: [55907.054542] Stack: [55907.054543] ffff88082363bcd8 [55907.054543] ffff8808262efac0 [55907.054544] ffffffffa08aa7c8 [55907.054544] ffffffffa08aa7c8 [55907.054545] ffffffffa08aa8b6 [55907.054545] ffffffffa08aa8b6 [55907.054546] ffff8804c1ba6800 [55907.054546] [55907.054546] 0000000000000000 [55907.054547] 0000000000000001 [55907.054548] ffff88083fb5e000 [55907.054548] ffff88082363bd08 [55907.054548] ffff8808262efb80 [55907.054549] ffffffffa09217bb [55907.054549] ffffffffa092c9b5 [55907.054550] ffff8804c1ba6800 [55907.054550] [55907.054551] 00000000000001a8 [55907.054551] 0000000000000001 [55907.054552] ffffc9000e331730 [55907.054552] ffff88082abba340 [55907.054553] ffff8808262efb68 [55907.054553] ffff88082abba348 [55907.054554] ffffffffa0ebd587 [55907.054554] ffff88082363bd70 [55907.054555] [55907.054555] ffffffffffffffff [55907.054555] Call Trace: [55907.054556] Call Trace: [55907.054561] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55907.054566] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55907.054571] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55907.054576] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55907.054582] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55907.054587] [] lnet_select_pathway+0x235/0x1140 [lnet] [55907.054593] [] lnet_finalize+0x1e9/0x690 [lnet] [55907.054597] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55907.054600] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55907.054606] [] lnet_send+0x51/0x180 [lnet] [55907.054609] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55907.054615] [] lnet_finalize+0x3f1/0x690 [lnet] [55907.054616] [] ? dequeue_task_fair+0x41e/0x660 [55907.054620] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55907.054622] [] ? wake_up_state+0x20/0x20 [55907.054624] [] ? __slab_free+0x81/0x2f0 [55907.054627] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55907.054633] [] lnet_ni_recv+0xc3/0x320 [lnet] [55907.054634] [] kthread+0xcf/0xe0 [55907.054640] [] lnet_parse+0x945/0xf90 [lnet] [55907.054641] [] ? kthread_create_on_node+0x140/0x140 [55907.054645] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55907.054647] [] ret_from_fork+0x58/0x90 [55907.054649] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55907.054651] [] ? kthread_create_on_node+0x140/0x140 [55907.054654] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55907.054655] Code: [55907.054657] [] ? dequeue_task_fair+0x41e/0x660 [55907.054658] 00 02 [55907.054659] [] ? wake_up_state+0x20/0x20 [55907.054660] 00 f0 [55907.054664] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55907.054664] 0f c1 [55907.054666] [] kthread+0xcf/0xe0 [55907.054667] 07 89 [55907.054669] [] ? kthread_create_on_node+0x140/0x140 [55907.054669] c2 c1 [55907.054671] [] ret_from_fork+0x58/0x90 [55907.054672] ea 10 [55907.054674] [] ? kthread_create_on_node+0x140/0x140 [55907.054674] 66 39 c2 [55907.054675] Code: [55907.054675] 75 [55907.054676] 00 [55907.054676] 01 [55907.054677] 02 [55907.054677] c3 [55907.054677] 00 [55907.054678] 55 [55907.054678] f0 [55907.054679] 83 [55907.054679] 0f [55907.054680] e2 [55907.054680] c1 [55907.054680] fe [55907.054681] 07 [55907.054681] 0f [55907.054682] 89 [55907.054682] b7 [55907.054682] c2 [55907.054683] f2 [55907.054683] c1 [55907.054684] 48 [55907.054684] ea [55907.054685] 89 [55907.054685] 10 [55907.054685] e5 [55907.054686] 66 [55907.054686] b8 [55907.054687] 39 [55907.054687] 00 [55907.054688] c2 [55907.054688] 80 [55907.054688] 75 [55907.054689] 00 [55907.054689] 01 [55907.054690] 00 [55907.054690] c3 [55907.054691] eb [55907.054691] 55 [55907.054691] 0d [55907.054692] 83 [55907.054692] 66 [55907.054693] e2 [55907.054693] 0f [55907.054694] fe [55907.054694] 1f [55907.054694] 0f [55907.054695] 44 [55907.054695] b7 [55907.054696] 00 [55907.054696] f2 [55907.054697] 00 [55907.054697] 48 [55907.054697] f3 [55907.054698] 89 [55907.054698] 90 [55907.054699] e5 [55907.054699] <83> [55907.054700] b8 [55907.054700] e8 [55907.054701] 00 [55907.054701] 01 [55907.054701] 80 [55907.054702] 74 [55907.054702] 00 [55907.054703] 0a [55907.054703] 00 [55907.054703] 0f [55907.054704] eb [55907.054704] b7 [55907.054705] 0d [55907.054705] 0f [55907.054706] 66 [55907.054706] 66 [55907.054706] 0f [55907.054707] 39 [55907.054707] 1f [55907.054708] ca [55907.054708] 44 [55907.054708] 75 [55907.054709] 00 [55907.054709] f1 [55907.054710] 00 [55907.054710] 5d [55907.054711] f3 [55907.054711] c3 [55907.054711] 90 [55907.054712] 66 [55907.054712] <83> [55907.054713] 66 [55907.054713] e8 [55907.054714] 66 [55907.054714] 01 [55907.054714] 90 [55907.054715] 74 [55907.054715] 66 [55907.054716] 0a [55907.054716] 66 [55907.054717] [55907.054719] 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55911.253524] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55911.490073] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55911.559789] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55911.595990] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55911.633348] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55911.658425] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55911.685822] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55911.703785] RAX: 0000000000002170 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55911.727794] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55911.751791] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55911.775848] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55911.799878] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55911.823907] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55911.851105] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55911.870592] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55911.894672] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55911.918774] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55911.942879] Stack: [55911.950113] ffff8808217bbcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba3200 [55911.975178] 0000000000000001 ffff8808217bbd08 ffffffffa09217bb ffff8804c1ba3200 [55912.000220] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff8808217bbd70 [55912.025281] Call Trace: [55912.034013] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55912.055909] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55912.078708] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55912.100914] [] lnet_finalize+0x1e9/0x690 [lnet] [55912.121981] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55912.144499] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55912.167900] [] ? dequeue_task_fair+0x41e/0x660 [55912.188746] [] ? wake_up_state+0x20/0x20 [55912.207861] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55912.230704] [] kthread+0xcf/0xe0 [55912.247561] [] ? kthread_create_on_node+0x140/0x140 [55912.269870] [] ret_from_fork+0x58/0x90 [55912.288471] [] ? kthread_create_on_node+0x140/0x140 [55912.310782] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55919.053179] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [55919.053181] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55919.053182] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55919.053184] Modules linked in: [55919.053184] Modules linked in: [55919.053185] ko2iblnd(OE) [55919.053185] ko2iblnd(OE) [55919.053186] lustre(OE) [55919.053186] lustre(OE) [55919.053186] lmv(OE) [55919.053187] lmv(OE) [55919.053187] fld(OE) [55919.053188] fld(OE) [55919.053188] mdc(OE) [55919.053189] mdc(OE) [55919.053189] fid(OE) [55919.053189] fid(OE) [55919.053190] lov(OE) [55919.053190] lov(OE) [55919.053191] ksocklnd(OE) [55919.053191] ksocklnd(OE) [55919.053192] ptlrpc(OE) [55919.053192] ptlrpc(OE) [55919.053193] obdclass(OE) [55919.053193] obdclass(OE) [55919.053193] lnet(OE) [55919.053194] lnet(OE) [55919.053194] libcfs(OE) [55919.053195] libcfs(OE) [55919.053195] rpcrdma [55919.053196] rpcrdma [55919.053196] ib_isert [55919.053197] ib_isert [55919.053197] iscsi_target_mod [55919.053198] iscsi_target_mod [55919.053198] ib_iser [55919.053198] ib_iser [55919.053199] libiscsi [55919.053199] libiscsi [55919.053200] scsi_transport_iscsi [55919.053200] scsi_transport_iscsi [55919.053201] ib_srpt [55919.053201] ib_srpt [55919.053202] target_core_mod [55919.053202] target_core_mod [55919.053202] ib_srp [55919.053203] ib_srp [55919.053203] scsi_transport_srp [55919.053204] scsi_transport_srp [55919.053204] scsi_tgt [55919.053205] scsi_tgt [55919.053205] ib_ipoib [55919.053206] ib_ipoib [55919.053206] rdma_ucm [55919.053206] rdma_ucm [55919.053207] ib_ucm [55919.053207] ib_ucm [55919.053208] ib_uverbs [55919.053208] ib_uverbs [55919.053209] ib_umad [55919.053209] ib_umad [55919.053209] rdma_cm [55919.053210] rdma_cm [55919.053210] ib_cm [55919.053211] ib_cm [55919.053211] iw_cm [55919.053212] iw_cm [55919.053212] mlx4_ib [55919.053213] mlx4_ib [55919.053213] intel_powerclamp [55919.053213] intel_powerclamp [55919.053214] coretemp [55919.053214] coretemp [55919.053215] intel_rapl [55919.053215] intel_rapl [55919.053216] iosf_mbi [55919.053216] iosf_mbi [55919.053216] kvm [55919.053217] kvm [55919.053217] irqbypass [55919.053218] irqbypass [55919.053218] crc32_pclmul [55919.053219] crc32_pclmul [55919.053219] ghash_clmulni_intel [55919.053220] ghash_clmulni_intel [55919.053220] aesni_intel [55919.053221] aesni_intel [55919.053221] ipmi_ssif [55919.053221] ipmi_ssif [55919.053222] lrw [55919.053222] lrw [55919.053223] gf128mul [55919.053223] gf128mul [55919.053224] glue_helper [55919.053224] glue_helper [55919.053225] hfi1 [55919.053225] hfi1 [55919.053225] ipmi_devintf [55919.053226] ipmi_devintf [55919.053226] rdmavt [55919.053227] rdmavt [55919.053227] ablk_helper [55919.053228] ablk_helper [55919.053228] cryptd [55919.053229] cryptd [55919.053229] iTCO_wdt [55919.053229] iTCO_wdt [55919.053230] ipmi_si [55919.053230] ipmi_si [55919.053231] sb_edac [55919.053231] sb_edac [55919.053232] sg [55919.053232] sg [55919.053232] iTCO_vendor_support [55919.053233] iTCO_vendor_support [55919.053233] ib_core [55919.053234] ib_core [55919.053234] ioatdma [55919.053235] ioatdma [55919.053235] i2c_i801 [55919.053235] i2c_i801 [55919.053236] mei_me [55919.053236] mei_me [55919.053237] mei [55919.053237] mei [55919.053238] edac_core [55919.053238] edac_core [55919.053239] ipmi_msghandler [55919.053239] ipmi_msghandler [55919.053239] lpc_ich [55919.053240] lpc_ich [55919.053240] ntb [55919.053241] ntb [55919.053241] wmi [55919.053242] wmi [55919.053242] shpchp [55919.053243] shpchp [55919.053243] pcspkr [55919.053243] pcspkr [55919.053244] nfsd [55919.053244] nfsd [55919.053245] nfs_acl [55919.053245] nfs_acl [55919.053246] lockd [55919.053246] lockd [55919.053247] grace [55919.053247] grace [55919.053247] auth_rpcgss [55919.053248] auth_rpcgss [55919.053248] sunrpc [55919.053249] sunrpc [55919.053249] ip_tables [55919.053250] ip_tables [55919.053250] ext4 [55919.053251] ext4 [55919.053251] mbcache [55919.053251] mbcache [55919.053252] jbd2 [55919.053252] jbd2 [55919.053253] sd_mod [55919.053253] sd_mod [55919.053254] crc_t10dif [55919.053254] crc_t10dif [55919.053255] crct10dif_generic [55919.053255] crct10dif_generic [55919.053255] mlx4_en [55919.053256] mlx4_en [55919.053256] mgag200 [55919.053257] mgag200 [55919.053257] drm_kms_helper [55919.053258] drm_kms_helper [55919.053258] syscopyarea [55919.053258] syscopyarea [55919.053259] sysfillrect [55919.053259] sysfillrect [55919.053260] sysimgblt [55919.053260] sysimgblt [55919.053261] fb_sys_fops [55919.053261] fb_sys_fops [55919.053261] isci [55919.053262] isci [55919.053262] ttm [55919.053263] ttm [55919.053263] igb [55919.053264] igb [55919.053264] crct10dif_pclmul [55919.053265] crct10dif_pclmul [55919.053265] crct10dif_common [55919.053265] crct10dif_common [55919.053266] ahci [55919.053266] ahci [55919.053267] ptp [55919.053267] ptp [55919.053268] crc32c_intel [55919.053268] crc32c_intel [55919.053269] libsas [55919.053269] libsas [55919.053269] libahci [55919.053270] libahci [55919.053270] pps_core [55919.053271] pps_core [55919.053271] drm [55919.053272] drm [55919.053272] mlx4_core [55919.053273] mlx4_core [55919.053273] mpt2sas [55919.053273] mpt2sas [55919.053274] dca [55919.053274] dca [55919.053275] libata [55919.053275] libata [55919.053276] raid_class [55919.053276] raid_class [55919.053276] i2c_algo_bit [55919.053277] i2c_algo_bit [55919.053277] scsi_transport_sas [55919.053278] scsi_transport_sas [55919.053278] devlink [55919.053279] devlink [55919.053279] i2c_core [55919.053280] i2c_core [55919.053280] fjes [55919.053281] [55919.053281] fjes [55919.053282] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55919.053283] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55919.053284] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55919.053284] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55919.053285] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55919.053286] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55919.053287] RIP: 0010:[] [55919.053288] [] _raw_spin_lock+0x32/0x50 [55919.053289] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55919.053290] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55919.053291] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [55919.053291] RAX: 0000000000006229 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55919.053292] RAX: 00000000000028ea RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55919.053293] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55919.053293] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55919.053294] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55919.053294] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55919.053295] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55919.053296] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55919.053296] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55919.053297] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55919.053298] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55919.053298] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55919.053299] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55919.053300] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55919.053301] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55919.053301] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55919.053302] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55919.053302] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55919.053303] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55919.053304] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55919.053304] Stack: [55919.053304] Stack: [55919.053305] ffff88082626bd08 [55919.053305] ffff880820b03c28 [55919.053306] ffffffffa08aa7c8 [55919.053306] ffffffffa08aa7c8 [55919.053307] ffff88082af11648 [55919.053307] ffff880820b03c28 [55919.053308] ffff8800b484ea00 [55919.053308] [55919.053308] 0000000000000000 [55919.053309] 0000000000000000 [55919.053309] ffff88083fb5e000 [55919.053310] ffff88082626bd70 [55919.053310] ffff880820b03c98 [55919.053311] ffffffffa0922006 [55919.053311] ffffffffa092c346 [55919.053312] ffff88082af10fb0 [55919.053312] [55919.053313] ffff880820b03c60 [55919.053313] ffff88082af10fb0 [55919.053314] 00000000000000b4 [55919.053314] ffff88082af10fb0 [55919.053315] 00000001033f89f4 [55919.053315] ffff88082626bd30 [55919.053316] ffff8800b4864274 [55919.053316] ffff88082626bd30 [55919.053316] [55919.053317] 0000000000000000 [55919.053317] Call Trace: [55919.053318] Call Trace: [55919.053323] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55919.053329] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55919.053334] [] lnet_finalize+0xf6/0x690 [lnet] [55919.053341] [] lnet_post_send_locked+0x446/0x880 [lnet] [55919.053344] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55919.053350] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55919.053353] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55919.053359] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55919.053361] [] ? dequeue_task_fair+0x41e/0x660 [55919.053366] [] lnet_finalize+0x1e9/0x690 [lnet] [55919.053367] [] ? wake_up_state+0x20/0x20 [55919.053371] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55919.053374] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55919.053378] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55919.053380] [] kthread+0xcf/0xe0 [55919.053382] [] ? dequeue_task_fair+0x41e/0x660 [55919.053383] [] ? kthread_create_on_node+0x140/0x140 [55919.053385] [] ? sched_clock_cpu+0x85/0xc0 [55919.053387] [] ret_from_fork+0x58/0x90 [55919.053388] [] ? wake_up_state+0x20/0x20 [55919.053390] [] ? kthread_create_on_node+0x140/0x140 [55919.053393] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55919.053394] Code: [55919.053396] [] kthread+0xcf/0xe0 [55919.053396] 00 02 [55919.053398] [] ? kthread_create_on_node+0x140/0x140 [55919.053399] 00 f0 [55919.053400] [] ret_from_fork+0x58/0x90 [55919.053401] 0f c1 [55919.053403] [] ? kthread_create_on_node+0x140/0x140 [55919.053403] 07 89 c2 [55919.053404] Code: [55919.053404] c1 [55919.053405] 00 [55919.053405] ea [55919.053406] 02 [55919.053406] 10 [55919.053406] 00 [55919.053407] 66 [55919.053407] f0 [55919.053408] 39 [55919.053408] 0f [55919.053408] c2 [55919.053409] c1 [55919.053409] 75 [55919.053410] 07 [55919.053410] 01 [55919.053410] 89 [55919.053411] c3 [55919.053411] c2 [55919.053412] 55 [55919.053412] c1 [55919.053412] 83 [55919.053413] ea [55919.053413] e2 [55919.053414] 10 [55919.053414] fe [55919.053414] 66 [55919.053415] 0f [55919.053415] 39 [55919.053416] b7 [55919.053416] c2 [55919.053416] f2 [55919.053417] 75 [55919.053417] 48 [55919.053418] 01 [55919.053418] 89 [55919.053418] c3 [55919.053419] e5 [55919.053419] 55 [55919.053420] b8 [55919.053420] 83 [55919.053420] 00 [55919.053421] e2 [55919.053421] 80 [55919.053422] fe [55919.053422] 00 [55919.053422] 0f [55919.053423] 00 [55919.053423] b7 [55919.053424] eb [55919.053424] f2 [55919.053424] 0d [55919.053425] 48 [55919.053425] 66 [55919.053426] 89 [55919.053426] 0f [55919.053426] e5 [55919.053427] 1f [55919.053427] b8 [55919.053428] 44 [55919.053428] 00 [55919.053428] 00 [55919.053429] 80 [55919.053429] 00 [55919.053430] 00 [55919.053430] f3 [55919.053431] 00 [55919.053431] 90 [55919.053431] eb [55919.053432] <83> [55919.053432] 0d [55919.053433] e8 [55919.053433] 66 [55919.053433] 01 [55919.053434] 0f [55919.053434] 74 [55919.053435] 1f [55919.053435] 0a [55919.053435] 44 [55919.053436] 0f [55919.053436] 00 [55919.053437] b7 [55919.053437] 00 [55919.053437] 0f [55919.053438] f3 [55919.053438] 66 [55919.053439] 90 [55919.053439] 39 [55919.053440] <83> [55919.053440] ca [55919.053440] e8 [55919.053441] 75 [55919.053441] 01 [55919.053442] f1 [55919.053442] 74 [55919.053443] 5d [55919.053443] 0a [55919.053443] c3 [55919.053444] 0f [55919.053444] 66 [55919.053445] b7 [55919.053445] 66 [55919.053445] 0f [55919.053446] 66 [55919.053446] 66 [55919.053447] 90 [55919.053447] 39 [55919.053447] 66 [55919.053448] ca [55919.053448] 66 [55919.053449] [55919.053450] 75 f1 5d c3 66 66 66 90 66 66 [55923.052767] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 21s! [kiblnd_sd_00_02:2898] [55923.052768] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [55923.052769] Modules linked in: [55923.052770] Modules linked in: [55923.052770] ko2iblnd(OE) [55923.052771] ko2iblnd(OE) [55923.052771] lustre(OE) [55923.052772] lustre(OE) [55923.052772] lmv(OE) [55923.052773] lmv(OE) [55923.052773] fld(OE) [55923.052774] fld(OE) [55923.052774] mdc(OE) [55923.052775] mdc(OE) [55923.052775] fid(OE) [55923.052776] fid(OE) [55923.052776] lov(OE) [55923.052776] lov(OE) [55923.052777] ksocklnd(OE) [55923.052777] ksocklnd(OE) [55923.052778] ptlrpc(OE) [55923.052778] ptlrpc(OE) [55923.052779] obdclass(OE) [55923.052779] obdclass(OE) [55923.052780] lnet(OE) [55923.052780] lnet(OE) [55923.052781] libcfs(OE) [55923.052781] libcfs(OE) [55923.052782] rpcrdma [55923.052782] rpcrdma [55923.052783] ib_isert [55923.052783] ib_isert [55923.052784] iscsi_target_mod [55923.052784] iscsi_target_mod [55923.052784] ib_iser [55923.052785] ib_iser [55923.052785] libiscsi [55923.052786] libiscsi [55923.052786] scsi_transport_iscsi [55923.052787] scsi_transport_iscsi [55923.052787] ib_srpt [55923.052788] ib_srpt [55923.052788] target_core_mod [55923.052789] target_core_mod [55923.052789] ib_srp [55923.052790] ib_srp [55923.052790] scsi_transport_srp [55923.052790] scsi_transport_srp [55923.052791] scsi_tgt [55923.052791] scsi_tgt [55923.052792] ib_ipoib [55923.052792] ib_ipoib [55923.052793] rdma_ucm [55923.052793] rdma_ucm [55923.052793] ib_ucm [55923.052794] ib_ucm [55923.052794] ib_uverbs [55923.052795] ib_uverbs [55923.052795] ib_umad [55923.052796] ib_umad [55923.052796] rdma_cm [55923.052797] rdma_cm [55923.052797] ib_cm [55923.052798] ib_cm [55923.052798] iw_cm [55923.052798] iw_cm [55923.052799] mlx4_ib [55923.052799] mlx4_ib [55923.052800] intel_powerclamp [55923.052800] intel_powerclamp [55923.052801] coretemp [55923.052801] coretemp [55923.052802] intel_rapl [55923.052802] intel_rapl [55923.052802] iosf_mbi [55923.052803] iosf_mbi [55923.052803] kvm [55923.052804] kvm [55923.052804] irqbypass [55923.052805] irqbypass [55923.052805] crc32_pclmul [55923.052806] crc32_pclmul [55923.052806] ghash_clmulni_intel [55923.052807] ghash_clmulni_intel [55923.052807] aesni_intel [55923.052808] aesni_intel [55923.052808] ipmi_ssif [55923.052808] ipmi_ssif [55923.052809] lrw [55923.052809] lrw [55923.052810] gf128mul [55923.052810] gf128mul [55923.052811] glue_helper [55923.052811] glue_helper [55923.052812] hfi1 [55923.052812] hfi1 [55923.052813] ipmi_devintf [55923.052813] ipmi_devintf [55923.052813] rdmavt [55923.052814] rdmavt [55923.052814] ablk_helper [55923.052815] ablk_helper [55923.052815] cryptd [55923.052816] cryptd [55923.052816] iTCO_wdt [55923.052817] iTCO_wdt [55923.052817] ipmi_si [55923.052818] ipmi_si [55923.052818] sb_edac [55923.052818] sb_edac [55923.052819] sg [55923.052819] sg [55923.052820] iTCO_vendor_support [55923.052820] iTCO_vendor_support [55923.052821] ib_core [55923.052821] ib_core [55923.052822] ioatdma [55923.052822] ioatdma [55923.052822] i2c_i801 [55923.052823] i2c_i801 [55923.052823] mei_me [55923.052824] mei_me [55923.052824] mei [55923.052825] mei [55923.052825] edac_core [55923.052826] edac_core [55923.052826] ipmi_msghandler [55923.052827] ipmi_msghandler [55923.052827] lpc_ich [55923.052828] lpc_ich [55923.052828] ntb [55923.052829] ntb [55923.052829] wmi [55923.052829] wmi [55923.052830] shpchp [55923.052830] shpchp [55923.052831] pcspkr [55923.052831] pcspkr [55923.052832] nfsd [55923.052832] nfsd [55923.052833] nfs_acl [55923.052833] nfs_acl [55923.052834] lockd [55923.052834] lockd [55923.052835] grace [55923.052835] grace [55923.052835] auth_rpcgss [55923.052836] auth_rpcgss [55923.052836] sunrpc [55923.052837] sunrpc [55923.052837] ip_tables [55923.052838] ip_tables [55923.052838] ext4 [55923.052839] ext4 [55923.052839] mbcache [55923.052839] mbcache [55923.052840] jbd2 [55923.052840] jbd2 [55923.052841] sd_mod [55923.052841] sd_mod [55923.052842] crc_t10dif [55923.052842] crc_t10dif [55923.052843] crct10dif_generic [55923.052843] crct10dif_generic [55923.052844] mlx4_en [55923.052844] mlx4_en [55923.052844] mgag200 [55923.052845] mgag200 [55923.052845] drm_kms_helper [55923.052846] drm_kms_helper [55923.052846] syscopyarea [55923.052847] syscopyarea [55923.052847] sysfillrect [55923.052848] sysfillrect [55923.052848] sysimgblt [55923.052849] sysimgblt [55923.052849] fb_sys_fops [55923.052849] fb_sys_fops [55923.052850] isci [55923.052850] isci [55923.052851] ttm [55923.052851] ttm [55923.052852] igb [55923.052852] igb [55923.052853] crct10dif_pclmul [55923.052853] crct10dif_pclmul [55923.052854] crct10dif_common [55923.052854] crct10dif_common [55923.052855] ahci [55923.052855] ahci [55923.052855] ptp [55923.052856] ptp [55923.052856] crc32c_intel [55923.052857] crc32c_intel [55923.052857] libsas [55923.052858] libsas [55923.052858] libahci [55923.052859] libahci [55923.052859] pps_core [55923.052860] pps_core [55923.052860] drm [55923.052860] drm [55923.052861] mlx4_core [55923.052861] mlx4_core [55923.052862] mpt2sas [55923.052862] mpt2sas [55923.052863] dca [55923.052863] dca [55923.052864] libata [55923.052864] libata [55923.052864] raid_class [55923.052865] raid_class [55923.052865] i2c_algo_bit [55923.052866] i2c_algo_bit [55923.052866] scsi_transport_sas [55923.052867] scsi_transport_sas [55923.052867] devlink [55923.052868] devlink [55923.052868] i2c_core [55923.052869] i2c_core [55923.052869] fjes [55923.052869] [55923.052870] fjes [55923.052871] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55923.052872] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55923.052873] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55923.052873] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55923.052874] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55923.052875] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55923.052876] RIP: 0010:[] [55923.052877] [] _raw_spin_lock+0x32/0x50 [55923.052879] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55923.052879] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [55923.052880] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55923.052881] RAX: 00000000000014dd RBX: 0000000100000000 RCX: 0000000000000a1e [55923.052881] RAX: 00000000000019ee RBX: 0000000000016c40 RCX: 0000000000000a1e [55923.052882] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55923.052883] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55923.052883] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55923.052884] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55923.052885] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55923.052885] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55923.052886] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55923.052887] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55923.052888] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55923.052888] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55923.052889] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55923.052890] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55923.052891] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55923.052891] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55923.052892] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55923.052893] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55923.052893] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55923.052894] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55923.052894] Stack: [55923.052894] Stack: [55923.052895] ffff88082aae7ba0 [55923.052896] ffff88082aa33ca0 [55923.052896] ffffffffa08aa7c8 [55923.052897] ffffffffa08aa7c8 [55923.052897] 000000000000763d [55923.052897] ffff88082aa33ca8 [55923.052898] ffff8803fe4d0400 [55923.052898] [55923.052899] 0000000000000000 [55923.052899] 0000000000000000 [55923.052900] 00050000c0a80184 [55923.052900] ffff88082aae7c08 [55923.052901] ffff88082aa33cc8 [55923.052901] ffffffffa0922006 [55923.052902] ffffffffa0916cd0 [55923.052902] ffff8804c1676dd0 [55923.052903] [55923.052903] ffff8800366a4530 [55923.052904] ffff8804c1676dd0 [55923.052904] ffff880821b758e8 [55923.052905] ffff8804c1676dd0 [55923.052905] ffff88083fb5e000 [55923.052906] 0000000000000078 [55923.052906] ffff88082aa33ce0 [55923.052906] ffff88082aae7c48 [55923.052907] [55923.052907] ffffffffa0eb8460 [55923.052908] Call Trace: [55923.052908] Call Trace: [55923.052913] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55923.052919] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55923.052925] [] lnet_finalize+0xf6/0x690 [lnet] [55923.052930] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55923.052934] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55923.052938] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55923.052939] [] ? enqueue_task_fair+0x208/0x6c0 [55923.052943] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55923.052944] [] ? sched_clock_cpu+0x85/0xc0 [55923.052946] [] ? lock_timer_base.isra.33+0x2b/0x50 [55923.052952] [] lnet_ni_recv+0xc3/0x320 [lnet] [55923.052953] [] ? try_to_del_timer_sync+0x5e/0x90 [55923.052959] [] lnet_parse+0x945/0xf90 [lnet] [55923.052963] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55923.052967] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55923.052968] [] ? schedule_timeout+0x17d/0x2d0 [55923.052969] [] ? __wake_up+0x44/0x50 [55923.052973] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55923.052977] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55923.052979] [] ? wake_up_state+0x20/0x20 [55923.052980] [] ? dequeue_task_fair+0x41e/0x660 [55923.052984] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55923.052985] [] ? wake_up_state+0x20/0x20 [55923.052987] [] kthread+0xcf/0xe0 [55923.052990] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55923.052992] [] ? kthread_create_on_node+0x140/0x140 [55923.052994] [] kthread+0xcf/0xe0 [55923.052996] [] ret_from_fork+0x58/0x90 [55923.052997] [] ? kthread_create_on_node+0x140/0x140 [55923.052999] [] ? kthread_create_on_node+0x140/0x140 [55923.053001] [] ret_from_fork+0x58/0x90 [55923.053002] Code: [55923.053003] [] ? kthread_create_on_node+0x140/0x140 [55923.053004] 00 02 00 [55923.053005] Code: [55923.053005] f0 [55923.053006] 00 [55923.053006] 0f [55923.053006] 02 [55923.053007] c1 [55923.053007] 00 [55923.053008] 07 [55923.053008] f0 [55923.053008] 89 [55923.053009] 0f [55923.053009] c2 [55923.053010] c1 [55923.053010] c1 [55923.053011] 07 [55923.053011] ea [55923.053011] 89 [55923.053012] 10 [55923.053012] c2 [55923.053013] 66 [55923.053013] c1 [55923.053014] 39 [55923.053014] ea [55923.053014] c2 [55923.053015] 10 [55923.053015] 75 [55923.053016] 66 [55923.053016] 01 [55923.053017] 39 [55923.053017] c3 [55923.053017] c2 [55923.053018] 55 [55923.053018] 75 [55923.053019] 83 [55923.053019] 01 [55923.053019] e2 [55923.053020] c3 [55923.053020] fe [55923.053021] 55 [55923.053021] 0f [55923.053022] 83 [55923.053022] b7 [55923.053022] e2 [55923.053023] f2 [55923.053023] fe [55923.053024] 48 [55923.053024] 0f [55923.053024] 89 [55923.053025] b7 [55923.053025] e5 [55923.053026] f2 [55923.053026] b8 [55923.053027] 48 [55923.053027] 00 [55923.053027] 89 [55923.053028] 80 [55923.053028] e5 [55923.053029] 00 [55923.053029] b8 [55923.053030] 00 [55923.053030] 00 [55923.053030] eb [55923.053031] 80 [55923.053031] 0d [55923.053032] 00 [55923.053032] 66 [55923.053032] 00 [55923.053033] 0f [55923.053033] eb [55923.053034] 1f [55923.053034] 0d [55923.053035] 44 [55923.053035] 66 [55923.053035] 00 [55923.053036] 0f [55923.053036] 00 [55923.053037] 1f [55923.053037] f3 [55923.053038] 44 [55923.053038] 90 [55923.053038] 00 [55923.053039] <83> [55923.053039] 00 [55923.053040] e8 [55923.053040] f3 [55923.053041] 01 [55923.053041] 90 [55923.053041] 74 [55923.053042] <83> [55923.053042] 0a [55923.053043] e8 [55923.053043] 0f [55923.053044] 01 [55923.053044] b7 [55923.053044] 74 [55923.053045] 0f [55923.053045] 0a [55923.053046] 66 [55923.053046] 0f [55923.053047] 39 [55923.053047] b7 [55923.053047] ca [55923.053048] 0f [55923.053048] 75 [55923.053049] 66 [55923.053049] f1 [55923.053049] 39 [55923.053050] 5d [55923.053050] ca [55923.053051] c3 [55923.053051] 75 [55923.053052] 66 [55923.053052] f1 [55923.053052] 66 [55923.053053] 5d [55923.053053] 66 [55923.053054] c3 [55923.053054] 90 [55923.053055] 66 [55923.053055] 66 [55923.053055] 66 [55923.053056] 66 [55923.053056] [55923.053057] 66 90 66 66 [55927.052355] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [kiblnd_sd_01_03:2903] [55927.052373] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55927.052378] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55927.052379] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55927.052380] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55927.052381] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55927.052383] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55927.052383] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [55927.052384] RAX: 0000000000006bf9 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55927.052384] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55927.052385] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55927.052385] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55927.052386] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55927.052387] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55927.052387] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55927.052388] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55927.052388] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55927.052389] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55927.052389] Stack: [55927.052390] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55927.052391] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [55927.052392] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [55927.052392] Call Trace: [55927.052397] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55927.052402] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55927.052408] [] lnet_select_pathway+0x235/0x1140 [lnet] [55927.052411] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55927.052417] [] lnet_send+0x51/0x180 [lnet] [55927.052422] [] lnet_finalize+0x3f1/0x690 [lnet] [55927.052426] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55927.052428] [] ? __slab_free+0x81/0x2f0 [55927.052433] [] lnet_ni_recv+0xc3/0x320 [lnet] [55927.052438] [] lnet_parse+0x945/0xf90 [lnet] [55927.052443] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55927.052447] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55927.052449] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55927.052453] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55927.052454] [] ? dequeue_task_fair+0x41e/0x660 [55927.052456] [] ? wake_up_state+0x20/0x20 [55927.052459] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55927.052461] [] kthread+0xcf/0xe0 [55927.052463] [] ? kthread_create_on_node+0x140/0x140 [55927.052464] [] ret_from_fork+0x58/0x90 [55927.052466] [] ? kthread_create_on_node+0x140/0x140 [55927.052476] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55928.679074] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55928.919161] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55928.990324] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55929.026662] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55929.064453] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55929.089941] RIP: 0010:[] [] _raw_spin_lock+0x30/0x50 [55929.117787] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [55929.136142] RAX: 000000000000106b RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55929.160490] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55929.184853] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55929.209218] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55929.233584] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55929.257959] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55929.285483] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55929.305290] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55929.329705] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55929.354126] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55929.378496] Stack: [55929.386001] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [55929.411276] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [55929.436556] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [55929.461879] Call Trace: [55929.470849] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55929.492980] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55929.515984] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55929.541819] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55929.563944] [] lnet_finalize+0x1e9/0x690 [lnet] [55929.585215] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55929.607915] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55929.631497] [] ? dequeue_task_fair+0x41e/0x660 [55929.652472] [] ? wake_up_state+0x20/0x20 [55929.671741] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55929.694724] [] kthread+0xcf/0xe0 [55929.711717] [] ? kthread_create_on_node+0x140/0x140 [55929.734149] [] ret_from_fork+0x58/0x90 [55929.752861] [] ? kthread_create_on_node+0x140/0x140 [55929.775280] Code: b8 00 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 90 83 e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 [55931.051943] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [55931.051945] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55931.051964] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55931.051969] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55931.051970] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55931.051971] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55931.051973] Workqueue: ib_cm cm_work_handler [ib_cm] [55931.051974] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55931.051975] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55931.051976] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000202 [55931.051976] RAX: 0000000000004504 RBX: 0000000000000000 RCX: 0000000000000a1e [55931.051977] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55931.051977] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55931.051978] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55931.051978] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55931.051979] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55931.051980] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55931.051980] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55931.051981] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55931.051981] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55931.051981] Stack: [55931.051982] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [55931.051983] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [55931.051984] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [55931.051985] Call Trace: [55931.051990] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55931.051995] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55931.051999] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55931.052001] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55931.052004] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55931.052006] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55931.052008] [] cma_req_handler+0x227/0x570 [rdma_cm] [55931.052010] [] cm_process_work+0x25/0x140 [ib_cm] [55931.052012] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55931.052014] [] cm_work_handler+0x185/0x1484 [ib_cm] [55931.052015] [] process_one_work+0x17b/0x470 [55931.052016] [] worker_thread+0x126/0x410 [55931.052017] [] ? rescuer_thread+0x460/0x460 [55931.052018] [] kthread+0xcf/0xe0 [55931.052020] [] ? kthread_create_on_node+0x140/0x140 [55931.052021] [] ret_from_fork+0x58/0x90 [55931.052023] [] ? kthread_create_on_node+0x140/0x140 [55931.052033] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55931.951863] INFO: rcu_sched self-detected stall on CPU { 11} (t=1140018 jiffies g=99188 c=99187 q=30420) [55931.951864] Task dump for CPU 1: [55931.951866] kiblnd_sd_00_00 R running task 0 2896 2 0x00000088 [55931.951867] ffff880820b03d30 000000003f673d40 ffffc90003ec02f0 ffff88041d63d600 [55931.951868] 0000000000000000 0000000000000000 ffff88042aa84a28 ffff880820b03da8 [55931.951869] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000002 [55931.951870] Call Trace: [55931.951874] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55931.951877] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55931.951879] [] ? dequeue_task_fair+0x41e/0x660 [55931.951880] [] ? sched_clock_cpu+0x85/0xc0 [55931.951882] [] ? wake_up_state+0x20/0x20 [55931.951886] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55931.951887] [] ? kthread+0xcf/0xe0 [55931.951889] [] ? kthread_create_on_node+0x140/0x140 [55931.951891] [] ? ret_from_fork+0x58/0x90 [55931.951892] [] ? kthread_create_on_node+0x140/0x140 [55931.951893] Task dump for CPU 5: [55931.951895] kiblnd_sd_00_02 R running task 0 2898 2 0x00000088 [55931.951896] ffff880415b13000 ffff8800abaa43a8 ffff88083fb5e000 0000000000000000 [55931.951897] ffff88082aae7da8 ffffffffa0ebedab ffff880181447188 ffff88082aae7da8 [55931.951898] ffffffff810bc064 ffff880181447180 ffff88042aa84a00 0000000000000001 [55931.951898] Call Trace: [55931.951901] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55931.951903] [] ? __wake_up+0x44/0x50 [55931.951906] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55931.951907] [] ? dequeue_task_fair+0x41e/0x660 [55931.951909] [] ? wake_up_state+0x20/0x20 [55931.951912] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55931.951913] [] ? kthread+0xcf/0xe0 [55931.951915] [] ? kthread_create_on_node+0x140/0x140 [55931.951916] [] ? ret_from_fork+0x58/0x90 [55931.951918] [] ? kthread_create_on_node+0x140/0x140 [55931.951918] Task dump for CPU 6: [55931.951919] kiblnd_sd_00_01 R running task 0 2897 2 0x00000088 [55931.951920] ffff88081f64fd30 00000000f1dbfe3f ffffc90003ebfde0 ffff88042dcd1e00 [55931.951921] 0000000000000000 0000000000000000 ffff88042aa84a28 ffff88081f64fda8 [55931.951922] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000012 [55931.951922] Call Trace: [55931.951925] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55931.951929] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55931.951930] [] ? dequeue_task_fair+0x41e/0x660 [55931.951931] [] ? wake_up_state+0x20/0x20 [55931.951935] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55931.951936] [] ? kthread+0xcf/0xe0 [55931.951937] [] ? kthread_create_on_node+0x140/0x140 [55931.951939] [] ? ret_from_fork+0x58/0x90 [55931.951940] [] ? kthread_create_on_node+0x140/0x140 [55931.951941] Task dump for CPU 7: [55931.951942] kiblnd_sd_00_03 R running task 0 2899 2 0x00000088 [55931.951943] ffff88082626bd30 00000000aaf7c743 ffffc90003ebe8c8 ffff8800b484ea00 [55931.951944] 0000000000000000 0000000000000000 ffff88042aa84a28 ffff88082626bda8 [55931.951945] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000001 [55931.951945] Call Trace: [55931.951948] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55931.951951] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55931.951952] [] ? dequeue_task_fair+0x41e/0x660 [55931.951954] [] ? wake_up_state+0x20/0x20 [55931.951957] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55931.951959] [] ? kthread+0xcf/0xe0 [55931.951960] [] ? kthread_create_on_node+0x140/0x140 [55931.951961] [] ? ret_from_fork+0x58/0x90 [55931.951963] [] ? kthread_create_on_node+0x140/0x140 [55931.951963] Task dump for CPU 10: [55931.951964] kiblnd_sd_01_01 R running task 0 2901 2 0x00000088 [55931.951965] ffff8807f886f000 ffffc9001327c860 ffff8808266e7a00 0000000000000000 [55931.951966] ffff8808262efda8 ffffffffa0ebedab ffff8808262efe90 ffff8808262efda8 [55931.951967] ffffffffa03e7d2e ffff88082b2fc3c0 ffff88082bda6c00 0000000000000017 [55931.951967] Call Trace: [55931.951971] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55931.951974] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55931.951977] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55931.951978] [] ? dequeue_task_fair+0x41e/0x660 [55931.951980] [] ? wake_up_state+0x20/0x20 [55931.951983] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55931.951985] [] ? kthread+0xcf/0xe0 [55931.951986] [] ? kthread_create_on_node+0x140/0x140 [55931.951988] [] ? ret_from_fork+0x58/0x90 [55931.951989] [] ? kthread_create_on_node+0x140/0x140 [55931.951990] Task dump for CPU 11: [55931.951991] kiblnd_sd_01_02 R running task 0 2902 2 0x00000088 [55931.951992] ffff880826310000 000000005cf0ae83 ffff88082d8c3db0 ffffffff810c42f8 [55931.951993] 000000000000000b ffffffff81a1a680 ffff88082d8c3dc8 ffffffff810c7b89 [55931.951993] 000000000000000c ffff88082d8c3df8 ffffffff811373f0 ffff88082d8d01c0 [55931.951994] Call Trace: [55931.951996] [] sched_show_task+0xa8/0x110 [55931.951997] [] dump_cpu_task+0x39/0x70 [55931.952000] [] rcu_dump_cpu_stacks+0x90/0xd0 [55931.952002] [] rcu_check_callbacks+0x442/0x720 [55931.952005] [] ? tick_sched_handle.isra.13+0x60/0x60 [55931.952009] [] update_process_times+0x47/0x80 [55931.952010] [] tick_sched_handle.isra.13+0x25/0x60 [55931.952011] [] tick_sched_timer+0x41/0x70 [55931.952012] [] __hrtimer_run_queues+0xd2/0x260 [55931.952013] [] hrtimer_interrupt+0xb0/0x1e0 [55931.952015] [] ? call_softirq+0x1c/0x30 [55931.952019] [] local_apic_timer_interrupt+0x37/0x60 [55931.952021] [] smp_apic_timer_interrupt+0x3f/0x60 [55931.952022] [] apic_timer_interrupt+0x6d/0x80 [55931.952023] [] ? _raw_spin_lock+0x3a/0x50 [55931.952028] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55931.952033] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55931.952039] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55931.952044] [] lnet_finalize+0x1e9/0x690 [lnet] [55931.952047] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55931.952050] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55931.952052] [] ? dequeue_task_fair+0x41e/0x660 [55931.952053] [] ? wake_up_state+0x20/0x20 [55931.952057] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55931.952058] [] kthread+0xcf/0xe0 [55931.952060] [] ? kthread_create_on_node+0x140/0x140 [55931.952061] [] ret_from_fork+0x58/0x90 [55931.952063] [] ? kthread_create_on_node+0x140/0x140 [55931.952063] Task dump for CPU 12: [55931.952064] kiblnd_sd_01_03 R running task 0 2903 2 0x00000088 [55931.952065] ffff880820a21000 ffffc90011f075b0 ffff8808266e7a00 0000000000000000 [55931.952066] ffff8808213b7da8 ffffffffa0ebedab ffff8808213b7e90 ffff8808213b7da8 [55931.952067] ffffffffa03e7d2e ffff88082b2fc3c0 ffff88082c5f3600 0000000000000011 [55931.952067] Call Trace: [55931.952070] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55931.952073] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55931.952077] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55931.952078] [] ? dequeue_task_fair+0x41e/0x660 [55931.952079] [] ? wake_up_state+0x20/0x20 [55931.952083] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55931.952084] [] ? kthread+0xcf/0xe0 [55931.952085] [] ? kthread_create_on_node+0x140/0x140 [55931.952087] [] ? ret_from_fork+0x58/0x90 [55931.952088] [] ? kthread_create_on_node+0x140/0x140 [55931.952089] Task dump for CPU 15: [55931.952090] kiblnd_sd_01_00 R running task 0 2900 2 0x00000088 [55931.952091] ffff8808217bbd30 00000000937dfd11 ffffc9000e335900 ffff8804c1ba3200 [55931.952092] 0000000000000000 0000000000000000 ffff88082bda6c28 ffff8808217bbda8 [55931.952093] ffffffffa0eba166 ffff88082b2fc3c0 ffff88082bda6c00 0000000000000014 [55931.952093] Call Trace: [55931.952097] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55931.952100] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55931.952101] [] ? dequeue_task_fair+0x41e/0x660 [55931.952103] [] ? wake_up_state+0x20/0x20 [55931.952106] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55931.952107] [] ? kthread+0xcf/0xe0 [55931.952109] [] ? kthread_create_on_node+0x140/0x140 [55931.952110] [] ? ret_from_fork+0x58/0x90 [55931.952112] [] ? kthread_create_on_node+0x140/0x140 [55935.051531] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55935.051532] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 21s! [kiblnd_sd_01_00:2900] [55935.051533] Modules linked in: [55935.051534] Modules linked in: [55935.051534] ko2iblnd(OE) [55935.051534] ko2iblnd(OE) [55935.051535] lustre(OE) [55935.051535] lustre(OE) [55935.051536] lmv(OE) [55935.051536] lmv(OE) [55935.051537] fld(OE) [55935.051537] fld(OE) [55935.051538] mdc(OE) [55935.051538] mdc(OE) [55935.051538] fid(OE) [55935.051539] fid(OE) [55935.051539] lov(OE) [55935.051540] lov(OE) [55935.051540] ksocklnd(OE) [55935.051541] ksocklnd(OE) [55935.051541] ptlrpc(OE) [55935.051541] ptlrpc(OE) [55935.051542] obdclass(OE) [55935.051542] obdclass(OE) [55935.051543] lnet(OE) [55935.051543] lnet(OE) [55935.051544] libcfs(OE) [55935.051544] libcfs(OE) [55935.051545] rpcrdma [55935.051545] rpcrdma [55935.051546] ib_isert [55935.051546] ib_isert [55935.051547] iscsi_target_mod [55935.051547] iscsi_target_mod [55935.051547] ib_iser [55935.051548] ib_iser [55935.051548] libiscsi [55935.051549] libiscsi [55935.051549] scsi_transport_iscsi [55935.051550] scsi_transport_iscsi [55935.051550] ib_srpt [55935.051551] ib_srpt [55935.051551] target_core_mod [55935.051551] target_core_mod [55935.051552] ib_srp [55935.051552] ib_srp [55935.051553] scsi_transport_srp [55935.051553] scsi_transport_srp [55935.051554] scsi_tgt [55935.051554] scsi_tgt [55935.051555] ib_ipoib [55935.051555] ib_ipoib [55935.051555] rdma_ucm [55935.051556] rdma_ucm [55935.051556] ib_ucm [55935.051557] ib_ucm [55935.051557] ib_uverbs [55935.051558] ib_uverbs [55935.051558] ib_umad [55935.051559] ib_umad [55935.051559] rdma_cm [55935.051559] rdma_cm [55935.051560] ib_cm [55935.051560] ib_cm [55935.051561] iw_cm [55935.051561] iw_cm [55935.051562] mlx4_ib [55935.051562] mlx4_ib [55935.051562] intel_powerclamp [55935.051563] intel_powerclamp [55935.051563] coretemp [55935.051564] coretemp [55935.051564] intel_rapl [55935.051565] intel_rapl [55935.051565] iosf_mbi [55935.051565] iosf_mbi [55935.051566] kvm [55935.051566] kvm [55935.051567] irqbypass [55935.051567] irqbypass [55935.051568] crc32_pclmul [55935.051568] crc32_pclmul [55935.051569] ghash_clmulni_intel [55935.051569] ghash_clmulni_intel [55935.051570] aesni_intel [55935.051570] aesni_intel [55935.051570] ipmi_ssif [55935.051571] ipmi_ssif [55935.051571] lrw [55935.051572] lrw [55935.051572] gf128mul [55935.051573] gf128mul [55935.051573] glue_helper [55935.051574] glue_helper [55935.051574] hfi1 [55935.051574] hfi1 [55935.051575] ipmi_devintf [55935.051575] ipmi_devintf [55935.051576] rdmavt [55935.051576] rdmavt [55935.051577] ablk_helper [55935.051577] ablk_helper [55935.051578] cryptd [55935.051578] cryptd [55935.051578] iTCO_wdt [55935.051579] iTCO_wdt [55935.051579] ipmi_si [55935.051580] ipmi_si [55935.051580] sb_edac [55935.051581] sb_edac [55935.051581] sg [55935.051582] sg [55935.051582] iTCO_vendor_support [55935.051583] iTCO_vendor_support [55935.051583] ib_core [55935.051583] ib_core [55935.051584] ioatdma [55935.051584] ioatdma [55935.051585] i2c_i801 [55935.051585] i2c_i801 [55935.051586] mei_me [55935.051586] mei_me [55935.051586] mei [55935.051587] mei [55935.051587] edac_core [55935.051588] edac_core [55935.051588] ipmi_msghandler [55935.051589] ipmi_msghandler [55935.051589] lpc_ich [55935.051590] lpc_ich [55935.051590] ntb [55935.051590] ntb [55935.051591] wmi [55935.051591] wmi [55935.051592] shpchp [55935.051592] shpchp [55935.051593] pcspkr [55935.051593] pcspkr [55935.051594] nfsd [55935.051594] nfsd [55935.051594] nfs_acl [55935.051595] nfs_acl [55935.051595] lockd [55935.051596] lockd [55935.051596] grace [55935.051597] grace [55935.051597] auth_rpcgss [55935.051597] auth_rpcgss [55935.051598] sunrpc [55935.051598] sunrpc [55935.051599] ip_tables [55935.051599] ip_tables [55935.051600] ext4 [55935.051600] ext4 [55935.051601] mbcache [55935.051601] mbcache [55935.051601] jbd2 [55935.051602] jbd2 [55935.051602] sd_mod [55935.051603] sd_mod [55935.051603] crc_t10dif [55935.051604] crc_t10dif [55935.051604] crct10dif_generic [55935.051605] crct10dif_generic [55935.051605] mlx4_en [55935.051605] mlx4_en [55935.051606] mgag200 [55935.051606] mgag200 [55935.051607] drm_kms_helper [55935.051607] drm_kms_helper [55935.051608] syscopyarea [55935.051608] syscopyarea [55935.051609] sysfillrect [55935.051609] sysfillrect [55935.051609] sysimgblt [55935.051610] sysimgblt [55935.051610] fb_sys_fops [55935.051611] fb_sys_fops [55935.051611] isci [55935.051612] isci [55935.051612] ttm [55935.051613] ttm [55935.051613] igb [55935.051613] igb [55935.051614] crct10dif_pclmul [55935.051614] crct10dif_pclmul [55935.051615] crct10dif_common [55935.051615] crct10dif_common [55935.051616] ahci [55935.051616] ahci [55935.051617] ptp [55935.051617] ptp [55935.051617] crc32c_intel [55935.051618] crc32c_intel [55935.051618] libsas [55935.051619] libsas [55935.051619] libahci [55935.051620] libahci [55935.051620] pps_core [55935.051621] pps_core [55935.051621] drm [55935.051622] drm [55935.051622] mlx4_core [55935.051622] mlx4_core [55935.051623] mpt2sas [55935.051623] mpt2sas [55935.051624] dca [55935.051624] dca [55935.051625] libata [55935.051625] libata [55935.051626] raid_class [55935.051626] raid_class [55935.051626] i2c_algo_bit [55935.051627] i2c_algo_bit [55935.051627] scsi_transport_sas [55935.051628] scsi_transport_sas [55935.051628] devlink [55935.051629] devlink [55935.051629] i2c_core [55935.051630] i2c_core [55935.051630] fjes [55935.051631] [55935.051631] fjes [55935.051632] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55935.051633] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55935.051634] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55935.051634] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55935.051635] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55935.051636] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55935.051637] RIP: 0010:[] [55935.051638] [] _raw_spin_lock+0x32/0x50 [55935.051640] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55935.051640] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [55935.051641] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55935.051642] RAX: 00000000000006e2 RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55935.051642] RAX: 00000000000030d9 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55935.051643] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55935.051644] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55935.051644] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55935.051645] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55935.051646] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55935.051646] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55935.051647] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55935.051648] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55935.051648] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55935.051649] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55935.051650] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55935.051651] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55935.051651] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55935.051652] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55935.051652] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55935.051653] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55935.051654] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55935.051654] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55935.051655] Stack: [55935.051655] Stack: [55935.051656] ffff8808262efac0 [55935.051656] ffff8808217bbcd8 [55935.051657] ffffffffa08aa7c8 [55935.051657] ffffffffa08aa7c8 [55935.051657] ffffffffa08aa8b6 [55935.051658] ffffffffa08aa8b6 [55935.051658] 0000000000000000 [55935.051659] [55935.051659] ffff8804c1ba3200 [55935.051660] ffff88083fb5e000 [55935.051660] 0000000000000001 [55935.051661] ffff8808262efb80 [55935.051661] ffff8808217bbd08 [55935.051662] ffffffffa092c9b5 [55935.051662] ffffffffa09217bb [55935.051663] 00000000000001a8 [55935.051663] [55935.051663] ffff8804c1ba3200 [55935.051664] ffffc9000e331730 [55935.051665] 0000000000000001 [55935.051665] ffff8808262efb68 [55935.051665] ffff88082abba340 [55935.051666] ffffffffa0ebd587 [55935.051666] ffff88082abba348 [55935.051667] ffffffffffffffff [55935.051667] [55935.051668] ffff8808217bbd70 [55935.051668] Call Trace: [55935.051668] Call Trace: [55935.051674] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55935.051679] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55935.051684] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55935.051689] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55935.051695] [] lnet_select_pathway+0x235/0x1140 [lnet] [55935.051701] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55935.051704] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55935.051710] [] lnet_finalize+0x1e9/0x690 [lnet] [55935.051715] [] lnet_send+0x51/0x180 [lnet] [55935.051719] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55935.051725] [] lnet_finalize+0x3f1/0x690 [lnet] [55935.051728] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55935.051732] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55935.051733] [] ? dequeue_task_fair+0x41e/0x660 [55935.051735] [] ? __slab_free+0x81/0x2f0 [55935.051737] [] ? wake_up_state+0x20/0x20 [55935.051742] [] lnet_ni_recv+0xc3/0x320 [lnet] [55935.051746] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55935.051752] [] lnet_parse+0x945/0xf90 [lnet] [55935.051753] [] kthread+0xcf/0xe0 [55935.051757] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55935.051758] [] ? kthread_create_on_node+0x140/0x140 [55935.051761] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55935.051762] [] ret_from_fork+0x58/0x90 [55935.051766] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55935.051767] [] ? kthread_create_on_node+0x140/0x140 [55935.051769] [] ? dequeue_task_fair+0x41e/0x660 [55935.051770] Code: [55935.051772] [] ? wake_up_state+0x20/0x20 [55935.051772] 00 02 [55935.051776] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55935.051776] 00 f0 [55935.051778] [] kthread+0xcf/0xe0 [55935.051779] 0f c1 [55935.051781] [] ? kthread_create_on_node+0x140/0x140 [55935.051781] 07 89 [55935.051783] [] ret_from_fork+0x58/0x90 [55935.051784] c2 c1 [55935.051785] [] ? kthread_create_on_node+0x140/0x140 [55935.051786] ea 10 66 [55935.051787] Code: [55935.051787] 39 [55935.051788] 00 [55935.051788] c2 [55935.051788] 02 [55935.051789] 75 [55935.051789] 00 [55935.051790] 01 [55935.051790] f0 [55935.051791] c3 [55935.051791] 0f [55935.051791] 55 [55935.051792] c1 [55935.051792] 83 [55935.051793] 07 [55935.051793] e2 [55935.051793] 89 [55935.051794] fe [55935.051794] c2 [55935.051795] 0f [55935.051795] c1 [55935.051795] b7 [55935.051796] ea [55935.051796] f2 [55935.051797] 10 [55935.051797] 48 [55935.051797] 66 [55935.051798] 89 [55935.051798] 39 [55935.051799] e5 [55935.051799] c2 [55935.051799] b8 [55935.051800] 75 [55935.051800] 00 [55935.051801] 01 [55935.051801] 80 [55935.051801] c3 [55935.051802] 00 [55935.051802] 55 [55935.051803] 00 [55935.051803] 83 [55935.051803] eb [55935.051804] e2 [55935.051804] 0d [55935.051805] fe [55935.051805] 66 [55935.051805] 0f [55935.051806] 0f [55935.051806] b7 [55935.051807] 1f [55935.051807] f2 [55935.051807] 44 [55935.051808] 48 [55935.051808] 00 [55935.051809] 89 [55935.051809] 00 [55935.051810] e5 [55935.051810] f3 [55935.051810] b8 [55935.051811] 90 [55935.051811] 00 [55935.051812] <83> [55935.051812] 80 [55935.051812] e8 [55935.051813] 00 [55935.051813] 01 [55935.051814] 00 [55935.051814] 74 [55935.051814] eb [55935.051815] 0a [55935.051815] 0d [55935.051816] 0f [55935.051816] 66 [55935.051817] b7 [55935.051817] 0f [55935.051817] 0f [55935.051818] 1f [55935.051818] 66 [55935.051819] 44 [55935.051819] 39 [55935.051819] 00 [55935.051820] ca [55935.051820] 00 [55935.051821] 75 [55935.051821] f3 [55935.051822] f1 [55935.051822] 90 [55935.051822] 5d [55935.051823] <83> [55935.051823] c3 [55935.051824] e8 [55935.051824] 66 [55935.051824] 01 [55935.051825] 66 [55935.051825] 74 [55935.051826] 66 [55935.051826] 0a [55935.051827] 90 [55935.051827] 0f [55935.051827] 66 [55935.051828] b7 [55935.051828] 66 [55935.051829] [55935.051831] 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55939.503495] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55939.740028] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55939.809750] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55939.842799] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55939.880152] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55939.905247] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55939.932621] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000202 [55939.950593] RAX: 0000000000003651 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55939.974584] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55939.998594] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55940.022623] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55940.046654] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55940.070682] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55940.097904] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55940.117392] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55940.141473] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55940.165578] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55940.189677] Stack: [55940.196905] ffff8807f639fd88 ffffffffa08aa831 0000000200000002 ffff8807f63a2980 [55940.221938] 0000000000000000 ffff8807f639fdc0 ffffffffa0942e30 00000000c0086540 [55940.246986] ffff8807f63a2900 0000000000000002 0000000000650aa0 0000000000000000 [55940.272048] Call Trace: [55940.280772] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55940.302686] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55940.324052] [] LNetCtl+0x116/0x800 [lnet] [55940.343428] [] lnet_ioctl+0x60/0x250 [lnet] [55940.363361] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55940.384737] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55940.407300] [] do_vfs_ioctl+0x2d5/0x4b0 [55940.426130] [] ? __fd_install+0x47/0x60 [55940.444969] [] SyS_ioctl+0xa1/0xc0 [55940.462401] [] system_call_fastpath+0x16/0x1b [55940.482973] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55947.050296] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55947.050297] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55947.050317] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55947.050322] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55947.050323] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55947.050324] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55947.050325] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55947.050326] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55947.050327] RSP: 0018:ffff880820b03c00 EFLAGS: 00000206 [55947.050327] RAX: 0000000000006b8e RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55947.050328] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55947.050329] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55947.050329] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55947.050330] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55947.050331] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55947.050331] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55947.050332] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55947.050332] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55947.050333] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55947.050333] Stack: [55947.050334] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [55947.050335] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [55947.050336] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [55947.050336] Call Trace: [55947.050342] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55947.050348] [] lnet_post_send_locked+0x446/0x880 [lnet] [55947.050354] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55947.050359] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55947.050364] [] lnet_finalize+0x1e9/0x690 [lnet] [55947.050368] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55947.050371] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55947.050373] [] ? dequeue_task_fair+0x41e/0x660 [55947.050374] [] ? sched_clock_cpu+0x85/0xc0 [55947.050376] [] ? wake_up_state+0x20/0x20 [55947.050379] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55947.050381] [] kthread+0xcf/0xe0 [55947.050382] [] ? kthread_create_on_node+0x140/0x140 [55947.050384] [] ret_from_fork+0x58/0x90 [55947.050385] [] ? kthread_create_on_node+0x140/0x140 [55947.050395] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55948.286600] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55948.527532] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55948.599044] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55948.635478] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55948.673346] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55948.698884] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55948.726725] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [55948.745133] RAX: 0000000000002285 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55948.769556] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55948.793967] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55948.818381] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55948.842796] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55948.867222] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55948.894807] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55948.914657] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55948.939055] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55948.963445] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55948.987832] Stack: [55948.995351] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [55949.020657] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [55949.045956] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [55949.071266] Call Trace: [55949.080221] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55949.102338] [] lnet_finalize+0xf6/0x690 [lnet] [55949.123320] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55949.146024] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55949.169562] [] ? dequeue_task_fair+0x41e/0x660 [55949.190552] [] ? wake_up_state+0x20/0x20 [55949.209810] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55949.232788] [] kthread+0xcf/0xe0 [55949.249749] [] ? kthread_create_on_node+0x140/0x140 [55949.272190] [] ret_from_fork+0x58/0x90 [55949.290905] [] ? kthread_create_on_node+0x140/0x140 [55949.313336] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55951.049884] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55951.049885] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [55951.049902] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55951.049908] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55951.049909] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55951.049909] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55951.049910] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55951.049912] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55951.049912] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55951.049913] RAX: 0000000000001e2e RBX: 0000000000016c40 RCX: 0000000000000a1e [55951.049913] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55951.049914] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55951.049915] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55951.049915] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55951.049916] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55951.049916] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55951.049917] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55951.049917] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55951.049918] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55951.049918] Stack: [55951.049919] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55951.049920] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55951.049921] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55951.049921] Call Trace: [55951.049927] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55951.049932] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55951.049935] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55951.049939] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55951.049940] [] ? lock_timer_base.isra.33+0x2b/0x50 [55951.049942] [] ? try_to_del_timer_sync+0x5e/0x90 [55951.049945] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55951.049947] [] ? schedule_timeout+0x17d/0x2d0 [55951.049950] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55951.049952] [] ? wake_up_state+0x20/0x20 [55951.049955] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55951.049957] [] kthread+0xcf/0xe0 [55951.049958] [] ? kthread_create_on_node+0x140/0x140 [55951.049960] [] ret_from_fork+0x58/0x90 [55951.049961] [] ? kthread_create_on_node+0x140/0x140 [55951.049971] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55952.288202] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55952.529131] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55952.600675] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55952.637124] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55952.674979] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55952.700573] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55952.728416] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000206 [55952.746801] RAX: 00000000000002c5 RBX: 0000000100000000 RCX: 0000000000000a1e [55952.771194] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55952.795606] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55952.820037] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55952.844428] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55952.868813] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55952.896350] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55952.916158] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55952.940585] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55952.964971] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55952.989381] Stack: [55952.996921] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [55953.022187] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [55953.047484] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [55953.072776] Call Trace: [55953.081726] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55953.103877] [] lnet_finalize+0xf6/0x690 [lnet] [55953.124839] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55953.146667] [] ? enqueue_task_fair+0x208/0x6c0 [55953.167631] [] ? sched_clock_cpu+0x85/0xc0 [55953.187494] [] lnet_ni_recv+0xc3/0x320 [lnet] [55953.208182] [] lnet_parse+0x945/0xf90 [lnet] [55953.228589] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55953.251876] [] ? __wake_up+0x44/0x50 [55953.269995] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55953.293529] [] ? dequeue_task_fair+0x41e/0x660 [55953.314490] [] ? wake_up_state+0x20/0x20 [55953.333748] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55953.356751] [] kthread+0xcf/0xe0 [55953.373704] [] ? kthread_create_on_node+0x140/0x140 [55953.396095] [] ret_from_fork+0x58/0x90 [55953.414780] [] ? kthread_create_on_node+0x140/0x140 [55953.437184] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55955.049472] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [kiblnd_sd_01_03:2903] [55955.049474] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 24s! [kiblnd_sd_00_01:2897] [55955.049494] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55955.049499] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55955.049501] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55955.049501] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55955.049502] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55955.049504] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55955.049504] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [55955.049505] RAX: 0000000000002a4d RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55955.049506] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55955.049506] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55955.049507] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55955.049507] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55955.049508] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55955.049508] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55955.049509] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55955.049509] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55955.049510] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55955.049510] Stack: [55955.049511] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [55955.049512] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [55955.049513] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [55955.049513] Call Trace: [55955.049519] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55955.049524] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55955.049530] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55955.049536] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55955.049541] [] lnet_finalize+0x1e9/0x690 [lnet] [55955.049544] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55955.049548] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55955.049549] [] ? dequeue_task_fair+0x41e/0x660 [55955.049551] [] ? wake_up_state+0x20/0x20 [55955.049554] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55955.049556] [] kthread+0xcf/0xe0 [55955.049557] [] ? kthread_create_on_node+0x140/0x140 [55955.049559] [] ret_from_fork+0x58/0x90 [55955.049561] [] ? kthread_create_on_node+0x140/0x140 [55955.049571] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55956.264991] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55956.506068] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55956.577642] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55956.614394] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55956.652291] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55956.677877] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55956.705764] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [55956.724175] RAX: 0000000000002488 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55956.748566] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55956.772977] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55956.797358] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55956.821762] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55956.846154] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55956.873729] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55956.893542] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55956.917936] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55956.942337] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55956.966755] Stack: [55956.974258] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55956.999566] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [55957.024870] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [55957.050205] Call Trace: [55957.059190] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55957.081327] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55957.104323] [] lnet_select_pathway+0x235/0x1140 [lnet] [55957.127611] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55957.150014] [] lnet_send+0x51/0x180 [lnet] [55957.169853] [] lnet_finalize+0x3f1/0x690 [lnet] [55957.191112] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55957.212951] [] ? __slab_free+0x81/0x2f0 [55957.231925] [] lnet_ni_recv+0xc3/0x320 [lnet] [55957.252604] [] lnet_parse+0x945/0xf90 [lnet] [55957.273012] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55957.295977] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55957.319243] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55957.340495] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55957.364055] [] ? dequeue_task_fair+0x41e/0x660 [55957.385019] [] ? wake_up_state+0x20/0x20 [55957.404297] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55957.427252] [] kthread+0xcf/0xe0 [55957.444199] [] ? kthread_create_on_node+0x140/0x140 [55957.466548] [] ret_from_fork+0x58/0x90 [55957.485174] [] ? kthread_create_on_node+0x140/0x140 [55957.507497] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55959.049060] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kiblnd_sd_01_02:2902] [55959.049061] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55959.049081] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55959.049086] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55959.049087] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55959.049088] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55959.049090] Workqueue: ib_cm cm_work_handler [ib_cm] [55959.049091] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55959.049092] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55959.049092] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000206 [55959.049093] RAX: 00000000000046d1 RBX: 0000000000000000 RCX: 0000000000000a1e [55959.049094] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55959.049094] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55959.049095] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55959.049095] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55959.049096] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55959.049097] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55959.049097] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55959.049098] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55959.049098] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55959.049099] Stack: [55959.049100] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [55959.049100] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [55959.049101] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [55959.049102] Call Trace: [55959.049107] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55959.049112] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55959.049116] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55959.049118] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55959.049121] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55959.049123] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55959.049125] [] cma_req_handler+0x227/0x570 [rdma_cm] [55959.049127] [] cm_process_work+0x25/0x140 [ib_cm] [55959.049129] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55959.049131] [] cm_work_handler+0x185/0x1484 [ib_cm] [55959.049132] [] process_one_work+0x17b/0x470 [55959.049133] [] worker_thread+0x126/0x410 [55959.049134] [] ? rescuer_thread+0x460/0x460 [55959.049135] [] kthread+0xcf/0xe0 [55959.049137] [] ? kthread_create_on_node+0x140/0x140 [55959.049138] [] ret_from_fork+0x58/0x90 [55959.049140] [] ? kthread_create_on_node+0x140/0x140 [55959.049150] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55960.344236] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55960.584825] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55960.656154] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55960.692805] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55960.730612] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55960.756159] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55960.783979] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [55960.802347] RAX: 000000000000284a RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55960.826736] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55960.851112] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55960.875493] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55960.899909] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55960.924274] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55960.951801] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55960.971636] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55960.996016] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55961.020415] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55961.044791] Stack: [55961.052314] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [55961.077603] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [55961.102886] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [55961.128161] Call Trace: [55961.137123] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55961.159227] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55961.182189] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55961.204612] [] lnet_finalize+0x1e9/0x690 [lnet] [55961.225881] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55961.248563] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55961.272077] [] ? dequeue_task_fair+0x41e/0x660 [55961.293046] [] ? wake_up_state+0x20/0x20 [55961.312282] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55961.335243] [] kthread+0xcf/0xe0 [55961.352202] [] ? kthread_create_on_node+0x140/0x140 [55961.374614] [] ret_from_fork+0x58/0x90 [55961.393309] [] ? kthread_create_on_node+0x140/0x140 [55961.415746] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55963.048648] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [kiblnd_sd_01_00:2900] [55963.048649] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 21s! [lctl:36467] [55963.048650] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55963.048652] Modules linked in: [55963.048652] Modules linked in: [55963.048653] ko2iblnd(OE) [55963.048653] ko2iblnd(OE) [55963.048654] lustre(OE) [55963.048654] lustre(OE) [55963.048654] lmv(OE) [55963.048655] lmv(OE) [55963.048655] fld(OE) [55963.048656] fld(OE) [55963.048656] mdc(OE) [55963.048657] mdc(OE) [55963.048657] fid(OE) [55963.048658] fid(OE) [55963.048658] lov(OE) [55963.048658] lov(OE) [55963.048659] ksocklnd(OE) [55963.048659] ksocklnd(OE) [55963.048660] ptlrpc(OE) [55963.048660] ptlrpc(OE) [55963.048661] obdclass(OE) [55963.048661] obdclass(OE) [55963.048661] lnet(OE) [55963.048662] lnet(OE) [55963.048662] libcfs(OE) [55963.048663] libcfs(OE) [55963.048663] rpcrdma [55963.048664] rpcrdma [55963.048664] ib_isert [55963.048665] ib_isert [55963.048665] iscsi_target_mod [55963.048665] iscsi_target_mod [55963.048666] ib_iser [55963.048666] ib_iser [55963.048667] libiscsi [55963.048667] libiscsi [55963.048668] scsi_transport_iscsi [55963.048668] scsi_transport_iscsi [55963.048668] ib_srpt [55963.048669] ib_srpt [55963.048669] target_core_mod [55963.048670] target_core_mod [55963.048670] ib_srp [55963.048671] ib_srp [55963.048671] scsi_transport_srp [55963.048671] scsi_transport_srp [55963.048672] scsi_tgt [55963.048672] scsi_tgt [55963.048673] ib_ipoib [55963.048673] ib_ipoib [55963.048674] rdma_ucm [55963.048674] rdma_ucm [55963.048674] ib_ucm [55963.048675] ib_ucm [55963.048675] ib_uverbs [55963.048676] ib_uverbs [55963.048676] ib_umad [55963.048676] ib_umad [55963.048677] rdma_cm [55963.048677] rdma_cm [55963.048678] ib_cm [55963.048678] ib_cm [55963.048679] iw_cm [55963.048679] iw_cm [55963.048679] mlx4_ib [55963.048680] mlx4_ib [55963.048680] intel_powerclamp [55963.048681] intel_powerclamp [55963.048681] coretemp [55963.048681] coretemp [55963.048682] intel_rapl [55963.048682] intel_rapl [55963.048683] iosf_mbi [55963.048683] iosf_mbi [55963.048683] kvm [55963.048684] kvm [55963.048684] irqbypass [55963.048685] irqbypass [55963.048685] crc32_pclmul [55963.048686] crc32_pclmul [55963.048686] ghash_clmulni_intel [55963.048686] ghash_clmulni_intel [55963.048687] aesni_intel [55963.048687] aesni_intel [55963.048688] ipmi_ssif [55963.048688] ipmi_ssif [55963.048689] lrw [55963.048689] lrw [55963.048689] gf128mul [55963.048690] gf128mul [55963.048690] glue_helper [55963.048691] glue_helper [55963.048691] hfi1 [55963.048692] hfi1 [55963.048692] ipmi_devintf [55963.048692] ipmi_devintf [55963.048693] rdmavt [55963.048693] rdmavt [55963.048694] ablk_helper [55963.048694] ablk_helper [55963.048695] cryptd [55963.048695] cryptd [55963.048695] iTCO_wdt [55963.048696] iTCO_wdt [55963.048696] ipmi_si [55963.048697] ipmi_si [55963.048697] sb_edac [55963.048698] sb_edac [55963.048698] sg [55963.048698] sg [55963.048699] iTCO_vendor_support [55963.048699] iTCO_vendor_support [55963.048700] ib_core [55963.048700] ib_core [55963.048701] ioatdma [55963.048701] ioatdma [55963.048701] i2c_i801 [55963.048702] i2c_i801 [55963.048702] mei_me [55963.048703] mei_me [55963.048703] mei [55963.048704] mei [55963.048704] edac_core [55963.048705] edac_core [55963.048705] ipmi_msghandler [55963.048705] ipmi_msghandler [55963.048706] lpc_ich [55963.048706] lpc_ich [55963.048707] ntb [55963.048707] ntb [55963.048708] wmi [55963.048708] wmi [55963.048708] shpchp [55963.048709] shpchp [55963.048709] pcspkr [55963.048710] pcspkr [55963.048710] nfsd [55963.048711] nfsd [55963.048711] nfs_acl [55963.048711] nfs_acl [55963.048712] lockd [55963.048712] lockd [55963.048713] grace [55963.048713] grace [55963.048714] auth_rpcgss [55963.048714] auth_rpcgss [55963.048714] sunrpc [55963.048715] sunrpc [55963.048715] ip_tables [55963.048716] ip_tables [55963.048716] ext4 [55963.048717] ext4 [55963.048717] mbcache [55963.048717] mbcache [55963.048718] jbd2 [55963.048718] jbd2 [55963.048719] sd_mod [55963.048719] sd_mod [55963.048720] crc_t10dif [55963.048720] crc_t10dif [55963.048721] crct10dif_generic [55963.048721] crct10dif_generic [55963.048721] mlx4_en [55963.048722] mlx4_en [55963.048722] mgag200 [55963.048723] mgag200 [55963.048723] drm_kms_helper [55963.048724] drm_kms_helper [55963.048724] syscopyarea [55963.048724] syscopyarea [55963.048725] sysfillrect [55963.048725] sysfillrect [55963.048726] sysimgblt [55963.048726] sysimgblt [55963.048727] fb_sys_fops [55963.048727] fb_sys_fops [55963.048728] isci [55963.048728] isci [55963.048729] ttm [55963.048729] ttm [55963.048729] igb [55963.048730] igb [55963.048730] crct10dif_pclmul [55963.048731] crct10dif_pclmul [55963.048731] crct10dif_common [55963.048732] crct10dif_common [55963.048732] ahci [55963.048733] ahci [55963.048733] ptp [55963.048733] ptp [55963.048734] crc32c_intel [55963.048734] crc32c_intel [55963.048735] libsas [55963.048735] libsas [55963.048736] libahci [55963.048736] libahci [55963.048736] pps_core [55963.048737] pps_core [55963.048737] drm [55963.048738] drm [55963.048738] mlx4_core [55963.048739] mlx4_core [55963.048739] mpt2sas [55963.048739] mpt2sas [55963.048740] dca [55963.048740] dca [55963.048741] libata [55963.048741] libata [55963.048742] raid_class [55963.048742] raid_class [55963.048742] i2c_algo_bit [55963.048743] i2c_algo_bit [55963.048743] scsi_transport_sas [55963.048744] scsi_transport_sas [55963.048744] devlink [55963.048745] devlink [55963.048745] i2c_core [55963.048745] i2c_core [55963.048746] fjes [55963.048746] [55963.048747] fjes [55963.048748] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55963.048749] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55963.048749] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55963.048750] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55963.048751] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55963.048751] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55963.048752] RIP: 0010:[] [55963.048754] [] _raw_spin_lock+0x32/0x50 [55963.048755] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55963.048756] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [55963.048756] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [55963.048757] RAX: 0000000000006fce RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55963.048758] RAX: 00000000000015eb RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55963.048758] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55963.048759] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55963.048759] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55963.048760] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55963.048761] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55963.048761] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55963.048762] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55963.048763] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55963.048763] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55963.048764] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55963.048765] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55963.048766] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55963.048766] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55963.048767] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55963.048767] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55963.048768] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55963.048769] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55963.048769] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55963.048770] Stack: [55963.048770] Stack: [55963.048771] ffff8807f639fd88 [55963.048771] ffff8808262efac0 [55963.048772] ffffffffa08aa831 [55963.048772] ffffffffa08aa7c8 [55963.048773] 0000000200000002 [55963.048773] ffffffffa08aa8b6 [55963.048774] ffff8807f63a2980 [55963.048774] [55963.048774] 0000000000000000 [55963.048775] 0000000000000000 [55963.048775] ffff88083fb5e000 [55963.048776] ffff8807f639fdc0 [55963.048776] ffff8808262efb80 [55963.048777] ffffffffa0942e30 [55963.048777] ffffffffa092c9b5 [55963.048778] 00000000c0086540 [55963.048778] [55963.048778] 00000000000001a8 [55963.048779] ffff8807f63a2900 [55963.048780] ffffc9000e331730 [55963.048780] 0000000000000002 [55963.048780] ffff8808262efb68 [55963.048781] 0000000000650aa0 [55963.048781] ffffffffa0ebd587 [55963.048782] 0000000000000000 [55963.048782] [55963.048783] ffffffffffffffff [55963.048783] Call Trace: [55963.048783] Call Trace: [55963.048789] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55963.048794] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55963.048800] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55963.048805] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55963.048811] [] LNetCtl+0x116/0x800 [lnet] [55963.048816] [] lnet_select_pathway+0x235/0x1140 [lnet] [55963.048822] [] lnet_ioctl+0x60/0x250 [lnet] [55963.048826] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55963.048830] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55963.048836] [] lnet_send+0x51/0x180 [lnet] [55963.048841] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55963.048846] [] lnet_finalize+0x3f1/0x690 [lnet] [55963.048848] [] do_vfs_ioctl+0x2d5/0x4b0 [55963.048851] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55963.048853] [] ? __fd_install+0x47/0x60 [55963.048855] [] ? __slab_free+0x81/0x2f0 [55963.048856] [] SyS_ioctl+0xa1/0xc0 [55963.048862] [] lnet_ni_recv+0xc3/0x320 [lnet] [55963.048863] [] system_call_fastpath+0x16/0x1b [55963.048869] [] lnet_parse+0x945/0xf90 [lnet] [55963.048869] Code: [55963.048873] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55963.048874] 00 02 00 f0 0f c1 [55963.048876] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55963.048877] 07 89 [55963.048880] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55963.048881] c2 c1 [55963.048883] [] ? dequeue_task_fair+0x41e/0x660 [55963.048883] ea 10 [55963.048885] [] ? wake_up_state+0x20/0x20 [55963.048886] 66 39 [55963.048889] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55963.048890] c2 75 [55963.048892] [] kthread+0xcf/0xe0 [55963.048892] 01 c3 [55963.048894] [] ? kthread_create_on_node+0x140/0x140 [55963.048894] 55 83 [55963.048896] [] ret_from_fork+0x58/0x90 [55963.048897] e2 fe [55963.048898] [] ? kthread_create_on_node+0x140/0x140 [55963.048899] 0f b7 f2 [55963.048900] Code: [55963.048900] 48 [55963.048900] 00 [55963.048901] 89 [55963.048901] 02 [55963.048902] e5 [55963.048902] 00 [55963.048903] b8 [55963.048903] f0 [55963.048903] 00 [55963.048904] 0f [55963.048904] 80 [55963.048905] c1 [55963.048905] 00 [55963.048905] 07 [55963.048906] 00 [55963.048906] 89 [55963.048906] eb [55963.048907] c2 [55963.048907] 0d [55963.048908] c1 [55963.048908] 66 [55963.048909] ea [55963.048909] 0f [55963.048909] 10 [55963.048910] 1f [55963.048910] 66 [55963.048911] 44 [55963.048911] 39 [55963.048911] 00 [55963.048912] c2 [55963.048912] 00 [55963.048912] 75 [55963.048913] f3 [55963.048913] 01 [55963.048914] 90 [55963.048914] c3 [55963.048915] <83> [55963.048915] 55 [55963.048915] e8 [55963.048916] 83 [55963.048916] 01 [55963.048916] e2 [55963.048917] 74 [55963.048917] fe [55963.048918] 0a [55963.048918] 0f [55963.048919] 0f [55963.048919] b7 [55963.048919] b7 [55963.048920] f2 [55963.048920] 0f [55963.048921] 48 [55963.048921] 66 [55963.048921] 89 [55963.048922] 39 [55963.048922] e5 [55963.048923] ca [55963.048923] b8 [55963.048923] 75 [55963.048924] 00 [55963.048924] f1 [55963.048925] 80 [55963.048925] 5d [55963.048925] 00 [55963.048926] c3 [55963.048926] 00 [55963.048927] 66 [55963.048927] eb [55963.048927] 66 [55963.048928] 0d [55963.048928] 66 [55963.048929] 66 [55963.048929] 90 [55963.048929] 0f [55963.048930] 66 [55963.048930] 1f [55963.048931] 66 [55963.048931] [55963.048935] 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55967.088811] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55967.325403] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55967.395145] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55967.431334] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55967.468672] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55967.493753] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55967.521159] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [55967.539151] RAX: 0000000000002a49 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55967.563167] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55967.587172] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55967.611215] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55967.635250] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55967.659284] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55967.686487] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55967.705993] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55967.730090] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55967.754189] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55967.778261] Stack: [55967.785485] ffff8808217bbcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba3200 [55967.810529] 0000000000000001 ffff8808217bbd08 ffffffffa09217bb ffff8804c1ba3200 [55967.835576] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff8808217bbd70 [55967.860635] Call Trace: [55967.869379] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55967.891281] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55967.914082] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55967.936312] [] lnet_finalize+0x1e9/0x690 [lnet] [55967.957383] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55967.979920] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55968.003332] [] ? dequeue_task_fair+0x41e/0x660 [55968.024183] [] ? wake_up_state+0x20/0x20 [55968.043269] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55968.066136] [] kthread+0xcf/0xe0 [55968.082980] [] ? kthread_create_on_node+0x140/0x140 [55968.105294] [] ret_from_fork+0x58/0x90 [55968.123899] [] ? kthread_create_on_node+0x140/0x140 [55968.146244] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55975.047412] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [55975.047414] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [55975.047434] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55975.047439] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55975.047440] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55975.047441] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55975.047442] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [55975.047444] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55975.047444] RSP: 0018:ffff88082626bce0 EFLAGS: 00000206 [55975.047445] RAX: 0000000000000fb1 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [55975.047445] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [55975.047446] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [55975.047446] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [55975.047447] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [55975.047448] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [55975.047448] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55975.047449] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [55975.047449] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55975.047450] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55975.047450] Stack: [55975.047451] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [55975.047452] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [55975.047453] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [55975.047453] Call Trace: [55975.047459] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55975.047465] [] lnet_finalize+0xf6/0x690 [lnet] [55975.047469] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55975.047472] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55975.047473] [] ? dequeue_task_fair+0x41e/0x660 [55975.047475] [] ? wake_up_state+0x20/0x20 [55975.047479] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55975.047480] [] kthread+0xcf/0xe0 [55975.047482] [] ? kthread_create_on_node+0x140/0x140 [55975.047483] [] ret_from_fork+0x58/0x90 [55975.047485] [] ? kthread_create_on_node+0x140/0x140 [55975.047495] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [55976.191938] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55976.433207] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55976.504923] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55976.541422] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55976.579348] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [55976.604961] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55976.632863] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [55976.651306] RAX: 0000000000004d62 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [55976.675767] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [55976.700223] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [55976.724671] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [55976.749081] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [55976.773508] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [55976.801082] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55976.820901] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55976.845296] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55976.869715] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55976.894107] Stack: [55976.901650] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [55976.926943] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [55976.952230] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [55976.977535] Call Trace: [55976.986536] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55977.008667] [] lnet_post_send_locked+0x446/0x880 [lnet] [55977.032243] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [55977.058113] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55977.080242] [] lnet_finalize+0x1e9/0x690 [lnet] [55977.101542] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55977.124259] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55977.147826] [] ? dequeue_task_fair+0x41e/0x660 [55977.168841] [] ? sched_clock_cpu+0x85/0xc0 [55977.188667] [] ? wake_up_state+0x20/0x20 [55977.207922] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55977.230919] [] kthread+0xcf/0xe0 [55977.247905] [] ? kthread_create_on_node+0x140/0x140 [55977.270344] [] ret_from_fork+0x58/0x90 [55977.289034] [] ? kthread_create_on_node+0x140/0x140 [55977.311442] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55979.047000] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [55979.047002] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [55979.047021] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55979.047027] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55979.047028] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55979.047028] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55979.047029] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [55979.047031] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55979.047031] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [55979.047032] RAX: 0000000000005f72 RBX: 0000000100000000 RCX: 0000000000000a1e [55979.047032] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [55979.047033] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [55979.047034] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [55979.047034] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [55979.047035] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [55979.047036] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55979.047036] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55979.047037] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55979.047037] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55979.047037] Stack: [55979.047039] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [55979.047040] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [55979.047041] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [55979.047041] Call Trace: [55979.047047] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55979.047053] [] lnet_finalize+0xf6/0x690 [lnet] [55979.047056] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55979.047058] [] ? enqueue_task_fair+0x208/0x6c0 [55979.047059] [] ? sched_clock_cpu+0x85/0xc0 [55979.047065] [] lnet_ni_recv+0xc3/0x320 [lnet] [55979.047071] [] lnet_parse+0x945/0xf90 [lnet] [55979.047074] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55979.047076] [] ? __wake_up+0x44/0x50 [55979.047079] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55979.047080] [] ? dequeue_task_fair+0x41e/0x660 [55979.047082] [] ? wake_up_state+0x20/0x20 [55979.047085] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55979.047087] [] kthread+0xcf/0xe0 [55979.047089] [] ? kthread_create_on_node+0x140/0x140 [55979.047090] [] ret_from_fork+0x58/0x90 [55979.047092] [] ? kthread_create_on_node+0x140/0x140 [55979.047102] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55980.312023] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55980.552590] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55980.623896] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55980.659397] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55980.697191] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [55980.722684] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55980.750508] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [55980.768855] RAX: 0000000000002016 RBX: 0000000000016c40 RCX: 0000000000000a1e [55980.793257] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [55980.817618] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [55980.841994] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [55980.866373] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [55980.890748] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [55980.918272] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55980.938075] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55980.962481] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55980.986877] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55981.011267] Stack: [55981.018758] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [55981.044047] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [55981.069372] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [55981.094657] Call Trace: [55981.103631] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55981.125761] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [55981.147037] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [55981.170616] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [55981.196436] [] ? lock_timer_base.isra.33+0x2b/0x50 [55981.218593] [] ? try_to_del_timer_sync+0x5e/0x90 [55981.240181] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [55981.264060] [] ? schedule_timeout+0x17d/0x2d0 [55981.284768] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [55981.306895] [] ? wake_up_state+0x20/0x20 [55981.326186] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [55981.350620] [] kthread+0xcf/0xe0 [55981.367610] [] ? kthread_create_on_node+0x140/0x140 [55981.390032] [] ret_from_fork+0x58/0x90 [55981.408753] [] ? kthread_create_on_node+0x140/0x140 [55981.431197] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55983.046589] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [kiblnd_sd_00_01:2897] [55983.046590] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [kiblnd_sd_01_03:2903] [55983.046612] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55983.046618] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55983.046619] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55983.046620] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55983.046621] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [55983.046622] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55983.046623] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [55983.046624] RAX: 0000000000004bff RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [55983.046624] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [55983.046625] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [55983.046625] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [55983.046626] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [55983.046626] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [55983.046627] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55983.046628] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55983.046628] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55983.046629] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55983.046629] Stack: [55983.046630] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55983.046631] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [55983.046632] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [55983.046632] Call Trace: [55983.046637] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55983.046642] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55983.046648] [] lnet_select_pathway+0x235/0x1140 [lnet] [55983.046652] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55983.046658] [] lnet_send+0x51/0x180 [lnet] [55983.046663] [] lnet_finalize+0x3f1/0x690 [lnet] [55983.046666] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55983.046668] [] ? __slab_free+0x81/0x2f0 [55983.046674] [] lnet_ni_recv+0xc3/0x320 [lnet] [55983.046679] [] lnet_parse+0x945/0xf90 [lnet] [55983.046684] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55983.046687] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55983.046690] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55983.046694] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55983.046695] [] ? dequeue_task_fair+0x41e/0x660 [55983.046697] [] ? wake_up_state+0x20/0x20 [55983.046701] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55983.046702] [] kthread+0xcf/0xe0 [55983.046704] [] ? kthread_create_on_node+0x140/0x140 [55983.046706] [] ret_from_fork+0x58/0x90 [55983.046707] [] ? kthread_create_on_node+0x140/0x140 [55983.046718] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55984.406662] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55984.646710] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55984.717872] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55984.754222] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55984.792033] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [55984.817519] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [55984.845324] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000206 [55984.863678] RAX: 0000000000001e6c RBX: ffff88083fb5e000 RCX: 0000000000000a1e [55984.888082] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [55984.912440] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [55984.936812] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [55984.961178] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [55984.985581] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [55985.013103] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55985.032903] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55985.057319] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55985.081733] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55985.106141] Stack: [55985.113664] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [55985.138977] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [55985.164264] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [55985.189545] Call Trace: [55985.198499] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55985.220616] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55985.243633] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [55985.269491] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [55985.291624] [] lnet_finalize+0x1e9/0x690 [lnet] [55985.312892] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55985.335597] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55985.359141] [] ? dequeue_task_fair+0x41e/0x660 [55985.380120] [] ? wake_up_state+0x20/0x20 [55985.399379] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55985.422376] [] kthread+0xcf/0xe0 [55985.439381] [] ? kthread_create_on_node+0x140/0x140 [55985.461824] [] ret_from_fork+0x58/0x90 [55985.480523] [] ? kthread_create_on_node+0x140/0x140 [55985.502928] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [55987.046176] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kiblnd_sd_01_02:2902] [55987.046178] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:27192] [55987.046197] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55987.046202] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55987.046203] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55987.046204] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55987.046206] Workqueue: ib_cm cm_work_handler [ib_cm] [55987.046206] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [55987.046208] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55987.046208] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000216 [55987.046209] RAX: 000000000000378f RBX: 0000000000000000 RCX: 0000000000000a1e [55987.046209] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [55987.046210] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [55987.046210] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [55987.046211] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [55987.046212] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [55987.046212] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55987.046213] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55987.046213] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55987.046214] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55987.046214] Stack: [55987.046215] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [55987.046216] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [55987.046217] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [55987.046217] Call Trace: [55987.046222] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55987.046227] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [55987.046231] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [55987.046233] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [55987.046237] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [55987.046239] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [55987.046241] [] cma_req_handler+0x227/0x570 [rdma_cm] [55987.046243] [] cm_process_work+0x25/0x140 [ib_cm] [55987.046244] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [55987.046246] [] cm_work_handler+0x185/0x1484 [ib_cm] [55987.046247] [] process_one_work+0x17b/0x470 [55987.046248] [] worker_thread+0x126/0x410 [55987.046249] [] ? rescuer_thread+0x460/0x460 [55987.046251] [] kthread+0xcf/0xe0 [55987.046252] [] ? kthread_create_on_node+0x140/0x140 [55987.046254] [] ret_from_fork+0x58/0x90 [55987.046255] [] ? kthread_create_on_node+0x140/0x140 [55987.046265] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55988.342980] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55988.583506] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55988.654858] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55988.691548] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55988.729398] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [55988.754946] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55988.782761] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [55988.801096] RAX: 000000000000702f RBX: 000001f280bac5e0 RCX: 0000000000000a1e [55988.825476] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [55988.849887] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [55988.874267] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [55988.898642] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [55988.923010] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [55988.950539] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55988.970343] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55988.994731] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55989.019117] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55989.043531] Stack: [55989.051029] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [55989.076357] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [55989.101639] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [55989.126913] Call Trace: [55989.135839] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55989.157968] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55989.180918] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55989.203311] [] lnet_finalize+0x1e9/0x690 [lnet] [55989.224583] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55989.247283] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55989.270811] [] ? dequeue_task_fair+0x41e/0x660 [55989.291773] [] ? wake_up_state+0x20/0x20 [55989.311018] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55989.333995] [] kthread+0xcf/0xe0 [55989.350968] [] ? kthread_create_on_node+0x140/0x140 [55989.373363] [] ret_from_fork+0x58/0x90 [55989.392066] [] ? kthread_create_on_node+0x140/0x140 [55989.414483] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [55991.045764] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [55991.045766] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 21s! [kiblnd_sd_01_00:2900] [55991.045767] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [55991.045768] Modules linked in: [55991.045769] Modules linked in: [55991.045769] ko2iblnd(OE) [55991.045770] ko2iblnd(OE) [55991.045770] lustre(OE) [55991.045771] lustre(OE) [55991.045771] lmv(OE) [55991.045772] lmv(OE) [55991.045772] fld(OE) [55991.045772] fld(OE) [55991.045773] mdc(OE) [55991.045773] mdc(OE) [55991.045774] fid(OE) [55991.045774] fid(OE) [55991.045775] lov(OE) [55991.045775] lov(OE) [55991.045775] ksocklnd(OE) [55991.045776] ksocklnd(OE) [55991.045776] ptlrpc(OE) [55991.045777] ptlrpc(OE) [55991.045777] obdclass(OE) [55991.045778] obdclass(OE) [55991.045778] lnet(OE) [55991.045778] lnet(OE) [55991.045779] libcfs(OE) [55991.045779] libcfs(OE) [55991.045780] rpcrdma [55991.045780] rpcrdma [55991.045781] ib_isert [55991.045781] ib_isert [55991.045782] iscsi_target_mod [55991.045782] iscsi_target_mod [55991.045782] ib_iser [55991.045783] ib_iser [55991.045783] libiscsi [55991.045784] libiscsi [55991.045784] scsi_transport_iscsi [55991.045784] scsi_transport_iscsi [55991.045785] ib_srpt [55991.045785] ib_srpt [55991.045786] target_core_mod [55991.045786] target_core_mod [55991.045787] ib_srp [55991.045787] ib_srp [55991.045788] scsi_transport_srp [55991.045788] scsi_transport_srp [55991.045788] scsi_tgt [55991.045789] scsi_tgt [55991.045789] ib_ipoib [55991.045790] ib_ipoib [55991.045790] rdma_ucm [55991.045791] rdma_ucm [55991.045791] ib_ucm [55991.045792] ib_ucm [55991.045792] ib_uverbs [55991.045792] ib_uverbs [55991.045793] ib_umad [55991.045793] ib_umad [55991.045794] rdma_cm [55991.045794] rdma_cm [55991.045795] ib_cm [55991.045795] ib_cm [55991.045795] iw_cm [55991.045796] iw_cm [55991.045796] mlx4_ib [55991.045797] mlx4_ib [55991.045797] intel_powerclamp [55991.045798] intel_powerclamp [55991.045798] coretemp [55991.045798] coretemp [55991.045799] intel_rapl [55991.045799] intel_rapl [55991.045800] iosf_mbi [55991.045800] iosf_mbi [55991.045801] kvm [55991.045801] kvm [55991.045801] irqbypass [55991.045802] irqbypass [55991.045802] crc32_pclmul [55991.045803] crc32_pclmul [55991.045803] ghash_clmulni_intel [55991.045804] ghash_clmulni_intel [55991.045804] aesni_intel [55991.045805] aesni_intel [55991.045805] ipmi_ssif [55991.045806] ipmi_ssif [55991.045806] lrw [55991.045806] lrw [55991.045807] gf128mul [55991.045807] gf128mul [55991.045808] glue_helper [55991.045808] glue_helper [55991.045809] hfi1 [55991.045809] hfi1 [55991.045810] ipmi_devintf [55991.045810] ipmi_devintf [55991.045810] rdmavt [55991.045811] rdmavt [55991.045811] ablk_helper [55991.045812] ablk_helper [55991.045812] cryptd [55991.045813] cryptd [55991.045813] iTCO_wdt [55991.045813] iTCO_wdt [55991.045814] ipmi_si [55991.045814] ipmi_si [55991.045815] sb_edac [55991.045815] sb_edac [55991.045816] sg [55991.045816] sg [55991.045817] iTCO_vendor_support [55991.045817] iTCO_vendor_support [55991.045817] ib_core [55991.045818] ib_core [55991.045818] ioatdma [55991.045819] ioatdma [55991.045819] i2c_i801 [55991.045820] i2c_i801 [55991.045820] mei_me [55991.045820] mei_me [55991.045821] mei [55991.045821] mei [55991.045822] edac_core [55991.045822] edac_core [55991.045823] ipmi_msghandler [55991.045823] ipmi_msghandler [55991.045823] lpc_ich [55991.045824] lpc_ich [55991.045824] ntb [55991.045825] ntb [55991.045825] wmi [55991.045826] wmi [55991.045826] shpchp [55991.045827] shpchp [55991.045827] pcspkr [55991.045827] pcspkr [55991.045828] nfsd [55991.045828] nfsd [55991.045829] nfs_acl [55991.045829] nfs_acl [55991.045830] lockd [55991.045830] lockd [55991.045830] grace [55991.045831] grace [55991.045831] auth_rpcgss [55991.045832] auth_rpcgss [55991.045832] sunrpc [55991.045833] sunrpc [55991.045833] ip_tables [55991.045834] ip_tables [55991.045834] ext4 [55991.045834] ext4 [55991.045835] mbcache [55991.045835] mbcache [55991.045836] jbd2 [55991.045836] jbd2 [55991.045837] sd_mod [55991.045837] sd_mod [55991.045837] crc_t10dif [55991.045838] crc_t10dif [55991.045838] crct10dif_generic [55991.045839] crct10dif_generic [55991.045839] mlx4_en [55991.045840] mlx4_en [55991.045840] mgag200 [55991.045841] mgag200 [55991.045841] drm_kms_helper [55991.045841] drm_kms_helper [55991.045842] syscopyarea [55991.045842] syscopyarea [55991.045843] sysfillrect [55991.045843] sysfillrect [55991.045844] sysimgblt [55991.045844] sysimgblt [55991.045844] fb_sys_fops [55991.045845] fb_sys_fops [55991.045845] isci [55991.045846] isci [55991.045846] ttm [55991.045847] ttm [55991.045847] igb [55991.045848] igb [55991.045848] crct10dif_pclmul [55991.045848] crct10dif_pclmul [55991.045849] crct10dif_common [55991.045849] crct10dif_common [55991.045850] ahci [55991.045850] ahci [55991.045851] ptp [55991.045851] ptp [55991.045851] crc32c_intel [55991.045852] crc32c_intel [55991.045852] libsas [55991.045853] libsas [55991.045853] libahci [55991.045854] libahci [55991.045854] pps_core [55991.045854] pps_core [55991.045855] drm [55991.045855] drm [55991.045856] mlx4_core [55991.045856] mlx4_core [55991.045856] mpt2sas [55991.045857] mpt2sas [55991.045857] dca [55991.045858] dca [55991.045858] libata [55991.045859] libata [55991.045859] raid_class [55991.045859] raid_class [55991.045860] i2c_algo_bit [55991.045860] i2c_algo_bit [55991.045861] scsi_transport_sas [55991.045861] scsi_transport_sas [55991.045862] devlink [55991.045862] devlink [55991.045862] i2c_core [55991.045863] i2c_core [55991.045863] fjes [55991.045864] [55991.045864] fjes [55991.045865] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55991.045866] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55991.045867] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55991.045867] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55991.045868] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [55991.045869] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [55991.045870] RIP: 0010:[] [55991.045871] [] _raw_spin_lock+0x37/0x50 [55991.045873] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [55991.045873] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000206 [55991.045874] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000216 [55991.045875] RAX: 00000000000022b4 RBX: 00000201e39c5101 RCX: 0000000000000a1e [55991.045875] RAX: 0000000000006f3f RBX: ffff8807f639fe00 RCX: 0000000000000a1e [55991.045876] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [55991.045877] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [55991.045877] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [55991.045878] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [55991.045879] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [55991.045879] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [55991.045880] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [55991.045880] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [55991.045881] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [55991.045882] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [55991.045883] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55991.045883] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55991.045884] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55991.045885] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [55991.045885] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55991.045886] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55991.045887] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55991.045887] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55991.045888] Stack: [55991.045888] Stack: [55991.045889] ffff8808217bbcd8 [55991.045889] ffff8807f639fd88 [55991.045890] ffffffffa08aa7c8 [55991.045890] ffffffffa08aa831 [55991.045890] ffffffffa08aa8b6 [55991.045891] 0000000200000002 [55991.045891] ffff8804c1ba3200 [55991.045892] [55991.045892] ffff8807f63a2980 [55991.045893] 0000000000000001 [55991.045893] 0000000000000000 [55991.045894] ffff8808217bbd08 [55991.045894] ffff8807f639fdc0 [55991.045895] ffffffffa09217bb [55991.045895] ffffffffa0942e30 [55991.045895] ffff8804c1ba3200 [55991.045896] [55991.045896] 00000000c0086540 [55991.045897] 0000000000000001 [55991.045897] ffff8807f63a2900 [55991.045898] ffff88082abba340 [55991.045898] 0000000000000002 [55991.045899] ffff88082abba348 [55991.045899] 0000000000650aa0 [55991.045899] ffff8808217bbd70 [55991.045900] [55991.045900] 0000000000000000 [55991.045900] Call Trace: [55991.045901] Call Trace: [55991.045906] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55991.045912] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [55991.045916] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55991.045923] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [55991.045928] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [55991.045933] [] LNetCtl+0x116/0x800 [lnet] [55991.045939] [] lnet_finalize+0x1e9/0x690 [lnet] [55991.045944] [] lnet_ioctl+0x60/0x250 [lnet] [55991.045948] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [55991.045953] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [55991.045956] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [55991.045961] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [55991.045963] [] ? dequeue_task_fair+0x41e/0x660 [55991.045964] [] do_vfs_ioctl+0x2d5/0x4b0 [55991.045966] [] ? wake_up_state+0x20/0x20 [55991.045967] [] ? __fd_install+0x47/0x60 [55991.045971] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55991.045972] [] SyS_ioctl+0xa1/0xc0 [55991.045974] [] kthread+0xcf/0xe0 [55991.045976] [] system_call_fastpath+0x16/0x1b [55991.045977] [] ? kthread_create_on_node+0x140/0x140 [55991.045978] Code: [55991.045980] [] ret_from_fork+0x58/0x90 [55991.045981] c2 c1 [55991.045982] [] ? kthread_create_on_node+0x140/0x140 [55991.045983] ea 10 66 [55991.045983] Code: [55991.045984] 39 [55991.045984] c1 [55991.045985] c2 [55991.045985] 07 [55991.045986] 75 [55991.045986] 89 [55991.045986] 01 [55991.045987] c2 [55991.045987] c3 [55991.045988] c1 [55991.045988] 55 [55991.045988] ea [55991.045989] 83 [55991.045989] 10 [55991.045990] e2 [55991.045990] 66 [55991.045990] fe [55991.045991] 39 [55991.045991] 0f [55991.045992] c2 [55991.045992] b7 [55991.045992] 75 [55991.045993] f2 [55991.045993] 01 [55991.045993] 48 [55991.045994] c3 [55991.045994] 89 [55991.045995] 55 [55991.045995] e5 [55991.045995] 83 [55991.045996] b8 [55991.045996] e2 [55991.045997] 00 [55991.045997] fe [55991.045997] 80 [55991.045998] 0f [55991.045998] 00 [55991.045999] b7 [55991.045999] 00 [55991.045999] f2 [55991.046000] eb [55991.046000] 48 [55991.046001] 0d [55991.046001] 89 [55991.046001] 66 [55991.046002] e5 [55991.046002] 0f [55991.046003] b8 [55991.046003] 1f [55991.046003] 00 [55991.046004] 44 [55991.046004] 80 [55991.046004] 00 [55991.046005] 00 [55991.046005] 00 [55991.046006] 00 [55991.046006] f3 [55991.046006] eb [55991.046007] 90 [55991.046007] 0d [55991.046008] 83 [55991.046008] 66 [55991.046008] e8 [55991.046009] 0f [55991.046009] 01 [55991.046010] 1f [55991.046010] 74 [55991.046011] 44 [55991.046011] 0a [55991.046011] 00 [55991.046012] 0f [55991.046012] 00 [55991.046012] b7 [55991.046013] f3 [55991.046013] 0f [55991.046014] 90 [55991.046014] <66> [55991.046015] 83 [55991.046015] 39 [55991.046015] e8 [55991.046016] ca [55991.046016] 01 [55991.046017] 75 [55991.046017] 74 [55991.046017] f1 [55991.046018] 0a [55991.046018] 5d [55991.046019] <0f> [55991.046019] c3 [55991.046019] b7 [55991.046020] 66 [55991.046020] 0f [55991.046021] 66 [55991.046021] 66 [55991.046021] 66 [55991.046022] 39 [55991.046022] 90 [55991.046023] ca [55991.046023] 66 [55991.046023] 75 [55991.046024] 66 [55991.046024] f1 [55991.046025] 90 [55991.046025] 5d [55991.046025] eb [55991.046026] c3 [55991.046026] d9 [55991.046026] 66 [55991.046027] 66 [55991.046027] 66 [55991.046028] 0f [55991.046028] 66 [55991.046028] 1f [55991.046029] 90 [55991.046029] 44 [55991.046030] 66 [55991.046030] 00 [55991.046030] [55991.046032] 66 90 eb d9 66 0f [55995.092646] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [55995.329145] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [55995.398841] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [55995.435019] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [55995.472368] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [55995.497439] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [55995.524853] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [55995.542826] RAX: 00000000000049cc RBX: ffff88082d2cded8 RCX: 0000000000000a1e [55995.566805] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [55995.590810] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [55995.614823] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [55995.638856] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [55995.662874] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [55995.690066] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [55995.709566] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [55995.733633] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [55995.757727] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [55995.781820] Stack: [55995.789044] ffff8808262efac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [55995.814065] ffff88083fb5e000 ffff8808262efb80 ffffffffa092c9b5 00000000000001a8 [55995.839100] ffffc9000e331730 ffff8808262efb68 ffffffffa0ebd587 ffffffffffffffff [55995.864158] Call Trace: [55995.872866] [] cfs_percpt_lock+0x58/0x110 [libcfs] [55995.894766] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [55995.917540] [] lnet_select_pathway+0x235/0x1140 [lnet] [55995.940592] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [55995.962815] [] lnet_send+0x51/0x180 [lnet] [55995.982478] [] lnet_finalize+0x3f1/0x690 [lnet] [55996.003586] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [55996.025271] [] ? __slab_free+0x81/0x2f0 [55996.044110] [] lnet_ni_recv+0xc3/0x320 [lnet] [55996.064683] [] lnet_parse+0x945/0xf90 [lnet] [55996.084950] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [55996.108109] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [55996.129260] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [55996.152711] [] ? dequeue_task_fair+0x41e/0x660 [55996.173574] [] ? wake_up_state+0x20/0x20 [55996.192782] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [55996.215722] [] kthread+0xcf/0xe0 [55996.232665] [] ? kthread_create_on_node+0x140/0x140 [55996.255027] [] ret_from_fork+0x58/0x90 [55996.273679] [] ? kthread_create_on_node+0x140/0x140 [55996.296015] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56003.044529] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [56003.044530] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [56003.044550] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56003.044556] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56003.044557] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56003.044558] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56003.044558] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [56003.044560] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56003.044561] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [56003.044561] RAX: 0000000000005460 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [56003.044562] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [56003.044562] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [56003.044563] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [56003.044564] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [56003.044564] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [56003.044565] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56003.044565] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56003.044566] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56003.044567] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56003.044567] Stack: [56003.044568] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [56003.044569] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [56003.044570] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [56003.044570] Call Trace: [56003.044576] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56003.044582] [] lnet_post_send_locked+0x446/0x880 [lnet] [56003.044588] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [56003.044593] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56003.044598] [] lnet_finalize+0x1e9/0x690 [lnet] [56003.044602] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56003.044606] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56003.044607] [] ? dequeue_task_fair+0x41e/0x660 [56003.044608] [] ? sched_clock_cpu+0x85/0xc0 [56003.044610] [] ? wake_up_state+0x20/0x20 [56003.044613] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56003.044615] [] kthread+0xcf/0xe0 [56003.044616] [] ? kthread_create_on_node+0x140/0x140 [56003.044618] [] ret_from_fork+0x58/0x90 [56003.044620] [] ? kthread_create_on_node+0x140/0x140 [56003.044630] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56004.278680] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56004.519578] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56004.591063] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56004.627506] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56004.665352] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [56004.690902] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56004.718759] RSP: 0018:ffff88082626bce0 EFLAGS: 00000206 [56004.737128] RAX: 00000000000073f6 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [56004.761550] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [56004.785929] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [56004.810313] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [56004.834702] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [56004.859089] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [56004.886667] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56004.906515] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [56004.930912] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56004.955336] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56004.979723] Stack: [56004.987238] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [56005.012577] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [56005.037877] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [56005.063177] Call Trace: [56005.072140] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56005.094259] [] lnet_finalize+0xf6/0x690 [lnet] [56005.115228] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56005.137917] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56005.161478] [] ? dequeue_task_fair+0x41e/0x660 [56005.182450] [] ? wake_up_state+0x20/0x20 [56005.201735] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56005.224720] [] kthread+0xcf/0xe0 [56005.241706] [] ? kthread_create_on_node+0x140/0x140 [56005.264122] [] ret_from_fork+0x58/0x90 [56005.282835] [] ? kthread_create_on_node+0x140/0x140 [56005.305239] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56007.044117] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [56007.044118] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [56007.044136] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56007.044141] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56007.044142] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56007.044143] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56007.044143] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [56007.044145] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56007.044146] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000202 [56007.044146] RAX: 00000000000070b3 RBX: 0000000100000000 RCX: 0000000000000a1e [56007.044147] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [56007.044147] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [56007.044148] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [56007.044149] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [56007.044149] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [56007.044150] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56007.044151] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56007.044151] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56007.044152] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56007.044152] Stack: [56007.044153] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [56007.044154] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [56007.044155] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [56007.044155] Call Trace: [56007.044160] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56007.044166] [] lnet_finalize+0xf6/0x690 [lnet] [56007.044170] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56007.044171] [] ? enqueue_task_fair+0x208/0x6c0 [56007.044173] [] ? sched_clock_cpu+0x85/0xc0 [56007.044178] [] lnet_ni_recv+0xc3/0x320 [lnet] [56007.044184] [] lnet_parse+0x945/0xf90 [lnet] [56007.044188] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56007.044189] [] ? __wake_up+0x44/0x50 [56007.044192] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56007.044193] [] ? dequeue_task_fair+0x41e/0x660 [56007.044195] [] ? wake_up_state+0x20/0x20 [56007.044198] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56007.044200] [] kthread+0xcf/0xe0 [56007.044201] [] ? kthread_create_on_node+0x140/0x140 [56007.044203] [] ret_from_fork+0x58/0x90 [56007.044204] [] ? kthread_create_on_node+0x140/0x140 [56007.044214] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56008.310619] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56008.551150] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56008.622415] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56008.657914] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56008.695707] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [56008.721232] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56008.749032] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [56008.767416] RAX: 0000000000004b9f RBX: 0000000000016c40 RCX: 0000000000000a1e [56008.791820] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [56008.816203] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [56008.840618] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [56008.865027] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [56008.889404] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [56008.916940] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56008.936783] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56008.961211] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56008.985606] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56009.010030] Stack: [56009.017528] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [56009.042823] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [56009.068114] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [56009.093443] Call Trace: [56009.102381] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56009.124503] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [56009.145765] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [56009.169308] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [56009.195157] [] ? lock_timer_base.isra.33+0x2b/0x50 [56009.217278] [] ? try_to_del_timer_sync+0x5e/0x90 [56009.238861] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [56009.262722] [] ? schedule_timeout+0x17d/0x2d0 [56009.283437] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [56009.305582] [] ? wake_up_state+0x20/0x20 [56009.324862] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [56009.349286] [] kthread+0xcf/0xe0 [56009.366287] [] ? kthread_create_on_node+0x140/0x140 [56009.388726] [] ret_from_fork+0x58/0x90 [56009.407467] [] ? kthread_create_on_node+0x140/0x140 [56009.429929] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56011.043705] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [kiblnd_sd_00_01:2897] [56011.043707] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [kiblnd_sd_01_03:2903] [56011.043727] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56011.043732] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56011.043733] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56011.043734] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56011.043735] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [56011.043736] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56011.043737] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [56011.043737] RAX: 00000000000012b9 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [56011.043738] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [56011.043739] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [56011.043739] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [56011.043740] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [56011.043740] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [56011.043741] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56011.043741] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56011.043742] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56011.043742] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56011.043743] Stack: [56011.043744] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [56011.043745] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [56011.043745] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [56011.043746] Call Trace: [56011.043751] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56011.043755] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56011.043761] [] lnet_select_pathway+0x235/0x1140 [lnet] [56011.043765] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56011.043771] [] lnet_send+0x51/0x180 [lnet] [56011.043776] [] lnet_finalize+0x3f1/0x690 [lnet] [56011.043780] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56011.043781] [] ? __slab_free+0x81/0x2f0 [56011.043787] [] lnet_ni_recv+0xc3/0x320 [lnet] [56011.043792] [] lnet_parse+0x945/0xf90 [lnet] [56011.043797] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56011.043800] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56011.043803] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56011.043806] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56011.043808] [] ? dequeue_task_fair+0x41e/0x660 [56011.043809] [] ? wake_up_state+0x20/0x20 [56011.043813] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56011.043814] [] kthread+0xcf/0xe0 [56011.043816] [] ? kthread_create_on_node+0x140/0x140 [56011.043817] [] ret_from_fork+0x58/0x90 [56011.043819] [] ? kthread_create_on_node+0x140/0x140 [56011.043829] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56012.403788] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56012.643835] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56012.714976] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56012.751308] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56012.789115] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [56012.814629] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56012.842435] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000202 [56012.860790] RAX: 0000000000007032 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [56012.885192] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [56012.909591] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [56012.933977] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [56012.958362] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [56012.982734] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [56013.010267] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56013.030069] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56013.054456] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56013.078827] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56013.103233] Stack: [56013.110714] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [56013.135997] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [56013.161275] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [56013.186576] Call Trace: [56013.195545] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56013.217663] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56013.240649] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [56013.266486] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56013.288639] [] lnet_finalize+0x1e9/0x690 [lnet] [56013.309941] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56013.332650] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56013.356222] [] ? dequeue_task_fair+0x41e/0x660 [56013.377218] [] ? wake_up_state+0x20/0x20 [56013.396504] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56013.419494] [] kthread+0xcf/0xe0 [56013.436490] [] ? kthread_create_on_node+0x140/0x140 [56013.458898] [] ret_from_fork+0x58/0x90 [56013.477624] [] ? kthread_create_on_node+0x140/0x140 [56013.500028] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56015.043293] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [kiblnd_sd_01_02:2902] [56015.043295] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [kworker/3:2:27192] [56015.043313] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56015.043319] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56015.043320] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56015.043321] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56015.043323] Workqueue: ib_cm cm_work_handler [ib_cm] [56015.043323] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [56015.043325] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [56015.043325] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000206 [56015.043326] RAX: 0000000000007277 RBX: 0000000000000000 RCX: 0000000000000a1e [56015.043326] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [56015.043327] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [56015.043328] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [56015.043328] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [56015.043329] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [56015.043329] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56015.043330] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56015.043330] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56015.043331] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56015.043331] Stack: [56015.043332] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [56015.043333] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [56015.043334] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [56015.043334] Call Trace: [56015.043339] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56015.043344] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [56015.043348] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [56015.043350] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [56015.043354] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [56015.043356] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [56015.043358] [] cma_req_handler+0x227/0x570 [rdma_cm] [56015.043360] [] cm_process_work+0x25/0x140 [ib_cm] [56015.043362] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [56015.043363] [] cm_work_handler+0x185/0x1484 [ib_cm] [56015.043365] [] process_one_work+0x17b/0x470 [56015.043365] [] worker_thread+0x126/0x410 [56015.043366] [] ? rescuer_thread+0x460/0x460 [56015.043368] [] kthread+0xcf/0xe0 [56015.043369] [] ? kthread_create_on_node+0x140/0x140 [56015.043371] [] ret_from_fork+0x58/0x90 [56015.043372] [] ? kthread_create_on_node+0x140/0x140 [56015.043382] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [56016.340365] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56016.580897] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56016.652237] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56016.688885] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56016.726728] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [56016.752273] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56016.780089] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [56016.798482] RAX: 0000000000000893 RBX: 000001f280bac5e0 RCX: 0000000000000a1e [56016.822886] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [56016.847303] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [56016.871691] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [56016.896069] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [56016.920433] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [56016.947994] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56016.967799] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56016.992177] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56017.016562] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56017.040970] Stack: [56017.048486] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [56017.073799] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [56017.099090] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [56017.124369] Call Trace: [56017.133309] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56017.155446] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56017.178417] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56017.200823] [] lnet_finalize+0x1e9/0x690 [lnet] [56017.222089] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56017.244770] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56017.268294] [] ? dequeue_task_fair+0x41e/0x660 [56017.289278] [] ? wake_up_state+0x20/0x20 [56017.308532] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56017.331484] [] kthread+0xcf/0xe0 [56017.348472] [] ? kthread_create_on_node+0x140/0x140 [56017.370871] [] ret_from_fork+0x58/0x90 [56017.389550] [] ? kthread_create_on_node+0x140/0x140 [56017.411942] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56019.042881] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [kiblnd_sd_01_00:2900] [56019.042882] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [56019.042883] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 21s! [kiblnd_sd_01_01:2901] [56019.042884] Modules linked in: [56019.042885] Modules linked in: [56019.042886] ko2iblnd(OE) [56019.042886] ko2iblnd(OE) [56019.042887] lustre(OE) [56019.042887] lustre(OE) [56019.042888] lmv(OE) [56019.042888] lmv(OE) [56019.042889] fld(OE) [56019.042889] fld(OE) [56019.042889] mdc(OE) [56019.042890] mdc(OE) [56019.042890] fid(OE) [56019.042891] fid(OE) [56019.042891] lov(OE) [56019.042892] lov(OE) [56019.042892] ksocklnd(OE) [56019.042892] ksocklnd(OE) [56019.042893] ptlrpc(OE) [56019.042893] ptlrpc(OE) [56019.042894] obdclass(OE) [56019.042894] obdclass(OE) [56019.042895] lnet(OE) [56019.042895] lnet(OE) [56019.042895] libcfs(OE) [56019.042896] libcfs(OE) [56019.042896] rpcrdma [56019.042897] rpcrdma [56019.042897] ib_isert [56019.042898] ib_isert [56019.042898] iscsi_target_mod [56019.042899] iscsi_target_mod [56019.042899] ib_iser [56019.042899] ib_iser [56019.042900] libiscsi [56019.042900] libiscsi [56019.042901] scsi_transport_iscsi [56019.042901] scsi_transport_iscsi [56019.042902] ib_srpt [56019.042902] ib_srpt [56019.042902] target_core_mod [56019.042903] target_core_mod [56019.042903] ib_srp [56019.042904] ib_srp [56019.042904] scsi_transport_srp [56019.042905] scsi_transport_srp [56019.042905] scsi_tgt [56019.042906] scsi_tgt [56019.042906] ib_ipoib [56019.042906] ib_ipoib [56019.042907] rdma_ucm [56019.042907] rdma_ucm [56019.042908] ib_ucm [56019.042908] ib_ucm [56019.042909] ib_uverbs [56019.042909] ib_uverbs [56019.042909] ib_umad [56019.042910] ib_umad [56019.042910] rdma_cm [56019.042911] rdma_cm [56019.042911] ib_cm [56019.042912] ib_cm [56019.042912] iw_cm [56019.042912] iw_cm [56019.042913] mlx4_ib [56019.042913] mlx4_ib [56019.042914] intel_powerclamp [56019.042914] intel_powerclamp [56019.042915] coretemp [56019.042915] coretemp [56019.042915] intel_rapl [56019.042916] intel_rapl [56019.042916] iosf_mbi [56019.042917] iosf_mbi [56019.042917] kvm [56019.042918] kvm [56019.042918] irqbypass [56019.042918] irqbypass [56019.042919] crc32_pclmul [56019.042919] crc32_pclmul [56019.042920] ghash_clmulni_intel [56019.042920] ghash_clmulni_intel [56019.042921] aesni_intel [56019.042921] aesni_intel [56019.042921] ipmi_ssif [56019.042922] ipmi_ssif [56019.042922] lrw [56019.042923] lrw [56019.042923] gf128mul [56019.042924] gf128mul [56019.042924] glue_helper [56019.042924] glue_helper [56019.042925] hfi1 [56019.042925] hfi1 [56019.042926] ipmi_devintf [56019.042926] ipmi_devintf [56019.042927] rdmavt [56019.042927] rdmavt [56019.042927] ablk_helper [56019.042928] ablk_helper [56019.042928] cryptd [56019.042929] cryptd [56019.042929] iTCO_wdt [56019.042930] iTCO_wdt [56019.042930] ipmi_si [56019.042930] ipmi_si [56019.042931] sb_edac [56019.042931] sb_edac [56019.042932] sg [56019.042932] sg [56019.042933] iTCO_vendor_support [56019.042933] iTCO_vendor_support [56019.042933] ib_core [56019.042934] ib_core [56019.042934] ioatdma [56019.042935] ioatdma [56019.042935] i2c_i801 [56019.042935] i2c_i801 [56019.042936] mei_me [56019.042936] mei_me [56019.042937] mei [56019.042937] mei [56019.042938] edac_core [56019.042938] edac_core [56019.042938] ipmi_msghandler [56019.042939] ipmi_msghandler [56019.042939] lpc_ich [56019.042940] lpc_ich [56019.042940] ntb [56019.042941] ntb [56019.042941] wmi [56019.042941] wmi [56019.042942] shpchp [56019.042942] shpchp [56019.042943] pcspkr [56019.042943] pcspkr [56019.042944] nfsd [56019.042944] nfsd [56019.042944] nfs_acl [56019.042945] nfs_acl [56019.042945] lockd [56019.042946] lockd [56019.042946] grace [56019.042947] grace [56019.042947] auth_rpcgss [56019.042947] auth_rpcgss [56019.042948] sunrpc [56019.042948] sunrpc [56019.042949] ip_tables [56019.042949] ip_tables [56019.042950] ext4 [56019.042950] ext4 [56019.042950] mbcache [56019.042951] mbcache [56019.042951] jbd2 [56019.042952] jbd2 [56019.042952] sd_mod [56019.042953] sd_mod [56019.042953] crc_t10dif [56019.042953] crc_t10dif [56019.042954] crct10dif_generic [56019.042954] crct10dif_generic [56019.042955] mlx4_en [56019.042955] mlx4_en [56019.042956] mgag200 [56019.042956] mgag200 [56019.042957] drm_kms_helper [56019.042957] drm_kms_helper [56019.042957] syscopyarea [56019.042958] syscopyarea [56019.042958] sysfillrect [56019.042959] sysfillrect [56019.042959] sysimgblt [56019.042960] sysimgblt [56019.042960] fb_sys_fops [56019.042960] fb_sys_fops [56019.042961] isci [56019.042961] isci [56019.042962] ttm [56019.042962] ttm [56019.042963] igb [56019.042963] igb [56019.042964] crct10dif_pclmul [56019.042964] crct10dif_pclmul [56019.042965] crct10dif_common [56019.042965] crct10dif_common [56019.042966] ahci [56019.042966] ahci [56019.042967] ptp [56019.042967] ptp [56019.042967] crc32c_intel [56019.042968] crc32c_intel [56019.042968] libsas [56019.042969] libsas [56019.042969] libahci [56019.042970] libahci [56019.042970] pps_core [56019.042971] pps_core [56019.042971] drm [56019.042972] drm [56019.042972] mlx4_core [56019.042972] mlx4_core [56019.042973] mpt2sas [56019.042973] mpt2sas [56019.042974] dca [56019.042974] dca [56019.042975] libata [56019.042975] libata [56019.042975] raid_class [56019.042976] raid_class [56019.042976] i2c_algo_bit [56019.042977] i2c_algo_bit [56019.042977] scsi_transport_sas [56019.042978] scsi_transport_sas [56019.042978] devlink [56019.042979] devlink [56019.042979] i2c_core [56019.042979] i2c_core [56019.042980] fjes [56019.042980] [56019.042981] fjes [56019.042982] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56019.042983] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56019.042983] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56019.042984] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56019.042985] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [56019.042986] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [56019.042987] RIP: 0010:[] [56019.042988] [] _raw_spin_lock+0x37/0x50 [56019.042989] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56019.042990] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000202 [56019.042990] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [56019.042991] RAX: 0000000000003883 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [56019.042992] RAX: 000000000000175a RBX: ffff88082d2cded8 RCX: 0000000000000a1e [56019.042992] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [56019.042993] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [56019.042994] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [56019.042994] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [56019.042995] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [56019.042995] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [56019.042996] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [56019.042997] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [56019.042997] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [56019.042998] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [56019.042999] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56019.043000] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56019.043000] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [56019.043001] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56019.043002] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56019.043002] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56019.043003] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56019.043003] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56019.043004] Stack: [56019.043004] Stack: [56019.043005] ffff8807f639fd88 [56019.043005] ffff8808262efac0 [56019.043006] ffffffffa08aa831 [56019.043006] ffffffffa08aa7c8 [56019.043007] 0000000200000002 [56019.043007] ffffffffa08aa8b6 [56019.043008] ffff8807f63a2980 [56019.043008] [56019.043008] 0000000000000000 [56019.043009] 0000000000000000 [56019.043010] ffff88083fb5e000 [56019.043010] ffff8807f639fdc0 [56019.043011] ffff8808262efb80 [56019.043011] ffffffffa0942e30 [56019.043012] ffffffffa092c9b5 [56019.043012] 00000000c0086540 [56019.043012] [56019.043013] 00000000000001a8 [56019.043014] ffff8807f63a2900 [56019.043014] ffffc9000e331730 [56019.043014] 0000000000000002 [56019.043015] ffff8808262efb68 [56019.043015] 0000000000650aa0 [56019.043016] ffffffffa0ebd587 [56019.043016] 0000000000000000 [56019.043017] [56019.043017] ffffffffffffffff [56019.043017] Call Trace: [56019.043018] Call Trace: [56019.043023] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [56019.043029] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56019.043035] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [56019.043040] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56019.043044] [] LNetCtl+0x116/0x800 [lnet] [56019.043051] [] lnet_select_pathway+0x235/0x1140 [lnet] [56019.043057] [] lnet_ioctl+0x60/0x250 [lnet] [56019.043060] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56019.043065] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [56019.043071] [] lnet_send+0x51/0x180 [lnet] [56019.043075] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [56019.043081] [] lnet_finalize+0x3f1/0x690 [lnet] [56019.043082] [] do_vfs_ioctl+0x2d5/0x4b0 [56019.043086] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56019.043087] [] ? __fd_install+0x47/0x60 [56019.043089] [] ? __slab_free+0x81/0x2f0 [56019.043090] [] SyS_ioctl+0xa1/0xc0 [56019.043096] [] lnet_ni_recv+0xc3/0x320 [lnet] [56019.043098] [] system_call_fastpath+0x16/0x1b [56019.043103] [] lnet_parse+0x945/0xf90 [lnet] [56019.043104] Code: c1 [56019.043108] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56019.043109] 07 89 c2 c1 ea 10 [56019.043111] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56019.043112] 66 39 [56019.043116] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56019.043116] c2 75 [56019.043118] [] ? dequeue_task_fair+0x41e/0x660 [56019.043119] 01 c3 [56019.043120] [] ? wake_up_state+0x20/0x20 [56019.043121] 55 83 [56019.043125] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56019.043125] e2 fe [56019.043127] [] kthread+0xcf/0xe0 [56019.043128] 0f b7 [56019.043129] [] ? kthread_create_on_node+0x140/0x140 [56019.043130] f2 48 [56019.043132] [] ret_from_fork+0x58/0x90 [56019.043132] 89 e5 [56019.043134] [] ? kthread_create_on_node+0x140/0x140 [56019.043135] b8 00 80 [56019.043135] Code: [56019.043136] 00 [56019.043136] 00 [56019.043136] 00 [56019.043137] 02 [56019.043137] eb [56019.043138] 00 [56019.043138] 0d [56019.043138] f0 [56019.043139] 66 [56019.043139] 0f [56019.043140] 0f [56019.043140] c1 [56019.043140] 1f [56019.043141] 07 [56019.043141] 44 [56019.043142] 89 [56019.043142] 00 [56019.043142] c2 [56019.043143] 00 [56019.043143] c1 [56019.043144] f3 [56019.043144] ea [56019.043144] 90 [56019.043145] 10 [56019.043145] 83 [56019.043146] 66 [56019.043146] e8 [56019.043146] 39 [56019.043147] 01 [56019.043147] c2 [56019.043148] 74 [56019.043148] 75 [56019.043148] 0a [56019.043149] 01 [56019.043149] <0f> [56019.043150] c3 [56019.043150] b7 [56019.043151] 55 [56019.043151] 0f [56019.043151] 83 [56019.043152] 66 [56019.043152] e2 [56019.043153] 39 [56019.043153] fe [56019.043153] ca [56019.043154] 0f [56019.043154] 75 [56019.043155] b7 [56019.043155] f1 [56019.043156] f2 [56019.043156] 5d [56019.043157] 48 [56019.043157] c3 [56019.043157] 89 [56019.043158] 66 [56019.043158] e5 [56019.043159] 66 [56019.043159] b8 [56019.043159] 66 [56019.043160] 00 [56019.043160] 90 [56019.043161] 80 [56019.043161] 66 [56019.043161] 00 [56019.043162] 66 [56019.043162] 00 [56019.043163] 90 [56019.043163] eb [56019.043163] eb [56019.043164] 0d [56019.043164] d9 [56019.043165] 66 [56019.043165] 66 [56019.043165] 0f [56019.043166] 0f [56019.043166] [56019.043171] 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56023.073038] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56023.309613] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56023.379354] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56023.415542] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56023.452905] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [56023.478005] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56023.505407] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [56023.523397] RAX: 00000000000007b2 RBX: 00000201e39c5101 RCX: 0000000000000a1e [56023.547391] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [56023.571406] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [56023.595431] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [56023.619456] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [56023.643501] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [56023.670701] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56023.690217] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56023.714325] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56023.738421] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56023.762525] Stack: [56023.769748] ffff8808217bbcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba3200 [56023.794786] 0000000000000001 ffff8808217bbd08 ffffffffa09217bb ffff8804c1ba3200 [56023.819829] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff8808217bbd70 [56023.844890] Call Trace: [56023.853624] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56023.875523] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56023.898323] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56023.920549] [] lnet_finalize+0x1e9/0x690 [lnet] [56023.941619] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56023.964133] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56023.987534] [] ? dequeue_task_fair+0x41e/0x660 [56024.008366] [] ? wake_up_state+0x20/0x20 [56024.027493] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56024.050370] [] kthread+0xcf/0xe0 [56024.067238] [] ? kthread_create_on_node+0x140/0x140 [56024.089533] [] ret_from_fork+0x58/0x90 [56024.108118] [] ? kthread_create_on_node+0x140/0x140 [56024.130471] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56031.041646] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kiblnd_sd_00_00:2896] [56031.041647] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [kiblnd_sd_00_03:2899] [56031.041667] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56031.041672] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56031.041674] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56031.041674] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56031.041675] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [56031.041676] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56031.041677] RSP: 0018:ffff88082626bce0 EFLAGS: 00000206 [56031.041677] RAX: 0000000000004cf9 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [56031.041678] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [56031.041678] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [56031.041679] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [56031.041680] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [56031.041680] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [56031.041681] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56031.041681] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [56031.041682] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56031.041683] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56031.041683] Stack: [56031.041684] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [56031.041685] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [56031.041686] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [56031.041686] Call Trace: [56031.041691] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56031.041697] [] lnet_finalize+0xf6/0x690 [lnet] [56031.041701] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56031.041704] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56031.041706] [] ? dequeue_task_fair+0x41e/0x660 [56031.041707] [] ? wake_up_state+0x20/0x20 [56031.041711] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56031.041712] [] kthread+0xcf/0xe0 [56031.041714] [] ? kthread_create_on_node+0x140/0x140 [56031.041715] [] ret_from_fork+0x58/0x90 [56031.041717] [] ? kthread_create_on_node+0x140/0x140 [56031.041727] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56032.186152] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56032.427363] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56032.499135] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56032.535630] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56032.573555] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [56032.599210] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56032.627150] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [56032.645615] RAX: 00000000000045cb RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [56032.670088] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [56032.694545] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [56032.718988] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [56032.743385] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [56032.767803] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [56032.795378] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56032.815194] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56032.839594] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56032.864031] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56032.888457] Stack: [56032.895958] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [56032.921260] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [56032.946554] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [56032.971891] Call Trace: [56032.980852] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56033.003021] [] lnet_post_send_locked+0x446/0x880 [lnet] [56033.026578] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [56033.052456] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56033.074595] [] lnet_finalize+0x1e9/0x690 [lnet] [56033.095896] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56033.118608] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56033.142169] [] ? dequeue_task_fair+0x41e/0x660 [56033.163163] [] ? sched_clock_cpu+0x85/0xc0 [56033.183013] [] ? wake_up_state+0x20/0x20 [56033.202265] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56033.225272] [] kthread+0xcf/0xe0 [56033.242239] [] ? kthread_create_on_node+0x140/0x140 [56033.264648] [] ret_from_fork+0x58/0x90 [56033.283343] [] ? kthread_create_on_node+0x140/0x140 [56033.305768] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56035.041234] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [56035.041235] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [56035.041254] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56035.041260] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56035.041261] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56035.041261] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56035.041262] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [56035.041264] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56035.041264] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [56035.041265] RAX: 0000000000007b0e RBX: 0000000000016c40 RCX: 0000000000000a1e [56035.041265] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [56035.041266] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [56035.041266] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [56035.041267] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [56035.041268] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [56035.041268] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56035.041269] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56035.041269] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56035.041270] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56035.041270] Stack: [56035.041271] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [56035.041272] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [56035.041273] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [56035.041273] Call Trace: [56035.041279] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56035.041284] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [56035.041287] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [56035.041291] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [56035.041292] [] ? lock_timer_base.isra.33+0x2b/0x50 [56035.041294] [] ? try_to_del_timer_sync+0x5e/0x90 [56035.041297] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [56035.041298] [] ? schedule_timeout+0x17d/0x2d0 [56035.041302] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [56035.041304] [] ? wake_up_state+0x20/0x20 [56035.041307] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [56035.041309] [] kthread+0xcf/0xe0 [56035.041310] [] ? kthread_create_on_node+0x140/0x140 [56035.041312] [] ret_from_fork+0x58/0x90 [56035.041313] [] ? kthread_create_on_node+0x140/0x140 [56035.041324] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56036.277922] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56036.518823] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56036.590338] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56036.626801] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56036.664702] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [56036.690251] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56036.718093] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [56036.736475] RAX: 0000000000007759 RBX: 0000000100000000 RCX: 0000000000000a1e [56036.760879] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [56036.785268] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [56036.809683] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [56036.834095] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [56036.858481] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [56036.885990] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56036.905809] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56036.930193] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56036.954570] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56036.978945] Stack: [56036.986445] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [56037.011739] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [56037.037070] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [56037.062366] Call Trace: [56037.071348] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56037.093475] [] lnet_finalize+0xf6/0x690 [lnet] [56037.114475] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56037.136304] [] ? enqueue_task_fair+0x208/0x6c0 [56037.157298] [] ? sched_clock_cpu+0x85/0xc0 [56037.177121] [] lnet_ni_recv+0xc3/0x320 [lnet] [56037.197808] [] lnet_parse+0x945/0xf90 [lnet] [56037.218233] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56037.241477] [] ? __wake_up+0x44/0x50 [56037.259596] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56037.283132] [] ? dequeue_task_fair+0x41e/0x660 [56037.304102] [] ? wake_up_state+0x20/0x20 [56037.323347] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56037.346340] [] kthread+0xcf/0xe0 [56037.363327] [] ? kthread_create_on_node+0x140/0x140 [56037.385719] [] ret_from_fork+0x58/0x90 [56037.404396] [] ? kthread_create_on_node+0x140/0x140 [56037.426793] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56039.040821] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [56039.040823] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [56039.040843] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56039.040849] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56039.040850] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56039.040851] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56039.040852] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [56039.040853] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56039.040854] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [56039.040854] RAX: 0000000000002f0f RBX: ffff88083fb5e000 RCX: 0000000000000a1e [56039.040855] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [56039.040856] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [56039.040856] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [56039.040857] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [56039.040857] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [56039.040858] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56039.040859] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56039.040859] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56039.040860] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56039.040860] Stack: [56039.040861] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [56039.040862] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [56039.040863] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [56039.040863] Call Trace: [56039.040868] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56039.040873] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56039.040879] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [56039.040885] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56039.040890] [] lnet_finalize+0x1e9/0x690 [lnet] [56039.040894] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56039.040897] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56039.040899] [] ? dequeue_task_fair+0x41e/0x660 [56039.040900] [] ? wake_up_state+0x20/0x20 [56039.040904] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56039.040905] [] kthread+0xcf/0xe0 [56039.040907] [] ? kthread_create_on_node+0x140/0x140 [56039.040908] [] ret_from_fork+0x58/0x90 [56039.040910] [] ? kthread_create_on_node+0x140/0x140 [56039.040920] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56040.256360] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56040.497399] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56040.568980] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56040.605741] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56040.643606] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [56040.669163] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56040.697017] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [56040.715401] RAX: 0000000000006d45 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [56040.739795] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [56040.764189] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [56040.788610] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [56040.813027] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [56040.837402] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [56040.864983] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56040.884803] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56040.909208] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56040.933620] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56040.958008] Stack: [56040.965521] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [56040.990831] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [56041.016140] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [56041.041448] Call Trace: [56041.050392] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56041.072524] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56041.095503] [] lnet_select_pathway+0x235/0x1140 [lnet] [56041.118791] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56041.141186] [] lnet_send+0x51/0x180 [lnet] [56041.161006] [] lnet_finalize+0x3f1/0x690 [lnet] [56041.182261] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56041.204119] [] ? __slab_free+0x81/0x2f0 [56041.223103] [] lnet_ni_recv+0xc3/0x320 [lnet] [56041.243777] [] lnet_parse+0x945/0xf90 [lnet] [56041.264197] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56041.287200] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56041.310465] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56041.331752] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56041.355333] [] ? dequeue_task_fair+0x41e/0x660 [56041.376314] [] ? wake_up_state+0x20/0x20 [56041.395553] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56041.418482] [] kthread+0xcf/0xe0 [56041.435414] [] ? kthread_create_on_node+0x140/0x140 [56041.457764] [] ret_from_fork+0x58/0x90 [56041.476370] [] ? kthread_create_on_node+0x140/0x140 [56041.498708] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56043.040409] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [kiblnd_sd_01_02:2902] [56043.040411] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [kworker/3:2:27192] [56043.040430] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56043.040436] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56043.040437] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56043.040437] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56043.040439] Workqueue: ib_cm cm_work_handler [ib_cm] [56043.040440] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [56043.040441] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56043.040442] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000216 [56043.040442] RAX: 0000000000001a71 RBX: 0000000000000000 RCX: 0000000000000a1e [56043.040443] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [56043.040444] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [56043.040444] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [56043.040445] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [56043.040445] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [56043.040446] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56043.040447] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56043.040447] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56043.040448] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56043.040448] Stack: [56043.040449] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [56043.040450] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [56043.040451] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [56043.040451] Call Trace: [56043.040456] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56043.040461] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [56043.040465] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [56043.040467] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [56043.040470] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [56043.040472] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [56043.040474] [] cma_req_handler+0x227/0x570 [rdma_cm] [56043.040476] [] cm_process_work+0x25/0x140 [ib_cm] [56043.040478] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [56043.040480] [] cm_work_handler+0x185/0x1484 [ib_cm] [56043.040481] [] process_one_work+0x17b/0x470 [56043.040482] [] worker_thread+0x126/0x410 [56043.040483] [] ? rescuer_thread+0x460/0x460 [56043.040484] [] kthread+0xcf/0xe0 [56043.040486] [] ? kthread_create_on_node+0x140/0x140 [56043.040487] [] ret_from_fork+0x58/0x90 [56043.040489] [] ? kthread_create_on_node+0x140/0x140 [56043.040499] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56044.335322] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56044.575912] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56044.647272] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56044.683924] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56044.721732] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [56044.747244] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56044.775095] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [56044.793489] RAX: 00000000000026e2 RBX: 000001f280bac5e0 RCX: 0000000000000a1e [56044.817899] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [56044.842317] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [56044.866721] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [56044.891096] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [56044.915469] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [56044.943009] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56044.962842] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56044.987259] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56045.011688] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56045.036082] Stack: [56045.043564] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [56045.068844] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [56045.094136] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [56045.119412] Call Trace: [56045.128375] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56045.150512] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56045.173464] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56045.195881] [] lnet_finalize+0x1e9/0x690 [lnet] [56045.217110] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56045.239820] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56045.263359] [] ? dequeue_task_fair+0x41e/0x660 [56045.284344] [] ? wake_up_state+0x20/0x20 [56045.303606] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56045.326572] [] kthread+0xcf/0xe0 [56045.343537] [] ? kthread_create_on_node+0x140/0x140 [56045.365931] [] ret_from_fork+0x58/0x90 [56045.384626] [] ? kthread_create_on_node+0x140/0x140 [56045.407046] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56047.039997] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [kiblnd_sd_01_01:2901] [56047.039998] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [56047.040000] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 21s! [kiblnd_sd_01_00:2900] [56047.040001] Modules linked in: [56047.040002] Modules linked in: [56047.040002] ko2iblnd(OE) [56047.040003] ko2iblnd(OE) [56047.040003] lustre(OE) [56047.040004] lustre(OE) [56047.040004] lmv(OE) [56047.040005] lmv(OE) [56047.040005] fld(OE) [56047.040005] fld(OE) [56047.040006] mdc(OE) [56047.040006] mdc(OE) [56047.040007] fid(OE) [56047.040007] fid(OE) [56047.040007] lov(OE) [56047.040008] lov(OE) [56047.040008] ksocklnd(OE) [56047.040009] ksocklnd(OE) [56047.040009] ptlrpc(OE) [56047.040010] ptlrpc(OE) [56047.040010] obdclass(OE) [56047.040010] obdclass(OE) [56047.040011] lnet(OE) [56047.040011] lnet(OE) [56047.040012] libcfs(OE) [56047.040012] libcfs(OE) [56047.040013] rpcrdma [56047.040013] rpcrdma [56047.040013] ib_isert [56047.040014] ib_isert [56047.040014] iscsi_target_mod [56047.040015] iscsi_target_mod [56047.040015] ib_iser [56047.040016] ib_iser [56047.040016] libiscsi [56047.040017] libiscsi [56047.040017] scsi_transport_iscsi [56047.040018] scsi_transport_iscsi [56047.040018] ib_srpt [56047.040018] ib_srpt [56047.040019] target_core_mod [56047.040019] target_core_mod [56047.040020] ib_srp [56047.040020] ib_srp [56047.040021] scsi_transport_srp [56047.040021] scsi_transport_srp [56047.040021] scsi_tgt [56047.040022] scsi_tgt [56047.040022] ib_ipoib [56047.040023] ib_ipoib [56047.040023] rdma_ucm [56047.040024] rdma_ucm [56047.040024] ib_ucm [56047.040025] ib_ucm [56047.040025] ib_uverbs [56047.040025] ib_uverbs [56047.040026] ib_umad [56047.040026] ib_umad [56047.040027] rdma_cm [56047.040027] rdma_cm [56047.040028] ib_cm [56047.040028] ib_cm [56047.040028] iw_cm [56047.040029] iw_cm [56047.040029] mlx4_ib [56047.040030] mlx4_ib [56047.040030] intel_powerclamp [56047.040031] intel_powerclamp [56047.040031] coretemp [56047.040031] coretemp [56047.040032] intel_rapl [56047.040032] intel_rapl [56047.040033] iosf_mbi [56047.040033] iosf_mbi [56047.040034] kvm [56047.040034] kvm [56047.040034] irqbypass [56047.040035] irqbypass [56047.040035] crc32_pclmul [56047.040036] crc32_pclmul [56047.040036] ghash_clmulni_intel [56047.040037] ghash_clmulni_intel [56047.040037] aesni_intel [56047.040038] aesni_intel [56047.040038] ipmi_ssif [56047.040038] ipmi_ssif [56047.040039] lrw [56047.040039] lrw [56047.040040] gf128mul [56047.040040] gf128mul [56047.040041] glue_helper [56047.040041] glue_helper [56047.040041] hfi1 [56047.040042] hfi1 [56047.040042] ipmi_devintf [56047.040043] ipmi_devintf [56047.040043] rdmavt [56047.040044] rdmavt [56047.040044] ablk_helper [56047.040045] ablk_helper [56047.040045] cryptd [56047.040045] cryptd [56047.040046] iTCO_wdt [56047.040046] iTCO_wdt [56047.040047] ipmi_si [56047.040047] ipmi_si [56047.040048] sb_edac [56047.040048] sb_edac [56047.040048] sg [56047.040049] sg [56047.040050] iTCO_vendor_support [56047.040050] iTCO_vendor_support [56047.040050] ib_core [56047.040051] ib_core [56047.040051] ioatdma [56047.040052] ioatdma [56047.040052] i2c_i801 [56047.040053] i2c_i801 [56047.040053] mei_me [56047.040053] mei_me [56047.040054] mei [56047.040054] mei [56047.040055] edac_core [56047.040055] edac_core [56047.040056] ipmi_msghandler [56047.040056] ipmi_msghandler [56047.040056] lpc_ich [56047.040057] lpc_ich [56047.040057] ntb [56047.040058] ntb [56047.040058] wmi [56047.040059] wmi [56047.040059] shpchp [56047.040060] shpchp [56047.040060] pcspkr [56047.040060] pcspkr [56047.040061] nfsd [56047.040061] nfsd [56047.040062] nfs_acl [56047.040062] nfs_acl [56047.040063] lockd [56047.040063] lockd [56047.040063] grace [56047.040064] grace [56047.040064] auth_rpcgss [56047.040065] auth_rpcgss [56047.040065] sunrpc [56047.040066] sunrpc [56047.040066] ip_tables [56047.040067] ip_tables [56047.040067] ext4 [56047.040067] ext4 [56047.040068] mbcache [56047.040068] mbcache [56047.040069] jbd2 [56047.040069] jbd2 [56047.040070] sd_mod [56047.040070] sd_mod [56047.040070] crc_t10dif [56047.040071] crc_t10dif [56047.040071] crct10dif_generic [56047.040072] crct10dif_generic [56047.040072] mlx4_en [56047.040073] mlx4_en [56047.040073] mgag200 [56047.040074] mgag200 [56047.040074] drm_kms_helper [56047.040074] drm_kms_helper [56047.040075] syscopyarea [56047.040075] syscopyarea [56047.040076] sysfillrect [56047.040076] sysfillrect [56047.040077] sysimgblt [56047.040077] sysimgblt [56047.040077] fb_sys_fops [56047.040078] fb_sys_fops [56047.040078] isci [56047.040079] isci [56047.040079] ttm [56047.040080] ttm [56047.040080] igb [56047.040081] igb [56047.040081] crct10dif_pclmul [56047.040081] crct10dif_pclmul [56047.040082] crct10dif_common [56047.040082] crct10dif_common [56047.040083] ahci [56047.040083] ahci [56047.040084] ptp [56047.040084] ptp [56047.040084] crc32c_intel [56047.040085] crc32c_intel [56047.040085] libsas [56047.040086] libsas [56047.040086] libahci [56047.040087] libahci [56047.040087] pps_core [56047.040087] pps_core [56047.040088] drm [56047.040088] drm [56047.040089] mlx4_core [56047.040089] mlx4_core [56047.040090] mpt2sas [56047.040090] mpt2sas [56047.040090] dca [56047.040091] dca [56047.040091] libata [56047.040092] libata [56047.040092] raid_class [56047.040093] raid_class [56047.040093] i2c_algo_bit [56047.040093] i2c_algo_bit [56047.040094] scsi_transport_sas [56047.040094] scsi_transport_sas [56047.040095] devlink [56047.040095] devlink [56047.040096] i2c_core [56047.040096] i2c_core [56047.040096] fjes [56047.040097] [56047.040097] fjes [56047.040098] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56047.040099] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56047.040100] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56047.040100] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56047.040101] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [56047.040102] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [56047.040103] RIP: 0010:[] [56047.040104] [] _raw_spin_lock+0x32/0x50 [56047.040106] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56047.040106] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [56047.040107] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [56047.040108] RAX: 00000000000033b2 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [56047.040108] RAX: 0000000000003c1b RBX: 00000201e39c5101 RCX: 0000000000000a1e [56047.040109] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [56047.040110] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [56047.040110] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [56047.040111] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [56047.040112] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [56047.040112] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [56047.040113] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [56047.040113] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [56047.040114] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [56047.040115] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [56047.040116] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56047.040116] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56047.040117] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [56047.040118] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56047.040118] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56047.040119] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56047.040120] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56047.040120] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56047.040120] Stack: [56047.040121] Stack: [56047.040121] ffff8807f639fd88 [56047.040122] ffff8808217bbcd8 [56047.040122] ffffffffa08aa831 [56047.040123] ffffffffa08aa7c8 [56047.040123] 0000000200000002 [56047.040124] ffffffffa08aa8b6 [56047.040124] ffff8807f63a2980 [56047.040124] [56047.040125] ffff8804c1ba3200 [56047.040125] 0000000000000000 [56047.040126] 0000000000000001 [56047.040126] ffff8807f639fdc0 [56047.040127] ffff8808217bbd08 [56047.040127] ffffffffa0942e30 [56047.040128] ffffffffa09217bb [56047.040128] 00000000c0086540 [56047.040128] [56047.040129] ffff8804c1ba3200 [56047.040129] ffff8807f63a2900 [56047.040130] 0000000000000001 [56047.040130] 0000000000000002 [56047.040131] ffff88082abba340 [56047.040131] 0000000000650aa0 [56047.040132] ffff88082abba348 [56047.040132] 0000000000000000 [56047.040132] [56047.040133] ffff8808217bbd70 [56047.040133] Call Trace: [56047.040133] Call Trace: [56047.040139] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [56047.040144] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56047.040150] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [56047.040155] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56047.040160] [] LNetCtl+0x116/0x800 [lnet] [56047.040166] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56047.040172] [] lnet_ioctl+0x60/0x250 [lnet] [56047.040177] [] lnet_finalize+0x1e9/0x690 [lnet] [56047.040182] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [56047.040186] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56047.040190] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [56047.040194] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56047.040195] [] do_vfs_ioctl+0x2d5/0x4b0 [56047.040197] [] ? dequeue_task_fair+0x41e/0x660 [56047.040198] [] ? __fd_install+0x47/0x60 [56047.040200] [] ? wake_up_state+0x20/0x20 [56047.040201] [] SyS_ioctl+0xa1/0xc0 [56047.040205] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56047.040207] [] system_call_fastpath+0x16/0x1b [56047.040208] [] kthread+0xcf/0xe0 [56047.040209] Code: [56047.040210] [] ? kthread_create_on_node+0x140/0x140 [56047.040211] 00 02 [56047.040213] [] ret_from_fork+0x58/0x90 [56047.040214] 00 f0 [56047.040215] [] ? kthread_create_on_node+0x140/0x140 [56047.040216] 0f c1 07 [56047.040217] Code: [56047.040217] 89 [56047.040217] 00 [56047.040218] c2 [56047.040218] 02 [56047.040219] c1 [56047.040219] 00 [56047.040219] ea [56047.040220] f0 [56047.040220] 10 [56047.040221] 0f [56047.040221] 66 [56047.040221] c1 [56047.040222] 39 [56047.040222] 07 [56047.040223] c2 [56047.040223] 89 [56047.040223] 75 [56047.040224] c2 [56047.040224] 01 [56047.040224] c1 [56047.040225] c3 [56047.040225] ea [56047.040226] 55 [56047.040226] 10 [56047.040226] 83 [56047.040227] 66 [56047.040227] e2 [56047.040228] 39 [56047.040228] fe [56047.040228] c2 [56047.040229] 0f [56047.040229] 75 [56047.040230] b7 [56047.040230] 01 [56047.040230] f2 [56047.040231] c3 [56047.040231] 48 [56047.040231] 55 [56047.040232] 89 [56047.040232] 83 [56047.040233] e5 [56047.040233] e2 [56047.040233] b8 [56047.040234] fe [56047.040234] 00 [56047.040235] 0f [56047.040235] 80 [56047.040235] b7 [56047.040236] 00 [56047.040236] f2 [56047.040237] 00 [56047.040237] 48 [56047.040237] eb [56047.040238] 89 [56047.040238] 0d [56047.040239] e5 [56047.040239] 66 [56047.040239] b8 [56047.040240] 0f [56047.040240] 00 [56047.040241] 1f [56047.040241] 80 [56047.040241] 44 [56047.040242] 00 [56047.040242] 00 [56047.040243] 00 [56047.040243] 00 [56047.040243] eb [56047.040244] f3 [56047.040244] 0d [56047.040245] 90 [56047.040245] 66 [56047.040245] <83> [56047.040246] 0f [56047.040246] e8 [56047.040247] 1f [56047.040247] 01 [56047.040247] 44 [56047.040248] 74 [56047.040248] 00 [56047.040249] 0a [56047.040249] 00 [56047.040249] 0f [56047.040250] f3 [56047.040250] b7 [56047.040250] 90 [56047.040251] 0f [56047.040251] <83> [56047.040252] 66 [56047.040252] e8 [56047.040252] 39 [56047.040253] 01 [56047.040253] ca [56047.040254] 74 [56047.040254] 75 [56047.040254] 0a [56047.040255] f1 [56047.040255] 0f [56047.040256] 5d [56047.040256] b7 [56047.040256] c3 [56047.040257] 0f [56047.040257] 66 [56047.040258] 66 [56047.040258] 66 [56047.040258] 39 [56047.040259] 66 [56047.040259] ca [56047.040260] 90 [56047.040260] 75 [56047.040260] 66 [56047.040261] f1 [56047.040261] 66 [56047.040262] [56047.040263] 5d c3 66 66 66 90 66 66 [56051.072937] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56051.309452] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56051.379144] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56051.415322] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56051.452679] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [56051.477755] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [56051.505150] RSP: 0018:ffff8808262efa98 EFLAGS: 00000212 [56051.523103] RAX: 000000000000332f RBX: ffff88082d2cded8 RCX: 0000000000000a1e [56051.547094] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [56051.571096] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [56051.595128] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [56051.619139] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [56051.643159] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [56051.670349] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56051.689827] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56051.713897] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56051.737978] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56051.762060] Stack: [56051.769274] ffff8808262efac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [56051.794304] ffff88083fb5e000 ffff8808262efb80 ffffffffa092c9b5 00000000000001a8 [56051.819339] ffffc9000e331730 ffff8808262efb68 ffffffffa0ebd587 ffffffffffffffff [56051.844384] Call Trace: [56051.853119] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56051.874990] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56051.897760] [] lnet_select_pathway+0x235/0x1140 [lnet] [56051.920825] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56051.943052] [] lnet_send+0x51/0x180 [lnet] [56051.962711] [] lnet_finalize+0x3f1/0x690 [lnet] [56051.983814] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56052.005496] [] ? __slab_free+0x81/0x2f0 [56052.024327] [] lnet_ni_recv+0xc3/0x320 [lnet] [56052.044876] [] lnet_parse+0x945/0xf90 [lnet] [56052.065181] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56052.088324] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56052.109475] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56052.132954] [] ? dequeue_task_fair+0x41e/0x660 [56052.153863] [] ? wake_up_state+0x20/0x20 [56052.173072] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56052.195985] [] kthread+0xcf/0xe0 [56052.212910] [] ? kthread_create_on_node+0x140/0x140 [56052.235294] [] ret_from_fork+0x58/0x90 [56052.253921] [] ? kthread_create_on_node+0x140/0x140 [56052.276262] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [56059.038762] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [kiblnd_sd_00_03:2899] [56059.038764] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [kiblnd_sd_00_00:2896] [56059.038783] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56059.038789] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56059.038790] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56059.038791] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56059.038791] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [56059.038793] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56059.038794] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [56059.038794] RAX: 0000000000000d03 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [56059.038795] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [56059.038795] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [56059.038796] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [56059.038797] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [56059.038797] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [56059.038798] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56059.038798] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56059.038799] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56059.038800] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56059.038800] Stack: [56059.038801] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [56059.038802] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [56059.038803] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [56059.038803] Call Trace: [56059.038809] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56059.038815] [] lnet_post_send_locked+0x446/0x880 [lnet] [56059.038821] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [56059.038826] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56059.038832] [] lnet_finalize+0x1e9/0x690 [lnet] [56059.038835] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56059.038839] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56059.038840] [] ? dequeue_task_fair+0x41e/0x660 [56059.038842] [] ? sched_clock_cpu+0x85/0xc0 [56059.038843] [] ? wake_up_state+0x20/0x20 [56059.038847] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56059.038848] [] kthread+0xcf/0xe0 [56059.038850] [] ? kthread_create_on_node+0x140/0x140 [56059.038851] [] ret_from_fork+0x58/0x90 [56059.038853] [] ? kthread_create_on_node+0x140/0x140 [56059.038863] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56060.273001] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56060.513871] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56060.585355] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56060.621789] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56060.659673] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [56060.685208] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [56060.713076] RSP: 0018:ffff88082626bce0 EFLAGS: 00000206 [56060.731485] RAX: 0000000000005447 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [56060.755891] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [56060.780290] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [56060.804671] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [56060.829055] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [56060.853475] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [56060.881058] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56060.900891] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [56060.925301] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56060.949690] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56060.974080] Stack: [56060.981591] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [56061.006892] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [56061.032201] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [56061.057499] Call Trace: [56061.066469] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56061.088588] [] lnet_finalize+0xf6/0x690 [lnet] [56061.109571] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56061.132266] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56061.155820] [] ? dequeue_task_fair+0x41e/0x660 [56061.176808] [] ? wake_up_state+0x20/0x20 [56061.196071] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56061.219081] [] kthread+0xcf/0xe0 [56061.236049] [] ? kthread_create_on_node+0x140/0x140 [56061.258462] [] ret_from_fork+0x58/0x90 [56061.277186] [] ? kthread_create_on_node+0x140/0x140 [56061.299610] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [56063.038350] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [56063.038351] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [56063.038370] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56063.038376] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56063.038377] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56063.038377] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56063.038378] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [56063.038380] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56063.038380] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000202 [56063.038381] RAX: 0000000000002b68 RBX: 0000000100000000 RCX: 0000000000000a1e [56063.038382] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [56063.038382] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [56063.038383] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [56063.038383] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [56063.038384] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [56063.038385] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56063.038385] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56063.038386] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56063.038386] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56063.038387] Stack: [56063.038388] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [56063.038389] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [56063.038390] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [56063.038390] Call Trace: [56063.038395] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56063.038401] [] lnet_finalize+0xf6/0x690 [lnet] [56063.038405] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56063.038406] [] ? enqueue_task_fair+0x208/0x6c0 [56063.038408] [] ? sched_clock_cpu+0x85/0xc0 [56063.038414] [] lnet_ni_recv+0xc3/0x320 [lnet] [56063.038419] [] lnet_parse+0x945/0xf90 [lnet] [56063.038423] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56063.038424] [] ? __wake_up+0x44/0x50 [56063.038427] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56063.038429] [] ? dequeue_task_fair+0x41e/0x660 [56063.038430] [] ? wake_up_state+0x20/0x20 [56063.038434] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56063.038435] [] kthread+0xcf/0xe0 [56063.038437] [] ? kthread_create_on_node+0x140/0x140 [56063.038438] [] ret_from_fork+0x58/0x90 [56063.038440] [] ? kthread_create_on_node+0x140/0x140 [56063.038450] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56064.304859] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56064.545389] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56064.616682] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56064.652183] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56064.690002] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [56064.715495] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56064.743337] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [56064.761658] RAX: 0000000000001387 RBX: 0000000000016c40 RCX: 0000000000000a1e [56064.786040] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [56064.810404] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [56064.834781] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [56064.859202] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [56064.883606] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [56064.911134] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56064.930935] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56064.955342] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56064.979772] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56065.004163] Stack: [56065.011659] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [56065.036948] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [56065.062251] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [56065.087566] Call Trace: [56065.096509] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56065.118624] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [56065.139879] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [56065.163424] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [56065.189258] [] ? lock_timer_base.isra.33+0x2b/0x50 [56065.211388] [] ? try_to_del_timer_sync+0x5e/0x90 [56065.232947] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [56065.256788] [] ? schedule_timeout+0x17d/0x2d0 [56065.277483] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [56065.299606] [] ? wake_up_state+0x20/0x20 [56065.318899] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [56065.343362] [] kthread+0xcf/0xe0 [56065.360363] [] ? kthread_create_on_node+0x140/0x140 [56065.382828] [] ret_from_fork+0x58/0x90 [56065.401548] [] ? kthread_create_on_node+0x140/0x140 [56065.423984] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56067.037938] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [56067.037940] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [56067.037959] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56067.037964] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56067.037966] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56067.037966] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56067.037967] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [56067.037968] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [56067.037969] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000202 [56067.037970] RAX: 000000000000778c RBX: ffff88083fb5e000 RCX: 0000000000000a1e [56067.037970] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [56067.037971] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [56067.037971] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [56067.037972] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [56067.037973] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [56067.037973] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56067.037974] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56067.037974] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56067.037975] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56067.037975] Stack: [56067.037976] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [56067.037977] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [56067.037978] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [56067.037978] Call Trace: [56067.037983] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56067.037988] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56067.037994] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [56067.037999] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56067.038005] [] lnet_finalize+0x1e9/0x690 [lnet] [56067.038008] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56067.038012] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56067.038013] [] ? dequeue_task_fair+0x41e/0x660 [56067.038015] [] ? wake_up_state+0x20/0x20 [56067.038018] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56067.038020] [] kthread+0xcf/0xe0 [56067.038021] [] ? kthread_create_on_node+0x140/0x140 [56067.038023] [] ret_from_fork+0x58/0x90 [56067.038024] [] ? kthread_create_on_node+0x140/0x140 [56067.038034] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [56068.254837] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56068.495903] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56068.567475] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56068.604203] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56068.642070] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [56068.667667] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56068.695518] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [56068.713945] RAX: 0000000000003a97 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [56068.738336] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [56068.762718] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [56068.787141] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [56068.811520] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [56068.835972] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [56068.863539] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56068.883398] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56068.907827] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56068.932220] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56068.956596] Stack: [56068.964136] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [56068.989441] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [56069.014739] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [56069.040076] Call Trace: [56069.049030] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56069.071165] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56069.094136] [] lnet_select_pathway+0x235/0x1140 [lnet] [56069.117428] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56069.139866] [] lnet_send+0x51/0x180 [lnet] [56069.159696] [] lnet_finalize+0x3f1/0x690 [lnet] [56069.180935] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56069.202725] [] ? __slab_free+0x81/0x2f0 [56069.221716] [] lnet_ni_recv+0xc3/0x320 [lnet] [56069.242391] [] lnet_parse+0x945/0xf90 [lnet] [56069.262820] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56069.285792] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56069.309045] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56069.330332] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56069.353888] [] ? dequeue_task_fair+0x41e/0x660 [56069.374880] [] ? wake_up_state+0x20/0x20 [56069.394167] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56069.417138] [] kthread+0xcf/0xe0 [56069.434085] [] ? kthread_create_on_node+0x140/0x140 [56069.456461] [] ret_from_fork+0x58/0x90 [56069.475074] [] ? kthread_create_on_node+0x140/0x140 [56069.497393] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56071.037526] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [kiblnd_sd_01_02:2902] [56071.037528] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [kworker/3:2:27192] [56071.037548] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56071.037553] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56071.037554] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56071.037555] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56071.037557] Workqueue: ib_cm cm_work_handler [ib_cm] [56071.037558] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [56071.037559] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [56071.037560] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000206 [56071.037560] RAX: 0000000000001378 RBX: 0000000000000000 RCX: 0000000000000a1e [56071.037561] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [56071.037561] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [56071.037562] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [56071.037562] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [56071.037563] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [56071.037563] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56071.037564] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56071.037565] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56071.037565] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56071.037565] Stack: [56071.037566] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [56071.037567] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [56071.037568] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [56071.037568] Call Trace: [56071.037574] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56071.037579] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [56071.037583] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [56071.037585] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [56071.037589] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [56071.037591] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [56071.037592] [] cma_req_handler+0x227/0x570 [rdma_cm] [56071.037594] [] cm_process_work+0x25/0x140 [ib_cm] [56071.037596] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [56071.037598] [] cm_work_handler+0x185/0x1484 [ib_cm] [56071.037599] [] process_one_work+0x17b/0x470 [56071.037600] [] worker_thread+0x126/0x410 [56071.037601] [] ? rescuer_thread+0x460/0x460 [56071.037602] [] kthread+0xcf/0xe0 [56071.037604] [] ? kthread_create_on_node+0x140/0x140 [56071.037606] [] ret_from_fork+0x58/0x90 [56071.037607] [] ? kthread_create_on_node+0x140/0x140 [56071.037617] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [56072.332399] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56072.572988] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56072.644299] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56072.680963] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56072.718803] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [56072.744311] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56072.772147] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000202 [56072.790540] RAX: 0000000000000a92 RBX: 000001f280bac5e0 RCX: 0000000000000a1e [56072.814920] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [56072.839330] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [56072.863741] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [56072.888115] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [56072.912479] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [56072.940031] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56072.959862] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56072.984277] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56073.008666] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56073.033051] Stack: [56073.040517] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [56073.065822] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [56073.091102] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [56073.116410] Call Trace: [56073.125373] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56073.147503] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56073.170481] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56073.192859] [] lnet_finalize+0x1e9/0x690 [lnet] [56073.214104] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56073.236789] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56073.260318] [] ? dequeue_task_fair+0x41e/0x660 [56073.281265] [] ? wake_up_state+0x20/0x20 [56073.300510] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56073.323472] [] kthread+0xcf/0xe0 [56073.340419] [] ? kthread_create_on_node+0x140/0x140 [56073.362809] [] ret_from_fork+0x58/0x90 [56073.381497] [] ? kthread_create_on_node+0x140/0x140 [56073.403905] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56075.037114] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [56075.037115] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 21s! [kiblnd_sd_01_01:2901] [56075.037116] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [kiblnd_sd_01_00:2900] [56075.037117] Modules linked in: [56075.037118] Modules linked in: [56075.037119] ko2iblnd(OE) [56075.037119] ko2iblnd(OE) [56075.037120] lustre(OE) [56075.037120] lustre(OE) [56075.037121] lmv(OE) [56075.037121] lmv(OE) [56075.037122] fld(OE) [56075.037122] fld(OE) [56075.037123] mdc(OE) [56075.037123] mdc(OE) [56075.037124] fid(OE) [56075.037124] fid(OE) [56075.037124] lov(OE) [56075.037125] lov(OE) [56075.037125] ksocklnd(OE) [56075.037126] ksocklnd(OE) [56075.037126] ptlrpc(OE) [56075.037127] ptlrpc(OE) [56075.037127] obdclass(OE) [56075.037128] obdclass(OE) [56075.037128] lnet(OE) [56075.037128] lnet(OE) [56075.037129] libcfs(OE) [56075.037129] libcfs(OE) [56075.037130] rpcrdma [56075.037130] rpcrdma [56075.037131] ib_isert [56075.037131] ib_isert [56075.037132] iscsi_target_mod [56075.037132] iscsi_target_mod [56075.037133] ib_iser [56075.037133] ib_iser [56075.037134] libiscsi [56075.037134] libiscsi [56075.037135] scsi_transport_iscsi [56075.037135] scsi_transport_iscsi [56075.037135] ib_srpt [56075.037136] ib_srpt [56075.037136] target_core_mod [56075.037137] target_core_mod [56075.037137] ib_srp [56075.037138] ib_srp [56075.037138] scsi_transport_srp [56075.037139] scsi_transport_srp [56075.037139] scsi_tgt [56075.037139] scsi_tgt [56075.037140] ib_ipoib [56075.037140] ib_ipoib [56075.037141] rdma_ucm [56075.037141] rdma_ucm [56075.037142] ib_ucm [56075.037142] ib_ucm [56075.037143] ib_uverbs [56075.037143] ib_uverbs [56075.037143] ib_umad [56075.037144] ib_umad [56075.037144] rdma_cm [56075.037145] rdma_cm [56075.037145] ib_cm [56075.037146] ib_cm [56075.037146] iw_cm [56075.037147] iw_cm [56075.037147] mlx4_ib [56075.037147] mlx4_ib [56075.037148] intel_powerclamp [56075.037148] intel_powerclamp [56075.037149] coretemp [56075.037149] coretemp [56075.037150] intel_rapl [56075.037150] intel_rapl [56075.037151] iosf_mbi [56075.037151] iosf_mbi [56075.037151] kvm [56075.037152] kvm [56075.037152] irqbypass [56075.037153] irqbypass [56075.037153] crc32_pclmul [56075.037154] crc32_pclmul [56075.037154] ghash_clmulni_intel [56075.037155] ghash_clmulni_intel [56075.037155] aesni_intel [56075.037156] aesni_intel [56075.037156] ipmi_ssif [56075.037156] ipmi_ssif [56075.037157] lrw [56075.037157] lrw [56075.037158] gf128mul [56075.037158] gf128mul [56075.037159] glue_helper [56075.037159] glue_helper [56075.037160] hfi1 [56075.037160] hfi1 [56075.037161] ipmi_devintf [56075.037161] ipmi_devintf [56075.037161] rdmavt [56075.037162] rdmavt [56075.037162] ablk_helper [56075.037163] ablk_helper [56075.037163] cryptd [56075.037164] cryptd [56075.037164] iTCO_wdt [56075.037165] iTCO_wdt [56075.037165] ipmi_si [56075.037165] ipmi_si [56075.037166] sb_edac [56075.037166] sb_edac [56075.037167] sg [56075.037167] sg [56075.037168] iTCO_vendor_support [56075.037168] iTCO_vendor_support [56075.037169] ib_core [56075.037169] ib_core [56075.037169] ioatdma [56075.037170] ioatdma [56075.037170] i2c_i801 [56075.037171] i2c_i801 [56075.037171] mei_me [56075.037172] mei_me [56075.037172] mei [56075.037173] mei [56075.037173] edac_core [56075.037173] edac_core [56075.037174] ipmi_msghandler [56075.037174] ipmi_msghandler [56075.037175] lpc_ich [56075.037175] lpc_ich [56075.037176] ntb [56075.037176] ntb [56075.037177] wmi [56075.037177] wmi [56075.037178] shpchp [56075.037178] shpchp [56075.037178] pcspkr [56075.037179] pcspkr [56075.037179] nfsd [56075.037180] nfsd [56075.037180] nfs_acl [56075.037181] nfs_acl [56075.037181] lockd [56075.037182] lockd [56075.037182] grace [56075.037182] grace [56075.037183] auth_rpcgss [56075.037183] auth_rpcgss [56075.037184] sunrpc [56075.037184] sunrpc [56075.037185] ip_tables [56075.037185] ip_tables [56075.037186] ext4 [56075.037186] ext4 [56075.037186] mbcache [56075.037187] mbcache [56075.037187] jbd2 [56075.037188] jbd2 [56075.037188] sd_mod [56075.037189] sd_mod [56075.037189] crc_t10dif [56075.037190] crc_t10dif [56075.037190] crct10dif_generic [56075.037190] crct10dif_generic [56075.037191] mlx4_en [56075.037191] mlx4_en [56075.037192] mgag200 [56075.037192] mgag200 [56075.037193] drm_kms_helper [56075.037193] drm_kms_helper [56075.037194] syscopyarea [56075.037194] syscopyarea [56075.037195] sysfillrect [56075.037195] sysfillrect [56075.037195] sysimgblt [56075.037196] sysimgblt [56075.037196] fb_sys_fops [56075.037197] fb_sys_fops [56075.037197] isci [56075.037198] isci [56075.037198] ttm [56075.037199] ttm [56075.037199] igb [56075.037199] igb [56075.037200] crct10dif_pclmul [56075.037200] crct10dif_pclmul [56075.037201] crct10dif_common [56075.037201] crct10dif_common [56075.037202] ahci [56075.037202] ahci [56075.037203] ptp [56075.037203] ptp [56075.037204] crc32c_intel [56075.037204] crc32c_intel [56075.037204] libsas [56075.037205] libsas [56075.037205] libahci [56075.037206] libahci [56075.037206] pps_core [56075.037207] pps_core [56075.037207] drm [56075.037208] drm [56075.037208] mlx4_core [56075.037208] mlx4_core [56075.037209] mpt2sas [56075.037209] mpt2sas [56075.037210] dca [56075.037210] dca [56075.037211] libata [56075.037211] libata [56075.037212] raid_class [56075.037212] raid_class [56075.037212] i2c_algo_bit [56075.037213] i2c_algo_bit [56075.037213] scsi_transport_sas [56075.037214] scsi_transport_sas [56075.037214] devlink [56075.037215] devlink [56075.037215] i2c_core [56075.037216] i2c_core [56075.037216] fjes [56075.037216] [56075.037217] fjes [56075.037218] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56075.037219] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56075.037220] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56075.037220] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56075.037221] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [56075.037222] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [56075.037223] RIP: 0010:[] [56075.037224] [] _raw_spin_lock+0x32/0x50 [56075.037225] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56075.037226] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [56075.037227] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000212 [56075.037227] RAX: 0000000000007c53 RBX: ffff88082d2cded8 RCX: 0000000000000a1e [56075.037228] RAX: 0000000000005751 RBX: 00000201e39c5101 RCX: 0000000000000a1e [56075.037229] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [56075.037230] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [56075.037230] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [56075.037231] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [56075.037231] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [56075.037232] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [56075.037233] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [56075.037233] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [56075.037234] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [56075.037235] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [56075.037236] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56075.037236] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56075.037237] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56075.037238] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56075.037238] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56075.037239] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56075.037240] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56075.037240] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56075.037241] Stack: [56075.037241] Stack: [56075.037242] ffff8808262efac0 [56075.037242] ffff8808217bbcd8 [56075.037243] ffffffffa08aa7c8 [56075.037243] ffffffffa08aa7c8 [56075.037243] ffffffffa08aa8b6 [56075.037244] ffffffffa08aa8b6 [56075.037244] 0000000000000000 [56075.037245] [56075.037245] ffff8804c1ba3200 [56075.037246] ffff88083fb5e000 [56075.037246] 0000000000000001 [56075.037247] ffff8808262efb80 [56075.037247] ffff8808217bbd08 [56075.037248] ffffffffa092c9b5 [56075.037248] ffffffffa09217bb [56075.037249] 00000000000001a8 [56075.037249] [56075.037249] ffff8804c1ba3200 [56075.037250] ffffc9000e331730 [56075.037251] 0000000000000001 [56075.037251] ffff8808262efb68 [56075.037252] ffff88082abba340 [56075.037252] ffffffffa0ebd587 [56075.037253] ffff88082abba348 [56075.037253] ffffffffffffffff [56075.037253] [56075.037254] ffff8808217bbd70 [56075.037254] Call Trace: [56075.037254] Call Trace: [56075.037260] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56075.037265] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56075.037270] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56075.037275] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56075.037281] [] lnet_select_pathway+0x235/0x1140 [lnet] [56075.037287] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56075.037291] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56075.037296] [] lnet_finalize+0x1e9/0x690 [lnet] [56075.037302] [] lnet_send+0x51/0x180 [lnet] [56075.037305] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56075.037311] [] lnet_finalize+0x3f1/0x690 [lnet] [56075.037314] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56075.037318] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56075.037319] [] ? dequeue_task_fair+0x41e/0x660 [56075.037322] [] ? __slab_free+0x81/0x2f0 [56075.037323] [] ? wake_up_state+0x20/0x20 [56075.037329] [] lnet_ni_recv+0xc3/0x320 [lnet] [56075.037332] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56075.037338] [] lnet_parse+0x945/0xf90 [lnet] [56075.037339] [] kthread+0xcf/0xe0 [56075.037343] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56075.037345] [] ? kthread_create_on_node+0x140/0x140 [56075.037347] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56075.037349] [] ret_from_fork+0x58/0x90 [56075.037353] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56075.037354] [] ? kthread_create_on_node+0x140/0x140 [56075.037355] [] ? dequeue_task_fair+0x41e/0x660 [56075.037356] Code: [56075.037358] [] ? wake_up_state+0x20/0x20 [56075.037358] 00 02 [56075.037362] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56075.037363] 00 f0 [56075.037365] [] kthread+0xcf/0xe0 [56075.037365] 0f c1 [56075.037367] [] ? kthread_create_on_node+0x140/0x140 [56075.037367] 07 89 [56075.037369] [] ret_from_fork+0x58/0x90 [56075.037370] c2 c1 [56075.037372] [] ? kthread_create_on_node+0x140/0x140 [56075.037372] ea 10 66 [56075.037373] Code: [56075.037373] 39 [56075.037374] 00 [56075.037374] c2 [56075.037375] 02 [56075.037375] 75 [56075.037376] 00 [56075.037376] 01 [56075.037376] f0 [56075.037377] c3 [56075.037377] 0f [56075.037378] 55 [56075.037378] c1 [56075.037378] 83 [56075.037379] 07 [56075.037379] e2 [56075.037380] 89 [56075.037380] fe [56075.037381] c2 [56075.037381] 0f [56075.037381] c1 [56075.037382] b7 [56075.037382] ea [56075.037383] f2 [56075.037383] 10 [56075.037383] 48 [56075.037384] 66 [56075.037384] 89 [56075.037385] 39 [56075.037385] e5 [56075.037385] c2 [56075.037386] b8 [56075.037386] 75 [56075.037387] 00 [56075.037387] 01 [56075.037387] 80 [56075.037388] c3 [56075.037388] 00 [56075.037389] 55 [56075.037389] 00 [56075.037389] 83 [56075.037390] eb [56075.037390] e2 [56075.037391] 0d [56075.037391] fe [56075.037391] 66 [56075.037392] 0f [56075.037392] 0f [56075.037393] b7 [56075.037393] 1f [56075.037394] f2 [56075.037394] 44 [56075.037394] 48 [56075.037395] 00 [56075.037395] 89 [56075.037395] 00 [56075.037396] e5 [56075.037396] f3 [56075.037397] b8 [56075.037397] 90 [56075.037398] 00 [56075.037398] <83> [56075.037398] 80 [56075.037399] e8 [56075.037399] 00 [56075.037400] 01 [56075.037400] 00 [56075.037400] 74 [56075.037401] eb [56075.037401] 0a [56075.037402] 0d [56075.037402] 0f [56075.037402] 66 [56075.037403] b7 [56075.037403] 0f [56075.037404] 0f [56075.037404] 1f [56075.037404] 66 [56075.037405] 44 [56075.037405] 39 [56075.037406] 00 [56075.037406] ca [56075.037407] 00 [56075.037407] 75 [56075.037407] f3 [56075.037408] f1 [56075.037408] 90 [56075.037409] 5d [56075.037409] <83> [56075.037409] c3 [56075.037410] e8 [56075.037410] 66 [56075.037411] 01 [56075.037411] 66 [56075.037411] 74 [56075.037412] 66 [56075.037412] 0a [56075.037413] 90 [56075.037413] 0f [56075.037413] 66 [56075.037414] b7 [56075.037414] 66 [56075.037415] [56075.037417] 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56079.246863] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56079.483345] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56079.553069] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56079.586100] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56079.623469] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [56079.648543] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56079.675943] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [56079.693908] RAX: 00000000000020a6 RBX: ffff8807f639fe00 RCX: 0000000000000a1e [56079.717914] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [56079.741912] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [56079.765925] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [56079.789956] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [56079.814006] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [56079.841226] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56079.860711] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [56079.884792] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56079.908884] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56079.932988] Stack: [56079.940223] ffff8807f639fd88 ffffffffa08aa831 0000000200000002 ffff8807f63a2980 [56079.965260] 0000000000000000 ffff8807f639fdc0 ffffffffa0942e30 00000000c0086540 [56079.990320] ffff8807f63a2900 0000000000000002 0000000000650aa0 0000000000000000 [56080.015394] Call Trace: [56080.024123] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [56080.046040] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [56080.067404] [] LNetCtl+0x116/0x800 [lnet] [56080.086763] [] lnet_ioctl+0x60/0x250 [lnet] [56080.106731] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [56080.128122] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [56080.150654] [] do_vfs_ioctl+0x2d5/0x4b0 [56080.169506] [] ? __fd_install+0x47/0x60 [56080.188327] [] SyS_ioctl+0xa1/0xc0 [56080.205727] [] system_call_fastpath+0x16/0x1b [56080.226310] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56087.035879] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [kiblnd_sd_00_03:2899] [56087.035880] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [kiblnd_sd_00_00:2896] [56087.035899] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56087.035904] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56087.035905] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56087.035906] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56087.035906] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [56087.035908] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56087.035909] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [56087.035909] RAX: 0000000000002723 RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [56087.035910] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [56087.035910] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [56087.035911] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [56087.035911] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [56087.035912] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [56087.035913] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56087.035913] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56087.035914] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56087.035914] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56087.035914] Stack: [56087.035915] ffff880820b03c28 ffffffffa08aa7c8 ffff880820b03c28 0000000000000000 [56087.035916] ffff88083fb5e000 ffff880820b03c98 ffffffffa092c346 ffff880820b03c60 [56087.035917] 00000000000000b4 00000001033f89f4 ffff8800b4864274 0000000000000000 [56087.035918] Call Trace: [56087.035923] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56087.035929] [] lnet_post_send_locked+0x446/0x880 [lnet] [56087.035935] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [56087.035940] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56087.035946] [] lnet_finalize+0x1e9/0x690 [lnet] [56087.035949] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56087.035953] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56087.035954] [] ? dequeue_task_fair+0x41e/0x660 [56087.035956] [] ? sched_clock_cpu+0x85/0xc0 [56087.035957] [] ? wake_up_state+0x20/0x20 [56087.035961] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56087.035962] [] kthread+0xcf/0xe0 [56087.035964] [] ? kthread_create_on_node+0x140/0x140 [56087.035965] [] ret_from_fork+0x58/0x90 [56087.035967] [] ? kthread_create_on_node+0x140/0x140 [56087.035977] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56088.272224] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56088.513069] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56088.584586] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56088.620994] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56088.658876] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [56088.684415] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56088.712246] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [56088.730649] RAX: 000000000000533d RBX: ffff88082626bcb0 RCX: 0000000000000a1e [56088.755072] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [56088.779451] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [56088.803868] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [56088.828289] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [56088.852719] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [56088.880269] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56088.900124] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [56088.924522] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56088.948949] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56088.973377] Stack: [56088.980932] ffff88082626bd08 ffffffffa08aa7c8 ffff88082af11648 ffff8800b484ea00 [56089.006239] 0000000000000000 ffff88082626bd70 ffffffffa0922006 ffff88082af10fb0 [56089.031516] ffff88082af10fb0 ffff88082af10fb0 ffff88082626bd30 ffff88082626bd30 [56089.056814] Call Trace: [56089.065785] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56089.087948] [] lnet_finalize+0xf6/0x690 [lnet] [56089.108921] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56089.131615] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56089.155199] [] ? dequeue_task_fair+0x41e/0x660 [56089.176207] [] ? wake_up_state+0x20/0x20 [56089.195458] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56089.218461] [] kthread+0xcf/0xe0 [56089.235424] [] ? kthread_create_on_node+0x140/0x140 [56089.257867] [] ret_from_fork+0x58/0x90 [56089.276559] [] ? kthread_create_on_node+0x140/0x140 [56089.298985] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56091.035467] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [56091.035468] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [56091.035487] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56091.035493] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56091.035494] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56091.035494] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56091.035495] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [56091.035497] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56091.035497] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [56091.035498] RAX: 00000000000058b4 RBX: 0000000000016c40 RCX: 0000000000000a1e [56091.035498] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [56091.035499] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [56091.035499] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [56091.035500] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [56091.035501] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [56091.035501] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56091.035502] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56091.035502] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56091.035503] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56091.035503] Stack: [56091.035504] ffff88082aa33ca0 ffffffffa08aa7c8 ffff88082aa33ca8 0000000000000000 [56091.035505] 00050000c0a80184 ffff88082aa33cc8 ffffffffa0916cd0 ffff8800366a4530 [56091.035506] ffff880821b758e8 ffff88083fb5e000 ffff88082aa33ce0 ffffffffa0eb8460 [56091.035506] Call Trace: [56091.035512] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56091.035517] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [56091.035521] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [56091.035524] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [56091.035526] [] ? lock_timer_base.isra.33+0x2b/0x50 [56091.035527] [] ? try_to_del_timer_sync+0x5e/0x90 [56091.035530] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [56091.035532] [] ? schedule_timeout+0x17d/0x2d0 [56091.035535] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [56091.035537] [] ? wake_up_state+0x20/0x20 [56091.035540] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [56091.035542] [] kthread+0xcf/0xe0 [56091.035543] [] ? kthread_create_on_node+0x140/0x140 [56091.035545] [] ret_from_fork+0x58/0x90 [56091.035546] [] ? kthread_create_on_node+0x140/0x140 [56091.035556] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56092.273635] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56092.514596] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56092.586077] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56092.622498] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56092.660352] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [56092.685923] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56092.713765] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000212 [56092.732176] RAX: 000000000000711f RBX: 0000000100000000 RCX: 0000000000000a1e [56092.756571] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [56092.780958] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [56092.805376] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [56092.829757] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [56092.854144] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [56092.881716] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56092.901557] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56092.925941] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56092.950320] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56092.974701] Stack: [56092.982209] ffff88082aae7ba0 ffffffffa08aa7c8 000000000000763d ffff8803fe4d0400 [56093.007529] 0000000000000000 ffff88082aae7c08 ffffffffa0922006 ffff8804c1676dd0 [56093.032859] ffff8804c1676dd0 ffff8804c1676dd0 0000000000000078 ffff88082aae7c48 [56093.058186] Call Trace: [56093.067169] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56093.089318] [] lnet_finalize+0xf6/0x690 [lnet] [56093.110295] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56093.132138] [] ? enqueue_task_fair+0x208/0x6c0 [56093.153123] [] ? sched_clock_cpu+0x85/0xc0 [56093.172944] [] lnet_ni_recv+0xc3/0x320 [lnet] [56093.193622] [] lnet_parse+0x945/0xf90 [lnet] [56093.214030] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56093.237273] [] ? __wake_up+0x44/0x50 [56093.255374] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56093.278952] [] ? dequeue_task_fair+0x41e/0x660 [56093.299922] [] ? wake_up_state+0x20/0x20 [56093.319182] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56093.342158] [] kthread+0xcf/0xe0 [56093.359112] [] ? kthread_create_on_node+0x140/0x140 [56093.381521] [] ret_from_fork+0x58/0x90 [56093.400193] [] ? kthread_create_on_node+0x140/0x140 [56093.422575] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56095.035055] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [56095.035057] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [56095.035078] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56095.035084] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56095.035085] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56095.035086] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56095.035086] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [56095.035088] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56095.035089] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000202 [56095.035089] RAX: 0000000000002645 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [56095.035090] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [56095.035090] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [56095.035091] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [56095.035091] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [56095.035092] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [56095.035092] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56095.035093] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56095.035094] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56095.035094] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56095.035095] Stack: [56095.035096] ffff8808213b7ac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [56095.035097] ffff88083fb5e000 ffff8808213b7b80 ffffffffa092c9b5 00000000000000c0 [56095.035098] ffffc9000e335900 ffff8808213b7b68 ffffffffa0ebd587 ffffffffffffffff [56095.035098] Call Trace: [56095.035103] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56095.035108] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56095.035114] [] lnet_select_pathway+0x235/0x1140 [lnet] [56095.035118] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56095.035124] [] lnet_send+0x51/0x180 [lnet] [56095.035129] [] lnet_finalize+0x3f1/0x690 [lnet] [56095.035133] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56095.035134] [] ? __slab_free+0x81/0x2f0 [56095.035140] [] lnet_ni_recv+0xc3/0x320 [lnet] [56095.035145] [] lnet_parse+0x945/0xf90 [lnet] [56095.035150] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56095.035153] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56095.035156] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56095.035160] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56095.035161] [] ? dequeue_task_fair+0x41e/0x660 [56095.035163] [] ? wake_up_state+0x20/0x20 [56095.035166] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56095.035168] [] kthread+0xcf/0xe0 [56095.035169] [] ? kthread_create_on_node+0x140/0x140 [56095.035171] [] ret_from_fork+0x58/0x90 [56095.035172] [] ? kthread_create_on_node+0x140/0x140 [56095.035183] Code: c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a <0f> b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f [56096.393955] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56096.634083] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56096.705241] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56096.741567] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56096.779379] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [56096.804867] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56096.832704] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000212 [56096.851095] RAX: 0000000000007a90 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [56096.875503] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [56096.899887] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [56096.924247] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [56096.948615] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [56096.973017] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [56097.000566] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56097.020396] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56097.044818] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56097.069239] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56097.093639] Stack: [56097.101164] ffff88081f64fc98 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff88042dcd1e00 [56097.126445] ffff8800b4864200 ffff88081f64fcd8 ffffffffa092e857 ffff8800b484d600 [56097.151702] ffff88042dcd1e00 0000000000000000 ffff88041d604a00 0000000000000000 [56097.176984] Call Trace: [56097.185918] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56097.208049] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56097.231060] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [56097.256900] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56097.279023] [] lnet_finalize+0x1e9/0x690 [lnet] [56097.300305] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56097.323002] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56097.346587] [] ? dequeue_task_fair+0x41e/0x660 [56097.367583] [] ? wake_up_state+0x20/0x20 [56097.386835] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56097.409845] [] kthread+0xcf/0xe0 [56097.426834] [] ? kthread_create_on_node+0x140/0x140 [56097.449248] [] ret_from_fork+0x58/0x90 [56097.467958] [] ? kthread_create_on_node+0x140/0x140 [56097.490378] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56099.034643] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [kiblnd_sd_01_02:2902] [56099.034645] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [kworker/3:2:27192] [56099.034664] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56099.034669] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56099.034670] CPU: 3 PID: 27192 Comm: kworker/3:2 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56099.034670] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56099.034672] Workqueue: ib_cm cm_work_handler [ib_cm] [56099.034673] task: ffff8803fc424e70 ti: ffff8803fea74000 task.ti: ffff8803fea74000 [56099.034675] RIP: 0010:[] [] _raw_spin_lock+0x3a/0x50 [56099.034675] RSP: 0018:ffff8803fea77ad8 EFLAGS: 00000206 [56099.034675] RAX: 000000000000548d RBX: 0000000000000000 RCX: 0000000000000a1e [56099.034676] RDX: 0000000000000a32 RSI: 0000000000000a32 RDI: ffff88042b272d00 [56099.034676] RBP: ffff8803fea77ad8 R08: ffff88082b276a60 R09: 0000000000000000 [56099.034677] R10: 0000000000000000 R11: 0000000000000012 R12: ffff8803fea77b18 [56099.034677] R13: ffffffff8132113e R14: ffff8803fea77b5e R15: ffffffff81a84224 [56099.034678] FS: 0000000000000000(0000) GS:ffff88042e0c0000(0000) knlGS:0000000000000000 [56099.034679] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56099.034679] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56099.034680] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56099.034680] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56099.034681] Stack: [56099.034682] ffff8803fea77b00 ffffffffa08aa7c8 ffff8803fea77bd0 00050000c0a80173 [56099.034683] ffff88083fb5de00 ffff8803fea77b18 ffffffffa0916e4b ffff8803fea77c68 [56099.034684] ffff8803fea77bc8 ffffffffa0ebfc24 ffff88041aa8a880 ffff8803fea77bd0 [56099.034684] Call Trace: [56099.034689] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56099.034694] [] lnet_nid2ni_addref+0x1b/0xa0 [lnet] [56099.034698] [] kiblnd_passive_connect+0x104/0x1890 [ko2iblnd] [56099.034700] [] ? _cma_attach_to_dev+0x5c/0x70 [rdma_cm] [56099.034703] [] kiblnd_cm_callback+0x755/0x22a0 [ko2iblnd] [56099.034705] [] ? cma_new_conn_id+0x157/0x510 [rdma_cm] [56099.034707] [] cma_req_handler+0x227/0x570 [rdma_cm] [56099.034709] [] cm_process_work+0x25/0x140 [ib_cm] [56099.034711] [] cm_req_handler+0x9cd/0xec0 [ib_cm] [56099.034713] [] cm_work_handler+0x185/0x1484 [ib_cm] [56099.034714] [] process_one_work+0x17b/0x470 [56099.034715] [] worker_thread+0x126/0x410 [56099.034716] [] ? rescuer_thread+0x460/0x460 [56099.034717] [] kthread+0xcf/0xe0 [56099.034719] [] ? kthread_create_on_node+0x140/0x140 [56099.034720] [] ret_from_fork+0x58/0x90 [56099.034722] [] ? kthread_create_on_node+0x140/0x140 [56099.034732] Code: c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 83 e8 01 74 0a 0f b7 0f <66> 39 ca 75 f1 5d c3 66 66 66 90 66 66 90 eb d9 66 0f 1f 44 00 [56100.331796] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56100.572398] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56100.643705] CPU: 11 PID: 2902 Comm: kiblnd_sd_01_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56100.680389] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56100.718170] task: ffff880826310000 ti: ffff880823638000 task.ti: ffff880823638000 [56100.743719] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56100.771571] RSP: 0018:ffff88082363bcb0 EFLAGS: 00000212 [56100.789969] RAX: 0000000000003700 RBX: 000001f280bac5e0 RCX: 0000000000000a1e [56100.814382] RDX: 0000000000000a22 RSI: 0000000000000a22 RDI: ffff88042b272d00 [56100.838781] RBP: ffff88082363bcb0 R08: ffff8804c1ba6810 R09: ffff880826310000 [56100.863198] R10: ffff88081f6b6ab8 R11: 0000000000000044 R12: ffff88082363bce0 [56100.887611] R13: ffffea00000002c8 R14: ffffffffa0ebe43a R15: ffff88082363bc88 [56100.911976] FS: 0000000000000000(0000) GS:ffff88082d8c0000(0000) knlGS:0000000000000000 [56100.939505] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56100.959328] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56100.983718] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56101.008141] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56101.032511] Stack: [56101.040012] ffff88082363bcd8 ffffffffa08aa7c8 ffffffffa08aa8b6 ffff8804c1ba6800 [56101.065294] 0000000000000001 ffff88082363bd08 ffffffffa09217bb ffff8804c1ba6800 [56101.090583] 0000000000000001 ffff88082abba340 ffff88082abba348 ffff88082363bd70 [56101.115893] Call Trace: [56101.124819] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56101.146942] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56101.169926] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56101.192325] [] lnet_finalize+0x1e9/0x690 [lnet] [56101.213589] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56101.236278] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56101.259800] [] ? dequeue_task_fair+0x41e/0x660 [56101.280744] [] ? wake_up_state+0x20/0x20 [56101.299979] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56101.322955] [] kthread+0xcf/0xe0 [56101.339907] [] ? kthread_create_on_node+0x140/0x140 [56101.362304] [] ret_from_fork+0x58/0x90 [56101.380972] [] ? kthread_create_on_node+0x140/0x140 [56101.403364] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56103.034231] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [kiblnd_sd_01_01:2901] [56103.034232] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [lctl:36467] [56103.034233] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [kiblnd_sd_01_00:2900] [56103.034234] Modules linked in: [56103.034235] Modules linked in: [56103.034235] ko2iblnd(OE) [56103.034236] ko2iblnd(OE) [56103.034236] lustre(OE) [56103.034237] lustre(OE) [56103.034237] lmv(OE) [56103.034238] lmv(OE) [56103.034238] fld(OE) [56103.034239] fld(OE) [56103.034239] mdc(OE) [56103.034239] mdc(OE) [56103.034240] fid(OE) [56103.034240] fid(OE) [56103.034241] lov(OE) [56103.034241] lov(OE) [56103.034242] ksocklnd(OE) [56103.034242] ksocklnd(OE) [56103.034242] ptlrpc(OE) [56103.034243] ptlrpc(OE) [56103.034243] obdclass(OE) [56103.034244] obdclass(OE) [56103.034244] lnet(OE) [56103.034245] lnet(OE) [56103.034245] libcfs(OE) [56103.034246] libcfs(OE) [56103.034246] rpcrdma [56103.034247] rpcrdma [56103.034247] ib_isert [56103.034247] ib_isert [56103.034248] iscsi_target_mod [56103.034248] iscsi_target_mod [56103.034249] ib_iser [56103.034249] ib_iser [56103.034250] libiscsi [56103.034250] libiscsi [56103.034251] scsi_transport_iscsi [56103.034251] scsi_transport_iscsi [56103.034251] ib_srpt [56103.034252] ib_srpt [56103.034252] target_core_mod [56103.034253] target_core_mod [56103.034253] ib_srp [56103.034254] ib_srp [56103.034254] scsi_transport_srp [56103.034255] scsi_transport_srp [56103.034255] scsi_tgt [56103.034255] scsi_tgt [56103.034256] ib_ipoib [56103.034256] ib_ipoib [56103.034257] rdma_ucm [56103.034257] rdma_ucm [56103.034258] ib_ucm [56103.034258] ib_ucm [56103.034259] ib_uverbs [56103.034259] ib_uverbs [56103.034259] ib_umad [56103.034260] ib_umad [56103.034260] rdma_cm [56103.034261] rdma_cm [56103.034261] ib_cm [56103.034262] ib_cm [56103.034262] iw_cm [56103.034262] iw_cm [56103.034263] mlx4_ib [56103.034263] mlx4_ib [56103.034264] intel_powerclamp [56103.034264] intel_powerclamp [56103.034265] coretemp [56103.034265] coretemp [56103.034265] intel_rapl [56103.034266] intel_rapl [56103.034266] iosf_mbi [56103.034267] iosf_mbi [56103.034267] kvm [56103.034268] kvm [56103.034268] irqbypass [56103.034268] irqbypass [56103.034269] crc32_pclmul [56103.034269] crc32_pclmul [56103.034270] ghash_clmulni_intel [56103.034270] ghash_clmulni_intel [56103.034271] aesni_intel [56103.034271] aesni_intel [56103.034271] ipmi_ssif [56103.034272] ipmi_ssif [56103.034272] lrw [56103.034273] lrw [56103.034273] gf128mul [56103.034274] gf128mul [56103.034274] glue_helper [56103.034274] glue_helper [56103.034275] hfi1 [56103.034275] hfi1 [56103.034276] ipmi_devintf [56103.034276] ipmi_devintf [56103.034277] rdmavt [56103.034277] rdmavt [56103.034278] ablk_helper [56103.034278] ablk_helper [56103.034278] cryptd [56103.034279] cryptd [56103.034279] iTCO_wdt [56103.034280] iTCO_wdt [56103.034280] ipmi_si [56103.034281] ipmi_si [56103.034281] sb_edac [56103.034282] sb_edac [56103.034282] sg [56103.034282] sg [56103.034283] iTCO_vendor_support [56103.034283] iTCO_vendor_support [56103.034284] ib_core [56103.034284] ib_core [56103.034285] ioatdma [56103.034285] ioatdma [56103.034286] i2c_i801 [56103.034286] i2c_i801 [56103.034286] mei_me [56103.034287] mei_me [56103.034287] mei [56103.034288] mei [56103.034288] edac_core [56103.034289] edac_core [56103.034289] ipmi_msghandler [56103.034290] ipmi_msghandler [56103.034290] lpc_ich [56103.034290] lpc_ich [56103.034291] ntb [56103.034291] ntb [56103.034292] wmi [56103.034292] wmi [56103.034293] shpchp [56103.034293] shpchp [56103.034293] pcspkr [56103.034294] pcspkr [56103.034294] nfsd [56103.034295] nfsd [56103.034295] nfs_acl [56103.034296] nfs_acl [56103.034296] lockd [56103.034297] lockd [56103.034297] grace [56103.034297] grace [56103.034298] auth_rpcgss [56103.034298] auth_rpcgss [56103.034299] sunrpc [56103.034299] sunrpc [56103.034300] ip_tables [56103.034300] ip_tables [56103.034301] ext4 [56103.034301] ext4 [56103.034301] mbcache [56103.034302] mbcache [56103.034302] jbd2 [56103.034303] jbd2 [56103.034303] sd_mod [56103.034304] sd_mod [56103.034304] crc_t10dif [56103.034304] crc_t10dif [56103.034305] crct10dif_generic [56103.034305] crct10dif_generic [56103.034306] mlx4_en [56103.034306] mlx4_en [56103.034307] mgag200 [56103.034307] mgag200 [56103.034308] drm_kms_helper [56103.034308] drm_kms_helper [56103.034308] syscopyarea [56103.034309] syscopyarea [56103.034309] sysfillrect [56103.034310] sysfillrect [56103.034310] sysimgblt [56103.034311] sysimgblt [56103.034311] fb_sys_fops [56103.034311] fb_sys_fops [56103.034312] isci [56103.034312] isci [56103.034313] ttm [56103.034313] ttm [56103.034314] igb [56103.034314] igb [56103.034315] crct10dif_pclmul [56103.034315] crct10dif_pclmul [56103.034315] crct10dif_common [56103.034316] crct10dif_common [56103.034316] ahci [56103.034317] ahci [56103.034317] ptp [56103.034318] ptp [56103.034318] crc32c_intel [56103.034319] crc32c_intel [56103.034319] libsas [56103.034319] libsas [56103.034320] libahci [56103.034320] libahci [56103.034321] pps_core [56103.034321] pps_core [56103.034322] drm [56103.034322] drm [56103.034322] mlx4_core [56103.034323] mlx4_core [56103.034323] mpt2sas [56103.034324] mpt2sas [56103.034324] dca [56103.034325] dca [56103.034325] libata [56103.034325] libata [56103.034326] raid_class [56103.034326] raid_class [56103.034327] i2c_algo_bit [56103.034327] i2c_algo_bit [56103.034328] scsi_transport_sas [56103.034328] scsi_transport_sas [56103.034329] devlink [56103.034329] devlink [56103.034329] i2c_core [56103.034330] i2c_core [56103.034330] fjes [56103.034331] [56103.034331] fjes [56103.034332] CPU: 9 PID: 36467 Comm: lctl Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56103.034333] CPU: 15 PID: 2900 Comm: kiblnd_sd_01_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56103.034334] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56103.034334] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56103.034335] task: ffff88082ab50000 ti: ffff8807f639c000 task.ti: ffff8807f639c000 [56103.034336] task: ffff88082aa3edd0 ti: ffff8808217b8000 task.ti: ffff8808217b8000 [56103.034337] RIP: 0010:[] [56103.034338] [] _raw_spin_lock+0x32/0x50 [56103.034339] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56103.034340] RSP: 0018:ffff8807f639fd60 EFLAGS: 00000212 [56103.034341] RSP: 0018:ffff8808217bbcb0 EFLAGS: 00000202 [56103.034341] RAX: 0000000000001c1a RBX: ffff8807f639fe00 RCX: 0000000000000a1e [56103.034342] RAX: 0000000000003457 RBX: 00000201e39c5101 RCX: 0000000000000a1e [56103.034343] RDX: 0000000000000a34 RSI: 0000000000000a34 RDI: ffff88042b272d00 [56103.034343] RDX: 0000000000000a26 RSI: 0000000000000a26 RDI: ffff88042b272d00 [56103.034344] RBP: ffff8807f639fd60 R08: 0000000000019b00 R09: ffff880187c07700 [56103.034345] RBP: ffff8808217bbcb0 R08: ffff8804c1ba3210 R09: ffff88082aa3edd0 [56103.034345] R10: ffffffffa0942cd3 R11: ffff8807f639fbbe R12: ffff8807f639fce0 [56103.034346] R10: ffff88082bda6cb8 R11: ffffea00208c0700 R12: ffff8808217bbce0 [56103.034347] R13: ffff88082ab50000 R14: ffff8807f639ffd8 R15: ffff8807f639fe50 [56103.034347] R13: ffffea0000000378 R14: ffffffffa0ebe43a R15: ffff8808217bbc88 [56103.034348] FS: 00007fd8ab488740(0000) GS:ffff88082d840000(0000) knlGS:0000000000000000 [56103.034349] FS: 0000000000000000(0000) GS:ffff88082d9c0000(0000) knlGS:0000000000000000 [56103.034349] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56103.034350] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56103.034351] CR2: 00007fd8aa99d614 CR3: 00000007f62d5000 CR4: 00000000000407e0 [56103.034352] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56103.034352] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56103.034353] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56103.034354] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56103.034354] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56103.034355] Stack: [56103.034355] Stack: [56103.034356] ffff8807f639fd88 [56103.034356] ffff8808217bbcd8 [56103.034357] ffffffffa08aa831 [56103.034357] ffffffffa08aa7c8 [56103.034358] 0000000200000002 [56103.034358] ffffffffa08aa8b6 [56103.034358] ffff8807f63a2980 [56103.034359] [56103.034359] ffff8804c1ba3200 [56103.034360] 0000000000000000 [56103.034360] 0000000000000001 [56103.034361] ffff8807f639fdc0 [56103.034361] ffff8808217bbd08 [56103.034362] ffffffffa0942e30 [56103.034362] ffffffffa09217bb [56103.034362] 00000000c0086540 [56103.034363] [56103.034363] ffff8804c1ba3200 [56103.034364] ffff8807f63a2900 [56103.034364] 0000000000000001 [56103.034365] 0000000000000002 [56103.034365] ffff88082abba340 [56103.034366] 0000000000650aa0 [56103.034366] ffff88082abba348 [56103.034367] 0000000000000000 [56103.034367] [56103.034367] ffff8808217bbd70 [56103.034368] Call Trace: [56103.034368] Call Trace: [56103.034373] [] cfs_percpt_lock+0xc1/0x110 [libcfs] [56103.034379] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56103.034385] [] lnet_fault_ctl+0x940/0xbc0 [lnet] [56103.034390] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56103.034395] [] LNetCtl+0x116/0x800 [lnet] [56103.034401] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56103.034407] [] lnet_ioctl+0x60/0x250 [lnet] [56103.034412] [] lnet_finalize+0x1e9/0x690 [lnet] [56103.034417] [] libcfs_ioctl+0x2ac/0x4c0 [libcfs] [56103.034420] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56103.034425] [] libcfs_psdev_ioctl+0x67/0xf0 [libcfs] [56103.034428] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56103.034430] [] do_vfs_ioctl+0x2d5/0x4b0 [56103.034431] [] ? dequeue_task_fair+0x41e/0x660 [56103.034433] [] ? __fd_install+0x47/0x60 [56103.034434] [] ? wake_up_state+0x20/0x20 [56103.034436] [] SyS_ioctl+0xa1/0xc0 [56103.034440] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56103.034441] [] system_call_fastpath+0x16/0x1b [56103.034443] [] kthread+0xcf/0xe0 [56103.034444] Code: [56103.034446] [] ? kthread_create_on_node+0x140/0x140 [56103.034446] 00 02 [56103.034448] [] ret_from_fork+0x58/0x90 [56103.034449] 00 f0 [56103.034450] [] ? kthread_create_on_node+0x140/0x140 [56103.034451] 0f c1 07 [56103.034452] Code: [56103.034452] 89 [56103.034453] c1 [56103.034453] c2 [56103.034453] 07 [56103.034454] c1 [56103.034454] 89 [56103.034455] ea [56103.034455] c2 [56103.034455] 10 [56103.034456] c1 [56103.034456] 66 [56103.034457] ea [56103.034457] 39 [56103.034457] 10 [56103.034458] c2 [56103.034458] 66 [56103.034458] 75 [56103.034459] 39 [56103.034459] 01 [56103.034460] c2 [56103.034460] c3 [56103.034460] 75 [56103.034461] 55 [56103.034461] 01 [56103.034462] 83 [56103.034462] c3 [56103.034462] e2 [56103.034463] 55 [56103.034463] fe [56103.034464] 83 [56103.034464] 0f [56103.034464] e2 [56103.034465] b7 [56103.034465] fe [56103.034466] f2 [56103.034466] 0f [56103.034466] 48 [56103.034467] b7 [56103.034467] 89 [56103.034468] f2 [56103.034468] e5 [56103.034468] 48 [56103.034469] b8 [56103.034469] 89 [56103.034470] 00 [56103.034470] e5 [56103.034470] 80 [56103.034471] b8 [56103.034471] 00 [56103.034472] 00 [56103.034472] 00 [56103.034472] 80 [56103.034473] eb [56103.034473] 00 [56103.034474] 0d [56103.034474] 00 [56103.034474] 66 [56103.034475] eb [56103.034475] 0f [56103.034475] 0d [56103.034476] 1f [56103.034476] 66 [56103.034477] 44 [56103.034477] 0f [56103.034477] 00 [56103.034478] 1f [56103.034478] 00 [56103.034479] 44 [56103.034479] f3 [56103.034479] 00 [56103.034480] 90 [56103.034480] 00 [56103.034481] <83> [56103.034481] f3 [56103.034482] e8 [56103.034482] 90 [56103.034482] 01 [56103.034483] 83 [56103.034483] 74 [56103.034484] e8 [56103.034484] 0a [56103.034484] 01 [56103.034485] 0f [56103.034485] 74 [56103.034486] b7 [56103.034486] 0a [56103.034486] 0f [56103.034487] <0f> [56103.034487] 66 [56103.034488] b7 [56103.034488] 39 [56103.034489] 0f [56103.034489] ca [56103.034489] 66 [56103.034490] 75 [56103.034490] 39 [56103.034491] f1 [56103.034491] ca [56103.034491] 5d [56103.034492] 75 [56103.034492] c3 [56103.034493] f1 [56103.034493] 66 [56103.034493] 5d [56103.034494] 66 [56103.034494] c3 [56103.034495] 66 [56103.034495] 66 [56103.034495] 90 [56103.034496] 66 [56103.034496] 66 [56103.034497] 66 [56103.034497] 66 [56103.034497] [56103.034499] 90 66 66 90 eb d9 66 0f [56107.067208] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56107.303696] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56107.373398] CPU: 10 PID: 2901 Comm: kiblnd_sd_01_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56107.409575] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56107.446944] task: ffff88082aa3de20 ti: ffff8808262ec000 task.ti: ffff8808262ec000 [56107.472013] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56107.499410] RSP: 0018:ffff8808262efa98 EFLAGS: 00000216 [56107.517365] RAX: 0000000000007847 RBX: ffff88082d2cded8 RCX: 0000000000000a1e [56107.541346] RDX: 0000000000000a2a RSI: 0000000000000a2a RDI: ffff88042b272d00 [56107.565338] RBP: ffff8808262efa98 R08: 0000000000000000 R09: 0001fd60baf0016e [56107.589343] R10: ffff8808262efc48 R11: 0000000000000000 R12: 0000000000000001 [56107.613377] R13: ffff88082d2cdea8 R14: 000000000000003f R15: ffff88041deb5f00 [56107.637399] FS: 0000000000000000(0000) GS:ffff88082d880000(0000) knlGS:0000000000000000 [56107.664595] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56107.684098] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56107.708166] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56107.732242] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56107.756328] Stack: [56107.763540] ffff8808262efac0 ffffffffa08aa7c8 ffffffffa08aa8b6 0000000000000000 [56107.788593] ffff88083fb5e000 ffff8808262efb80 ffffffffa092c9b5 00000000000001a8 [56107.813651] ffffc9000e331730 ffff8808262efb68 ffffffffa0ebd587 ffffffffffffffff [56107.838690] Call Trace: [56107.847396] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56107.869288] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56107.892058] [] lnet_select_pathway+0x235/0x1140 [lnet] [56107.915143] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56107.937352] [] lnet_send+0x51/0x180 [lnet] [56107.956992] [] lnet_finalize+0x3f1/0x690 [lnet] [56107.978096] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56107.999758] [] ? __slab_free+0x81/0x2f0 [56108.018590] [] lnet_ni_recv+0xc3/0x320 [lnet] [56108.039157] [] lnet_parse+0x945/0xf90 [lnet] [56108.059420] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56108.082584] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56108.103735] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56108.127223] [] ? dequeue_task_fair+0x41e/0x660 [56108.148120] [] ? wake_up_state+0x20/0x20 [56108.167330] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56108.190280] [] kthread+0xcf/0xe0 [56108.207192] [] ? kthread_create_on_node+0x140/0x140 [56108.229590] [] ret_from_fork+0x58/0x90 [56108.248212] [] ? kthread_create_on_node+0x140/0x140 [56108.270554] Code: 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 01 c3 55 83 e2 fe 0f b7 f2 48 89 e5 b8 00 80 00 00 eb 0d 66 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56111.936314] INFO: rcu_sched self-detected stall on CPU [56111.936315] INFO: rcu_sched self-detected stall on CPU [56111.936315] { [56111.936316] 11 [56111.936316] } [56111.936326] (t=1320021 jiffies g=99188 c=99187 q=34167) [56111.936327] Task dump for CPU 1: [56111.936327] kiblnd_sd_00_00 R [56111.936327] running task [56111.936328] 0 2896 2 0x00000088 [56111.936329] ffff880820b03d30 [56111.936329] 000000003f673d40 [56111.936329] ffffc90003ec02f0 [56111.936330] ffff88041d63d600 [56111.936330] [56111.936330] 0000000000000000 [56111.936330] 0000000000000000 [56111.936330] ffff88042aa84a28 [56111.936331] ffff880820b03da8 [56111.936331] [56111.936331] ffffffffa0eba166 [56111.936331] ffff880181447180 [56111.936332] ffff88042aa84a00 [56111.936332] 0000000000000002 [56111.936332] [56111.936332] Call Trace: [56111.936333] [56111.936336] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56111.936337] [56111.936340] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56111.936340] [56111.936341] [] ? dequeue_task_fair+0x41e/0x660 [56111.936342] [56111.936343] [] ? sched_clock_cpu+0x85/0xc0 [56111.936343] [56111.936345] [] ? wake_up_state+0x20/0x20 [56111.936345] [56111.936348] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56111.936349] [56111.936350] [] ? kthread+0xcf/0xe0 [56111.936350] [56111.936352] [] ? kthread_create_on_node+0x140/0x140 [56111.936352] [56111.936353] [] ? ret_from_fork+0x58/0x90 [56111.936354] [56111.936355] [] ? kthread_create_on_node+0x140/0x140 [56111.936356] Task dump for CPU 5: [56111.936356] kiblnd_sd_00_02 R [56111.936356] running task [56111.936357] 0 2898 2 0x00000088 [56111.936357] ffff880415b13000 [56111.936357] ffff8800abaa43a8 [56111.936357] ffff88083fb5e000 [56111.936358] 0000000000000000 [56111.936358] [56111.936358] ffff88082aae7da8 [56111.936358] ffffffffa0ebedab [56111.936359] ffff880181447188 [56111.936359] ffff88082aae7da8 [56111.936359] [56111.936359] ffffffff810bc064 [56111.936359] ffff880181447180 [56111.936360] ffff88042aa84a00 [56111.936360] 0000000000000001 [56111.936360] [56111.936360] Call Trace: [56111.936360] [56111.936364] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56111.936364] [56111.936365] [] ? __wake_up+0x44/0x50 [56111.936365] [56111.936368] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56111.936368] [56111.936369] [] ? dequeue_task_fair+0x41e/0x660 [56111.936369] [56111.936371] [] ? wake_up_state+0x20/0x20 [56111.936371] [56111.936374] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56111.936374] [56111.936375] [] ? kthread+0xcf/0xe0 [56111.936376] [56111.936377] [] ? kthread_create_on_node+0x140/0x140 [56111.936377] [56111.936378] [] ? ret_from_fork+0x58/0x90 [56111.936379] [56111.936380] [] ? kthread_create_on_node+0x140/0x140 [56111.936380] Task dump for CPU 6: [56111.936381] kiblnd_sd_00_01 R [56111.936381] running task [56111.936382] 0 2897 2 0x00000088 [56111.936382] ffff88081f64fd30 [56111.936382] 00000000f1dbfe3f [56111.936382] ffffc90003ebfde0 [56111.936382] ffff88042dcd1e00 [56111.936383] [56111.936383] 0000000000000000 [56111.936383] 0000000000000000 [56111.936383] ffff88042aa84a28 [56111.936384] ffff88081f64fda8 [56111.936384] [56111.936384] ffffffffa0eba166 [56111.936384] ffff880181447180 [56111.936384] ffff88042aa84a00 [56111.936385] 0000000000000012 [56111.936385] [56111.936385] Call Trace: [56111.936385] [56111.936388] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56111.936389] [56111.936392] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56111.936392] [56111.936393] [] ? dequeue_task_fair+0x41e/0x660 [56111.936393] [56111.936394] [] ? wake_up_state+0x20/0x20 [56111.936395] [56111.936398] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56111.936398] [56111.936399] [] ? kthread+0xcf/0xe0 [56111.936399] [56111.936401] [] ? kthread_create_on_node+0x140/0x140 [56111.936401] [56111.936402] [] ? ret_from_fork+0x58/0x90 [56111.936402] [56111.936403] [] ? kthread_create_on_node+0x140/0x140 [56111.936404] Task dump for CPU 7: [56111.936404] kiblnd_sd_00_03 R [56111.936404] running task [56111.936405] 0 2899 2 0x00000088 [56111.936405] ffff88082626bd30 [56111.936405] 00000000aaf7c743 [56111.936406] ffffc90003ebe8c8 [56111.936406] ffff8800b484ea00 [56111.936406] [56111.936406] 0000000000000000 [56111.936407] 0000000000000000 [56111.936407] ffff88042aa84a28 [56111.936407] ffff88082626bda8 [56111.936407] [56111.936407] ffffffffa0eba166 [56111.936408] ffff880181447180 [56111.936408] ffff88042aa84a00 [56111.936408] 0000000000000001 [56111.936408] [56111.936408] Call Trace: [56111.936409] [56111.936412] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56111.936412] [56111.936415] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56111.936415] [56111.936416] [] ? dequeue_task_fair+0x41e/0x660 [56111.936417] [56111.936418] [] ? wake_up_state+0x20/0x20 [56111.936418] [56111.936421] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56111.936421] [56111.936422] [] ? kthread+0xcf/0xe0 [56111.936423] [56111.936424] [] ? kthread_create_on_node+0x140/0x140 [56111.936424] [56111.936425] [] ? ret_from_fork+0x58/0x90 [56111.936426] [56111.936427] [] ? kthread_create_on_node+0x140/0x140 [56111.936427] Task dump for CPU 10: [56111.936428] kiblnd_sd_01_01 R [56111.936428] running task [56111.936428] 0 2901 2 0x00000088 [56111.936429] ffff8807f886f000 [56111.936429] ffffc9001327c860 [56111.936429] ffff8808266e7a00 [56111.936429] 0000000000000000 [56111.936429] [56111.936430] ffff8808262efda8 [56111.936430] ffffffffa0ebedab [56111.936430] ffff8808262efe90 [56111.936430] ffff8808262efda8 [56111.936431] [56111.936431] ffffffffa03e7d2e [56111.936431] ffff88082b2fc3c0 [56111.936431] ffff88082bda6c00 [56111.936431] 0000000000000017 [56111.936432] [56111.936432] Call Trace: [56111.936432] [56111.936435] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56111.936437] [56111.936439] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56111.936439] [56111.936442] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56111.936443] [56111.936444] [] ? dequeue_task_fair+0x41e/0x660 [56111.936444] [56111.936445] [] ? wake_up_state+0x20/0x20 [56111.936445] [56111.936449] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56111.936449] [56111.936450] [] ? kthread+0xcf/0xe0 [56111.936450] [56111.936451] [] ? kthread_create_on_node+0x140/0x140 [56111.936452] [56111.936453] [] ? ret_from_fork+0x58/0x90 [56111.936453] [56111.936454] [] ? kthread_create_on_node+0x140/0x140 [56111.936455] Task dump for CPU 11: [56111.936455] kiblnd_sd_01_02 R [56111.936455] running task [56111.936456] 0 2902 2 0x00000088 [56111.936456] ffff880826310000 [56111.936456] 000000005cf0ae83 [56111.936456] ffff88082d8c3db0 [56111.936457] ffffffff810c42f8 [56111.936457] [56111.936457] 000000000000000b [56111.936457] ffffffff81a1a680 [56111.936458] ffff88082d8c3dc8 [56111.936458] ffffffff810c7b89 [56111.936458] [56111.936458] 000000000000000c [56111.936458] ffff88082d8c3df8 [56111.936459] ffffffff811373f0 [56111.936459] ffff88082d8d01c0 [56111.936459] [56111.936459] Call Trace: [56111.936460] [56111.936460] [56111.936461] [] sched_show_task+0xa8/0x110 [56111.936461] [56111.936463] [] dump_cpu_task+0x39/0x70 [56111.936463] [56111.936464] [] rcu_dump_cpu_stacks+0x90/0xd0 [56111.936464] [56111.936465] [] rcu_check_callbacks+0x442/0x720 [56111.936466] [56111.936467] [] ? tick_sched_handle.isra.13+0x60/0x60 [56111.936467] [56111.936469] [] update_process_times+0x47/0x80 [56111.936469] [56111.936470] [] tick_sched_handle.isra.13+0x25/0x60 [56111.936470] [56111.936472] [] tick_sched_timer+0x41/0x70 [56111.936472] [56111.936473] [] __hrtimer_run_queues+0xd2/0x260 [56111.936473] [56111.936474] [] hrtimer_interrupt+0xb0/0x1e0 [56111.936474] [56111.936475] [] ? call_softirq+0x1c/0x30 [56111.936475] [56111.936477] [] local_apic_timer_interrupt+0x37/0x60 [56111.936478] [56111.936479] [] smp_apic_timer_interrupt+0x3f/0x60 [56111.936479] [56111.936480] [] apic_timer_interrupt+0x6d/0x80 [56111.936480] [56111.936480] [56111.936481] [] ? _raw_spin_lock+0x32/0x50 [56111.936482] [56111.936487] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56111.936487] [56111.936492] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56111.936492] [56111.936498] [] lnet_msg_decommit+0x11b/0x6c0 [lnet] [56111.936498] [56111.936503] [] lnet_finalize+0x1e9/0x690 [lnet] [56111.936504] [56111.936507] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56111.936507] [56111.936510] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56111.936510] [56111.936511] [] ? dequeue_task_fair+0x41e/0x660 [56111.936512] [56111.936513] [] ? wake_up_state+0x20/0x20 [56111.936513] [56111.936516] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56111.936516] [56111.936518] [] kthread+0xcf/0xe0 [56111.936518] [56111.936519] [] ? kthread_create_on_node+0x140/0x140 [56111.936519] [56111.936521] [] ret_from_fork+0x58/0x90 [56111.936521] [56111.936522] [] ? kthread_create_on_node+0x140/0x140 [56111.936523] Task dump for CPU 12: [56111.936523] kiblnd_sd_01_03 R [56111.936523] running task [56111.936524] 0 2903 2 0x00000088 [56111.936524] ffff880820a21000 [56111.936524] ffffc90011f075b0 [56111.936524] ffff8808266e7a00 [56111.936525] 0000000000000000 [56111.936525] [56111.936525] ffff8808213b7da8 [56111.936525] ffffffffa0ebedab [56111.936526] ffff8808213b7e90 [56111.936526] ffff8808213b7da8 [56111.936526] [56111.936526] ffffffffa03e7d2e [56111.936526] ffff88082b2fc3c0 [56111.936527] ffff88082c5f3600 [56111.936527] 0000000000000011 [56111.936527] [56111.936527] Call Trace: [56111.936528] [56111.936531] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56111.936532] [56111.936534] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56111.936534] [56111.936537] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56111.936537] [56111.936538] [] ? dequeue_task_fair+0x41e/0x660 [56111.936538] [56111.936540] [] ? wake_up_state+0x20/0x20 [56111.936540] [56111.936543] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56111.936543] [56111.936544] [] ? kthread+0xcf/0xe0 [56111.936545] [56111.936546] [] ? kthread_create_on_node+0x140/0x140 [56111.936546] [56111.936547] [] ? ret_from_fork+0x58/0x90 [56111.936548] [56111.936549] [] ? kthread_create_on_node+0x140/0x140 [56111.936549] Task dump for CPU 15: [56111.936550] kiblnd_sd_01_00 R [56111.936550] running task [56111.936550] 0 2900 2 0x00000088 [56111.936550] ffff8808217bbd30 [56111.936551] 00000000937dfd11 [56111.936551] ffffc9000e335900 [56111.936551] ffff8804c1ba3200 [56111.936551] [56111.936551] 0000000000000000 [56111.936552] 0000000000000000 [56111.936552] ffff88082bda6c28 [56111.936552] ffff8808217bbda8 [56111.936552] [56111.936553] ffffffffa0eba166 [56111.936553] ffff88082b2fc3c0 [56111.936553] ffff88082bda6c00 [56111.936553] 0000000000000014 [56111.936553] [56111.936554] Call Trace: [56111.936554] [56111.936557] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56111.936557] [56111.936560] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56111.936560] [56111.936561] [] ? dequeue_task_fair+0x41e/0x660 [56111.936562] [56111.936563] [] ? wake_up_state+0x20/0x20 [56111.936563] [56111.936566] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56111.936567] [56111.936568] [] ? kthread+0xcf/0xe0 [56111.936568] [56111.936569] [] ? kthread_create_on_node+0x140/0x140 [56111.936570] [56111.936571] [] ? ret_from_fork+0x58/0x90 [56111.936571] [56111.936573] [] ? kthread_create_on_node+0x140/0x140 [56114.933312] { [56114.939415] 15} (t=1323025 jiffies g=99188 c=99187 q=34167) [56114.954712] Task dump for CPU 1: [56114.965940] kiblnd_sd_00_00 R running task 0 2896 2 0x00000088 [56114.989793] ffff880820b03d30 000000003f673d40 ffffc90003ec02f0 ffff88041d63d600 [56115.032995] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [kiblnd_sd_00_03:2899] [56115.032997] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [kiblnd_sd_00_00:2896] [56115.014814] 0000000000000000 0000000000000000 ffff88042aa84a28 [56115.032997] Modules linked in: [56115.032998] Modules linked in: [56115.032998] ko2iblnd(OE) [56115.032999] ko2iblnd(OE) [56115.032999] lustre(OE) [56115.032999] lustre(OE) [56115.033000] lmv(OE) [56115.033000] lmv(OE) [56115.033000] fld(OE) [56115.033001] fld(OE) [56115.033001] mdc(OE) [56115.033001] mdc(OE) [56115.033002] fid(OE) [56115.033002] fid(OE) [56115.033002] lov(OE) [56115.033003] lov(OE) [56115.033003] ksocklnd(OE) [56115.033003] ksocklnd(OE) [56115.033004] ptlrpc(OE) [56115.033004] ptlrpc(OE) [56115.033004] obdclass(OE) [56115.033005] obdclass(OE) [56115.033005] lnet(OE) [56115.033006] lnet(OE) [56115.033006] libcfs(OE) [56115.033006] libcfs(OE) [56115.033007] rpcrdma [56115.033007] rpcrdma [56115.033007] ib_isert [56115.033008] ib_isert [56115.033008] iscsi_target_mod [56115.033008] iscsi_target_mod [56115.033009] ib_iser [56115.033009] ib_iser [56115.033009] libiscsi [56115.033010] libiscsi [56115.033010] scsi_transport_iscsi [56115.033010] scsi_transport_iscsi [56115.033011] ib_srpt [56115.033011] ib_srpt [56115.033012] target_core_mod [56115.033012] target_core_mod [56115.033012] ib_srp [56115.033013] ib_srp [56115.033013] scsi_transport_srp [56115.033013] scsi_transport_srp [56115.033014] scsi_tgt [56115.033014] scsi_tgt [56115.033014] ib_ipoib [56115.033015] ib_ipoib [56115.033015] rdma_ucm [56115.033015] rdma_ucm [56115.033016] ib_ucm [56115.033016] ib_ucm [56115.033016] ib_uverbs [56115.033017] ib_uverbs [56115.033017] ib_umad [56115.033017] ib_umad [56115.033018] rdma_cm [56115.033018] rdma_cm [56115.033018] ib_cm [56115.033019] ib_cm [56115.033019] iw_cm [56115.033019] iw_cm [56115.033020] mlx4_ib [56115.033020] mlx4_ib [56115.033021] intel_powerclamp [56115.033021] intel_powerclamp [56115.033021] coretemp [56115.033022] coretemp [56115.033022] intel_rapl [56115.033022] intel_rapl [56115.033023] iosf_mbi [56115.033023] iosf_mbi [56115.033023] kvm [56115.033024] kvm [56115.033024] irqbypass [56115.033024] irqbypass [56115.033025] crc32_pclmul [56115.033025] crc32_pclmul [56115.033025] ghash_clmulni_intel [56115.033026] ghash_clmulni_intel [56115.033026] aesni_intel [56115.033026] aesni_intel [56115.033027] ipmi_ssif [56115.033027] ipmi_ssif [56115.033027] lrw [56115.033028] lrw [56115.033028] gf128mul [56115.033028] gf128mul [56115.033029] glue_helper [56115.033029] glue_helper [56115.033029] hfi1 [56115.033030] hfi1 [56115.033030] ipmi_devintf [56115.033030] ipmi_devintf [56115.033031] rdmavt [56115.033031] rdmavt [56115.033031] ablk_helper [56115.033032] ablk_helper [56115.033032] cryptd [56115.033032] cryptd [56115.033033] iTCO_wdt [56115.033033] iTCO_wdt [56115.033033] ipmi_si [56115.033034] ipmi_si [56115.033034] sb_edac [56115.033034] sb_edac [56115.033035] sg [56115.033035] sg [56115.033035] iTCO_vendor_support [56115.033036] iTCO_vendor_support [56115.033036] ib_core [56115.033037] ib_core [56115.033037] ioatdma [56115.033037] ioatdma [56115.033038] i2c_i801 [56115.033038] i2c_i801 [56115.033038] mei_me [56115.033039] mei_me [56115.033039] mei [56115.033039] mei [56115.033040] edac_core [56115.033040] edac_core [56115.033040] ipmi_msghandler [56115.033041] ipmi_msghandler [56115.033041] lpc_ich [56115.033041] lpc_ich [56115.033042] ntb [56115.033042] ntb [56115.033042] wmi [56115.033043] wmi [56115.033043] shpchp [56115.033043] shpchp [56115.033044] pcspkr [56115.033044] pcspkr [56115.033045] nfsd [56115.033045] nfsd [56115.033045] nfs_acl [56115.033046] nfs_acl [56115.033046] lockd [56115.033046] lockd [56115.033047] grace [56115.033047] grace [56115.033047] auth_rpcgss [56115.033048] auth_rpcgss [56115.033048] sunrpc [56115.033048] sunrpc [56115.033049] ip_tables [56115.033049] ip_tables [56115.033049] ext4 [56115.033050] ext4 [56115.033050] mbcache [56115.033050] mbcache [56115.033051] jbd2 [56115.033051] jbd2 [56115.033051] sd_mod [56115.033052] sd_mod [56115.033052] crc_t10dif [56115.033052] crc_t10dif [56115.033053] crct10dif_generic [56115.033053] crct10dif_generic [56115.033053] mlx4_en [56115.033054] mlx4_en [56115.033054] mgag200 [56115.033055] mgag200 [56115.033055] drm_kms_helper [56115.033055] drm_kms_helper [56115.033056] syscopyarea [56115.033056] syscopyarea [56115.033056] sysfillrect [56115.033057] sysfillrect [56115.033057] sysimgblt [56115.033057] sysimgblt [56115.033058] fb_sys_fops [56115.033058] fb_sys_fops [56115.033058] isci [56115.033059] isci [56115.033059] ttm [56115.033059] ttm [56115.033060] igb [56115.033060] igb [56115.033060] crct10dif_pclmul [56115.033061] crct10dif_pclmul [56115.033061] crct10dif_common [56115.033062] crct10dif_common [56115.033062] ahci [56115.033062] ahci [56115.033063] ptp [56115.033063] ptp [56115.033063] crc32c_intel [56115.033064] crc32c_intel [56115.033064] libsas [56115.033064] libsas [56115.033065] libahci [56115.033065] libahci [56115.033065] pps_core [56115.033066] pps_core [56115.033066] drm [56115.033066] drm [56115.033067] mlx4_core [56115.033067] mlx4_core [56115.033067] mpt2sas [56115.033068] mpt2sas [56115.033068] dca [56115.033068] dca [56115.033069] libata [56115.033069] libata [56115.033069] raid_class [56115.033070] raid_class [56115.033070] i2c_algo_bit [56115.033071] i2c_algo_bit [56115.033071] scsi_transport_sas [56115.033071] scsi_transport_sas [56115.033072] devlink [56115.033072] devlink [56115.033072] i2c_core [56115.033073] i2c_core [56115.033073] fjes [56115.033073] fjes [56115.033074] [56115.033074] [56115.033075] CPU: 7 PID: 2899 Comm: kiblnd_sd_00_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56115.033076] CPU: 1 PID: 2896 Comm: kiblnd_sd_00_00 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56115.033077] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56115.033077] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56115.033078] task: ffff88082af10fb0 ti: ffff880826268000 task.ti: ffff880826268000 [56115.033079] task: ffff8804c1673ec0 ti: ffff880820b00000 task.ti: ffff880820b00000 [56115.033079] RIP: 0010:[] [56115.033080] RIP: 0010:[] [56115.033081] [] _raw_spin_lock+0x30/0x50 [56115.033083] [] _raw_spin_lock+0x32/0x50 [56115.033083] RSP: 0018:ffff88082626bce0 EFLAGS: 00000216 [56115.033084] RSP: 0018:ffff880820b03c00 EFLAGS: 00000202 [56115.033085] RAX: 0000000000003401 RBX: ffff88082626bcb0 RCX: 0000000000000a1e [56115.033085] RAX: 0000000000001a9d RBX: ffffc90003ebfc30 RCX: 0000000000000a1e [56115.033086] RDX: 0000000000000a24 RSI: 0000000000000a24 RDI: ffff88042b272d00 [56115.033086] RDX: 0000000000000a2e RSI: 0000000000000a2e RDI: ffff88042b272d00 [56115.033087] RBP: ffff88082626bce0 R08: ffffc90003ebf570 R09: 0000000000000008 [56115.033088] RBP: ffff880820b03c00 R08: ffff8803ee33f180 R09: 0000000000000002 [56115.033088] R10: ffff88042aa84ab8 R11: 0000000000000000 R12: ffff88042e156cc0 [56115.033089] R10: ffff88042aa84ab8 R11: ffff880819fc0858 R12: 00000000000006b8 [56115.033090] R13: ffff88082af11018 R14: ffffffff810ca97e R15: ffff88082626bc60 [56115.033091] R13: 0000000000000001 R14: ffff8800b4848200 R15: 000000003f673d40 [56115.033091] FS: 0000000000000000(0000) GS:ffff88042e1c0000(0000) knlGS:0000000000000000 [56115.033092] FS: 0000000000000000(0000) GS:ffff88042e040000(0000) knlGS:0000000000000000 [56115.033093] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56115.033094] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56115.033094] CR2: 00007f1cb4805430 CR3: 00000000019ba000 CR4: 00000000000407e0 [56115.033095] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56115.033096] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56115.033096] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56115.033097] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56115.033098] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56115.033098] Stack: [56115.033098] Stack: [56115.033099] ffff88082626bd08 [56115.033099] ffff880820b03c28 [56115.033099] ffffffffa08aa7c8 [56115.033100] ffffffffa08aa7c8 [56115.033100] ffff88082af11648 [56115.033100] ffff880820b03c28 [56115.033101] ffff8800b484ea00 [56115.033101] 0000000000000000 [56115.033101] [56115.033102] [56115.033102] 0000000000000000 [56115.033102] ffff88083fb5e000 [56115.033103] ffff88082626bd70 [56115.033103] ffff880820b03c98 [56115.033104] ffffffffa0922006 [56115.033104] ffffffffa092c346 [56115.033104] ffff88082af10fb0 [56115.033105] ffff880820b03c60 [56115.033105] [56115.033105] [56115.033106] ffff88082af10fb0 [56115.033106] 00000000000000b4 [56115.033106] ffff88082af10fb0 [56115.033107] 00000001033f89f4 [56115.033107] ffff88082626bd30 [56115.033107] ffff8800b4864274 [56115.033108] ffff88082626bd30 [56115.033108] 0000000000000000 [56115.033108] [56115.033109] [56115.033109] Call Trace: [56115.033109] Call Trace: [56115.033110] [56115.033115] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56115.033115] [56115.033115] [56115.033121] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56115.033126] [] lnet_finalize+0xf6/0x690 [lnet] [56115.033127] [56115.033127] [56115.033133] [] lnet_post_send_locked+0x446/0x880 [lnet] [56115.033136] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56115.033137] [56115.033137] [56115.033143] [] lnet_return_tx_credits_locked+0x1e9/0x480 [lnet] [56115.033146] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56115.033147] [56115.033147] [56115.033152] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56115.033154] [] ? dequeue_task_fair+0x41e/0x660 [56115.033154] [56115.033154] [56115.033159] [] lnet_finalize+0x1e9/0x690 [lnet] [56115.033160] [] ? wake_up_state+0x20/0x20 [56115.033161] [56115.033161] [56115.033164] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56115.033168] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56115.033168] [56115.033168] [56115.033172] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56115.033173] [] kthread+0xcf/0xe0 [56115.033174] [56115.033174] [56115.033175] [] ? dequeue_task_fair+0x41e/0x660 [56115.033177] [] ? kthread_create_on_node+0x140/0x140 [56115.033177] [56115.033177] [56115.033179] [] ? sched_clock_cpu+0x85/0xc0 [56115.033180] [] ret_from_fork+0x58/0x90 [56115.033181] [56115.033181] [56115.033182] [] ? wake_up_state+0x20/0x20 [56115.033184] [] ? kthread_create_on_node+0x140/0x140 [56115.033184] [56115.033187] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56115.033188] Code: [56115.033188] [56115.033188] b8 [56115.033190] [] kthread+0xcf/0xe0 [56115.033190] 00 [56115.033190] [56115.033191] 00 [56115.033192] [] ? kthread_create_on_node+0x140/0x140 [56115.033192] 02 [56115.033193] [56115.033193] 00 [56115.033195] [] ret_from_fork+0x58/0x90 [56115.033195] f0 [56115.033195] [56115.033196] 0f [56115.033197] [] ? kthread_create_on_node+0x140/0x140 [56115.033197] c1 [56115.033198] Code: [56115.033198] 07 [56115.033198] 00 [56115.033199] 89 [56115.033199] 02 [56115.033199] c2 [56115.033199] 00 [56115.033200] c1 [56115.033200] f0 [56115.033200] ea [56115.033201] 0f [56115.033201] 10 [56115.033201] c1 [56115.033202] 66 [56115.033202] 07 [56115.033202] 39 [56115.033203] 89 [56115.033203] c2 [56115.033203] c2 [56115.033204] 75 [56115.033204] c1 [56115.033204] 01 [56115.033204] ea [56115.033205] c3 [56115.033205] 10 [56115.033205] 55 [56115.033206] 66 [56115.033206] 83 [56115.033206] 39 [56115.033207] e2 [56115.033207] c2 [56115.033207] fe [56115.033208] 75 [56115.033208] 0f [56115.033208] 01 [56115.033209] b7 [56115.033209] c3 [56115.033209] f2 [56115.033210] 55 [56115.033210] 48 [56115.033210] 83 [56115.033211] 89 [56115.033211] e2 [56115.033211] e5 [56115.033212] fe [56115.033212] b8 [56115.033212] 0f [56115.033212] 00 [56115.033213] b7 [56115.033213] 80 [56115.033213] f2 [56115.033214] 00 [56115.033214] 48 [56115.033214] 00 [56115.033215] 89 [56115.033215] eb [56115.033215] e5 [56115.033216] 0d [56115.033216] b8 [56115.033216] 66 [56115.033217] 00 [56115.033217] 0f [56115.033217] 80 [56115.033217] 1f [56115.033218] 00 [56115.033218] 44 [56115.033219] 00 [56115.033219] 00 [56115.033219] eb [56115.033219] 00 [56115.033220] 0d [56115.033220] [56115.033221] 66 [56115.033221] 90 [56115.033221] 0f [56115.033221] 83 [56115.033222] 1f [56115.033222] e8 [56115.033222] 44 [56115.033223] 01 [56115.033223] 00 [56115.033223] 74 [56115.033224] 00 [56115.033224] 0a [56115.033224] f3 [56115.033225] 0f [56115.033225] 90 [56115.033225] b7 [56115.033226] <83> [56115.033226] 0f [56115.033226] e8 [56115.033227] 66 [56115.033227] 01 [56115.033227] 39 [56115.033228] 74 [56115.033228] ca [56115.033228] 0a [56115.033228] 75 [56115.033229] 0f [56115.033229] f1 [56115.033229] b7 [56115.033230] 5d [56115.033230] 0f [56115.033230] c3 [56115.033231] 66 [56115.033231] 66 [56115.033231] 39 [56115.033232] 66 [56115.033232] ca [56115.033232] 66 [56115.033233] 75 [56115.033233] 90 [56115.033233] f1 [56115.033233] [56115.033234] 5d [56115.033234] c3 [56115.033234] 66 [56115.033234] 66 [56115.033235] 66 [56115.033235] 90 [56115.033235] 66 [56115.033235] 66 [56115.033235] [56117.357462] ffff880820b03da8 [56117.367895] [56117.369196] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000002 [56117.394286] Call Trace: [56117.403056] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56117.426124] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56117.450056] [] ? dequeue_task_fair+0x41e/0x660 [56117.470839] [] ? sched_clock_cpu+0x85/0xc0 [56117.490471] [] ? wake_up_state+0x20/0x20 [56117.509522] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56117.532279] [] ? kthread+0xcf/0xe0 [56117.549574] [] ? kthread_create_on_node+0x140/0x140 [56117.571750] [] ? ret_from_fork+0x58/0x90 [56117.590750] [] ? kthread_create_on_node+0x140/0x140 [56117.612867] Task dump for CPU 5: [56117.624137] kiblnd_sd_00_02 R running task 0 2898 2 0x00000088 [56117.648023] ffff880415b13000 ffff8800abaa43a8 ffff88083fb5e000 0000000000000000 [56117.673103] ffff88082aae7da8 ffffffffa0ebedab ffff880181447188 ffff88082aae7da8 [56117.698156] ffffffff810bc064 ffff880181447180 ffff88042aa84a00 0000000000000001 [56117.723222] Call Trace: [56117.731951] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56117.755588] [] ? __wake_up+0x44/0x50 [56117.773473] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56117.797382] [] ? dequeue_task_fair+0x41e/0x660 [56117.818157] [] ? wake_up_state+0x20/0x20 [56117.837218] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56117.859978] [] ? kthread+0xcf/0xe0 [56117.877300] [] ? kthread_create_on_node+0x140/0x140 [56117.899506] [] ? ret_from_fork+0x58/0x90 [56117.918550] [] ? kthread_create_on_node+0x140/0x140 [56117.940758] Task dump for CPU 6: [56117.952068] kiblnd_sd_00_01 R running task 0 2897 2 0x00000088 [56117.976010] ffff88081f64fd30 00000000f1dbfe3f ffffc90003ebfde0 ffff88042dcd1e00 [56118.001126] 0000000000000000 0000000000000000 ffff88042aa84a28 ffff88081f64fda8 [56118.026234] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000012 [56118.051351] Call Trace: [56118.060138] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56118.083247] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56118.107186] [] ? dequeue_task_fair+0x41e/0x660 [56118.127977] [] ? wake_up_state+0x20/0x20 [56118.147057] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56118.169867] [] ? kthread+0xcf/0xe0 [56118.187221] [] ? kthread_create_on_node+0x140/0x140 [56118.209453] [] ? ret_from_fork+0x58/0x90 [56118.228527] [] ? kthread_create_on_node+0x140/0x140 [56118.250706] Task dump for CPU 7: [56118.262013] kiblnd_sd_00_03 R running task 0 2899 2 0x00000088 [56118.285907] ffff88082626bd30 00000000aaf7c743 ffffc90003ebe8c8 ffff8800b484ea00 [56118.310990] 0000000000000000 0000000000000000 ffff88042aa84a28 ffff88082626bda8 [56118.336090] ffffffffa0eba166 ffff880181447180 ffff88042aa84a00 0000000000000001 [56118.361158] Call Trace: [56118.369899] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56118.392949] [] ? kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56118.416852] [] ? dequeue_task_fair+0x41e/0x660 [56118.437613] [] ? wake_up_state+0x20/0x20 [56118.456628] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56118.479390] [] ? kthread+0xcf/0xe0 [56118.496711] [] ? kthread_create_on_node+0x140/0x140 [56118.518903] [] ? ret_from_fork+0x58/0x90 [56118.537918] [] ? kthread_create_on_node+0x140/0x140 [56118.560099] Task dump for CPU 10: [56118.571676] kiblnd_sd_01_01 R running task 0 2901 2 0x00000088 [56118.595581] ffff8807f886f000 ffffc9001327c860 ffff8808266e7a00 0000000000000000 [56118.620659] ffff8808262efda8 ffffffffa0ebedab ffff8808262efe90 ffff8808262efda8 [56118.645760] ffffffffa03e7d2e ffff88082b2fc3c0 ffff88082bda6c00 0000000000000017 [56118.670849] Call Trace: [56118.679573] [] ? kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56118.703202] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56118.724253] [] ? kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56118.748157] [] ? dequeue_task_fair+0x41e/0x660 [56118.768894] [] ? wake_up_state+0x20/0x20 [56118.787917] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56118.810649] [] ? kthread+0xcf/0xe0 [56118.827976] [] ? kthread_create_on_node+0x140/0x140 [56118.850164] [] ? ret_from_fork+0x58/0x90 [56118.869188] [] ? kthread_create_on_node+0x140/0x140 [56118.891375] Task dump for CPU 11: [56118.902966] kiblnd_sd_01_02 R running task 0 2902 2 0x00000088 [56118.926890] ffff88082363bd30 000000005cf0ae83 ffffc900061c2218 ffff8804c1ba6800 [56118.951969] 0000000000000000 0000000000000000 ffff88081f6b6a28 ffff88082363bda8 [56118.977049] ffffffffa0eba166 ffff88082b2fc3c0 ffff88081f6b6a00 000000000000000b [56119.002135] Call Trace: [56119.010878] [] ? kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56119.032583] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kiblnd_connd:2895] [56119.032585] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [kiblnd_sd_00_02:2898] [56119.032585] Modules linked in: [56119.032586] Modules linked in: [56119.032587] ko2iblnd(OE) [56119.032587] ko2iblnd(OE) [56119.032588] lustre(OE) [56119.032588] lustre(OE) [56119.032588] lmv(OE) [56119.032589] lmv(OE) [56119.032589] fld(OE) [56119.032590] fld(OE) [56119.032590] mdc(OE) [56119.032591] mdc(OE) [56119.032591] fid(OE) [56119.032591] fid(OE) [56119.032592] lov(OE) [56119.032592] lov(OE) [56119.032593] ksocklnd(OE) [56119.032593] ksocklnd(OE) [56119.032594] ptlrpc(OE) [56119.032594] ptlrpc(OE) [56119.032595] obdclass(OE) [56119.032595] obdclass(OE) [56119.032596] lnet(OE) [56119.032596] lnet(OE) [56119.032597] libcfs(OE) [56119.032597] libcfs(OE) [56119.032598] rpcrdma [56119.032598] rpcrdma [56119.032598] ib_isert [56119.032599] ib_isert [56119.032599] iscsi_target_mod [56119.032600] iscsi_target_mod [56119.032600] ib_iser [56119.032601] ib_iser [56119.032601] libiscsi [56119.032602] libiscsi [56119.032602] scsi_transport_iscsi [56119.032602] scsi_transport_iscsi [56119.032603] ib_srpt [56119.032603] ib_srpt [56119.032604] target_core_mod [56119.032604] target_core_mod [56119.032605] ib_srp [56119.032605] ib_srp [56119.032606] scsi_transport_srp [56119.032606] scsi_transport_srp [56119.032607] scsi_tgt [56119.032607] scsi_tgt [56119.032608] ib_ipoib [56119.032608] ib_ipoib [56119.032608] rdma_ucm [56119.032609] rdma_ucm [56119.032609] ib_ucm [56119.032610] ib_ucm [56119.032610] ib_uverbs [56119.032611] ib_uverbs [56119.032611] ib_umad [56119.032612] ib_umad [56119.032612] rdma_cm [56119.032613] rdma_cm [56119.032613] ib_cm [56119.032614] ib_cm [56119.032614] iw_cm [56119.032615] iw_cm [56119.032615] mlx4_ib [56119.032615] mlx4_ib [56119.032616] intel_powerclamp [56119.032616] intel_powerclamp [56119.032617] coretemp [56119.032617] coretemp [56119.032618] intel_rapl [56119.032618] intel_rapl [56119.032619] iosf_mbi [56119.032619] iosf_mbi [56119.032620] kvm [56119.032620] kvm [56119.032620] irqbypass [56119.032621] irqbypass [56119.032621] crc32_pclmul [56119.032622] crc32_pclmul [56119.032622] ghash_clmulni_intel [56119.032623] ghash_clmulni_intel [56119.032623] aesni_intel [56119.032624] aesni_intel [56119.032624] ipmi_ssif [56119.032625] ipmi_ssif [56119.032625] lrw [56119.032626] lrw [56119.032626] gf128mul [56119.032626] gf128mul [56119.032627] glue_helper [56119.032627] glue_helper [56119.032628] hfi1 [56119.032628] hfi1 [56119.032629] ipmi_devintf [56119.032629] ipmi_devintf [56119.032630] rdmavt [56119.032630] rdmavt [56119.032631] ablk_helper [56119.032631] ablk_helper [56119.032632] cryptd [56119.032632] cryptd [56119.032632] iTCO_wdt [56119.032633] iTCO_wdt [56119.032633] ipmi_si [56119.032634] ipmi_si [56119.032634] sb_edac [56119.032635] sb_edac [56119.032635] sg [56119.032636] sg [56119.032636] iTCO_vendor_support [56119.032637] iTCO_vendor_support [56119.032637] ib_core [56119.032638] ib_core [56119.032638] ioatdma [56119.032638] ioatdma [56119.032639] i2c_i801 [56119.032639] i2c_i801 [56119.032640] mei_me [56119.032640] mei_me [56119.032641] mei [56119.032641] mei [56119.032642] edac_core [56119.032642] edac_core [56119.032643] ipmi_msghandler [56119.032643] ipmi_msghandler [56119.032643] lpc_ich [56119.032644] lpc_ich [56119.032644] ntb [56119.032645] ntb [56119.032645] wmi [56119.032646] wmi [56119.032646] shpchp [56119.032647] shpchp [56119.032647] pcspkr [56119.032647] pcspkr [56119.032648] nfsd [56119.032648] nfsd [56119.032649] nfs_acl [56119.032649] nfs_acl [56119.032650] lockd [56119.032650] lockd [56119.032651] grace [56119.032651] grace [56119.032652] auth_rpcgss [56119.032652] auth_rpcgss [56119.032652] sunrpc [56119.032653] sunrpc [56119.032653] ip_tables [56119.032654] ip_tables [56119.032654] ext4 [56119.032655] ext4 [56119.032655] mbcache [56119.032656] mbcache [56119.032656] jbd2 [56119.032656] jbd2 [56119.032657] sd_mod [56119.032657] sd_mod [56119.032658] crc_t10dif [56119.032658] crc_t10dif [56119.032659] crct10dif_generic [56119.032659] crct10dif_generic [56119.032660] mlx4_en [56119.032660] mlx4_en [56119.032661] mgag200 [56119.032661] mgag200 [56119.032662] drm_kms_helper [56119.032662] drm_kms_helper [56119.032662] syscopyarea [56119.032663] syscopyarea [56119.032663] sysfillrect [56119.032664] sysfillrect [56119.032664] sysimgblt [56119.032665] sysimgblt [56119.032665] fb_sys_fops [56119.032666] fb_sys_fops [56119.032666] isci [56119.032667] isci [56119.032667] ttm [56119.032667] ttm [56119.032668] igb [56119.032668] igb [56119.032669] crct10dif_pclmul [56119.032669] crct10dif_pclmul [56119.032670] crct10dif_common [56119.032670] crct10dif_common [56119.032671] ahci [56119.032671] ahci [56119.032672] ptp [56119.032672] ptp [56119.032673] crc32c_intel [56119.032673] crc32c_intel [56119.032673] libsas [56119.032674] libsas [56119.032674] libahci [56119.032675] libahci [56119.032675] pps_core [56119.032676] pps_core [56119.032676] drm [56119.032677] drm [56119.032677] mlx4_core [56119.032677] mlx4_core [56119.032678] mpt2sas [56119.032678] mpt2sas [56119.032679] dca [56119.032679] dca [56119.032680] libata [56119.032680] libata [56119.032681] raid_class [56119.032681] raid_class [56119.032682] i2c_algo_bit [56119.032682] i2c_algo_bit [56119.032683] scsi_transport_sas [56119.032683] scsi_transport_sas [56119.032683] devlink [56119.032684] devlink [56119.032684] i2c_core [56119.032685] i2c_core [56119.032685] fjes [56119.032686] [56119.032686] fjes [56119.032687] CPU: 2 PID: 2895 Comm: kiblnd_connd Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56119.032688] CPU: 5 PID: 2898 Comm: kiblnd_sd_00_02 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56119.032689] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56119.032690] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56119.032691] task: ffff8804c1672f10 ti: ffff88082aa30000 task.ti: ffff88082aa30000 [56119.032691] task: ffff8804c1676dd0 ti: ffff88082aae4000 task.ti: ffff88082aae4000 [56119.032692] RIP: 0010:[] [56119.032694] [] _raw_spin_lock+0x32/0x50 [56119.032695] RIP: 0010:[] [] _raw_spin_lock+0x37/0x50 [56119.032696] RSP: 0018:ffff88082aa33c78 EFLAGS: 00000216 [56119.032697] RSP: 0018:ffff88082aae7b78 EFLAGS: 00000206 [56119.032697] RAX: 0000000000002dd0 RBX: 0000000000016c40 RCX: 0000000000000a1e [56119.032698] RAX: 0000000000005000 RBX: 0000000100000000 RCX: 0000000000000a1e [56119.032699] RDX: 0000000000000a30 RSI: 0000000000000a30 RDI: ffff88042b272d00 [56119.032699] RDX: 0000000000000a20 RSI: 0000000000000a20 RDI: ffff88042b272d00 [56119.032700] RBP: ffff88082aa33c78 R08: ffff880821b75878 R09: 0000000000000008 [56119.032701] RBP: ffff88082aae7b78 R08: 0000000000000138 R09: 00000000000000c0 [56119.032701] R10: ffff880821b758b8 R11: 0000000000000000 R12: 0000000000000000 [56119.032702] R10: ffff88082aae7c48 R11: 0000000000000000 R12: 0000000100000004 [56119.032703] R13: ffff88082aa33ccc R14: ffff880187c18800 R15: ffff8801810be408 [56119.032703] R13: 0000000000002000 R14: 0000000000000400 R15: 0000000000001000 [56119.032704] FS: 0000000000000000(0000) GS:ffff88042e080000(0000) knlGS:0000000000000000 [56119.032705] FS: 0000000000000000(0000) GS:ffff88042e140000(0000) knlGS:0000000000000000 [56119.032706] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56119.032706] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56119.032707] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56119.032708] CR2: 00007f8bd01cf000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56119.032708] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56119.032709] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56119.032710] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56119.032710] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56119.032711] Stack: [56119.032711] Stack: [56119.032712] ffff88082aa33ca0 [56119.032712] ffff88082aae7ba0 [56119.032713] ffffffffa08aa7c8 [56119.032713] ffffffffa08aa7c8 [56119.032714] ffff88082aa33ca8 [56119.032714] 000000000000763d [56119.032715] 0000000000000000 [56119.032715] [56119.032716] ffff8803fe4d0400 [56119.032716] 00050000c0a80184 [56119.032717] 0000000000000000 [56119.032717] ffff88082aa33cc8 [56119.032718] ffff88082aae7c08 [56119.032718] ffffffffa0916cd0 [56119.032719] ffffffffa0922006 [56119.032719] ffff8800366a4530 [56119.032719] [56119.032720] ffff8804c1676dd0 [56119.032720] ffff880821b758e8 [56119.032721] ffff8804c1676dd0 [56119.032721] ffff88083fb5e000 [56119.032722] ffff8804c1676dd0 [56119.032722] ffff88082aa33ce0 [56119.032723] 0000000000000078 [56119.032723] ffffffffa0eb8460 [56119.032724] [56119.032724] ffff88082aae7c48 [56119.032724] Call Trace: [56119.032725] Call Trace: [56119.032730] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56119.032736] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56119.032741] [] lnet_cpt_of_nid+0x40/0x70 [lnet] [56119.032747] [] lnet_finalize+0xf6/0x690 [lnet] [56119.032751] [] kiblnd_get_idle_tx+0x20/0x260 [ko2iblnd] [56119.032755] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56119.032758] [] kiblnd_check_sends_locked+0x95a/0xe50 [ko2iblnd] [56119.032760] [] ? enqueue_task_fair+0x208/0x6c0 [56119.032762] [] ? lock_timer_base.isra.33+0x2b/0x50 [56119.032764] [] ? sched_clock_cpu+0x85/0xc0 [56119.032765] [] ? try_to_del_timer_sync+0x5e/0x90 [56119.032771] [] lnet_ni_recv+0xc3/0x320 [lnet] [56119.032775] [] kiblnd_check_conns+0x620/0x840 [ko2iblnd] [56119.032781] [] lnet_parse+0x945/0xf90 [lnet] [56119.032782] [] ? schedule_timeout+0x17d/0x2d0 [56119.032786] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56119.032790] [] kiblnd_connd+0x7ab/0x900 [ko2iblnd] [56119.032791] [] ? __wake_up+0x44/0x50 [56119.032793] [] ? wake_up_state+0x20/0x20 [56119.032796] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56119.032800] [] ? kiblnd_check_conns+0x840/0x840 [ko2iblnd] [56119.032801] [] ? dequeue_task_fair+0x41e/0x660 [56119.032803] [] kthread+0xcf/0xe0 [56119.032805] [] ? wake_up_state+0x20/0x20 [56119.032806] [] ? kthread_create_on_node+0x140/0x140 [56119.032810] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56119.032812] [] ret_from_fork+0x58/0x90 [56119.032813] [] kthread+0xcf/0xe0 [56119.032815] [] ? kthread_create_on_node+0x140/0x140 [56119.032817] [] ? kthread_create_on_node+0x140/0x140 [56119.032817] Code: [56119.032819] [] ret_from_fork+0x58/0x90 [56119.032820] 00 02 [56119.032822] [] ? kthread_create_on_node+0x140/0x140 [56119.032822] 00 f0 0f [56119.032823] Code: [56119.032823] c1 [56119.032824] c1 [56119.032824] 07 [56119.032825] 07 [56119.032825] 89 [56119.032825] 89 [56119.032826] c2 [56119.032826] c2 [56119.032827] c1 [56119.032827] c1 [56119.032828] ea [56119.032828] ea [56119.032828] 10 [56119.032829] 10 [56119.032829] 66 [56119.032830] 66 [56119.032830] 39 [56119.032830] 39 [56119.032831] c2 [56119.032831] c2 [56119.032832] 75 [56119.032832] 75 [56119.032833] 01 [56119.032833] 01 [56119.032833] c3 [56119.032834] c3 [56119.032834] 55 [56119.032835] 55 [56119.032835] 83 [56119.032835] 83 [56119.032836] e2 [56119.032836] e2 [56119.032837] fe [56119.032837] fe [56119.032838] 0f [56119.032838] 0f [56119.032838] b7 [56119.032839] b7 [56119.032839] f2 [56119.032840] f2 [56119.032840] 48 [56119.032841] 48 [56119.032841] 89 [56119.032841] 89 [56119.032842] e5 [56119.032842] e5 [56119.032843] b8 [56119.032843] b8 [56119.032843] 00 [56119.032844] 00 [56119.032844] 80 [56119.032845] 80 [56119.032845] 00 [56119.032846] 00 [56119.032846] 00 [56119.032846] 00 [56119.032847] eb [56119.032847] eb [56119.032848] 0d [56119.032848] 0d [56119.032849] 66 [56119.032849] 66 [56119.032849] 0f [56119.032850] 0f [56119.032850] 1f [56119.032851] 1f [56119.032851] 44 [56119.032851] 44 [56119.032852] 00 [56119.032852] 00 [56119.032853] 00 [56119.032853] 00 [56119.032854] f3 [56119.032854] f3 [56119.032854] 90 [56119.032855] 90 [56119.032855] <83> [56119.032856] 83 [56119.032856] e8 [56119.032857] e8 [56119.032857] 01 [56119.032857] 01 [56119.032858] 74 [56119.032858] 74 [56119.032859] 0a [56119.032859] 0a [56119.032860] 0f [56119.032860] <0f> [56119.032860] b7 [56119.032861] b7 [56119.032861] 0f [56119.032862] 0f [56119.032862] 66 [56119.032863] 66 [56119.032863] 39 [56119.032863] 39 [56119.032864] ca [56119.032864] ca [56119.032865] 75 [56119.032865] 75 [56119.032866] f1 [56119.032866] f1 [56119.032866] 5d [56119.032867] 5d [56119.032867] c3 [56119.032868] c3 [56119.032868] 66 [56119.032869] 66 [56119.032869] 66 [56119.032869] 66 [56119.032870] 66 [56119.032870] 66 [56119.032871] 90 [56119.032871] 90 [56119.032872] 66 [56119.032872] 66 [56119.032872] 66 [56119.032873] [56119.032874] 66 90 eb d9 66 0f [56123.032172] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kiblnd_sd_00_01:2897] [56123.032173] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [kiblnd_sd_01_03:2903] [56123.032175] Modules linked in: [56123.032176] Modules linked in: [56123.032178] ko2iblnd(OE) [56123.032179] ko2iblnd(OE) [56123.032180] lustre(OE) [56123.032181] lustre(OE) [56123.032182] lmv(OE) [56123.032183] lmv(OE) [56123.032184] fld(OE) [56123.032185] fld(OE) [56123.032186] mdc(OE) [56123.032187] mdc(OE) [56123.032188] fid(OE) [56123.032189] fid(OE) [56123.032190] lov(OE) [56123.032191] lov(OE) [56123.032192] ksocklnd(OE) [56123.032193] ksocklnd(OE) [56123.032194] ptlrpc(OE) [56123.032195] ptlrpc(OE) [56123.032197] obdclass(OE) [56123.032198] obdclass(OE) [56123.032199] lnet(OE) [56123.032200] lnet(OE) [56123.032201] libcfs(OE) [56123.032202] libcfs(OE) [56123.032203] rpcrdma [56123.032204] rpcrdma [56123.032205] ib_isert [56123.032206] ib_isert [56123.032207] iscsi_target_mod [56123.032208] iscsi_target_mod [56123.032209] ib_iser [56123.032210] ib_iser [56123.032212] libiscsi [56123.032213] libiscsi [56123.032214] scsi_transport_iscsi [56123.032215] scsi_transport_iscsi [56123.032216] ib_srpt [56123.032217] ib_srpt [56123.032218] target_core_mod [56123.032220] target_core_mod [56123.032221] ib_srp [56123.032222] ib_srp [56123.032223] scsi_transport_srp [56123.032224] scsi_transport_srp [56123.032225] scsi_tgt [56123.032226] scsi_tgt [56123.032227] ib_ipoib [56123.032228] ib_ipoib [56123.032229] rdma_ucm [56123.032230] rdma_ucm [56123.032231] ib_ucm [56123.032232] ib_ucm [56123.032233] ib_uverbs [56123.032234] ib_uverbs [56123.032235] ib_umad [56123.032236] ib_umad [56123.032237] rdma_cm [56123.032239] rdma_cm [56123.032240] ib_cm [56123.032241] ib_cm [56123.032243] iw_cm [56123.032244] iw_cm [56123.032245] mlx4_ib [56123.032246] mlx4_ib [56123.032247] intel_powerclamp [56123.032248] intel_powerclamp [56123.032249] coretemp [56123.032250] coretemp [56123.032251] intel_rapl [56123.032253] intel_rapl [56123.032254] iosf_mbi [56123.032255] iosf_mbi [56123.032256] kvm [56123.032257] kvm [56123.032259] irqbypass [56123.032260] irqbypass [56123.032261] crc32_pclmul [56123.032262] crc32_pclmul [56123.032263] ghash_clmulni_intel [56123.032264] ghash_clmulni_intel [56123.032265] aesni_intel [56123.032267] aesni_intel [56123.032268] ipmi_ssif [56123.032269] ipmi_ssif [56123.032270] lrw [56123.032271] lrw [56123.032272] gf128mul [56123.032273] gf128mul [56123.032274] glue_helper [56123.032276] glue_helper [56123.032277] hfi1 [56123.032278] hfi1 [56123.032279] ipmi_devintf [56123.032280] ipmi_devintf [56123.032282] rdmavt [56123.032283] rdmavt [56123.032284] ablk_helper [56123.032285] ablk_helper [56123.032286] cryptd [56123.032287] cryptd [56123.032288] iTCO_wdt [56123.032289] iTCO_wdt [56123.032290] ipmi_si [56123.032291] ipmi_si [56123.032292] sb_edac [56123.032293] sb_edac [56123.032294] sg [56123.032295] sg [56123.032296] iTCO_vendor_support [56123.032297] iTCO_vendor_support [56123.032298] ib_core [56123.032299] ib_core [56123.032300] ioatdma [56123.032301] ioatdma [56123.032302] i2c_i801 [56123.032303] i2c_i801 [56123.032304] mei_me [56123.032305] mei_me [56123.032307] mei [56123.032308] mei [56123.032309] edac_core [56123.032310] edac_core [56123.032311] ipmi_msghandler [56123.032312] ipmi_msghandler [56123.032313] lpc_ich [56123.032314] lpc_ich [56123.032315] ntb [56123.032317] ntb [56123.032318] wmi [56123.032319] wmi [56123.032320] shpchp [56123.032321] shpchp [56123.032322] pcspkr [56123.032323] pcspkr [56123.032325] nfsd [56123.032326] nfsd [56123.032327] nfs_acl [56123.032328] nfs_acl [56123.032329] lockd [56123.032330] lockd [56123.032331] grace [56123.032332] grace [56123.032333] auth_rpcgss [56123.032334] auth_rpcgss [56123.032335] sunrpc [56123.032337] sunrpc [56123.032338] ip_tables [56123.032339] ip_tables [56123.032340] ext4 [56123.032341] ext4 [56123.032342] mbcache [56123.032343] mbcache [56123.032344] jbd2 [56123.032345] jbd2 [56123.032346] sd_mod [56123.032348] sd_mod [56123.032349] crc_t10dif [56123.032350] crc_t10dif [56123.032351] crct10dif_generic [56123.032352] crct10dif_generic [56123.032353] mlx4_en [56123.032354] mlx4_en [56123.032355] mgag200 [56123.032356] mgag200 [56123.032358] drm_kms_helper [56123.032359] drm_kms_helper [56123.032360] syscopyarea [56123.032361] syscopyarea [56123.032362] sysfillrect [56123.032363] sysfillrect [56123.032364] sysimgblt [56123.032366] sysimgblt [56123.032367] fb_sys_fops [56123.032368] fb_sys_fops [56123.032369] isci [56123.032370] isci [56123.032371] ttm [56123.032372] ttm [56123.032373] igb [56123.032375] igb [56123.032375] crct10dif_pclmul [56123.032377] crct10dif_pclmul [56123.032377] crct10dif_common [56123.032379] crct10dif_common [56123.032380] ahci [56123.032381] ahci [56123.032382] ptp [56123.032383] ptp [56123.032385] crc32c_intel [56123.032386] crc32c_intel [56123.032387] libsas [56123.032388] libsas [56123.032389] libahci [56123.032390] libahci [56123.032391] pps_core [56123.032392] pps_core [56123.032393] drm [56123.032394] drm [56123.032395] mlx4_core [56123.032397] mlx4_core [56123.032398] mpt2sas [56123.032399] mpt2sas [56123.032400] dca [56123.032401] dca [56123.032402] libata [56123.032404] libata [56123.032404] raid_class [56123.032406] raid_class [56123.032407] i2c_algo_bit [56123.032408] i2c_algo_bit [56123.032409] scsi_transport_sas [56123.032410] scsi_transport_sas [56123.032411] devlink [56123.032412] devlink [56123.032413] i2c_core [56123.032414] i2c_core [56123.032416] fjes [56123.032417] [56123.032418] fjes [56123.032420] CPU: 6 PID: 2897 Comm: kiblnd_sd_00_01 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56123.032422] CPU: 12 PID: 2903 Comm: kiblnd_sd_01_03 Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56123.032423] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56123.032424] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56123.032426] task: ffff8804c1675e20 ti: ffff88081f64c000 task.ti: ffff88081f64c000 [56123.032427] task: ffff880826310fb0 ti: ffff8808213b4000 task.ti: ffff8808213b4000 [56123.032430] RIP: 0010:[] [56123.032432] [] _raw_spin_lock+0x3a/0x50 [56123.032434] RIP: 0010:[] [] _raw_spin_lock+0x32/0x50 [56123.032435] RSP: 0018:ffff88081f64fc70 EFLAGS: 00000206 [56123.032436] RSP: 0018:ffff8808213b7a98 EFLAGS: 00000216 [56123.032437] RAX: 0000000000000822 RBX: ffff88083fb5e000 RCX: 0000000000000a1e [56123.032438] RAX: 00000000000062f1 RBX: ffff88082c5f36d8 RCX: 0000000000000a1e [56123.032440] RDX: 0000000000000a2c RSI: 0000000000000a2c RDI: ffff88042b272d00 [56123.032441] RDX: 0000000000000a28 RSI: 0000000000000a28 RDI: ffff88042b272d00 [56123.032442] RBP: ffff88081f64fc70 R08: ffff8803ee33ed80 R09: 0000000000000002 [56123.032443] RBP: ffff8808213b7a98 R08: 0000000000000000 R09: 0001fd60baf0016e [56123.032445] R10: ffff88042aa84ab8 R11: ffff880819f7ff98 R12: 00000000f1dbfe3f [56123.032446] R10: ffff8808213b7c48 R11: 0000000000000000 R12: 0000000000000001 [56123.032447] R13: 0000000000001000 R14: ffff88042d8fe000 R15: 00050000c0a8016e [56123.032449] R13: ffff88082c5f36a8 R14: ffff88082c933800 R15: ffff8808213b7a78 [56123.032450] FS: 0000000000000000(0000) GS:ffff88042e180000(0000) knlGS:0000000000000000 [56123.032451] FS: 0000000000000000(0000) GS:ffff88082d900000(0000) knlGS:0000000000000000 [56123.032453] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56123.032454] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [56123.032456] CR2: 00007f5a85968000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56123.032457] CR2: 00007f03566ca000 CR3: 00000000019ba000 CR4: 00000000000407e0 [56123.032458] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56123.032459] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [56123.032461] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56123.032462] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [56123.032463] Stack: [56123.032463] Stack: [56123.032465] ffff88081f64fc98 [56123.032466] ffff8808213b7ac0 [56123.032468] ffffffffa08aa7c8 [56123.032469] ffffffffa08aa7c8 [56123.032470] ffffffffa08aa8b6 [56123.032471] ffffffffa08aa8b6 [56123.032472] ffff88042dcd1e00 [56123.032473] [56123.032474] 0000000000000000 [56123.032476] ffff8800b4864200 [56123.032477] ffff88083fb5e000 [56123.032478] ffff88081f64fcd8 [56123.032479] ffff8808213b7b80 [56123.032480] ffffffffa092e857 [56123.032481] ffffffffa092c9b5 [56123.032482] ffff8800b484d600 [56123.032483] [56123.032484] 00000000000000c0 [56123.032486] ffff88042dcd1e00 [56123.032487] ffffc9000e335900 [56123.032488] 0000000000000000 [56123.032489] ffff8808213b7b68 [56123.032490] ffff88041d604a00 [56123.032492] ffffffffa0ebd587 [56123.032492] 0000000000000000 [56123.032493] [56123.032494] ffffffffffffffff [56123.032495] Call Trace: [56123.032496] Call Trace: [56123.032504] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56123.032510] [] cfs_percpt_lock+0x58/0x110 [libcfs] [56123.032517] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56123.032522] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56123.032530] [] lnet_return_tx_credits_locked+0x227/0x480 [lnet] [56123.032537] [] lnet_select_pathway+0x235/0x1140 [lnet] [56123.032544] [] lnet_msg_decommit+0xd0/0x6c0 [lnet] [56123.032549] [] ? kiblnd_send+0x357/0xa10 [ko2iblnd] [56123.032556] [] lnet_finalize+0x1e9/0x690 [lnet] [56123.032562] [] lnet_send+0x51/0x180 [lnet] [56123.032568] [] kiblnd_tx_done+0x106/0x3d0 [ko2iblnd] [56123.032574] [] lnet_finalize+0x3f1/0x690 [lnet] [56123.032579] [] kiblnd_scheduler+0xd90/0x1130 [ko2iblnd] [56123.032583] [] kiblnd_recv+0x1cd/0x7b0 [ko2iblnd] [56123.032586] [] ? dequeue_task_fair+0x41e/0x660 [56123.032589] [] ? __slab_free+0x81/0x2f0 [56123.032592] [] ? wake_up_state+0x20/0x20 [56123.032598] [] lnet_ni_recv+0xc3/0x320 [lnet] [56123.032604] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56123.032610] [] lnet_parse+0x945/0xf90 [lnet] [56123.032614] [] kthread+0xcf/0xe0 [56123.032619] [] ? cfs_percpt_unlock+0x36/0xc0 [libcfs] [56123.032622] [] ? kthread_create_on_node+0x140/0x140 [56123.032626] [] kiblnd_handle_rx+0x20b/0x690 [ko2iblnd] [56123.032629] [] ret_from_fork+0x58/0x90 [56123.032633] [] ? rvt_poll_cq+0xde/0xf0 [rdmavt] [56123.032635] [] ? kthread_create_on_node+0x140/0x140 [56123.032641] [] kiblnd_scheduler+0xef3/0x1130 [ko2iblnd] [56123.032642] Code: c2 [56123.032645] [] ? dequeue_task_fair+0x41e/0x660 [56123.032647] c1 ea [56123.032650] [] ? wake_up_state+0x20/0x20 [56123.032651] 10 66 [56123.032656] [] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd] [56123.032658] 39 c2 [56123.032661] [] kthread+0xcf/0xe0 [56123.032663] 75 01 [56123.032665] [] ? kthread_create_on_node+0x140/0x140 [56123.032667] c3 55 [56123.032671] [] ret_from_fork+0x58/0x90 [56123.032672] 83 e2 [56123.032675] [] ? kthread_create_on_node+0x140/0x140 [56123.032677] fe 0f [56123.032678] Code: [56123.032679] b7 [56123.032680] 00 [56123.032681] f2 [56123.032682] 02 [56123.032683] 48 [56123.032684] 00 [56123.032685] 89 [56123.032686] f0 [56123.032687] e5 [56123.032688] 0f [56123.032689] b8 [56123.032690] c1 [56123.032691] 00 [56123.032692] 07 [56123.032693] 80 [56123.032694] 89 [56123.032696] 00 [56123.032697] c2 [56123.032698] 00 [56123.032699] c1 [56123.032700] eb [56123.032701] ea [56123.032702] 0d [56123.032703] 10 [56123.032704] 66 [56123.032705] 66 [56123.032706] 0f [56123.032707] 39 [56123.032708] 1f [56123.032709] c2 [56123.032710] 44 [56123.032711] 75 [56123.032712] 00 [56123.032713] 01 [56123.032713] 00 [56123.032714] c3 [56123.032715] f3 [56123.032716] 55 [56123.032717] 90 [56123.032718] 83 [56123.032719] 83 [56123.032720] e2 [56123.032721] e8 [56123.032722] fe [56123.032723] 01 [56123.032724] 0f [56123.032725] 74 [56123.032726] b7 [56123.032728] 0a [56123.032729] f2 [56123.032730] 0f [56123.032731] 48 [56123.032732] b7 [56123.032733] 89 [56123.032734] 0f [56123.032735] e5 [56123.032736] <66> [56123.032737] b8 [56123.032738] 39 [56123.032739] 00 [56123.032740] ca [56123.032741] 80 [56123.032742] 75 [56123.032743] 00 [56123.032744] f1 [56123.032745] 00 [56123.032746] 5d [56123.032747] eb [56123.032748] c3 [56123.032750] 0d [56123.032751] 66 [56123.032752] 66 [56123.032753] 66 [56123.032754] 0f [56123.032755] 66 [56123.032756] 1f [56123.032757] 90 [56123.032758] 44 [56123.032759] 66 [56123.032760] 00 [56123.032761] 66 [56123.032762] 00 [56123.032763] 90 [56123.032764] f3 [56123.032765] eb [56123.032766] 90 [56123.032767] d9 [56123.032768] <83> [56123.032769] 66 [56123.032770] e8 [56123.032771] 0f [56123.032772] 01 [56123.032773] 1f [56123.032774] 74 [56123.032775] 44 [56123.032776] 0a [56123.032777] 00 [56123.032778] [56123.032782] 0f b7 0f 66 39 ca 75 f1 5d c3 66 66 66 90 66 66 [56126.997161] NMI watchdog: Watchdog detected hard LOCKUP on cpu 8 [56126.997179] Modules linked in: ko2iblnd(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel ipmi_ssif lrw gf128mul glue_helper hfi1 ipmi_devintf rdmavt ablk_helper cryptd iTCO_wdt ipmi_si sb_edac sg iTCO_vendor_support ib_core ioatdma i2c_i801 mei_me mei edac_core ipmi_msghandler lpc_ich ntb wmi shpchp pcspkr nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en mgag200 drm_kms_helper [56126.997185] syscopyarea sysfillrect sysimgblt fb_sys_fops isci ttm igb crct10dif_pclmul crct10dif_common ahci ptp crc32c_intel libsas libahci pps_core drm mlx4_core mpt2sas dca libata raid_class i2c_algo_bit scsi_transport_sas devlink i2c_core fjes [56126.997187] CPU: 8 PID: 10 Comm: rcu_sched Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56126.997187] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56126.997189] 0000000000000000 000000008d1421ea ffff88082d805ba8 ffffffff816862ac [56126.997190] ffff88082d805bc0 ffffffff8112f217 ffff8804c1508400 ffff88082d805bf8 [56126.997191] ffffffff811747ae 0000000000000001 ffff88082d80a280 ffff8804c1508400 [56126.997191] Call Trace: [56126.997195] [] dump_stack+0x19/0x1b [56126.997198] [] watchdog_overflow_callback+0xd7/0x100 [56126.997201] [] __perf_event_overflow+0x8e/0x1f0 [56126.997203] [] perf_event_overflow+0x14/0x20 [56126.997207] [] intel_pmu_handle_irq+0x1f8/0x4e0 [56126.997210] [] ? ioremap_page_range+0x27c/0x3e0 [56126.997212] [] ? vunmap_page_range+0x1c4/0x310 [56126.997213] [] ? unmap_kernel_range_noflush+0x11/0x20 [56126.997217] [] ? ghes_copy_tofrom_phys+0x124/0x210 [56126.997219] [] ? ghes_read_estatus+0xa0/0x190 [56126.997220] [] perf_event_nmi_handler+0x2b/0x50 [56126.997221] [] nmi_handle.isra.0+0x69/0xb0 [56126.997223] [] do_nmi+0x133/0x410 [56126.997224] [] end_repeat_nmi+0x1e/0x2e [56126.997225] [] ? _raw_spin_lock_irqsave+0x47/0x60 [56126.997226] [] ? _raw_spin_lock_irqsave+0x47/0x60 [56126.997227] [] ? _raw_spin_lock_irqsave+0x47/0x60 [56126.997229] <> [] force_qs_rnp+0x54/0x170 [56126.997230] [] ? rcu_eqs_exit+0xa0/0xa0 [56126.997231] [] rcu_gp_fqs+0x39/0xb0 [56126.997232] [] rcu_gp_kthread+0x3b1/0x7e0 [56126.997234] [] ? wake_up_atomic_t+0x30/0x30 [56126.997235] [] ? rcu_gp_fqs+0xb0/0xb0 [56126.997237] [] kthread+0xcf/0xe0 [56126.997238] [] ? kthread_create_on_node+0x140/0x140 [56126.997240] [] ret_from_fork+0x58/0x90 [56126.997241] [] ? kthread_create_on_node+0x140/0x140 [56126.997242] Kernel panic - not syncing: Hard LOCKUP [56126.997243] CPU: 8 PID: 10 Comm: rcu_sched Tainted: G OEL ------------ 3.10.0-514.6.1.el7.x86_64 #1 [56126.997244] Hardware name: Intel Corporation S2600GZ ........../S2600GZ, BIOS SE5C600.86B.01.08.0003.022620131521 02/26/2013 [56126.997245] ffffffff818d9828 000000008d1421ea ffff88082d805b18 ffffffff816862ac [56126.997246] ffff88082d805b98 ffffffff8167f6b3 0000000b00000010 ffff88082d805ba8 [56126.997247] ffff88082d805b48 000000008d1421ea ffff88082d805b88 ffffffff818d952e [56126.997247] Call Trace: [56126.997248] [] dump_stack+0x19/0x1b [56126.997251] [] panic+0xe3/0x1f2 [56126.997253] [] nmi_panic+0x3f/0x40 [56126.997255] [] watchdog_overflow_callback+0xf6/0x100 [56126.997256] [] __perf_event_overflow+0x8e/0x1f0 [56126.997257] [] perf_event_overflow+0x14/0x20 [56126.997259] [] intel_pmu_handle_irq+0x1f8/0x4e0 [56126.997260] [] ? ioremap_page_range+0x27c/0x3e0 [56126.997261] [] ? vunmap_page_range+0x1c4/0x310 [56126.997262] [] ? unmap_kernel_range_noflush+0x11/0x20 [56126.997264] [] ? ghes_copy_tofrom_phys+0x124/0x210 [56126.997265] [] ? ghes_read_estatus+0xa0/0x190 [56126.997266] [] perf_event_nmi_handler+0x2b/0x50 [56126.997267] [] nmi_handle.isra.0+0x69/0xb0 [56126.997268] [] do_nmi+0x133/0x410 [56126.997269] [] end_repeat_nmi+0x1e/0x2e [56126.997271] [] ? _raw_spin_lock_irqsave+0x47/0x60 [56126.997271] [] ? _raw_spin_lock_irqsave+0x47/0x60 [56126.997272] [] ? _raw_spin_lock_irqsave+0x47/0x60 [56126.997274] <> [] force_qs_rnp+0x54/0x170 [56126.997274] [] ? rcu_eqs_exit+0xa0/0xa0 [56126.997276] [] rcu_gp_fqs+0x39/0xb0 [56126.997277] [] rcu_gp_kthread+0x3b1/0x7e0 [56126.997278] [] ? wake_up_atomic_t+0x30/0x30 [56126.997279] [] ? rcu_gp_fqs+0xb0/0xb0 [56126.997280] [] kthread+0xcf/0xe0 [56126.997282] [] ? kthread_create_on_node+0x140/0x140 [56126.997283] [] ret_from_fork+0x58/0x90 [56126.997285] [] ? kthread_create_on_node+0x140/0x140