[83147.112641] crct10dif_common [83147.112641] mlx4_core [83147.112642] mpt3sas [83147.112642] drm [83147.112642] libahci [83147.112643] crc32c_intel [83147.112643] ptp [83147.112643] raid_class [83147.112643] libata [83147.112644] megaraid_sas [83147.112644] devlink [83147.112645] i2c_core [83147.112645] scsi_transport_sas [83147.112645] pps_core [83147.112646] [83147.112647] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83147.112647] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83147.112648] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83147.112649] RIP: 0010:[] [83147.112650] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83147.112651] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83147.112652] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83147.112652] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83147.112653] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83147.112654] R10: ffff882021a69800 R11: 0000000000000001 R12: ffff88203d0b5f48 [83147.112654] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83147.112655] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83147.112656] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83147.112656] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83147.112657] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83147.112657] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83147.112658] Stack: [83147.112658] ffff881ffba83b80 [83147.112658] ffffffff8169e61f [83147.112658] ffff881ffba83b90 [83147.112659] ffffffff816abb70 [83147.112659] [83147.112659] ffff881ffba83bd0 [83147.112660] ffffffffc0b72198 [83147.112660] 0000000000000000 [83147.112660] ffff882021a69800 [83147.112660] [83147.112661] ffff881ffba83c18 [83147.112661] ffff882021a69860 [83147.112661] ffff882021a69800 [83147.112661] ffff8810175b8000 [83147.112662] [83147.112662] Call Trace: [83147.112662] [83147.112664] [] queued_spin_lock_slowpath+0xb/0xf [83147.112664] [83147.112665] [] _raw_spin_lock+0x20/0x30 [83147.112666] [83147.112686] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83147.112687] [83147.112706] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83147.112706] [83147.112725] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83147.112725] [83147.112742] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83147.112743] [83147.112760] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83147.112761] [83147.112778] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83147.112778] [83147.112783] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83147.112783] [83147.112801] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83147.112801] [83147.112818] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83147.112819] [83147.112824] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83147.112824] [83147.112841] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83147.112842] [83147.112859] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83147.112860] [83147.112881] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83147.112882] [83147.112883] [] ? __schedule+0x39d/0x8b0 [83147.112883] [83147.112903] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83147.112903] [83147.112904] [] kthread+0xcf/0xe0 [83147.112905] [83147.112906] [] ? insert_kthread_work+0x40/0x40 [83147.112906] [83147.112907] [] ret_from_fork+0x58/0x90 [83147.112907] [83147.112908] [] ? insert_kthread_work+0x40/0x40 [83147.112909] Code: [83147.112909] 0d [83147.112909] 48 [83147.112910] 98 [83147.112910] 83 [83147.112910] e2 [83147.112910] 30 [83147.112911] 48 [83147.112911] 81 [83147.112911] c2 [83147.112911] 80 [83147.112912] 78 [83147.112912] 01 [83147.112912] 00 [83147.112912] 48 [83147.112913] 03 [83147.112913] 14 [83147.112913] c5 [83147.112913] e0 [83147.112914] fd [83147.112914] b0 [83147.112914] 81 [83147.112914] 4c [83147.112915] 89 [83147.112915] 02 [83147.112915] 41 [83147.112915] 8b [83147.112916] 40 [83147.112916] 08 [83147.112916] 85 [83147.112917] c0 [83147.112917] 75 [83147.112917] 0f [83147.112917] 0f [83147.112918] 1f [83147.112918] 44 [83147.112918] 00 [83147.112918] 00 [83147.112919] f3 [83147.112919] 90 [83147.112919] 41 [83147.112919] 8b [83147.112920] 40 [83147.112920] 08 [83147.112920] <85> [83147.112920] c0 [83147.112921] 74 [83147.112921] f6 [83147.112921] 4d [83147.112921] 8b [83147.112922] 08 [83147.112922] 4d [83147.112922] 85 [83147.112922] c9 [83147.112923] 74 [83147.112923] 04 [83147.112923] 41 [83147.112923] 0f [83147.112924] 18 [83147.112924] 09 [83147.112924] 8b [83147.112924] 17 [83147.112925] 0f [83147.112925] b7 [83147.112925] c2 [83147.112925] [83147.118608] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ldlm_bl_20:33193] [83147.118608] Modules linked in: [83147.118608] mgs(OE) [83147.118609] mgc(OE) [83147.118609] osd_ldiskfs(OE) [83147.118610] lquota(OE) [83147.118610] ldiskfs(OE) [83147.118610] lustre(OE) [83147.118611] lmv(OE) [83147.118611] mdc(OE) [83147.118611] lov(OE) [83147.118612] fid(OE) [83147.118612] fld(OE) [83147.118612] ko2iblnd(OE) [83147.118613] ptlrpc(OE) [83147.118613] obdclass(OE) [83147.118613] lnet(OE) [83147.118614] libcfs(OE) [83147.118614] vfat [83147.118614] fat [83147.118615] uas [83147.118615] usb_storage [83147.118615] mpt2sas [83147.118615] mptctl [83147.118616] mptbase [83147.118616] rpcsec_gss_krb5 [83147.118616] dell_rbu [83147.118617] nfsv4 [83147.118617] dns_resolver [83147.118617] nfs [83147.118618] fscache [83147.118618] ib_ipoib [83147.118618] rdma_ucm [83147.118619] ib_ucm [83147.118619] ib_uverbs [83147.118619] ib_umad [83147.118619] rdma_cm [83147.118620] ib_cm [83147.118620] iw_cm [83147.118620] mlx4_ib [83147.118621] ib_core [83147.118621] sb_edac [83147.118621] edac_core [83147.118622] intel_powerclamp [83147.118622] coretemp [83147.118622] intel_rapl [83147.118622] iosf_mbi [83147.118623] dm_service_time [83147.118623] kvm_intel [83147.118624] kvm [83147.118624] irqbypass [83147.118624] crc32_pclmul [83147.118625] ghash_clmulni_intel [83147.118625] aesni_intel [83147.118625] lrw [83147.118625] gf128mul [83147.118626] glue_helper [83147.118626] ablk_helper [83147.118626] cryptd [83147.118627] iTCO_wdt [83147.118627] dcdbas [83147.118628] iTCO_vendor_support [83147.118628] ipmi_si [83147.118628] ipmi_devintf [83147.118628] mxm_wmi [83147.118629] dm_round_robin [83147.118629] pcspkr [83147.118629] sg [83147.118630] ipmi_msghandler [83147.118631] acpi_power_meter [83147.118631] wmi [83147.118631] mei_me [83147.118631] mei [83147.118632] shpchp [83147.118632] lpc_ich [83147.118632] nfsd [83147.118633] auth_rpcgss [83147.118633] dm_multipath [83147.118633] dm_mod [83147.118634] nfs_acl [83147.118634] lockd [83147.118634] grace [83147.118635] sunrpc [83147.118635] ip_tables [83147.118635] ext4 [83147.118635] mbcache [83147.118636] jbd2 [83147.118636] sd_mod [83147.118637] crc_t10dif [83147.118637] crct10dif_generic [83147.118637] mlx4_en [83147.118638] i2c_algo_bit [83147.118638] drm_kms_helper [83147.118638] syscopyarea [83147.118639] sysfillrect [83147.118639] sysimgblt [83147.118639] fb_sys_fops [83147.118640] ttm [83147.118640] tg3 [83147.118640] ahci [83147.118641] crct10dif_pclmul [83147.118641] crct10dif_common [83147.118641] mlx4_core [83147.118642] mpt3sas [83147.118642] drm [83147.118642] libahci [83147.118643] crc32c_intel [83147.118643] ptp [83147.118643] raid_class [83147.118644] libata [83147.118644] megaraid_sas [83147.118644] devlink [83147.118644] i2c_core [83147.118645] scsi_transport_sas [83147.118645] pps_core [83147.118646] [83147.118647] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83147.118647] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83147.118648] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83147.118649] RIP: 0010:[] [83147.118651] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83147.118651] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83147.118652] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83147.118653] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83147.118653] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83147.118654] R10: ffff882021a50400 R11: 0000000000000000 R12: ffff88103ce96cc0 [83147.118654] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83147.118655] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83147.118656] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83147.118656] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83147.118657] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83147.118657] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83147.118658] Stack: [83147.118658] ffff881ff8e2fb80 [83147.118658] ffffffff8169e61f [83147.118659] ffff881ff8e2fb90 [83147.118659] ffffffff816abb70 [83147.118659] [83147.118659] ffff881ff8e2fbd0 [83147.118660] ffffffffc0b72198 [83147.118660] 0000000000000000 [83147.118660] ffff882021a50400 [83147.118660] [83147.118661] ffff881ff8e2fc18 [83147.118661] ffff882021a50460 [83147.118661] ffff882021a50400 [83147.118662] ffff8810175b8000 [83147.118662] [83147.118662] Call Trace: [83147.118662] [83147.118664] [] queued_spin_lock_slowpath+0xb/0xf [83147.118664] [83147.118666] [] _raw_spin_lock+0x20/0x30 [83147.118666] [83147.118684] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83147.118685] [83147.118703] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83147.118704] [83147.118722] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83147.118723] [83147.118740] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83147.118741] [83147.118758] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83147.118758] [83147.118775] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83147.118776] [83147.118781] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83147.118781] [83147.118798] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83147.118799] [83147.118816] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83147.118816] [83147.118821] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83147.118822] [83147.118839] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83147.118839] [83147.118857] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83147.118857] [83147.118879] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83147.118879] [83147.118880] [] ? __schedule+0x39d/0x8b0 [83147.118881] [83147.118901] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83147.118901] [83147.118902] [] kthread+0xcf/0xe0 [83147.118902] [83147.118903] [] ? insert_kthread_work+0x40/0x40 [83147.118904] [83147.118905] [] ret_from_fork+0x58/0x90 [83147.118905] [83147.118906] [] ? insert_kthread_work+0x40/0x40 [83147.118906] Code: [83147.118907] 0d [83147.118907] 48 [83147.118907] 98 [83147.118908] 83 [83147.118908] e2 [83147.118908] 30 [83147.118908] 48 [83147.118909] 81 [83147.118909] c2 [83147.118909] 80 [83147.118909] 78 [83147.118910] 01 [83147.118910] 00 [83147.118910] 48 [83147.118910] 03 [83147.118911] 14 [83147.118911] c5 [83147.118911] e0 [83147.118911] fd [83147.118912] b0 [83147.118912] 81 [83147.118912] 4c [83147.118912] 89 [83147.118913] 02 [83147.118913] 41 [83147.118913] 8b [83147.118913] 40 [83147.118914] 08 [83147.118914] 85 [83147.118914] c0 [83147.118914] 75 [83147.118915] 0f [83147.118915] 0f [83147.118915] 1f [83147.118915] 44 [83147.118916] 00 [83147.118916] 00 [83147.118916] f3 [83147.118916] 90 [83147.118917] 41 [83147.118917] 8b [83147.118917] 40 [83147.118917] 08 [83147.118918] <85> [83147.118918] c0 [83147.118918] 74 [83147.118919] f6 [83147.118919] 4d [83147.118919] 8b [83147.118919] 08 [83147.118920] 4d [83147.118920] 85 [83147.118920] c9 [83147.118920] 74 [83147.118921] 04 [83147.118921] 41 [83147.118921] 0f [83147.118921] 18 [83147.118922] 09 [83147.118922] 8b [83147.118922] 17 [83147.118922] 0f [83147.118923] b7 [83147.118923] c2 [83147.118923] [83147.124607] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ldlm_bl_04:16006] [83147.124608] Modules linked in: [83147.124608] mgs(OE) [83147.124608] mgc(OE) [83147.124609] osd_ldiskfs(OE) [83147.124609] lquota(OE) [83147.124609] ldiskfs(OE) [83147.124610] lustre(OE) [83147.124610] lmv(OE) [83147.124610] mdc(OE) [83147.124611] lov(OE) [83147.124611] fid(OE) [83147.124612] fld(OE) [83147.124612] ko2iblnd(OE) [83147.124612] ptlrpc(OE) [83147.124613] obdclass(OE) [83147.124613] lnet(OE) [83147.124613] libcfs(OE) [83147.124614] vfat [83147.124614] fat [83147.124614] uas [83147.124615] usb_storage [83147.124615] mpt2sas [83147.124615] mptctl [83147.124615] mptbase [83147.124616] rpcsec_gss_krb5 [83147.124616] dell_rbu [83147.124616] nfsv4 [83147.124617] dns_resolver [83147.124617] nfs [83147.124617] fscache [83147.124618] ib_ipoib [83147.124618] rdma_ucm [83147.124619] ib_ucm [83147.124619] ib_uverbs [83147.124619] ib_umad [83147.124620] rdma_cm [83147.124620] ib_cm [83147.124620] iw_cm [83147.124621] mlx4_ib [83147.124621] ib_core [83147.124621] sb_edac [83147.124622] edac_core [83147.124622] intel_powerclamp [83147.124622] coretemp [83147.124623] intel_rapl [83147.124623] iosf_mbi [83147.124623] dm_service_time [83147.124624] kvm_intel [83147.124624] kvm [83147.124624] irqbypass [83147.124625] crc32_pclmul [83147.124625] ghash_clmulni_intel [83147.124625] aesni_intel [83147.124626] lrw [83147.124626] gf128mul [83147.124626] glue_helper [83147.124627] ablk_helper [83147.124627] cryptd [83147.124627] iTCO_wdt [83147.124628] dcdbas [83147.124628] iTCO_vendor_support [83147.124629] ipmi_si [83147.124629] ipmi_devintf [83147.124629] mxm_wmi [83147.124630] dm_round_robin [83147.124630] pcspkr [83147.124630] sg [83147.124630] ipmi_msghandler [83147.124631] acpi_power_meter [83147.124631] wmi [83147.124632] mei_me [83147.124632] mei [83147.124632] shpchp [83147.124632] lpc_ich [83147.124633] nfsd [83147.124633] auth_rpcgss [83147.124633] dm_multipath [83147.124634] dm_mod [83147.124634] nfs_acl [83147.124634] lockd [83147.124635] grace [83147.124635] sunrpc [83147.124635] ip_tables [83147.124636] ext4 [83147.124636] mbcache [83147.124636] jbd2 [83147.124637] sd_mod [83147.124637] crc_t10dif [83147.124637] crct10dif_generic [83147.124638] mlx4_en [83147.124638] i2c_algo_bit [83147.124638] drm_kms_helper [83147.124638] syscopyarea [83147.124639] sysfillrect [83147.124639] sysimgblt [83147.124639] fb_sys_fops [83147.124640] ttm [83147.124640] tg3 [83147.124640] ahci [83147.124641] crct10dif_pclmul [83147.124641] crct10dif_common [83147.124641] mlx4_core [83147.124642] mpt3sas [83147.124642] drm [83147.124642] libahci [83147.124643] crc32c_intel [83147.124643] ptp [83147.124643] raid_class [83147.124644] libata [83147.124644] megaraid_sas [83147.124644] devlink [83147.124645] i2c_core [83147.124645] scsi_transport_sas [83147.124645] pps_core [83147.124646] [83147.124647] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83147.124647] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83147.124648] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83147.124649] RIP: 0010:[] [83147.124650] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83147.124651] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83147.124652] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83147.124652] RDX: ffff88103d057880 RSI: 0000000000910000 RDI: ffff88102137981c [83147.124653] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83147.124654] R10: ffff882021aef000 R11: 0000000000000000 R12: ffff88201c627c58 [83147.124654] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83147.124655] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83147.124656] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83147.124656] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83147.124657] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83147.124658] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83147.124658] Stack: [83147.124658] ffff88201c627b80 [83147.124658] ffffffff8169e61f [83147.124659] ffff88201c627b90 [83147.124659] ffffffff816abb70 [83147.124659] [83147.124660] ffff88201c627bd0 [83147.124660] ffffffffc0b72198 [83147.124660] 0000000000000000 [83147.124660] ffff882021aef000 [83147.124661] [83147.124661] ffff88201c627c18 [83147.124661] ffff882021aef060 [83147.124661] ffff882021aef000 [83147.124662] ffff8810175b8000 [83147.124662] [83147.124662] Call Trace: [83147.124663] [83147.124664] [] queued_spin_lock_slowpath+0xb/0xf [83147.124665] [83147.124666] [] _raw_spin_lock+0x20/0x30 [83147.124666] [83147.124684] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83147.124685] [83147.124706] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83147.124706] [83147.124724] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83147.124725] [83147.124742] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83147.124742] [83147.124760] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83147.124760] [83147.124777] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83147.124778] [83147.124783] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83147.124783] [83147.124801] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83147.124801] [83147.124818] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83147.124819] [83147.124824] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83147.124824] [83147.124841] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83147.124842] [83147.124859] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83147.124860] [83147.124881] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83147.124882] [83147.124883] [] ? wake_up_state+0x20/0x20 [83147.124884] [83147.124904] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83147.124904] [83147.124905] [] kthread+0xcf/0xe0 [83147.124905] [83147.124906] [] ? insert_kthread_work+0x40/0x40 [83147.124907] [83147.124908] [] ret_from_fork+0x58/0x90 [83147.124908] [83147.124909] [] ? insert_kthread_work+0x40/0x40 [83147.124909] Code: [83147.124910] 0d [83147.124910] 48 [83147.124910] 98 [83147.124911] 83 [83147.124911] e2 [83147.124911] 30 [83147.124911] 48 [83147.124912] 81 [83147.124912] c2 [83147.124912] 80 [83147.124912] 78 [83147.124913] 01 [83147.124913] 00 [83147.124913] 48 [83147.124913] 03 [83147.124914] 14 [83147.124914] c5 [83147.124914] e0 [83147.124915] fd [83147.124915] b0 [83147.124915] 81 [83147.124915] 4c [83147.124916] 89 [83147.124916] 02 [83147.124916] 41 [83147.124916] 8b [83147.124917] 40 [83147.124917] 08 [83147.124917] 85 [83147.124917] c0 [83147.124918] 75 [83147.124918] 0f [83147.124918] 0f [83147.124918] 1f [83147.124919] 44 [83147.124919] 00 [83147.124919] 00 [83147.124919] f3 [83147.124920] 90 [83147.124920] 41 [83147.124920] 8b [83147.124921] 40 [83147.124921] 08 [83147.124921] <85> [83147.124921] c0 [83147.124922] 74 [83147.124922] f6 [83147.124922] 4d [83147.124922] 8b [83147.124923] 08 [83147.124923] 4d [83147.124923] 85 [83147.124923] c9 [83147.124924] 74 [83147.124924] 04 [83147.124924] 41 [83147.124925] 0f [83147.124925] 18 [83147.124925] 09 [83147.124925] 8b [83147.124926] 17 [83147.124926] 0f [83147.124926] b7 [83147.124926] c2 [83147.124927] [83151.064522] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ldlm_bl_16:33189] [83151.064523] Modules linked in: [83151.064523] mgs(OE) [83151.064524] mgc(OE) [83151.064524] osd_ldiskfs(OE) [83151.064525] lquota(OE) [83151.064525] ldiskfs(OE) [83151.064525] lustre(OE) [83151.064526] lmv(OE) [83151.064526] mdc(OE) [83151.064526] lov(OE) [83151.064527] fid(OE) [83151.064527] fld(OE) [83151.064527] ko2iblnd(OE) [83151.064528] ptlrpc(OE) [83151.064528] obdclass(OE) [83151.064528] lnet(OE) [83151.064529] libcfs(OE) [83151.064529] vfat [83151.064529] fat [83151.064530] uas [83151.064530] usb_storage [83151.064530] mpt2sas [83151.064531] mptctl [83151.064531] mptbase [83151.064531] rpcsec_gss_krb5 [83151.064532] dell_rbu [83151.064532] nfsv4 [83151.064532] dns_resolver [83151.064532] nfs [83151.064533] fscache [83151.064533] ib_ipoib [83151.064533] rdma_ucm [83151.064534] ib_ucm [83151.064534] ib_uverbs [83151.064534] ib_umad [83151.064535] rdma_cm [83151.064535] ib_cm [83151.064535] iw_cm [83151.064535] mlx4_ib [83151.064536] ib_core [83151.064536] sb_edac [83151.064537] edac_core [83151.064537] intel_powerclamp [83151.064537] coretemp [83151.064538] intel_rapl [83151.064538] iosf_mbi [83151.064538] dm_service_time [83151.064539] kvm_intel [83151.064539] kvm [83151.064539] irqbypass [83151.064539] crc32_pclmul [83151.064540] ghash_clmulni_intel [83151.064540] aesni_intel [83151.064540] lrw [83151.064541] gf128mul [83151.064541] glue_helper [83151.064541] ablk_helper [83151.064542] cryptd [83151.064542] iTCO_wdt [83151.064542] dcdbas [83151.064542] iTCO_vendor_support [83151.064543] ipmi_si [83151.064543] ipmi_devintf [83151.064543] mxm_wmi [83151.064544] dm_round_robin [83151.064544] pcspkr [83151.064544] sg [83151.064545] ipmi_msghandler [83151.064545] acpi_power_meter [83151.064545] wmi [83151.064546] mei_me [83151.064546] mei [83151.064546] shpchp [83151.064546] lpc_ich [83151.064547] nfsd [83151.064547] auth_rpcgss [83151.064547] dm_multipath [83151.064548] dm_mod [83151.064548] nfs_acl [83151.064548] lockd [83151.064549] grace [83151.064549] sunrpc [83151.064549] ip_tables [83151.064550] ext4 [83151.064550] mbcache [83151.064550] jbd2 [83151.064551] sd_mod [83151.064551] crc_t10dif [83151.064551] crct10dif_generic [83151.064552] mlx4_en [83151.064552] i2c_algo_bit [83151.064552] drm_kms_helper [83151.064553] syscopyarea [83151.064553] sysfillrect [83151.064553] sysimgblt [83151.064554] fb_sys_fops [83151.064554] ttm [83151.064554] tg3 [83151.064555] ahci [83151.064555] crct10dif_pclmul [83151.064555] crct10dif_common [83151.064556] mlx4_core [83151.064556] mpt3sas [83151.064556] drm [83151.064557] libahci [83151.064557] crc32c_intel [83151.064557] ptp [83151.064558] raid_class [83151.064558] libata [83151.064558] megaraid_sas [83151.064558] devlink [83151.064559] i2c_core [83151.064559] scsi_transport_sas [83151.064559] pps_core [83151.064560] [83151.064561] CPU: 3 PID: 33189 Comm: ldlm_bl_16 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83151.064561] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83151.064562] task: ffff88102c376eb0 ti: ffff881fefb44000 task.ti: ffff881fefb44000 [83151.064563] RIP: 0010:[] [83151.064564] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83151.064565] RSP: 0018:ffff881fefb47b70 EFLAGS: 00000246 [83151.064566] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000190000 [83151.064566] RDX: ffff88203c8d7880 RSI: 0000000000b90001 RDI: ffff88102137981c [83151.064567] RBP: ffff881fefb47b70 R08: ffff88203c657880 R09: 0000000000000000 [83151.064568] R10: ffff882021a45400 R11: 0000000000000000 R12: ffff88102c376f18 [83151.064568] R13: ffff88203c2b92a0 R14: ffff881000000141 R15: ffff88207ffd8008 [83151.064569] FS: 0000000000000000(0000) GS:ffff88203c640000(0000) knlGS:0000000000000000 [83151.064570] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83151.064570] CR2: 000055ce548c9220 CR3: 00000000019f2000 CR4: 00000000001407e0 [83151.064571] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83151.064571] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83151.064572] Stack: [83151.064572] ffff881fefb47b80 [83151.064572] ffffffff8169e61f [83151.064573] ffff881fefb47b90 [83151.064573] ffffffff816abb70 [83151.064573] [83151.064574] ffff881fefb47bd0 [83151.064574] ffffffffc0b72198 [83151.064574] 0000000000000000 [83151.064575] ffff882021a45400 [83151.064575] [83151.064575] ffff881fefb47c18 [83151.064576] ffff882021a45460 [83151.064576] ffff882021a45400 [83151.064576] ffff8810175b8000 [83151.064576] [83151.064577] Call Trace: [83151.064577] [83151.064579] [] queued_spin_lock_slowpath+0xb/0xf [83151.064579] [83151.064580] [] _raw_spin_lock+0x20/0x30 [83151.064581] [83151.064601] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83151.064601] [83151.064622] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83151.064622] [83151.064641] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83151.064641] [83151.064659] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83151.064659] [83151.064676] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83151.064677] [83151.064694] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83151.064695] [83151.064699] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83151.064700] [83151.064717] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83151.064718] [83151.064735] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83151.064735] [83151.064740] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83151.064741] [83151.064760] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83151.064761] [83151.064778] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83151.064778] [83151.064798] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83151.064798] [83151.064799] [] ? __schedule+0x39d/0x8b0 [83151.064800] [83151.064820] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83151.064820] [83151.064821] [] kthread+0xcf/0xe0 [83151.064822] [83151.064823] [] ? insert_kthread_work+0x40/0x40 [83151.064823] [83151.064824] [] ret_from_fork+0x58/0x90 [83151.064825] [83151.064826] [] ? insert_kthread_work+0x40/0x40 [83151.064826] Code: [83151.064826] 0d [83151.064827] 48 [83151.064827] 98 [83151.064827] 83 [83151.064827] e2 [83151.064828] 30 [83151.064828] 48 [83151.064828] 81 [83151.064828] c2 [83151.064829] 80 [83151.064829] 78 [83151.064829] 01 [83151.064829] 00 [83151.064830] 48 [83151.064830] 03 [83151.064830] 14 [83151.064830] c5 [83151.064831] e0 [83151.064831] fd [83151.064831] b0 [83151.064831] 81 [83151.064832] 4c [83151.064832] 89 [83151.064832] 02 [83151.064832] 41 [83151.064833] 8b [83151.064833] 40 [83151.064833] 08 [83151.064833] 85 [83151.064834] c0 [83151.064834] 75 [83151.064834] 0f [83151.064835] 0f [83151.064835] 1f [83151.064835] 44 [83151.064835] 00 [83151.064836] 00 [83151.064836] f3 [83151.064836] 90 [83151.064836] 41 [83151.064837] 8b [83151.064837] 40 [83151.064837] 08 [83151.064837] <85> [83151.064838] c0 [83151.064838] 74 [83151.064838] f6 [83151.064838] 4d [83151.064839] 8b [83151.064839] 08 [83151.064839] 4d [83151.064839] 85 [83151.064840] c9 [83151.064840] 74 [83151.064840] 04 [83151.064840] 41 [83151.064841] 0f [83151.064841] 18 [83151.064841] 09 [83151.064841] 8b [83151.064842] 17 [83151.064842] 0f [83151.064842] b7 [83151.064842] c2 [83151.064843] [83151.067522] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ldlm_bl_09:16011] [83151.067522] Modules linked in: [83151.067523] mgs(OE) [83151.067523] mgc(OE) [83151.067524] osd_ldiskfs(OE) [83151.067524] lquota(OE) [83151.067524] ldiskfs(OE) [83151.067525] lustre(OE) [83151.067525] lmv(OE) [83151.067525] mdc(OE) [83151.067526] lov(OE) [83151.067526] fid(OE) [83151.067526] fld(OE) [83151.067526] ko2iblnd(OE) [83151.067527] ptlrpc(OE) [83151.067527] obdclass(OE) [83151.067528] lnet(OE) [83151.067528] libcfs(OE) [83151.067528] vfat [83151.067529] fat [83151.067529] uas [83151.067529] usb_storage [83151.067530] mpt2sas [83151.067530] mptctl [83151.067530] mptbase [83151.067531] rpcsec_gss_krb5 [83151.067531] dell_rbu [83151.067531] nfsv4 [83151.067532] dns_resolver [83151.067532] nfs [83151.067532] fscache [83151.067533] ib_ipoib [83151.067533] rdma_ucm [83151.067533] ib_ucm [83151.067533] ib_uverbs [83151.067534] ib_umad [83151.067534] rdma_cm [83151.067534] ib_cm [83151.067535] iw_cm [83151.067535] mlx4_ib [83151.067535] ib_core [83151.067536] sb_edac [83151.067536] edac_core [83151.067536] intel_powerclamp [83151.067537] coretemp [83151.067537] intel_rapl [83151.067538] iosf_mbi [83151.067538] dm_service_time [83151.067538] kvm_intel [83151.067538] kvm [83151.067539] irqbypass [83151.067539] crc32_pclmul [83151.067539] ghash_clmulni_intel [83151.067540] aesni_intel [83151.067540] lrw [83151.067540] gf128mul [83151.067541] glue_helper [83151.067541] ablk_helper [83151.067541] cryptd [83151.067542] iTCO_wdt [83151.067542] dcdbas [83151.067543] iTCO_vendor_support [83151.067543] ipmi_si [83151.067543] ipmi_devintf [83151.067544] mxm_wmi [83151.067544] dm_round_robin [83151.067544] pcspkr [83151.067545] sg [83151.067545] ipmi_msghandler [83151.067545] acpi_power_meter [83151.067546] wmi [83151.067546] mei_me [83151.067546] mei [83151.067547] shpchp [83151.067547] lpc_ich [83151.067548] nfsd [83151.067548] auth_rpcgss [83151.067548] dm_multipath [83151.067548] dm_mod [83151.067549] nfs_acl [83151.067549] lockd [83151.067550] grace [83151.067550] sunrpc [83151.067550] ip_tables [83151.067551] ext4 [83151.067551] mbcache [83151.067551] jbd2 [83151.067552] sd_mod [83151.067552] crc_t10dif [83151.067552] crct10dif_generic [83151.067553] mlx4_en [83151.067553] i2c_algo_bit [83151.067553] drm_kms_helper [83151.067554] syscopyarea [83151.067554] sysfillrect [83151.067555] sysimgblt [83151.067555] fb_sys_fops [83151.067556] ttm [83151.067556] tg3 [83151.067556] ahci [83151.067557] crct10dif_pclmul [83151.067557] crct10dif_common [83151.067557] mlx4_core [83151.067558] mpt3sas [83151.067558] drm [83151.067558] libahci [83151.067558] crc32c_intel [83151.067559] ptp [83151.067559] raid_class [83151.067559] libata [83151.067560] megaraid_sas [83151.067560] devlink [83151.067560] i2c_core [83151.067561] scsi_transport_sas [83151.067561] pps_core [83151.067561] [83151.067563] CPU: 4 PID: 16011 Comm: ldlm_bl_09 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83151.067563] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83151.067564] task: ffff88203ad88fd0 ti: ffff88201c7fc000 task.ti: ffff88201c7fc000 [83151.067564] RIP: 0010:[] [83151.067567] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83151.067567] RSP: 0018:ffff88201c7ffb70 EFLAGS: 00000246 [83151.067568] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000210000 [83151.067568] RDX: ffff88103cf97880 RSI: 0000000000610000 RDI: ffff88102137981c [83151.067569] RBP: ffff88201c7ffb70 R08: ffff88103ce97880 R09: 0000000000000000 [83151.067569] R10: ffff882021be9a00 R11: 0000000000000000 R12: ffff88201c7ffc58 [83151.067570] R13: 0000000000000001 R14: ffff88201c7ffb28 R15: ffffffff81322c35 [83151.067570] FS: 0000000000000000(0000) GS:ffff88103ce80000(0000) knlGS:0000000000000000 [83151.067571] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83151.067572] CR2: 00007f8905462248 CR3: 00000000019f2000 CR4: 00000000001407e0 [83151.067572] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83151.067572] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83151.067573] Stack: [83151.067573] ffff88201c7ffb80 [83151.067573] ffffffff8169e61f [83151.067573] ffff88201c7ffb90 [83151.067574] ffffffff816abb70 [83151.067574] [83151.067574] ffff88201c7ffbd0 [83151.067574] ffffffffc0b72198 [83151.067575] 0000000000000000 [83151.067575] ffff882021be9a00 [83151.067575] [83151.067575] ffff88201c7ffc18 [83151.067576] ffff882021be9a60 [83151.067576] ffff882021be9a00 [83151.067576] ffff8810175b8000 [83151.067576] [83151.067577] Call Trace: [83151.067577] [83151.067579] [] queued_spin_lock_slowpath+0xb/0xf [83151.067579] [83151.067580] [] _raw_spin_lock+0x20/0x30 [83151.067580] [83151.067603] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83151.067604] [83151.067622] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83151.067623] [83151.067640] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83151.067641] [83151.067657] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83151.067657] [83151.067673] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83151.067674] [83151.067690] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83151.067690] [83151.067695] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83151.067696] [83151.067712] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83151.067712] [83151.067728] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83151.067729] [83151.067733] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83151.067734] [83151.067750] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83151.067750] [83151.067767] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83151.067767] [83151.067786] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83151.067786] [83151.067787] [] ? wake_up_state+0x20/0x20 [83151.067788] [83151.067807] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83151.067807] [83151.067808] [] kthread+0xcf/0xe0 [83151.067808] [83151.067809] [] ? insert_kthread_work+0x40/0x40 [83151.067809] [83151.067810] [] ret_from_fork+0x58/0x90 [83151.067811] [83151.067812] [] ? insert_kthread_work+0x40/0x40 [83151.067812] Code: [83151.067812] 13 [83151.067812] 48 [83151.067813] c1 [83151.067813] ea [83151.067813] 0d [83151.067813] 48 [83151.067814] 98 [83151.067814] 83 [83151.067814] e2 [83151.067814] 30 [83151.067814] 48 [83151.067815] 81 [83151.067815] c2 [83151.067815] 80 [83151.067815] 78 [83151.067816] 01 [83151.067816] 00 [83151.067816] 48 [83151.067816] 03 [83151.067816] 14 [83151.067817] c5 [83151.067817] e0 [83151.067817] fd [83151.067817] b0 [83151.067817] 81 [83151.067818] 4c [83151.067818] 89 [83151.067818] 02 [83151.067818] 41 [83151.067819] 8b [83151.067819] 40 [83151.067819] 08 [83151.067819] 85 [83151.067819] c0 [83151.067820] 75 [83151.067820] 0f [83151.067820] 0f [83151.067820] 1f [83151.067821] 44 [83151.067821] 00 [83151.067821] 00 [83151.067821] f3 [83151.067821] 90 [83151.067822] <41> [83151.067822] 8b [83151.067822] 40 [83151.067822] 08 [83151.067823] 85 [83151.067823] c0 [83151.067823] 74 [83151.067823] f6 [83151.067824] 4d [83151.067824] 8b [83151.067824] 08 [83151.067824] 4d [83151.067824] 85 [83151.067824] c9 [83151.067825] 74 [83151.067825] 04 [83151.067825] 41 [83151.067825] 0f [83151.067826] 18 [83151.067826] 09 [83151.067826] 8b [83151.067826] [83151.070522] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ldlm_bl_11:16013] [83151.070523] Modules linked in: [83151.070523] mgs(OE) [83151.070524] mgc(OE) [83151.070524] osd_ldiskfs(OE) [83151.070525] lquota(OE) [83151.070525] ldiskfs(OE) [83151.070525] lustre(OE) [83151.070526] lmv(OE) [83151.070526] mdc(OE) [83151.070526] lov(OE) [83151.070527] fid(OE) [83151.070527] fld(OE) [83151.070528] ko2iblnd(OE) [83151.070528] ptlrpc(OE) [83151.070528] obdclass(OE) [83151.070529] lnet(OE) [83151.070529] libcfs(OE) [83151.070529] vfat [83151.070530] fat [83151.070530] uas [83151.070530] usb_storage [83151.070531] mpt2sas [83151.070531] mptctl [83151.070531] mptbase [83151.070531] rpcsec_gss_krb5 [83151.070532] dell_rbu [83151.070532] nfsv4 [83151.070532] dns_resolver [83151.070533] nfs [83151.070533] fscache [83151.070533] ib_ipoib [83151.070534] rdma_ucm [83151.070534] ib_ucm [83151.070534] ib_uverbs [83151.070534] ib_umad [83151.070535] rdma_cm [83151.070535] ib_cm [83151.070535] iw_cm [83151.070536] mlx4_ib [83151.070536] ib_core [83151.070536] sb_edac [83151.070537] edac_core [83151.070537] intel_powerclamp [83151.070537] coretemp [83151.070538] intel_rapl [83151.070538] iosf_mbi [83151.070538] dm_service_time [83151.070539] kvm_intel [83151.070539] kvm [83151.070539] irqbypass [83151.070540] crc32_pclmul [83151.070540] ghash_clmulni_intel [83151.070540] aesni_intel [83151.070541] lrw [83151.070541] gf128mul [83151.070541] glue_helper [83151.070542] ablk_helper [83151.070542] cryptd [83151.070542] iTCO_wdt [83151.070543] dcdbas [83151.070543] iTCO_vendor_support [83151.070543] ipmi_si [83151.070543] ipmi_devintf [83151.070544] mxm_wmi [83151.070544] dm_round_robin [83151.070545] pcspkr [83151.070545] sg [83151.070545] ipmi_msghandler [83151.070546] acpi_power_meter [83151.070546] wmi [83151.070546] mei_me [83151.070547] mei [83151.070547] shpchp [83151.070547] lpc_ich [83151.070547] nfsd [83151.070548] auth_rpcgss [83151.070548] dm_multipath [83151.070548] dm_mod [83151.070549] nfs_acl [83151.070549] lockd [83151.070549] grace [83151.070550] sunrpc [83151.070550] ip_tables [83151.070550] ext4 [83151.070551] mbcache [83151.070551] jbd2 [83151.070551] sd_mod [83151.070552] crc_t10dif [83151.070552] crct10dif_generic [83151.070552] mlx4_en [83151.070553] i2c_algo_bit [83151.070553] drm_kms_helper [83151.070553] syscopyarea [83151.070554] sysfillrect [83151.070554] sysimgblt [83151.070555] fb_sys_fops [83151.070555] ttm [83151.070555] tg3 [83151.070555] ahci [83151.070556] crct10dif_pclmul [83151.070556] crct10dif_common [83151.070556] mlx4_core [83151.070557] mpt3sas [83151.070557] drm [83151.070557] libahci [83151.070558] crc32c_intel [83151.070558] ptp [83151.070558] raid_class [83151.070559] libata [83151.070559] megaraid_sas [83151.070559] devlink [83151.070560] i2c_core [83151.070560] scsi_transport_sas [83151.070560] pps_core [83151.070560] [83151.070562] CPU: 5 PID: 16013 Comm: ldlm_bl_11 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83151.070562] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83151.070563] task: ffff88203ad88000 ti: ffff88201c804000 task.ti: ffff88201c804000 [83151.070564] RIP: 0010:[] [83151.070566] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83151.070566] RSP: 0018:ffff88201c807b70 EFLAGS: 00000246 [83151.070567] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000290000 [83151.070568] RDX: ffff88103d017880 RSI: 0000000000810001 RDI: ffff88102137981c [83151.070568] RBP: ffff88201c807b70 R08: ffff88203c697880 R09: 0000000000000000 [83151.070569] R10: ffff882021afca00 R11: 0000000000000000 R12: ffff88201c807c58 [83151.070569] R13: 0000000000000001 R14: ffff88201c807b28 R15: ffffffff81322c35 [83151.070570] FS: 0000000000000000(0000) GS:ffff88203c680000(0000) knlGS:0000000000000000 [83151.070571] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83151.070571] CR2: 00007fd7ddcd8090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83151.070572] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83151.070572] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83151.070573] Stack: [83151.070573] ffff88201c807b80 [83151.070573] ffffffff8169e61f [83151.070574] ffff88201c807b90 [83151.070574] ffffffff816abb70 [83151.070574] [83151.070574] ffff88201c807bd0 [83151.070575] ffffffffc0b72198 [83151.070575] 0000000000000000 [83151.070575] ffff882021afca00 [83151.070576] [83151.070576] ffff88201c807c18 [83151.070576] ffff882021afca60 [83151.070577] ffff882021afca00 [83151.070577] ffff8810175b8000 [83151.070577] [83151.070577] Call Trace: [83151.070578] [83151.070580] [] queued_spin_lock_slowpath+0xb/0xf [83151.070580] [83151.070581] [] _raw_spin_lock+0x20/0x30 [83151.070581] [83151.070600] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83151.070600] [83151.070621] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83151.070621] [83151.070640] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83151.070640] [83151.070658] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83151.070658] [83151.070675] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83151.070676] [83151.070693] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83151.070693] [83151.070698] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83151.070699] [83151.070716] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83151.070717] [83151.070733] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83151.070734] [83151.070739] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83151.070739] [83151.070758] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83151.070759] [83151.070776] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83151.070777] [83151.070799] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83151.070799] [83151.070800] [] ? wake_up_state+0x20/0x20 [83151.070801] [83151.070821] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83151.070821] [83151.070822] [] kthread+0xcf/0xe0 [83151.070823] [83151.070824] [] ? insert_kthread_work+0x40/0x40 [83151.070824] [83151.070825] [] ret_from_fork+0x58/0x90 [83151.070825] [83151.070826] [] ? insert_kthread_work+0x40/0x40 [83151.070827] Code: [83151.070827] 13 [83151.070827] 48 [83151.070827] c1 [83151.070828] ea [83151.070828] 0d [83151.070828] 48 [83151.070828] 98 [83151.070829] 83 [83151.070829] e2 [83151.070829] 30 [83151.070830] 48 [83151.070830] 81 [83151.070830] c2 [83151.070830] 80 [83151.070831] 78 [83151.070831] 01 [83151.070831] 00 [83151.070831] 48 [83151.070832] 03 [83151.070832] 14 [83151.070832] c5 [83151.070832] e0 [83151.070833] fd [83151.070833] b0 [83151.070833] 81 [83151.070833] 4c [83151.070834] 89 [83151.070834] 02 [83151.070834] 41 [83151.070834] 8b [83151.070835] 40 [83151.070835] 08 [83151.070835] 85 [83151.070835] c0 [83151.070836] 75 [83151.070836] 0f [83151.070836] 0f [83151.070836] 1f [83151.070837] 44 [83151.070837] 00 [83151.070837] 00 [83151.070837] f3 [83151.070838] 90 [83151.070838] <41> [83151.070838] 8b [83151.070839] 40 [83151.070839] 08 [83151.070839] 85 [83151.070839] c0 [83151.070840] 74 [83151.070840] f6 [83151.070840] 4d [83151.070840] 8b [83151.070841] 08 [83151.070841] 4d [83151.070841] 85 [83151.070841] c9 [83151.070842] 74 [83151.070842] 04 [83151.070842] 41 [83151.070842] 0f [83151.070843] 18 [83151.070843] 09 [83151.070843] 8b [83151.070843] [83151.076522] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ldlm_bl_17:33190] [83151.076523] Modules linked in: [83151.076523] mgs(OE) [83151.076523] mgc(OE) [83151.076524] osd_ldiskfs(OE) [83151.076524] lquota(OE) [83151.076525] ldiskfs(OE) [83151.076525] lustre(OE) [83151.076525] lmv(OE) [83151.076526] mdc(OE) [83151.076526] lov(OE) [83151.076526] fid(OE) [83151.076527] fld(OE) [83151.076527] ko2iblnd(OE) [83151.076527] ptlrpc(OE) [83151.076528] obdclass(OE) [83151.076528] lnet(OE) [83151.076528] libcfs(OE) [83151.076529] vfat [83151.076529] fat [83151.076529] uas [83151.076530] usb_storage [83151.076530] mpt2sas [83151.076530] mptctl [83151.076531] mptbase [83151.076531] rpcsec_gss_krb5 [83151.076531] dell_rbu [83151.076531] nfsv4 [83151.076532] dns_resolver [83151.076532] nfs [83151.076532] fscache [83151.076533] ib_ipoib [83151.076533] rdma_ucm [83151.076533] ib_ucm [83151.076534] ib_uverbs [83151.076534] ib_umad [83151.076534] rdma_cm [83151.076535] ib_cm [83151.076535] iw_cm [83151.076535] mlx4_ib [83151.076535] ib_core [83151.076536] sb_edac [83151.076536] edac_core [83151.076536] intel_powerclamp [83151.076537] coretemp [83151.076537] intel_rapl [83151.076537] iosf_mbi [83151.076538] dm_service_time [83151.076538] kvm_intel [83151.076538] kvm [83151.076539] irqbypass [83151.076539] crc32_pclmul [83151.076540] ghash_clmulni_intel [83151.076540] aesni_intel [83151.076540] lrw [83151.076541] gf128mul [83151.076541] glue_helper [83151.076541] ablk_helper [83151.076542] cryptd [83151.076542] iTCO_wdt [83151.076542] dcdbas [83151.076543] iTCO_vendor_support [83151.076543] ipmi_si [83151.076543] ipmi_devintf [83151.076543] mxm_wmi [83151.076544] dm_round_robin [83151.076544] pcspkr [83151.076544] sg [83151.076545] ipmi_msghandler [83151.076545] acpi_power_meter [83151.076545] wmi [83151.076546] mei_me [83151.076546] mei [83151.076546] shpchp [83151.076547] lpc_ich [83151.076547] nfsd [83151.076547] auth_rpcgss [83151.076548] dm_multipath [83151.076548] dm_mod [83151.076548] nfs_acl [83151.076549] lockd [83151.076549] grace [83151.076549] sunrpc [83151.076550] ip_tables [83151.076550] ext4 [83151.076550] mbcache [83151.076550] jbd2 [83151.076551] sd_mod [83151.076551] crc_t10dif [83151.076552] crct10dif_generic [83151.076552] mlx4_en [83151.076552] i2c_algo_bit [83151.076553] drm_kms_helper [83151.076553] syscopyarea [83151.076553] sysfillrect [83151.076554] sysimgblt [83151.076554] fb_sys_fops [83151.076554] ttm [83151.076554] tg3 [83151.076555] ahci [83151.076555] crct10dif_pclmul [83151.076556] crct10dif_common [83151.076556] mlx4_core [83151.076556] mpt3sas [83151.076556] drm [83151.076557] libahci [83151.076557] crc32c_intel [83151.076558] ptp [83151.076558] raid_class [83151.076558] libata [83151.076558] megaraid_sas [83151.076559] devlink [83151.076559] i2c_core [83151.076559] scsi_transport_sas [83151.076560] pps_core [83151.076560] [83151.076561] CPU: 7 PID: 33190 Comm: ldlm_bl_17 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83151.076562] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83151.076562] task: ffff88102d281fa0 ti: ffff881fffd34000 task.ti: ffff881fffd34000 [83151.076563] RIP: 0010:[] [83151.076565] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83151.076566] RSP: 0018:ffff881fffd37b70 EFLAGS: 00000246 [83151.076566] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000390000 [83151.076567] RDX: ffff88103cf17880 RSI: 0000000000410000 RDI: ffff88102137981c [83151.076568] RBP: ffff881fffd37b70 R08: ffff88203c6d7880 R09: 0000000000000000 [83151.076568] R10: ffff882021a6aa00 R11: 0000000000000000 R12: ffff88102d282008 [83151.076569] R13: ffff881fffd37b00 R14: ffff881f00000141 R15: ffff88207ffd8008 [83151.076570] FS: 0000000000000000(0000) GS:ffff88203c6c0000(0000) knlGS:0000000000000000 [83151.076570] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83151.076571] CR2: 00007f240470f978 CR3: 00000000019f2000 CR4: 00000000001407e0 [83151.076571] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83151.076572] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83151.076572] Stack: [83151.076572] ffff881fffd37b80 [83151.076573] ffffffff8169e61f [83151.076573] ffff881fffd37b90 [83151.076573] ffffffff816abb70 [83151.076574] [83151.076574] ffff881fffd37bd0 [83151.076574] ffffffffc0b72198 [83151.076575] 0000000000000000 [83151.076575] ffff882021a6aa00 [83151.076575] [83151.076576] ffff881fffd37c18 [83151.076576] ffff882021a6aa60 [83151.076576] ffff882021a6aa00 [83151.076577] ffff8810175b8000 [83151.076577] [83151.076577] Call Trace: [83151.076578] [83151.076579] [] queued_spin_lock_slowpath+0xb/0xf [83151.076580] [83151.076581] [] _raw_spin_lock+0x20/0x30 [83151.076581] [83151.076601] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83151.076602] [83151.076623] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83151.076623] [83151.076642] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83151.076642] [83151.076660] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83151.076660] [83151.076677] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83151.076678] [83151.076695] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83151.076696] [83151.076700] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83151.076701] [83151.076718] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83151.076719] [83151.076735] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83151.076736] [83151.076741] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83151.076741] [83151.076758] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83151.076759] [83151.076776] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83151.076777] [83151.076798] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83151.076799] [83151.076800] [] ? __schedule+0x39d/0x8b0 [83151.076801] [83151.076820] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83151.076821] [83151.076822] [] kthread+0xcf/0xe0 [83151.076822] [83151.076823] [] ? insert_kthread_work+0x40/0x40 [83151.076824] [83151.076825] [] ret_from_fork+0x58/0x90 [83151.076825] [83151.076826] [] ? insert_kthread_work+0x40/0x40 [83151.076826] Code: [83151.076827] 0d [83151.076827] 48 [83151.076827] 98 [83151.076827] 83 [83151.076828] e2 [83151.076828] 30 [83151.076828] 48 [83151.076828] 81 [83151.076829] c2 [83151.076829] 80 [83151.076829] 78 [83151.076830] 01 [83151.076830] 00 [83151.076830] 48 [83151.076830] 03 [83151.076831] 14 [83151.076831] c5 [83151.076831] e0 [83151.076831] fd [83151.076832] b0 [83151.076832] 81 [83151.076832] 4c [83151.076832] 89 [83151.076833] 02 [83151.076833] 41 [83151.076833] 8b [83151.076833] 40 [83151.076834] 08 [83151.076834] 85 [83151.076834] c0 [83151.076834] 75 [83151.076835] 0f [83151.076835] 0f [83151.076835] 1f [83151.076835] 44 [83151.076836] 00 [83151.076836] 00 [83151.076836] f3 [83151.076836] 90 [83151.076837] 41 [83151.076837] 8b [83151.076837] 40 [83151.076837] 08 [83151.076838] <85> [83151.076838] c0 [83151.076838] 74 [83151.076839] f6 [83151.076839] 4d [83151.076839] 8b [83151.076839] 08 [83151.076840] 4d [83151.076840] 85 [83151.076840] c9 [83151.076840] 74 [83151.076841] 04 [83151.076841] 41 [83151.076841] 0f [83151.076841] 18 [83151.076842] 09 [83151.076842] 8b [83151.076842] 17 [83151.076842] 0f [83151.076843] b7 [83151.076843] c2 [83151.076843] [83155.660072] { [83155.661931] 16} (t=258345 jiffies g=368094 c=368093 q=29514) [83155.667014] Task dump for CPU 0: [83155.670612] ldlm_bl_13 R running task 0 16015 2 0x00000088 [83155.678489] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83155.686773] ffff88203d33bf40 ffff88201c80fde8 ffffffffc0b72b3c ffff88200be50c00 [83155.695060] ffff88201c80fe18 ffffffffc0b737cc ffff88200be50c00 ffff882000000019 [83155.703345] Call Trace: [83155.706074] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83155.714343] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83155.723003] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83155.731373] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83155.739238] [] ? wake_up_state+0x20/0x20 [83155.745474] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83155.753726] [] ? kthread+0xcf/0xe0 [83155.759361] [] ? insert_kthread_work+0x40/0x40 [83155.766159] [] ? ret_from_fork+0x58/0x90 [83155.772373] [] ? insert_kthread_work+0x40/0x40 [83155.779170] Task dump for CPU 1: [83155.782768] ldlm_bl_08 R running task 0 16010 2 0x00000088 [83155.790648] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83155.798933] ffff8810e9242f70 ffff88201c7fbde8 ffffffffc0b72b3c ffff881017a74c00 [83155.807220] ffff88201c7fbe18 ffffffffc0b737cc ffff881017a74c00 ffff882000000111 [83155.815505] Call Trace: [83155.818235] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83155.826495] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83155.835154] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83155.843525] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83155.851389] [] ? wake_up_state+0x20/0x20 [83155.857623] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83155.865875] [] ? kthread+0xcf/0xe0 [83155.871508] [] ? insert_kthread_work+0x40/0x40 [83155.878304] [] ? ret_from_fork+0x58/0x90 [83155.884519] [] ? insert_kthread_work+0x40/0x40 [83155.891315] Task dump for CPU 2: [83155.894913] ldlm_bl_06 R running task 0 16008 2 0x00000088 [83155.902792] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83155.911076] ffff8810247eeeb0 ffff88201c7ebde8 ffffffffc0b72b3c ffff88102af4e400 [83155.919360] ffff88201c7ebe18 ffffffffc0b737cc ffff88102af4e400 ffff88200000000d [83155.927645] Call Trace: [83155.930373] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83155.938642] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83155.947301] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83155.955671] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83155.963535] [] ? wake_up_state+0x20/0x20 [83155.969769] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83155.978021] [] ? kthread+0xcf/0xe0 [83155.983654] [] ? insert_kthread_work+0x40/0x40 [83155.990452] [] ? ret_from_fork+0x58/0x90 [83155.996667] [] ? insert_kthread_work+0x40/0x40 [83156.003464] Task dump for CPU 8: [83156.007062] ldlm_bl_10 R running task 0 16012 2 0x00000088 [83156.014944] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83156.023229] ffff88203ad89fa0 ffff88201c803de8 ffffffffc0b72b3c ffff882012e84400 [83156.031513] ffff88201c803e18 ffffffffc0b737cc ffff882012e84400 ffff88200000000d [83156.039799] Call Trace: [83156.042527] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83156.050797] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83156.059456] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83156.067827] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83156.075690] [] ? wake_up_state+0x20/0x20 [83156.081924] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83156.090176] [] ? kthread+0xcf/0xe0 [83156.095809] [] ? insert_kthread_work+0x40/0x40 [83156.102607] [] ? ret_from_fork+0x58/0x90 [83156.108823] [] ? insert_kthread_work+0x40/0x40 [83156.115618] Task dump for CPU 10: [83156.119311] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83156.127192] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83156.135477] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83156.143762] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83156.152048] Call Trace: [83156.154776] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83156.163047] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83156.171706] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83156.180074] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83156.187939] [] ? wake_up_state+0x20/0x20 [83156.194172] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83156.202424] [] ? kthread+0xcf/0xe0 [83156.208057] [] ? insert_kthread_work+0x40/0x40 [83156.214853] [] ? ret_from_fork+0x58/0x90 [83156.221068] [] ? insert_kthread_work+0x40/0x40 [83156.227865] Task dump for CPU 12: [83156.231557] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83156.239440] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83156.247724] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83156.256011] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83156.264295] Call Trace: [83156.267024] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83156.275293] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83156.283951] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83156.292321] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83156.300184] [] ? wake_up_state+0x20/0x20 [83156.306419] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83156.314671] [] ? kthread+0xcf/0xe0 [83156.320303] [] ? insert_kthread_work+0x40/0x40 [83156.327102] [] ? ret_from_fork+0x58/0x90 [83156.333316] [] ? insert_kthread_work+0x40/0x40 [83156.340114] Task dump for CPU 16: [83156.343808] ldlm_bl_12 R running task 0 16014 2 0x00000088 [83156.351690] ffff88203d33af70 000000004f7e2e39 ffff88103d003db8 ffffffff810c3a98 [83156.359975] 0000000000000010 ffffffff81a552c0 ffff88103d003dd0 ffffffff810c7429 [83156.368263] 0000000000000001 ffff88103d003e00 ffffffff81137b10 ffff88103d010260 [83156.376549] Call Trace: [83156.379274] [] sched_show_task+0xa8/0x110 [83156.386280] [] dump_cpu_task+0x39/0x70 [83156.392302] [] rcu_dump_cpu_stacks+0x90/0xd0 [83156.398906] [] rcu_check_callbacks+0x442/0x730 [83156.405704] [] ? tick_sched_do_timer+0x50/0x50 [83156.412501] [] update_process_times+0x46/0x80 [83156.419201] [] tick_sched_handle+0x30/0x70 [83156.425611] [] tick_sched_timer+0x39/0x80 [83156.431923] [] __hrtimer_run_queues+0xd4/0x260 [83156.438721] [] hrtimer_interrupt+0xaf/0x1d0 [83156.445229] [] local_apic_timer_interrupt+0x35/0x60 [83156.452511] [] smp_apic_timer_interrupt+0x3d/0x50 [83156.459600] [] apic_timer_interrupt+0x6d/0x80 [83156.466300] [] ? native_queued_spin_lock_slowpath+0x116/0x1e0 [83156.475245] [] queued_spin_lock_slowpath+0xb/0xf [83156.482236] [] _raw_spin_lock+0x20/0x30 [83156.488372] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83156.496547] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83156.504624] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83156.512021] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83156.519804] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83156.527587] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83156.534971] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83156.543046] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83156.550926] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83156.558793] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83156.566870] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83156.575336] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83156.583511] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83156.591182] [] ? wake_up_state+0x20/0x20 [83156.597416] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83156.605667] [] kthread+0xcf/0xe0 [83156.611108] [] ? insert_kthread_work+0x40/0x40 [83156.617906] [] ret_from_fork+0x58/0x90 [83156.623927] [] ? insert_kthread_work+0x40/0x40 [83156.630725] Task dump for CPU 18: [83156.634417] ldlm_bl_02 R running task 0 15061 2 0x00000088 [83156.642296] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83156.650581] ffff882012edeeb0 ffff88203d3bfde8 ffffffffc0b72b3c ffff88200f638000 [83156.658865] ffff88203d3bfe18 ffffffffc0b737cc ffff88200f638000 ffff88200000000d [83156.667152] Call Trace: [83156.669882] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83156.678152] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83156.686810] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83156.695179] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83156.703043] [] ? wake_up_state+0x20/0x20 [83156.709276] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83156.717529] [] ? kthread+0xcf/0xe0 [83156.723162] [] ? insert_kthread_work+0x40/0x40 [83156.729960] [] ? ret_from_fork+0x58/0x90 [83156.736174] [] ? insert_kthread_work+0x40/0x40 [83156.742972] Task dump for CPU 20: [83156.746664] ldlm_bl_01 R running task 0 15060 2 0x00000088 [83156.754546] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83156.762830] ffff882012eddee0 ffff88203d3bbde8 ffffffffc0b72b3c ffff881039384c00 [83156.771114] ffff88203d3bbe18 ffffffffc0b737cc ffff881039384c00 ffff88200000003d [83156.779398] Call Trace: [83156.782128] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83156.790399] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83156.799056] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83156.807426] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83156.815290] [] ? wake_up_state+0x20/0x20 [83156.821524] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83156.829776] [] ? kthread+0xcf/0xe0 [83156.835409] [] ? insert_kthread_work+0x40/0x40 [83156.842208] [] ? ret_from_fork+0x58/0x90 [83156.848423] [] ? insert_kthread_work+0x40/0x40 [83156.855218] Task dump for CPU 22: [83156.858911] ldlm_bl_07 R running task 0 16009 2 0x00000088 [83156.866790] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83156.875076] ffff8810e9240fd0 ffff88201c7efde8 ffffffffc0b72b3c ffff88200f63f400 [83156.883360] ffff88201c7efe18 ffffffffc0b737cc ffff88200f63f400 ffff88200000000d [83156.891647] Call Trace: [83156.894376] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83156.902645] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83156.911304] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83156.919674] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83156.927539] [] ? wake_up_state+0x20/0x20 [83156.933773] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83156.942025] [] ? kthread+0xcf/0xe0 [83156.947657] [] ? insert_kthread_work+0x40/0x40 [83156.954456] [] ? ret_from_fork+0x58/0x90 [83156.960672] [] ? insert_kthread_work+0x40/0x40 [83163.057264] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ldlm_bl_08:16010] [83163.058265] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ldlm_bl_13:16015] [83163.058304] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.058315] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.058317] CPU: 0 PID: 16015 Comm: ldlm_bl_13 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.058317] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.058318] task: ffff88203d33bf40 ti: ffff88201c80c000 task.ti: ffff88201c80c000 [83163.058324] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83163.058324] RSP: 0018:ffff88201c80fb70 EFLAGS: 00000246 [83163.058325] RAX: 0000000000000000 RBX: 000000013ce56cc0 RCX: 0000000000010000 [83163.058326] RDX: ffff88103d0d7880 RSI: 0000000000b10000 RDI: ffff88102137981c [83163.058326] RBP: ffff88201c80fb70 R08: ffff88103ce17880 R09: 0000000000000000 [83163.058327] R10: ffff882022efc200 R11: 0000000000000005 R12: ffff88201c80fc58 [83163.058328] R13: 0000000000000001 R14: ffff88201c80fb28 R15: ffffffff81322c35 [83163.058329] FS: 0000000000000000(0000) GS:ffff88103ce00000(0000) knlGS:0000000000000000 [83163.058329] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.058330] CR2: 00007f17e6a8b248 CR3: 00000000019f2000 CR4: 00000000001407f0 [83163.058330] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.058331] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.058331] Stack: [83163.058333] ffff88201c80fb80 ffffffff8169e61f ffff88201c80fb90 ffffffff816abb70 [83163.058334] ffff88201c80fbd0 ffffffffc0b72198 0000000000000000 ffff882022efc200 [83163.058335] ffff88201c80fc18 ffff882022efc260 ffff882022efc200 ffff8810175b8000 [83163.058335] Call Trace: [83163.058339] [] queued_spin_lock_slowpath+0xb/0xf [83163.058342] [] _raw_spin_lock+0x20/0x30 [83163.058384] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.058408] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.058428] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.058445] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.058463] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.058481] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.058489] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.058507] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.058524] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.058530] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.058550] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.058568] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.058591] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.058593] [] ? wake_up_state+0x20/0x20 [83163.058614] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.058615] [] kthread+0xcf/0xe0 [83163.058617] [] ? insert_kthread_work+0x40/0x40 [83163.058618] [] ret_from_fork+0x58/0x90 [83163.058620] [] ? insert_kthread_work+0x40/0x40 [83163.058633] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83163.061265] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ldlm_bl_06:16008] [83163.061283] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.061288] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.061289] CPU: 2 PID: 16008 Comm: ldlm_bl_06 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.061289] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.061290] task: ffff8810247eeeb0 ti: ffff88201c7e8000 task.ti: ffff88201c7e8000 [83163.061293] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83163.061294] RSP: 0018:ffff88201c7ebb70 EFLAGS: 00000246 [83163.061294] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000110000 [83163.061295] RDX: ffff88203c897880 RSI: 0000000000a90001 RDI: ffff88102137981c [83163.061295] RBP: ffff88201c7ebb70 R08: ffff88103ce57880 R09: 0000000000000000 [83163.061296] R10: ffff882022f00400 R11: 0000000000000005 R12: ffff88201c7ebc58 [83163.061296] R13: 0000000000000001 R14: ffff88201c7ebb28 R15: ffffffff81322c35 [83163.061297] FS: 0000000000000000(0000) GS:ffff88103ce40000(0000) knlGS:0000000000000000 [83163.061297] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.061297] CR2: 0000000002577010 CR3: 00000000019f2000 CR4: 00000000001407e0 [83163.061298] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.061298] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.061298] Stack: [83163.061299] ffff88201c7ebb80 ffffffff8169e61f ffff88201c7ebb90 ffffffff816abb70 [83163.061300] ffff88201c7ebbd0 ffffffffc0b72198 0000000000000000 ffff882022f00400 [83163.061301] ffff88201c7ebc18 ffff882022f00460 ffff882022f00400 ffff8810175b8000 [83163.061301] Call Trace: [83163.061303] [] queued_spin_lock_slowpath+0xb/0xf [83163.061304] [] _raw_spin_lock+0x20/0x30 [83163.061321] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.061337] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.061351] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.061365] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.061378] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.061391] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.061395] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.061409] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.061422] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.061426] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.061440] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.061453] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.061469] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.061470] [] ? wake_up_state+0x20/0x20 [83163.061485] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.061486] [] kthread+0xcf/0xe0 [83163.061487] [] ? insert_kthread_work+0x40/0x40 [83163.061488] [] ret_from_fork+0x58/0x90 [83163.061489] [] ? insert_kthread_work+0x40/0x40 [83163.061497] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83163.073264] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ldlm_bl_14:16016] [83163.073286] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.073294] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.073295] CPU: 6 PID: 16016 Comm: ldlm_bl_14 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.073296] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.073296] task: ffff88203d338000 ti: ffff88201c810000 task.ti: ffff88201c810000 [83163.073299] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83163.073300] RSP: 0018:ffff88201c813b70 EFLAGS: 00000246 [83163.073301] RAX: 0000000000000000 RBX: 000000011c813be8 RCX: 0000000000310000 [83163.073301] RDX: ffff88103ce57880 RSI: 0000000000110000 RDI: ffff88102137981c [83163.073302] RBP: ffff88201c813b70 R08: ffff88103ced7880 R09: 0000000000000000 [83163.073302] R10: ffff882022f00800 R11: 0000000000000000 R12: ffff88201c813c58 [83163.073303] R13: 0000000000000001 R14: ffff88201c813b28 R15: ffffffff81322c35 [83163.073304] FS: 0000000000000000(0000) GS:ffff88103cec0000(0000) knlGS:0000000000000000 [83163.073305] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.073305] CR2: 0000000002079728 CR3: 00000000019f2000 CR4: 00000000001407e0 [83163.073306] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.073306] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.073307] Stack: [83163.073308] ffff88201c813b80 ffffffff8169e61f ffff88201c813b90 ffffffff816abb70 [83163.073309] ffff88201c813bd0 ffffffffc0b72198 0000000000000000 ffff882022f00800 [83163.073310] ffff88201c813c18 ffff882022f00860 ffff882022f00800 ffff8810175b8000 [83163.073311] Call Trace: [83163.073313] [] queued_spin_lock_slowpath+0xb/0xf [83163.073314] [] _raw_spin_lock+0x20/0x30 [83163.073336] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.073358] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.073378] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.073395] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.073412] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.073430] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.073436] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.073455] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.073472] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.073478] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.073495] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.073513] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.073535] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.073537] [] ? wake_up_state+0x20/0x20 [83163.073557] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.073558] [] kthread+0xcf/0xe0 [83163.073560] [] ? insert_kthread_work+0x40/0x40 [83163.073561] [] ret_from_fork+0x58/0x90 [83163.073562] [] ? insert_kthread_work+0x40/0x40 [83163.073576] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83163.079264] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ldlm_bl_10:16012] [83163.079286] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.079293] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.079294] CPU: 8 PID: 16012 Comm: ldlm_bl_10 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.079294] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.079295] task: ffff88203ad89fa0 ti: ffff88201c800000 task.ti: ffff88201c800000 [83163.079298] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83163.079299] RSP: 0018:ffff88201c803b70 EFLAGS: 00000246 [83163.079299] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000410000 [83163.079300] RDX: ffff88203c617880 RSI: 0000000000090000 RDI: ffff88102137981c [83163.079301] RBP: ffff88201c803b70 R08: ffff88103cf17880 R09: 0000000000000000 [83163.079301] R10: ffff882022f00a00 R11: 0000000000000005 R12: ffff88201c803c58 [83163.079302] R13: 0000000000000001 R14: ffff88201c803b28 R15: ffffffff81322c35 [83163.079303] FS: 0000000000000000(0000) GS:ffff88103cf00000(0000) knlGS:0000000000000000 [83163.079303] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.079304] CR2: 00007fca10cf5000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83163.079305] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.079305] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.079305] Stack: [83163.079307] ffff88201c803b80 ffffffff8169e61f ffff88201c803b90 ffffffff816abb70 [83163.079308] ffff88201c803bd0 ffffffffc0b72198 0000000000000000 ffff882022f00a00 [83163.079309] ffff88201c803c18 ffff882022f00a60 ffff882022f00a00 ffff8810175b8000 [83163.079309] Call Trace: [83163.079311] [] queued_spin_lock_slowpath+0xb/0xf [83163.079313] [] _raw_spin_lock+0x20/0x30 [83163.079331] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.079352] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.079371] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.079389] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.079407] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.079424] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.079430] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.079447] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.079465] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.079470] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.079490] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.079508] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.079530] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.079531] [] ? wake_up_state+0x20/0x20 [83163.079552] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.079553] [] kthread+0xcf/0xe0 [83163.079554] [] ? insert_kthread_work+0x40/0x40 [83163.079556] [] ret_from_fork+0x58/0x90 [83163.079557] [] ? insert_kthread_work+0x40/0x40 [83163.079570] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83163.085263] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ldlm_bl_05:16007] [83163.085285] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.085292] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.085294] CPU: 10 PID: 16007 Comm: ldlm_bl_05 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.085295] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.085295] task: ffff8810247ecf10 ti: ffff88201c64c000 task.ti: ffff88201c64c000 [83163.085298] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83163.085299] RSP: 0018:ffff88201c64fb70 EFLAGS: 00000246 [83163.085299] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000510000 [83163.085300] RDX: ffff88103ce17880 RSI: 0000000000010000 RDI: ffff88102137981c [83163.085300] RBP: ffff88201c64fb70 R08: ffff88103cf57880 R09: 0000000000000000 [83163.085301] R10: ffff882022e18200 R11: 0000000000000005 R12: ffff88201c64fc58 [83163.085301] R13: 0000000000000001 R14: ffff88201c64fb28 R15: ffffffff81322c35 [83163.085302] FS: 0000000000000000(0000) GS:ffff88103cf40000(0000) knlGS:0000000000000000 [83163.085303] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.085303] CR2: 0000000001dd94b8 CR3: 00000000019f2000 CR4: 00000000001407e0 [83163.085304] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.085304] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.085305] Stack: [83163.085306] ffff88201c64fb80 ffffffff8169e61f ffff88201c64fb90 ffffffff816abb70 [83163.085307] ffff88201c64fbd0 ffffffffc0b72198 0000000000000000 ffff882022e18200 [83163.085308] ffff88201c64fc18 ffff882022e18260 ffff882022e18200 ffff8810175b8000 [83163.085309] Call Trace: [83163.085311] [] queued_spin_lock_slowpath+0xb/0xf [83163.085312] [] _raw_spin_lock+0x20/0x30 [83163.085330] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.085349] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.085369] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.085386] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.085404] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.085421] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.085427] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.085445] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.085462] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.085468] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.085485] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.085502] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.085522] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.085524] [] ? wake_up_state+0x20/0x20 [83163.085544] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.085545] [] kthread+0xcf/0xe0 [83163.085547] [] ? insert_kthread_work+0x40/0x40 [83163.085548] [] ret_from_fork+0x58/0x90 [83163.085549] [] ? insert_kthread_work+0x40/0x40 [83163.085563] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83163.091263] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ldlm_bl_15:16017] [83163.091285] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.091292] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.091293] CPU: 12 PID: 16017 Comm: ldlm_bl_15 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.091294] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.091295] task: ffff88203d338fd0 ti: ffff88201c814000 task.ti: ffff88201c814000 [83163.091297] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83163.091298] RSP: 0018:ffff88201c817b70 EFLAGS: 00000246 [83163.091298] RAX: 0000000000000000 RBX: 00000001f8c12595 RCX: 0000000000610000 [83163.091299] RDX: ffff88203c757880 RSI: 0000000000590001 RDI: ffff88102137981c [83163.091299] RBP: ffff88201c817b70 R08: ffff88103cf97880 R09: 0000000000000000 [83163.091300] R10: ffff882022f00a00 R11: 0000000000000005 R12: ffff88201c817c58 [83163.091301] R13: 0000000000000001 R14: ffff88201c817b28 R15: ffffffff81322c35 [83163.091301] FS: 0000000000000000(0000) GS:ffff88103cf80000(0000) knlGS:0000000000000000 [83163.091302] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.091302] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83163.091303] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.091304] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.091304] Stack: [83163.091305] ffff88201c817b80 ffffffff8169e61f ffff88201c817b90 ffffffff816abb70 [83163.091306] ffff88201c817bd0 ffffffffc0b72198 0000000000000000 ffff882022f00a00 [83163.091307] ffff88201c817c18 ffff882022f00a60 ffff882022f00a00 ffff8810175b8000 [83163.091308] Call Trace: [83163.091309] [] queued_spin_lock_slowpath+0xb/0xf [83163.091311] [] _raw_spin_lock+0x20/0x30 [83163.091329] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.091350] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.091369] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.091386] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.091404] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.091422] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.091427] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.091445] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.091462] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.091468] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.091487] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.091505] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.091527] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.091529] [] ? wake_up_state+0x20/0x20 [83163.091549] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.091550] [] kthread+0xcf/0xe0 [83163.091552] [] ? insert_kthread_work+0x40/0x40 [83163.091553] [] ret_from_fork+0x58/0x90 [83163.091554] [] ? insert_kthread_work+0x40/0x40 [83163.091568] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83163.109263] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ldlm_bl_02:15061] [83163.109284] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.109291] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.109292] CPU: 18 PID: 15061 Comm: ldlm_bl_02 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.109292] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.109293] task: ffff882012edeeb0 ti: ffff88203d3bc000 task.ti: ffff88203d3bc000 [83163.109295] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83163.109296] RSP: 0018:ffff88203d3bfb70 EFLAGS: 00000246 [83163.109297] RAX: 0000000000000000 RBX: 000000013cf16cc0 RCX: 0000000000910000 [83163.109297] RDX: ffff88203c7d7880 RSI: 0000000000790000 RDI: ffff88102137981c [83163.109298] RBP: ffff88203d3bfb70 R08: ffff88103d057880 R09: 0000000000000000 [83163.109298] R10: ffff882022f01400 R11: 0000000000000005 R12: ffff88203d3bfc58 [83163.109299] R13: 0000000000000001 R14: ffff88203d3bfb28 R15: ffffffff81322c35 [83163.109300] FS: 0000000000000000(0000) GS:ffff88103d040000(0000) knlGS:0000000000000000 [83163.109300] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.109301] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83163.109301] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.109302] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.109302] Stack: [83163.109303] ffff88203d3bfb80 ffffffff8169e61f ffff88203d3bfb90 ffffffff816abb70 [83163.109304] ffff88203d3bfbd0 ffffffffc0b72198 0000000000000000 ffff882022f01400 [83163.109306] ffff88203d3bfc18 ffff882022f01460 ffff882022f01400 ffff8810175b8000 [83163.109306] Call Trace: [83163.109308] [] queued_spin_lock_slowpath+0xb/0xf [83163.109309] [] _raw_spin_lock+0x20/0x30 [83163.109328] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.109349] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.109368] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.109386] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.109404] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.109421] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.109427] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.109444] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.109462] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.109467] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.109485] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.109503] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.109525] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.109526] [] ? wake_up_state+0x20/0x20 [83163.109547] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.109548] [] kthread+0xcf/0xe0 [83163.109550] [] ? insert_kthread_work+0x40/0x40 [83163.109551] [] ret_from_fork+0x58/0x90 [83163.109552] [] ? insert_kthread_work+0x40/0x40 [83163.109565] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83163.115262] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ldlm_bl_01:15060] [83163.115284] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.115291] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.115292] CPU: 20 PID: 15060 Comm: ldlm_bl_01 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.115293] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.115293] task: ffff882012eddee0 ti: ffff88203d3b8000 task.ti: ffff88203d3b8000 [83163.115296] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83163.115296] RSP: 0018:ffff88203d3bbb70 EFLAGS: 00000246 [83163.115297] RAX: 0000000000000000 RBX: 0000000180597933 RCX: 0000000000a10000 [83163.115298] RDX: ffff88203c857880 RSI: 0000000000990000 RDI: ffff88102137981c [83163.115298] RBP: ffff88203d3bbb70 R08: ffff88103d097880 R09: 0000000000000000 [83163.115299] R10: ffff882022ef1000 R11: 0000000000000005 R12: ffff88203d3bbc58 [83163.115299] R13: 0000000000000001 R14: ffff88203d3bbb28 R15: ffffffff81322c35 [83163.115300] FS: 0000000000000000(0000) GS:ffff88103d080000(0000) knlGS:0000000000000000 [83163.115301] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.115301] CR2: 00007f02c314e4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83163.115302] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.115303] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.115303] Stack: [83163.115304] ffff88203d3bbb80 ffffffff8169e61f ffff88203d3bbb90 ffffffff816abb70 [83163.115305] ffff88203d3bbbd0 ffffffffc0b72198 0000000000000000 ffff882022ef1000 [83163.115306] ffff88203d3bbc18 ffff882022ef1060 ffff882022ef1000 ffff8810175b8000 [83163.115306] Call Trace: [83163.115308] [] queued_spin_lock_slowpath+0xb/0xf [83163.115310] [] _raw_spin_lock+0x20/0x30 [83163.115330] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.115351] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.115370] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.115388] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.115405] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.115423] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.115428] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.115446] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.115463] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.115469] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.115489] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.115506] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.115529] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.115530] [] ? wake_up_state+0x20/0x20 [83163.115551] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.115552] [] kthread+0xcf/0xe0 [83163.115553] [] ? insert_kthread_work+0x40/0x40 [83163.115555] [] ret_from_fork+0x58/0x90 [83163.115556] [] ? insert_kthread_work+0x40/0x40 [83163.115569] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83163.121262] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ldlm_bl_07:16009] [83163.121284] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83163.121291] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83163.121292] CPU: 22 PID: 16009 Comm: ldlm_bl_07 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83163.121293] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83163.121293] task: ffff8810e9240fd0 ti: ffff88201c7ec000 task.ti: ffff88201c7ec000 [83163.121296] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83163.121296] RSP: 0018:ffff88201c7efb70 EFLAGS: 00000246 [83163.121297] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000b10000 [83163.121298] RDX: ffff88103ce97880 RSI: 0000000000210000 RDI: ffff88102137981c [83163.121298] RBP: ffff88201c7efb70 R08: ffff88103d0d7880 R09: 0000000000000000 [83163.121299] R10: ffff882022f01400 R11: 0000000000000005 R12: ffff88201c7efc58 [83163.121299] R13: 0000000000000001 R14: ffff88201c7efb28 R15: ffffffff81322c35 [83163.121300] FS: 0000000000000000(0000) GS:ffff88103d0c0000(0000) knlGS:0000000000000000 [83163.121301] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83163.121301] CR2: 00007f262c4764f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83163.121302] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83163.121303] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83163.121303] Stack: [83163.121304] ffff88201c7efb80 ffffffff8169e61f ffff88201c7efb90 ffffffff816abb70 [83163.121305] ffff88201c7efbd0 ffffffffc0b72198 0000000000000000 ffff882022f01400 [83163.121307] ffff88201c7efc18 ffff882022f01460 ffff882022f01400 ffff8810175b8000 [83163.121307] Call Trace: [83163.121309] [] queued_spin_lock_slowpath+0xb/0xf [83163.121310] [] _raw_spin_lock+0x20/0x30 [83163.121329] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83163.121350] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83163.121369] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83163.121386] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83163.121404] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83163.121422] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83163.121427] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83163.121445] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.121463] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83163.121468] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83163.121486] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83163.121504] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83163.121526] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83163.121527] [] ? wake_up_state+0x20/0x20 [83163.121547] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83163.121549] [] kthread+0xcf/0xe0 [83163.121550] [] ? insert_kthread_work+0x40/0x40 [83163.121551] [] ret_from_fork+0x58/0x90 [83163.121552] [] ? insert_kthread_work+0x40/0x40 [83163.121566] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83167.094138] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83167.173614] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83167.202482] CPU: 1 PID: 16010 Comm: ldlm_bl_08 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83167.215192] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83167.223540] task: ffff8810e9242f70 ti: ffff88201c7f8000 task.ti: ffff88201c7f8000 [83167.231888] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83167.242960] RSP: 0018:ffff88201c7fbb70 EFLAGS: 00000246 [83167.248885] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000090000 [83167.256845] RDX: ffff88103cf57880 RSI: 0000000000510000 RDI: ffff88102137981c [83167.264805] RBP: ffff88201c7fbb70 R08: ffff88203c617880 R09: 0000000000000000 [83167.272765] R10: ffff882022e05800 R11: 0000000000000005 R12: ffff88201c7fbc58 [83167.280725] R13: 0000000000000001 R14: ffff88201c7fbb28 R15: ffffffff81322c35 [83167.288686] FS: 0000000000000000(0000) GS:ffff88203c600000(0000) knlGS:0000000000000000 [83167.297711] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83167.304121] CR2: 00007f11b0c8c000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83167.312081] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83167.320041] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83167.328001] Stack: [83167.330241] ffff88201c7fbb80 ffffffff8169e61f ffff88201c7fbb90 ffffffff816abb70 [83167.338528] ffff88201c7fbbd0 ffffffffc0b72198 0000000000000000 ffff882022e05800 [83167.346815] ffff88201c7fbc18 ffff882022e05860 ffff882022e05800 ffff8810175b8000 [83167.355102] Call Trace: [83167.357830] [] queued_spin_lock_slowpath+0xb/0xf [83167.364821] [] _raw_spin_lock+0x20/0x30 [83167.370960] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83167.379138] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83167.387216] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83167.394613] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83167.402395] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83167.410178] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83167.417562] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83167.425636] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83167.433517] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83167.441385] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83167.449463] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83167.457926] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83167.466103] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83167.473765] [] ? wake_up_state+0x20/0x20 [83167.480000] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83167.488251] [] kthread+0xcf/0xe0 [83167.493691] [] ? insert_kthread_work+0x40/0x40 [83167.500489] [] ret_from_fork+0x58/0x90 [83167.506512] [] ? insert_kthread_work+0x40/0x40 [83167.513309] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83175.082005] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [ldlm_bl_03:15915] [83175.088004] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [ldlm_bl_22:33195] [83175.088027] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83175.088035] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83175.088036] CPU: 11 PID: 33195 Comm: ldlm_bl_22 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83175.088037] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83175.088038] task: ffff88203d0b1fa0 ti: ffff88203d4b8000 task.ti: ffff88203d4b8000 [83175.088041] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83175.088041] RSP: 0018:ffff88203d4bbb70 EFLAGS: 00000246 [83175.088042] RAX: 0000000000000000 RBX: 5a5a5a5a5a5a5a5a RCX: 0000000000590000 [83175.088043] RDX: ffff88103ced7880 RSI: 0000000000310000 RDI: ffff88102137981c [83175.088043] RBP: ffff88203d4bbb70 R08: ffff88203c757880 R09: 0000000000000000 [83175.088044] R10: ffff882021bf2a00 R11: 0000000000000001 R12: 5a5a5a5a5a5a5a5a [83175.088045] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83175.088046] FS: 0000000000000000(0000) GS:ffff88203c740000(0000) knlGS:0000000000000000 [83175.088046] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83175.088047] CR2: 00007fef29abd090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83175.088047] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83175.088048] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83175.088048] Stack: [83175.088050] ffff88203d4bbb80 ffffffff8169e61f ffff88203d4bbb90 ffffffff816abb70 [83175.088051] ffff88203d4bbbd0 ffffffffc0b72198 0000000000000000 ffff882021bf2a00 [83175.088052] ffff88203d4bbc18 ffff882021bf2a60 ffff882021bf2a00 ffff8810175b8000 [83175.088052] Call Trace: [83175.088054] [] queued_spin_lock_slowpath+0xb/0xf [83175.088056] [] _raw_spin_lock+0x20/0x30 [83175.088084] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83175.088107] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83175.088127] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83175.088145] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83175.088162] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83175.088180] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83175.088186] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83175.088205] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.088222] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.088228] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83175.088246] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83175.088263] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83175.088285] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83175.088286] [] ? __schedule+0x39d/0x8b0 [83175.088307] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83175.088308] [] kthread+0xcf/0xe0 [83175.088310] [] ? insert_kthread_work+0x40/0x40 [83175.088311] [] ret_from_fork+0x58/0x90 [83175.088312] [] ? insert_kthread_work+0x40/0x40 [83175.088326] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83175.094005] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [ldlm_bl_19:33192] [83175.094027] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83175.094034] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83175.094036] CPU: 13 PID: 33192 Comm: ldlm_bl_19 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83175.094036] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83175.094037] task: ffff88203d0b2f70 ti: ffff881ff8e20000 task.ti: ffff881ff8e20000 [83175.094040] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83175.094041] RSP: 0018:ffff881ff8e23b70 EFLAGS: 00000246 [83175.094041] RAX: 0000000000000000 RBX: ffff881ff8e23be8 RCX: 0000000000690000 [83175.094042] RDX: ffff88203c697880 RSI: 0000000000290001 RDI: ffff88102137981c [83175.094043] RBP: ffff881ff8e23b70 R08: ffff88203c797880 R09: 0000000000000000 [83175.094043] R10: ffff882022e04600 R11: 0000000000000000 R12: ffff88103ce96cc0 [83175.094044] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83175.094044] FS: 0000000000000000(0000) GS:ffff88203c780000(0000) knlGS:0000000000000000 [83175.094045] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83175.094046] CR2: 00007f352c5974f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83175.094046] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83175.094047] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83175.094047] Stack: [83175.094048] ffff881ff8e23b80 ffffffff8169e61f ffff881ff8e23b90 ffffffff816abb70 [83175.094049] ffff881ff8e23bd0 ffffffffc0b72198 0000000000000000 ffff882022e04600 [83175.094051] ffff881ff8e23c18 ffff882022e04660 ffff882022e04600 ffff8810175b8000 [83175.094051] Call Trace: [83175.094053] [] queued_spin_lock_slowpath+0xb/0xf [83175.094054] [] _raw_spin_lock+0x20/0x30 [83175.094073] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83175.094094] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83175.094114] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83175.094131] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83175.094149] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83175.094166] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83175.094172] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83175.094189] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.094207] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.094212] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83175.094232] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83175.094249] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83175.094271] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83175.094273] [] ? __schedule+0x39d/0x8b0 [83175.094293] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83175.094295] [] kthread+0xcf/0xe0 [83175.094296] [] ? insert_kthread_work+0x40/0x40 [83175.094297] [] ret_from_fork+0x58/0x90 [83175.094298] [] ? insert_kthread_work+0x40/0x40 [83175.094312] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83175.100004] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [ldlm_bl_23:33196] [83175.100027] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83175.100034] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83175.100035] CPU: 15 PID: 33196 Comm: ldlm_bl_23 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83175.100036] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83175.100037] task: ffff88203d0b0000 ti: ffff882015ae0000 task.ti: ffff882015ae0000 [83175.100039] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83175.100040] RSP: 0018:ffff882015ae3b70 EFLAGS: 00000246 [83175.100041] RAX: 0000000000000000 RBX: ffff88203c756cc0 RCX: 0000000000790000 [83175.100041] RDX: ffff88203c6d7880 RSI: 0000000000390000 RDI: ffff88102137981c [83175.100042] RBP: ffff882015ae3b70 R08: ffff88203c7d7880 R09: 0000000000000000 [83175.100042] R10: ffff882021b71400 R11: 0000000000000001 R12: ffff88203d0b0068 [83175.100043] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83175.100044] FS: 0000000000000000(0000) GS:ffff88203c7c0000(0000) knlGS:0000000000000000 [83175.100044] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83175.100045] CR2: 00007faebc9e03cc CR3: 00000000019f2000 CR4: 00000000001407e0 [83175.100045] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83175.100046] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83175.100046] Stack: [83175.100047] ffff882015ae3b80 ffffffff8169e61f ffff882015ae3b90 ffffffff816abb70 [83175.100048] ffff882015ae3bd0 ffffffffc0b72198 0000000000000000 ffff882021b71400 [83175.100050] ffff882015ae3c18 ffff882021b71460 ffff882021b71400 ffff8810175b8000 [83175.100050] Call Trace: [83175.100052] [] queued_spin_lock_slowpath+0xb/0xf [83175.100053] [] _raw_spin_lock+0x20/0x30 [83175.100072] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83175.100093] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83175.100112] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83175.100130] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83175.100147] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83175.100165] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83175.100171] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83175.100189] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.100206] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.100212] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83175.100231] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83175.100249] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83175.100271] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83175.100273] [] ? __schedule+0x39d/0x8b0 [83175.100293] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83175.100295] [] kthread+0xcf/0xe0 [83175.100296] [] ? insert_kthread_work+0x40/0x40 [83175.100297] [] ret_from_fork+0x58/0x90 [83175.100299] [] ? insert_kthread_work+0x40/0x40 [83175.100312] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83175.106004] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [ldlm_bl_21:33194] [83175.106025] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83175.106032] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83175.106033] CPU: 17 PID: 33194 Comm: ldlm_bl_21 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83175.106034] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83175.106034] task: ffff88203d0b6eb0 ti: ffff882012eac000 task.ti: ffff882012eac000 [83175.106037] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83175.106038] RSP: 0018:ffff882012eafb70 EFLAGS: 00000246 [83175.106038] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000890000 [83175.106039] RDX: ffff88203c797880 RSI: 0000000000690000 RDI: ffff88102137981c [83175.106039] RBP: ffff882012eafb70 R08: ffff88203c817880 R09: 0000000000000000 [83175.106040] R10: ffff882022e03600 R11: 0000000000000001 R12: 0000000000000000 [83175.106041] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c816cc0 [83175.106042] FS: 0000000000000000(0000) GS:ffff88203c800000(0000) knlGS:0000000000000000 [83175.106042] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83175.106043] CR2: 00007f7ce52f4550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83175.106043] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83175.106044] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83175.106044] Stack: [83175.106045] ffff882012eafb80 ffffffff8169e61f ffff882012eafb90 ffffffff816abb70 [83175.106047] ffff882012eafbd0 ffffffffc0b72198 0000000000000000 ffff882022e03600 [83175.106048] ffff882012eafc18 ffff882022e03660 ffff882022e03600 ffff8810175b8000 [83175.106048] Call Trace: [83175.106050] [] queued_spin_lock_slowpath+0xb/0xf [83175.106051] [] _raw_spin_lock+0x20/0x30 [83175.106070] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83175.106092] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83175.106111] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83175.106128] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83175.106146] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83175.106163] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83175.106169] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83175.106186] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.106204] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.106209] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83175.106229] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83175.106247] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83175.106269] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83175.106270] [] ? __schedule+0x39d/0x8b0 [83175.106291] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83175.106292] [] kthread+0xcf/0xe0 [83175.106293] [] ? insert_kthread_work+0x40/0x40 [83175.106295] [] ret_from_fork+0x58/0x90 [83175.106296] [] ? insert_kthread_work+0x40/0x40 [83175.106309] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83175.112004] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ldlm_bl_18:33191] [83175.112025] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83175.112032] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83175.112034] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83175.112034] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83175.112035] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83175.112037] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83175.112038] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83175.112038] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83175.112039] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83175.112039] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83175.112040] R10: ffff882022e12600 R11: 0000000000000001 R12: ffff88203d0b5f48 [83175.112040] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83175.112041] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83175.112042] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83175.112043] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83175.112043] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83175.112044] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83175.112044] Stack: [83175.112045] ffff881ffba83b80 ffffffff8169e61f ffff881ffba83b90 ffffffff816abb70 [83175.112046] ffff881ffba83bd0 ffffffffc0b72198 0000000000000000 ffff882022e12600 [83175.112047] ffff881ffba83c18 ffff882022e12660 ffff882022e12600 ffff8810175b8000 [83175.112048] Call Trace: [83175.112050] [] queued_spin_lock_slowpath+0xb/0xf [83175.112051] [] _raw_spin_lock+0x20/0x30 [83175.112069] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83175.112088] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83175.112108] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83175.112125] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83175.112143] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83175.112160] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83175.112166] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83175.112183] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.112201] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.112206] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83175.112224] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83175.112242] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83175.112264] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83175.112265] [] ? __schedule+0x39d/0x8b0 [83175.112286] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83175.112287] [] kthread+0xcf/0xe0 [83175.112288] [] ? insert_kthread_work+0x40/0x40 [83175.112290] [] ret_from_fork+0x58/0x90 [83175.112291] [] ? insert_kthread_work+0x40/0x40 [83175.112304] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83175.118004] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ldlm_bl_20:33193] [83175.118024] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83175.118030] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83175.118031] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83175.118031] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83175.118032] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83175.118035] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83175.118035] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83175.118036] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83175.118036] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83175.118037] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83175.118037] R10: ffff882022e04200 R11: 0000000000000000 R12: ffff88103ce96cc0 [83175.118038] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83175.118038] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83175.118039] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83175.118040] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83175.118040] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83175.118040] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83175.118041] Stack: [83175.118042] ffff881ff8e2fb80 ffffffff8169e61f ffff881ff8e2fb90 ffffffff816abb70 [83175.118043] ffff881ff8e2fbd0 ffffffffc0b72198 0000000000000000 ffff882022e04200 [83175.118044] ffff881ff8e2fc18 ffff882022e04260 ffff882022e04200 ffff8810175b8000 [83175.118044] Call Trace: [83175.118046] [] queued_spin_lock_slowpath+0xb/0xf [83175.118047] [] _raw_spin_lock+0x20/0x30 [83175.118065] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83175.118083] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83175.118101] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83175.118118] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83175.118134] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83175.118151] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83175.118156] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83175.118172] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.118189] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.118194] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83175.118211] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83175.118227] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83175.118247] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83175.118248] [] ? __schedule+0x39d/0x8b0 [83175.118267] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83175.118269] [] kthread+0xcf/0xe0 [83175.118270] [] ? insert_kthread_work+0x40/0x40 [83175.118271] [] ret_from_fork+0x58/0x90 [83175.118272] [] ? insert_kthread_work+0x40/0x40 [83175.118283] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83175.124004] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ldlm_bl_04:16006] [83175.124026] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83175.124033] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83175.124034] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83175.124035] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83175.124035] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83175.124037] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83175.124038] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83175.124039] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83175.124039] RDX: ffff88103d057880 RSI: 0000000000910001 RDI: ffff88102137981c [83175.124040] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83175.124040] R10: ffff882022f06200 R11: 0000000000000000 R12: ffff88201c627c58 [83175.124041] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83175.124042] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83175.124042] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83175.124043] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83175.124044] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83175.124044] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83175.124044] Stack: [83175.124046] ffff88201c627b80 ffffffff8169e61f ffff88201c627b90 ffffffff816abb70 [83175.124047] ffff88201c627bd0 ffffffffc0b72198 0000000000000000 ffff882022f06200 [83175.124048] ffff88201c627c18 ffff882022f06260 ffff882022f06200 ffff8810175b8000 [83175.124048] Call Trace: [83175.124050] [] queued_spin_lock_slowpath+0xb/0xf [83175.124052] [] _raw_spin_lock+0x20/0x30 [83175.124072] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83175.124093] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83175.124112] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83175.124130] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83175.124147] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83175.124165] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83175.124170] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83175.124188] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.124206] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83175.124211] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83175.124231] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83175.124249] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83175.124269] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83175.124271] [] ? wake_up_state+0x20/0x20 [83175.124291] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83175.124292] [] kthread+0xcf/0xe0 [83175.124294] [] ? insert_kthread_work+0x40/0x40 [83175.124295] [] ret_from_fork+0x58/0x90 [83175.124296] [] ? insert_kthread_work+0x40/0x40 [83175.124310] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83178.223728] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83178.303202] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83178.332069] CPU: 9 PID: 15915 Comm: ldlm_bl_03 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83178.344781] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83178.353129] task: ffff88202c648000 ti: ffff88201763c000 task.ti: ffff88201763c000 [83178.361477] RIP: 0010:[] [] ldlm_process_plain_lock+0x361/0xb30 [ptlrpc] [83178.372569] RSP: 0018:ffff88201763fbe0 EFLAGS: 00000287 [83178.378493] RAX: ffff88203c4dabc8 RBX: ffff88201763fb70 RCX: 0000000000000010 [83178.386453] RDX: 0000000000000000 RSI: ffff882022e15800 RDI: ffff8810175b8000 [83178.394413] RBP: ffff88201763fc58 R08: ffff88201763fcd0 R09: ffff88103d017880 [83178.402373] R10: ffff882022e15800 R11: 7fffffffffffffff R12: 0000000000000010 [83178.410333] R13: ffffffff810fa356 R14: ffffffffffffff10 R15: ffff88102137981c [83178.418294] FS: 0000000000000000(0000) GS:ffff88203c700000(0000) knlGS:0000000000000000 [83178.427320] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83178.433729] CR2: 00007ffa010a6000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83178.441689] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83178.449649] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83178.457609] Stack: [83178.459849] ffff88201763fc7c ffff88201763fcd0 ffff88201763fc80 ffff881021379800 [83178.468136] ffff881021379820 0000001000000001 ffff881021379840 ffff88201763fc18 [83178.476423] ffff88201763fc18 000000006cba29d7 0000000000000002 ffff881021379840 [83178.484710] Call Trace: [83178.487455] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83178.494851] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83178.502636] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83178.510419] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83178.517801] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83178.525876] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83178.533756] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83178.541624] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83178.549701] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83178.558164] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83178.566341] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83178.574012] [] ? wake_up_state+0x20/0x20 [83178.580246] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83178.588497] [] kthread+0xcf/0xe0 [83178.593937] [] ? insert_kthread_work+0x40/0x40 [83178.600735] [] ret_from_fork+0x58/0x90 [83178.606758] [] ? insert_kthread_work+0x40/0x40 [83178.613555] Code: a0 0f 84 e9 01 00 00 4c 39 fb ba 01 00 00 00 4d 89 fe 0f 84 cd 03 00 00 66 90 49 8b 86 d0 01 00 00 8b 4d b4 4c 8b a8 98 fe ff ff <41> 8b 86 98 00 00 00 85 0c 85 60 d2 c9 c0 75 5c 49 83 be a8 00 [83179.063919] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ldlm_bl_16:33189] [83179.066919] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ldlm_bl_09:16011] [83179.066949] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83179.066958] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83179.066960] CPU: 4 PID: 16011 Comm: ldlm_bl_09 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83179.066960] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83179.066961] task: ffff88203ad88fd0 ti: ffff88201c7fc000 task.ti: ffff88201c7fc000 [83179.066964] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83179.066965] RSP: 0018:ffff88201c7ffb70 EFLAGS: 00000246 [83179.066965] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000210000 [83179.066966] RDX: ffff88103cf97880 RSI: 0000000000610001 RDI: ffff88102137981c [83179.066967] RBP: ffff88201c7ffb70 R08: ffff88103ce97880 R09: 0000000000000000 [83179.066967] R10: ffff88202323e400 R11: 0000000000000000 R12: ffff88201c7ffc58 [83179.066968] R13: 0000000000000001 R14: ffff88201c7ffb28 R15: ffffffff81322c35 [83179.066969] FS: 0000000000000000(0000) GS:ffff88103ce80000(0000) knlGS:0000000000000000 [83179.066969] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83179.066970] CR2: 00007f8905462248 CR3: 00000000019f2000 CR4: 00000000001407e0 [83179.066971] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83179.066971] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83179.066971] Stack: [83179.066973] ffff88201c7ffb80 ffffffff8169e61f ffff88201c7ffb90 ffffffff816abb70 [83179.066974] ffff88201c7ffbd0 ffffffffc0b72198 0000000000000000 ffff88202323e400 [83179.066975] ffff88201c7ffc18 ffff88202323e460 ffff88202323e400 ffff8810175b8000 [83179.066975] Call Trace: [83179.066978] [] queued_spin_lock_slowpath+0xb/0xf [83179.066979] [] _raw_spin_lock+0x20/0x30 [83179.067011] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83179.067034] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83179.067055] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83179.067073] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83179.067090] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83179.067108] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83179.067115] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83179.067134] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83179.067151] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83179.067157] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83179.067177] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83179.067194] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83179.067216] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83179.067218] [] ? wake_up_state+0x20/0x20 [83179.067239] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83179.067240] [] kthread+0xcf/0xe0 [83179.067242] [] ? insert_kthread_work+0x40/0x40 [83179.067243] [] ret_from_fork+0x58/0x90 [83179.067244] [] ? insert_kthread_work+0x40/0x40 [83179.067258] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83179.069919] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ldlm_bl_11:16013] [83179.069942] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83179.069950] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83179.069951] CPU: 5 PID: 16013 Comm: ldlm_bl_11 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83179.069952] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83179.069952] task: ffff88203ad88000 ti: ffff88201c804000 task.ti: ffff88201c804000 [83179.069955] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83179.069956] RSP: 0018:ffff88201c807b70 EFLAGS: 00000246 [83179.069956] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000290000 [83179.069957] RDX: ffff88103d017880 RSI: 0000000000810001 RDI: ffff88102137981c [83179.069957] RBP: ffff88201c807b70 R08: ffff88203c697880 R09: 0000000000000000 [83179.069958] R10: ffff882022f37c00 R11: 0000000000000000 R12: ffff88201c807c58 [83179.069958] R13: 0000000000000001 R14: ffff88201c807b28 R15: ffffffff81322c35 [83179.069959] FS: 0000000000000000(0000) GS:ffff88203c680000(0000) knlGS:0000000000000000 [83179.069960] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83179.069960] CR2: 00007fd7ddcd8090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83179.069961] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83179.069961] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83179.069962] Stack: [83179.069963] ffff88201c807b80 ffffffff8169e61f ffff88201c807b90 ffffffff816abb70 [83179.069964] ffff88201c807bd0 ffffffffc0b72198 0000000000000000 ffff882022f37c00 [83179.069965] ffff88201c807c18 ffff882022f37c60 ffff882022f37c00 ffff8810175b8000 [83179.069966] Call Trace: [83179.069968] [] queued_spin_lock_slowpath+0xb/0xf [83179.069969] [] _raw_spin_lock+0x20/0x30 [83179.069993] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83179.070015] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83179.070035] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83179.070053] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83179.070070] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83179.070088] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83179.070094] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83179.070112] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83179.070130] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83179.070136] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83179.070155] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83179.070173] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83179.070195] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83179.070196] [] ? wake_up_state+0x20/0x20 [83179.070217] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83179.070218] [] kthread+0xcf/0xe0 [83179.070219] [] ? insert_kthread_work+0x40/0x40 [83179.070221] [] ret_from_fork+0x58/0x90 [83179.070222] [] ? insert_kthread_work+0x40/0x40 [83179.070236] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83179.075918] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ldlm_bl_17:33190] [83179.075940] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83179.075947] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83179.075949] CPU: 7 PID: 33190 Comm: ldlm_bl_17 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83179.075949] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83179.075950] task: ffff88102d281fa0 ti: ffff881fffd34000 task.ti: ffff881fffd34000 [83179.075952] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83179.075953] RSP: 0018:ffff881fffd37b70 EFLAGS: 00000246 [83179.075954] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000390000 [83179.075954] RDX: ffff88103cf17880 RSI: 0000000000410000 RDI: ffff88102137981c [83179.075955] RBP: ffff881fffd37b70 R08: ffff88203c6d7880 R09: 0000000000000000 [83179.075955] R10: ffff882022e18c00 R11: 0000000000000000 R12: ffff88102d282008 [83179.075956] R13: ffff881fffd37b00 R14: ffff881f00000141 R15: ffff88207ffd8008 [83179.075956] FS: 0000000000000000(0000) GS:ffff88203c6c0000(0000) knlGS:0000000000000000 [83179.075957] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83179.075958] CR2: 00007f240470f978 CR3: 00000000019f2000 CR4: 00000000001407e0 [83179.075958] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83179.075959] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83179.075959] Stack: [83179.075960] ffff881fffd37b80 ffffffff8169e61f ffff881fffd37b90 ffffffff816abb70 [83179.075961] ffff881fffd37bd0 ffffffffc0b72198 0000000000000000 ffff882022e18c00 [83179.075962] ffff881fffd37c18 ffff882022e18c60 ffff882022e18c00 ffff8810175b8000 [83179.075963] Call Trace: [83179.075965] [] queued_spin_lock_slowpath+0xb/0xf [83179.075966] [] _raw_spin_lock+0x20/0x30 [83179.075986] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83179.076006] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83179.076025] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83179.076043] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83179.076060] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83179.076078] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83179.076083] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83179.076101] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83179.076118] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83179.076123] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83179.076141] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83179.076159] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83179.076179] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83179.076180] [] ? __schedule+0x39d/0x8b0 [83179.076200] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83179.076202] [] kthread+0xcf/0xe0 [83179.076203] [] ? insert_kthread_work+0x40/0x40 [83179.076205] [] ret_from_fork+0x58/0x90 [83179.076206] [] ? insert_kthread_work+0x40/0x40 [83179.076219] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83179.102918] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 21s! [ldlm_bl_12:16014] [83179.102941] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83179.102948] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83179.102949] CPU: 16 PID: 16014 Comm: ldlm_bl_12 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83179.102950] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83179.102951] task: ffff88203d33af70 ti: ffff88201c808000 task.ti: ffff88201c808000 [83179.102953] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83179.102954] RSP: 0018:ffff88201c80bb70 EFLAGS: 00000246 [83179.102954] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000810000 [83179.102955] RDX: ffff88203c717880 RSI: 0000000000490000 RDI: ffff88102137981c [83179.102956] RBP: ffff88201c80bb70 R08: ffff88103d017880 R09: 0000000000000000 [83179.102956] R10: ffff88202322f800 R11: 0000000000000005 R12: ffff88201c80bc58 [83179.102957] R13: 0000000000000001 R14: ffff88201c80bb28 R15: ffffffff81322c35 [83179.102958] FS: 0000000000000000(0000) GS:ffff88103d000000(0000) knlGS:0000000000000000 [83179.102958] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83179.102959] CR2: 00007f361a3bb4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83179.102959] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83179.102960] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83179.102960] Stack: [83179.102961] ffff88201c80bb80 ffffffff8169e61f ffff88201c80bb90 ffffffff816abb70 [83179.102962] ffff88201c80bbd0 ffffffffc0b72198 0000000000000000 ffff88202322f800 [83179.102964] ffff88201c80bc18 ffff88202322f860 ffff88202322f800 ffff8810175b8000 [83179.102964] Call Trace: [83179.102966] [] queued_spin_lock_slowpath+0xb/0xf [83179.102967] [] _raw_spin_lock+0x20/0x30 [83179.102988] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83179.103009] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83179.103028] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83179.103046] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83179.103063] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83179.103081] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83179.103086] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83179.103104] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83179.103121] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83179.103127] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83179.103147] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83179.103164] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83179.103186] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83179.103188] [] ? wake_up_state+0x20/0x20 [83179.103209] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83179.103210] [] kthread+0xcf/0xe0 [83179.103211] [] ? insert_kthread_work+0x40/0x40 [83179.103213] [] ret_from_fork+0x58/0x90 [83179.103214] [] ? insert_kthread_work+0x40/0x40 [83179.103227] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83180.862577] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83180.942049] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83180.970916] CPU: 3 PID: 33189 Comm: ldlm_bl_16 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83180.983628] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83180.991977] task: ffff88102c376eb0 ti: ffff881fefb44000 task.ti: ffff881fefb44000 [83181.000324] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83181.011396] RSP: 0018:ffff881fefb47b70 EFLAGS: 00000246 [83181.017320] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000190000 [83181.025281] RDX: ffff88203c8d7880 RSI: 0000000000b90000 RDI: ffff88102137981c [83181.033241] RBP: ffff881fefb47b70 R08: ffff88203c657880 R09: 0000000000000000 [83181.041201] R10: ffff882022e0ac00 R11: 0000000000000000 R12: ffff88102c376f18 [83181.049161] R13: ffff88203c2b92a0 R14: ffff881000000141 R15: ffff88207ffd8008 [83181.057122] FS: 0000000000000000(0000) GS:ffff88203c640000(0000) knlGS:0000000000000000 [83181.066147] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83181.072557] CR2: 000055ce548c9220 CR3: 00000000019f2000 CR4: 00000000001407e0 [83181.080517] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83181.088477] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83181.096437] Stack: [83181.098677] ffff881fefb47b80 ffffffff8169e61f ffff881fefb47b90 ffffffff816abb70 [83181.106964] ffff881fefb47bd0 ffffffffc0b72198 0000000000000000 ffff882022e0ac00 [83181.115249] ffff881fefb47c18 ffff882022e0ac60 ffff882022e0ac00 ffff8810175b8000 [83181.123534] Call Trace: [83181.126261] [] queued_spin_lock_slowpath+0xb/0xf [83181.133252] [] _raw_spin_lock+0x20/0x30 [83181.139388] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83181.147565] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83181.155642] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83181.163038] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83181.170822] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83181.178606] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83181.185989] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83181.194064] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83181.201944] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83181.209812] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83181.217887] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83181.226352] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83181.234527] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83181.242198] [] ? __schedule+0x39d/0x8b0 [83181.248336] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83181.256587] [] kthread+0xcf/0xe0 [83181.262027] [] ? insert_kthread_work+0x40/0x40 [83181.268825] [] ret_from_fork+0x58/0x90 [83181.274848] [] ? insert_kthread_work+0x40/0x40 [83181.281645] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83191.056660] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ldlm_bl_08:16010] [83191.057660] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ldlm_bl_13:16015] [83191.057694] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.057704] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.057705] CPU: 0 PID: 16015 Comm: ldlm_bl_13 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.057706] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.057707] task: ffff88203d33bf40 ti: ffff88201c80c000 task.ti: ffff88201c80c000 [83191.057711] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83191.057712] RSP: 0018:ffff88201c80fb70 EFLAGS: 00000246 [83191.057712] RAX: 0000000000000000 RBX: 000000013ce56cc0 RCX: 0000000000010000 [83191.057713] RDX: ffff88103d0d7880 RSI: 0000000000b10000 RDI: ffff88102137981c [83191.057714] RBP: ffff88201c80fb70 R08: ffff88103ce17880 R09: 0000000000000000 [83191.057714] R10: ffff8820233eb000 R11: 0000000000000005 R12: ffff88201c80fc58 [83191.057715] R13: 0000000000000001 R14: ffff88201c80fb28 R15: ffffffff81322c35 [83191.057716] FS: 0000000000000000(0000) GS:ffff88103ce00000(0000) knlGS:0000000000000000 [83191.057716] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.057717] CR2: 00007f17e6a8b248 CR3: 00000000019f2000 CR4: 00000000001407f0 [83191.057717] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.057718] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.057718] Stack: [83191.057720] ffff88201c80fb80 ffffffff8169e61f ffff88201c80fb90 ffffffff816abb70 [83191.057721] ffff88201c80fbd0 ffffffffc0b72198 0000000000000000 ffff8820233eb000 [83191.057722] ffff88201c80fc18 ffff8820233eb060 ffff8820233eb000 ffff8810175b8000 [83191.057723] Call Trace: [83191.057725] [] queued_spin_lock_slowpath+0xb/0xf [83191.057727] [] _raw_spin_lock+0x20/0x30 [83191.057762] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.057786] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.057807] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.057824] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.057842] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.057860] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.057866] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.057886] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.057903] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.057909] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.057929] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.057947] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.057969] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.057971] [] ? wake_up_state+0x20/0x20 [83191.057992] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.057994] [] kthread+0xcf/0xe0 [83191.057995] [] ? insert_kthread_work+0x40/0x40 [83191.057997] [] ret_from_fork+0x58/0x90 [83191.057998] [] ? insert_kthread_work+0x40/0x40 [83191.058012] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83191.060660] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ldlm_bl_06:16008] [83191.060679] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.060683] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.060685] CPU: 2 PID: 16008 Comm: ldlm_bl_06 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.060685] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.060686] task: ffff8810247eeeb0 ti: ffff88201c7e8000 task.ti: ffff88201c7e8000 [83191.060688] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83191.060689] RSP: 0018:ffff88201c7ebb70 EFLAGS: 00000246 [83191.060689] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000110000 [83191.060690] RDX: ffff88203c897880 RSI: 0000000000a90000 RDI: ffff88102137981c [83191.060690] RBP: ffff88201c7ebb70 R08: ffff88103ce57880 R09: 0000000000000000 [83191.060691] R10: ffff8820233ee000 R11: 0000000000000005 R12: ffff88201c7ebc58 [83191.060691] R13: 0000000000000001 R14: ffff88201c7ebb28 R15: ffffffff81322c35 [83191.060692] FS: 0000000000000000(0000) GS:ffff88103ce40000(0000) knlGS:0000000000000000 [83191.060692] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.060693] CR2: 0000000002577010 CR3: 00000000019f2000 CR4: 00000000001407e0 [83191.060693] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.060694] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.060694] Stack: [83191.060695] ffff88201c7ebb80 ffffffff8169e61f ffff88201c7ebb90 ffffffff816abb70 [83191.060695] ffff88201c7ebbd0 ffffffffc0b72198 0000000000000000 ffff8820233ee000 [83191.060696] ffff88201c7ebc18 ffff8820233ee060 ffff8820233ee000 ffff8810175b8000 [83191.060696] Call Trace: [83191.060699] [] queued_spin_lock_slowpath+0xb/0xf [83191.060700] [] _raw_spin_lock+0x20/0x30 [83191.060715] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.060730] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.060745] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.060758] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.060771] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.060784] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.060789] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.060802] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.060815] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.060820] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.060833] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.060846] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.060862] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.060863] [] ? wake_up_state+0x20/0x20 [83191.060878] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.060879] [] kthread+0xcf/0xe0 [83191.060880] [] ? insert_kthread_work+0x40/0x40 [83191.060881] [] ret_from_fork+0x58/0x90 [83191.060882] [] ? insert_kthread_work+0x40/0x40 [83191.060890] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83191.072660] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ldlm_bl_14:16016] [83191.072681] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.072689] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.072690] CPU: 6 PID: 16016 Comm: ldlm_bl_14 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.072691] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.072692] task: ffff88203d338000 ti: ffff88201c810000 task.ti: ffff88201c810000 [83191.072694] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83191.072695] RSP: 0018:ffff88201c813b70 EFLAGS: 00000246 [83191.072696] RAX: 0000000000000000 RBX: 000000011c813be8 RCX: 0000000000310000 [83191.072697] RDX: ffff88103ce57880 RSI: 0000000000110000 RDI: ffff88102137981c [83191.072697] RBP: ffff88201c813b70 R08: ffff88103ced7880 R09: 0000000000000000 [83191.072698] R10: ffff8820233eea00 R11: 0000000000000000 R12: ffff88201c813c58 [83191.072698] R13: 0000000000000001 R14: ffff88201c813b28 R15: ffffffff81322c35 [83191.072699] FS: 0000000000000000(0000) GS:ffff88103cec0000(0000) knlGS:0000000000000000 [83191.072700] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.072701] CR2: 0000000002079728 CR3: 00000000019f2000 CR4: 00000000001407e0 [83191.072701] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.072702] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.072702] Stack: [83191.072703] ffff88201c813b80 ffffffff8169e61f ffff88201c813b90 ffffffff816abb70 [83191.072704] ffff88201c813bd0 ffffffffc0b72198 0000000000000000 ffff8820233eea00 [83191.072706] ffff88201c813c18 ffff8820233eea60 ffff8820233eea00 ffff8810175b8000 [83191.072706] Call Trace: [83191.072708] [] queued_spin_lock_slowpath+0xb/0xf [83191.072709] [] _raw_spin_lock+0x20/0x30 [83191.072728] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.072750] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.072769] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.072786] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.072804] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.072822] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.072827] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.072845] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.072862] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.072868] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.072885] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.072903] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.072923] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.072925] [] ? wake_up_state+0x20/0x20 [83191.072945] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.072947] [] kthread+0xcf/0xe0 [83191.072948] [] ? insert_kthread_work+0x40/0x40 [83191.072949] [] ret_from_fork+0x58/0x90 [83191.072951] [] ? insert_kthread_work+0x40/0x40 [83191.072964] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83191.078659] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ldlm_bl_10:16012] [83191.078681] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.078688] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.078690] CPU: 8 PID: 16012 Comm: ldlm_bl_10 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.078690] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.078691] task: ffff88203ad89fa0 ti: ffff88201c800000 task.ti: ffff88201c800000 [83191.078693] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x148/0x1e0 [83191.078694] RSP: 0018:ffff88201c803b70 EFLAGS: 00000202 [83191.078694] RAX: 0000000000000001 RBX: 000000013c796cc0 RCX: 0000000000410000 [83191.078695] RDX: 0000000000510001 RSI: 0000000000090000 RDI: ffff88102137981c [83191.078695] RBP: ffff88201c803b70 R08: ffff88103cf17880 R09: ffff88203c6d7880 [83191.078696] R10: ffff882022f58c00 R11: 0000000000000005 R12: ffff88201c803c58 [83191.078696] R13: 0000000000000001 R14: ffff88201c803b28 R15: ffffffff81322c35 [83191.078697] FS: 0000000000000000(0000) GS:ffff88103cf00000(0000) knlGS:0000000000000000 [83191.078698] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.078698] CR2: 00007fca10cf5000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83191.078699] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.078700] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.078700] Stack: [83191.078701] ffff88201c803b80 ffffffff8169e61f ffff88201c803b90 ffffffff816abb70 [83191.078702] ffff88201c803bd0 ffffffffc0b72198 0000000000000000 ffff882022f58c00 [83191.078703] ffff88201c803c18 ffff882022f58c60 ffff882022f58c00 ffff8810175b8000 [83191.078704] Call Trace: [83191.078706] [] queued_spin_lock_slowpath+0xb/0xf [83191.078707] [] _raw_spin_lock+0x20/0x30 [83191.078725] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.078745] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.078764] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.078781] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.078799] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.078817] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.078822] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.078840] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.078857] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.078862] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.078882] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.078899] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.078922] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.078923] [] ? wake_up_state+0x20/0x20 [83191.078943] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.078945] [] kthread+0xcf/0xe0 [83191.078946] [] ? insert_kthread_work+0x40/0x40 [83191.078947] [] ret_from_fork+0x58/0x90 [83191.078949] [] ? insert_kthread_work+0x40/0x40 [83191.078962] Code: 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 8b 17 <0f> b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 00 00 [83191.084659] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ldlm_bl_05:16007] [83191.084680] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.084688] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.084689] CPU: 10 PID: 16007 Comm: ldlm_bl_05 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.084690] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.084691] task: ffff8810247ecf10 ti: ffff88201c64c000 task.ti: ffff88201c64c000 [83191.084693] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83191.084694] RSP: 0018:ffff88201c64fb70 EFLAGS: 00000246 [83191.084694] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000510000 [83191.084695] RDX: ffff88103ce17880 RSI: 0000000000010000 RDI: ffff88102137981c [83191.084695] RBP: ffff88201c64fb70 R08: ffff88103cf57880 R09: 0000000000000000 [83191.084696] R10: ffff88201077e800 R11: 0000000000000005 R12: ffff88201c64fc58 [83191.084697] R13: 0000000000000001 R14: ffff88201c64fb28 R15: ffffffff81322c35 [83191.084697] FS: 0000000000000000(0000) GS:ffff88103cf40000(0000) knlGS:0000000000000000 [83191.084698] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.084698] CR2: 0000000001dd94b8 CR3: 00000000019f2000 CR4: 00000000001407e0 [83191.084699] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.084700] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.084700] Stack: [83191.084701] ffff88201c64fb80 ffffffff8169e61f ffff88201c64fb90 ffffffff816abb70 [83191.084702] ffff88201c64fbd0 ffffffffc0b72198 0000000000000000 ffff88201077e800 [83191.084703] ffff88201c64fc18 ffff88201077e860 ffff88201077e800 ffff8810175b8000 [83191.084704] Call Trace: [83191.084706] [] queued_spin_lock_slowpath+0xb/0xf [83191.084707] [] _raw_spin_lock+0x20/0x30 [83191.084725] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.084745] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.084764] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.084782] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.084799] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.084817] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.084822] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.084840] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.084858] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.084863] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.084883] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.084900] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.084920] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.084922] [] ? wake_up_state+0x20/0x20 [83191.084942] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.084943] [] kthread+0xcf/0xe0 [83191.084945] [] ? insert_kthread_work+0x40/0x40 [83191.084946] [] ret_from_fork+0x58/0x90 [83191.084947] [] ? insert_kthread_work+0x40/0x40 [83191.084961] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83191.090659] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ldlm_bl_15:16017] [83191.090680] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.090687] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.090688] CPU: 12 PID: 16017 Comm: ldlm_bl_15 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.090689] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.090690] task: ffff88203d338fd0 ti: ffff88201c814000 task.ti: ffff88201c814000 [83191.090692] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83191.090692] RSP: 0018:ffff88201c817b70 EFLAGS: 00000246 [83191.090693] RAX: 0000000000000000 RBX: 00000001f8c12595 RCX: 0000000000610000 [83191.090694] RDX: ffff88203c757880 RSI: 0000000000590000 RDI: ffff88102137981c [83191.090694] RBP: ffff88201c817b70 R08: ffff88103cf97880 R09: 0000000000000000 [83191.090695] R10: ffff882022f58c00 R11: 0000000000000005 R12: ffff88201c817c58 [83191.090695] R13: 0000000000000001 R14: ffff88201c817b28 R15: ffffffff81322c35 [83191.090696] FS: 0000000000000000(0000) GS:ffff88103cf80000(0000) knlGS:0000000000000000 [83191.090697] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.090697] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83191.090698] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.090698] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.090698] Stack: [83191.090700] ffff88201c817b80 ffffffff8169e61f ffff88201c817b90 ffffffff816abb70 [83191.090701] ffff88201c817bd0 ffffffffc0b72198 0000000000000000 ffff882022f58c00 [83191.090702] ffff88201c817c18 ffff882022f58c60 ffff882022f58c00 ffff8810175b8000 [83191.090702] Call Trace: [83191.090704] [] queued_spin_lock_slowpath+0xb/0xf [83191.090705] [] _raw_spin_lock+0x20/0x30 [83191.090724] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.090743] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.090762] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.090779] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.090797] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.090815] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.090820] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.090838] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.090855] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.090860] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.090878] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.090896] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.090918] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.090920] [] ? wake_up_state+0x20/0x20 [83191.090940] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.090941] [] kthread+0xcf/0xe0 [83191.090942] [] ? insert_kthread_work+0x40/0x40 [83191.090943] [] ret_from_fork+0x58/0x90 [83191.090945] [] ? insert_kthread_work+0x40/0x40 [83191.090958] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83191.108659] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ldlm_bl_02:15061] [83191.108680] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.108688] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.108689] CPU: 18 PID: 15061 Comm: ldlm_bl_02 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.108689] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.108690] task: ffff882012edeeb0 ti: ffff88203d3bc000 task.ti: ffff88203d3bc000 [83191.108693] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83191.108693] RSP: 0018:ffff88203d3bfb70 EFLAGS: 00000246 [83191.108694] RAX: 0000000000000000 RBX: 000000013cf16cc0 RCX: 0000000000910000 [83191.108694] RDX: ffff88203c7d7880 RSI: 0000000000790000 RDI: ffff88102137981c [83191.108695] RBP: ffff88203d3bfb70 R08: ffff88103d057880 R09: 0000000000000000 [83191.108695] R10: ffff882022f59800 R11: 0000000000000005 R12: ffff88203d3bfc58 [83191.108696] R13: 0000000000000001 R14: ffff88203d3bfb28 R15: ffffffff81322c35 [83191.108697] FS: 0000000000000000(0000) GS:ffff88103d040000(0000) knlGS:0000000000000000 [83191.108697] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.108698] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83191.108698] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.108699] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.108699] Stack: [83191.108701] ffff88203d3bfb80 ffffffff8169e61f ffff88203d3bfb90 ffffffff816abb70 [83191.108702] ffff88203d3bfbd0 ffffffffc0b72198 0000000000000000 ffff882022f59800 [83191.108703] ffff88203d3bfc18 ffff882022f59860 ffff882022f59800 ffff8810175b8000 [83191.108703] Call Trace: [83191.108705] [] queued_spin_lock_slowpath+0xb/0xf [83191.108707] [] _raw_spin_lock+0x20/0x30 [83191.108727] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.108748] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.108767] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.108785] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.108803] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.108821] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.108826] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.108843] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.108861] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.108866] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.108884] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.108902] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.108923] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.108925] [] ? wake_up_state+0x20/0x20 [83191.108945] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.108947] [] kthread+0xcf/0xe0 [83191.108948] [] ? insert_kthread_work+0x40/0x40 [83191.108949] [] ret_from_fork+0x58/0x90 [83191.108951] [] ? insert_kthread_work+0x40/0x40 [83191.108964] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83191.114658] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ldlm_bl_01:15060] [83191.114680] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.114687] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.114688] CPU: 20 PID: 15060 Comm: ldlm_bl_01 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.114689] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.114690] task: ffff882012eddee0 ti: ffff88203d3b8000 task.ti: ffff88203d3b8000 [83191.114692] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83191.114693] RSP: 0018:ffff88203d3bbb70 EFLAGS: 00000246 [83191.114693] RAX: 0000000000000000 RBX: 0000000180597933 RCX: 0000000000a10000 [83191.114694] RDX: ffff88203c857880 RSI: 0000000000990000 RDI: ffff88102137981c [83191.114694] RBP: ffff88203d3bbb70 R08: ffff88103d097880 R09: 0000000000000000 [83191.114695] R10: ffff8820233d4800 R11: 0000000000000005 R12: ffff88203d3bbc58 [83191.114696] R13: 0000000000000001 R14: ffff88203d3bbb28 R15: ffffffff81322c35 [83191.114696] FS: 0000000000000000(0000) GS:ffff88103d080000(0000) knlGS:0000000000000000 [83191.114697] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.114697] CR2: 00007f02c314e4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83191.114698] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.114699] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.114699] Stack: [83191.114700] ffff88203d3bbb80 ffffffff8169e61f ffff88203d3bbb90 ffffffff816abb70 [83191.114701] ffff88203d3bbbd0 ffffffffc0b72198 0000000000000000 ffff8820233d4800 [83191.114702] ffff88203d3bbc18 ffff8820233d4860 ffff8820233d4800 ffff8810175b8000 [83191.114703] Call Trace: [83191.114705] [] queued_spin_lock_slowpath+0xb/0xf [83191.114706] [] _raw_spin_lock+0x20/0x30 [83191.114725] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.114745] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.114765] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.114783] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.114802] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.114820] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.114825] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.114845] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.114863] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.114868] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.114887] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.114906] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.114927] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.114928] [] ? wake_up_state+0x20/0x20 [83191.114950] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.114951] [] kthread+0xcf/0xe0 [83191.114952] [] ? insert_kthread_work+0x40/0x40 [83191.114954] [] ret_from_fork+0x58/0x90 [83191.114955] [] ? insert_kthread_work+0x40/0x40 [83191.114969] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83191.120658] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ldlm_bl_07:16009] [83191.120679] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83191.120687] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83191.120688] CPU: 22 PID: 16009 Comm: ldlm_bl_07 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83191.120688] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83191.120689] task: ffff8810e9240fd0 ti: ffff88201c7ec000 task.ti: ffff88201c7ec000 [83191.120691] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83191.120692] RSP: 0018:ffff88201c7efb70 EFLAGS: 00000246 [83191.120693] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000b10000 [83191.120693] RDX: ffff88103ce97880 RSI: 0000000000210001 RDI: ffff88102137981c [83191.120694] RBP: ffff88201c7efb70 R08: ffff88103d0d7880 R09: 0000000000000000 [83191.120694] R10: ffff882022f59800 R11: 0000000000000005 R12: ffff88201c7efc58 [83191.120695] R13: 0000000000000001 R14: ffff88201c7efb28 R15: ffffffff81322c35 [83191.120696] FS: 0000000000000000(0000) GS:ffff88103d0c0000(0000) knlGS:0000000000000000 [83191.120696] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83191.120697] CR2: 00007f262c4764f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83191.120698] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83191.120698] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83191.120698] Stack: [83191.120700] ffff88201c7efb80 ffffffff8169e61f ffff88201c7efb90 ffffffff816abb70 [83191.120701] ffff88201c7efbd0 ffffffffc0b72198 0000000000000000 ffff882022f59800 [83191.120702] ffff88201c7efc18 ffff882022f59860 ffff882022f59800 ffff8810175b8000 [83191.120702] Call Trace: [83191.120705] [] queued_spin_lock_slowpath+0xb/0xf [83191.120706] [] _raw_spin_lock+0x20/0x30 [83191.120726] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83191.120746] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83191.120765] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83191.120782] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83191.120800] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83191.120818] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83191.120823] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83191.120841] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.120858] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83191.120864] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83191.120881] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83191.120899] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83191.120921] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83191.120923] [] ? wake_up_state+0x20/0x20 [83191.120943] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83191.120944] [] kthread+0xcf/0xe0 [83191.120946] [] ? insert_kthread_work+0x40/0x40 [83191.120947] [] ret_from_fork+0x58/0x90 [83191.120948] [] ? insert_kthread_work+0x40/0x40 [83191.120962] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83195.093548] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83195.173019] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83195.201883] CPU: 1 PID: 16010 Comm: ldlm_bl_08 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83195.214595] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83195.222944] task: ffff8810e9242f70 ti: ffff88201c7f8000 task.ti: ffff88201c7f8000 [83195.231291] RIP: 0010:[] [] ldlm_process_plain_lock+0x357/0xb30 [ptlrpc] [83195.242386] RSP: 0018:ffff88201c7fbbe0 EFLAGS: 00000287 [83195.248311] RAX: ffff88202e338fc8 RBX: ffff88201c7fbb70 RCX: 0000000000000010 [83195.256271] RDX: 0000000000000000 RSI: ffff8820232b1e00 RDI: ffff8810175b8000 [83195.264231] RBP: ffff88201c7fbc58 R08: ffff88201c7fbcd0 R09: ffff88103cf17880 [83195.272192] R10: ffff8820232b1e00 R11: 0000000000000005 R12: 0000000000000010 [83195.280152] R13: ffffffff810fa326 R14: ffffffffffffff10 R15: ffff88102137981c [83195.288113] FS: 0000000000000000(0000) GS:ffff88203c600000(0000) knlGS:0000000000000000 [83195.297139] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83195.303548] CR2: 00007f11b0c8c000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83195.311508] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83195.319469] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83195.327429] Stack: [83195.329669] ffff88201c7fbc7c ffff88201c7fbcd0 ffff88201c7fbc80 ffff881021379800 [83195.337956] ffff881021379820 0000001000000001 ffff881021379840 ffff88201c7fbc18 [83195.346241] ffff88201c7fbc18 00000000a6f785ac 0000000000000002 ffff881021379840 [83195.354528] Call Trace: [83195.357275] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83195.364671] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83195.372457] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83195.380240] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83195.387623] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83195.395699] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83195.403580] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83195.411448] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83195.419525] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83195.427988] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83195.436163] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83195.443833] [] ? wake_up_state+0x20/0x20 [83195.450067] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83195.458319] [] kthread+0xcf/0xe0 [83195.463759] [] ? insert_kthread_work+0x40/0x40 [83195.470557] [] ret_from_fork+0x58/0x90 [83195.476580] [] ? insert_kthread_work+0x40/0x40 [83195.483377] Code: 8b 40 40 48 39 45 b8 4c 8d 78 a0 0f 84 e9 01 00 00 4c 39 fb ba 01 00 00 00 4d 89 fe 0f 84 cd 03 00 00 66 90 49 8b 86 d0 01 00 00 <8b> 4d b4 4c 8b a8 98 fe ff ff 41 8b 86 98 00 00 00 85 0c 85 60 [83203.081401] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [ldlm_bl_03:15915] [83203.087400] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [ldlm_bl_22:33195] [83203.087423] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83203.087431] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83203.087433] CPU: 11 PID: 33195 Comm: ldlm_bl_22 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83203.087433] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83203.087434] task: ffff88203d0b1fa0 ti: ffff88203d4b8000 task.ti: ffff88203d4b8000 [83203.087438] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83203.087438] RSP: 0018:ffff88203d4bbb70 EFLAGS: 00000246 [83203.087439] RAX: 0000000000000000 RBX: 5a5a5a5a5a5a5a5a RCX: 0000000000590000 [83203.087440] RDX: ffff88103ced7880 RSI: 0000000000310000 RDI: ffff88102137981c [83203.087440] RBP: ffff88203d4bbb70 R08: ffff88203c757880 R09: 0000000000000000 [83203.087441] R10: ffff88202329b200 R11: 0000000000000001 R12: 5a5a5a5a5a5a5a5a [83203.087441] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83203.087442] FS: 0000000000000000(0000) GS:ffff88203c740000(0000) knlGS:0000000000000000 [83203.087443] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83203.087443] CR2: 00007fef29abd090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83203.087444] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83203.087444] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83203.087445] Stack: [83203.087446] ffff88203d4bbb80 ffffffff8169e61f ffff88203d4bbb90 ffffffff816abb70 [83203.087447] ffff88203d4bbbd0 ffffffffc0b72198 0000000000000000 ffff88202329b200 [83203.087448] ffff88203d4bbc18 ffff88202329b260 ffff88202329b200 ffff8810175b8000 [83203.087449] Call Trace: [83203.087451] [] queued_spin_lock_slowpath+0xb/0xf [83203.087452] [] _raw_spin_lock+0x20/0x30 [83203.087481] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83203.087505] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83203.087526] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83203.087544] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83203.087561] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83203.087579] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83203.087585] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83203.087603] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.087620] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.087626] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83203.087643] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83203.087661] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83203.087683] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83203.087685] [] ? __schedule+0x39d/0x8b0 [83203.087705] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83203.087707] [] kthread+0xcf/0xe0 [83203.087708] [] ? insert_kthread_work+0x40/0x40 [83203.087710] [] ret_from_fork+0x58/0x90 [83203.087711] [] ? insert_kthread_work+0x40/0x40 [83203.087725] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83203.093400] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [ldlm_bl_19:33192] [83203.093422] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83203.093430] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83203.093431] CPU: 13 PID: 33192 Comm: ldlm_bl_19 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83203.093432] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83203.093433] task: ffff88203d0b2f70 ti: ffff881ff8e20000 task.ti: ffff881ff8e20000 [83203.093436] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83203.093436] RSP: 0018:ffff881ff8e23b70 EFLAGS: 00000246 [83203.093437] RAX: 0000000000000000 RBX: ffff881ff8e23be8 RCX: 0000000000690000 [83203.093437] RDX: ffff88203c697880 RSI: 0000000000290001 RDI: ffff88102137981c [83203.093438] RBP: ffff881ff8e23b70 R08: ffff88203c797880 R09: 0000000000000000 [83203.093438] R10: ffff8820232ae800 R11: 0000000000000000 R12: ffff88103ce96cc0 [83203.093439] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83203.093440] FS: 0000000000000000(0000) GS:ffff88203c780000(0000) knlGS:0000000000000000 [83203.093440] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83203.093441] CR2: 00007f352c5974f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83203.093441] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83203.093442] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83203.093442] Stack: [83203.093443] ffff881ff8e23b80 ffffffff8169e61f ffff881ff8e23b90 ffffffff816abb70 [83203.093445] ffff881ff8e23bd0 ffffffffc0b72198 0000000000000000 ffff8820232ae800 [83203.093446] ffff881ff8e23c18 ffff8820232ae860 ffff8820232ae800 ffff8810175b8000 [83203.093446] Call Trace: [83203.093448] [] queued_spin_lock_slowpath+0xb/0xf [83203.093450] [] _raw_spin_lock+0x20/0x30 [83203.093468] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83203.093489] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83203.093509] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83203.093526] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83203.093544] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83203.093561] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83203.093567] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83203.093585] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.093602] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.093608] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83203.093628] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83203.093645] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83203.093667] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83203.093669] [] ? __schedule+0x39d/0x8b0 [83203.093689] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83203.093691] [] kthread+0xcf/0xe0 [83203.093692] [] ? insert_kthread_work+0x40/0x40 [83203.093693] [] ret_from_fork+0x58/0x90 [83203.093694] [] ? insert_kthread_work+0x40/0x40 [83203.093708] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83203.099400] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [ldlm_bl_23:33196] [83203.099422] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83203.099430] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83203.099431] CPU: 15 PID: 33196 Comm: ldlm_bl_23 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83203.099432] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83203.099432] task: ffff88203d0b0000 ti: ffff882015ae0000 task.ti: ffff882015ae0000 [83203.099435] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83203.099436] RSP: 0018:ffff882015ae3b70 EFLAGS: 00000246 [83203.099437] RAX: 0000000000000000 RBX: ffff88203c756cc0 RCX: 0000000000790000 [83203.099437] RDX: ffff88203c6d7880 RSI: 0000000000390001 RDI: ffff88102137981c [83203.099438] RBP: ffff882015ae3b70 R08: ffff88203c7d7880 R09: 0000000000000000 [83203.099438] R10: ffff882023214e00 R11: 0000000000000001 R12: ffff88203d0b0068 [83203.099439] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83203.099440] FS: 0000000000000000(0000) GS:ffff88203c7c0000(0000) knlGS:0000000000000000 [83203.099440] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83203.099441] CR2: 00007faebc9e03cc CR3: 00000000019f2000 CR4: 00000000001407e0 [83203.099441] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83203.099442] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83203.099442] Stack: [83203.099444] ffff882015ae3b80 ffffffff8169e61f ffff882015ae3b90 ffffffff816abb70 [83203.099445] ffff882015ae3bd0 ffffffffc0b72198 0000000000000000 ffff882023214e00 [83203.099446] ffff882015ae3c18 ffff882023214e60 ffff882023214e00 ffff8810175b8000 [83203.099446] Call Trace: [83203.099448] [] queued_spin_lock_slowpath+0xb/0xf [83203.099449] [] _raw_spin_lock+0x20/0x30 [83203.099470] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83203.099491] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83203.099510] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83203.099528] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83203.099545] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83203.099563] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83203.099569] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83203.099586] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.099603] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.099609] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83203.099627] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83203.099644] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83203.099666] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83203.099667] [] ? __schedule+0x39d/0x8b0 [83203.099687] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83203.099689] [] kthread+0xcf/0xe0 [83203.099690] [] ? insert_kthread_work+0x40/0x40 [83203.099691] [] ret_from_fork+0x58/0x90 [83203.099693] [] ? insert_kthread_work+0x40/0x40 [83203.099706] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83203.105400] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [ldlm_bl_21:33194] [83203.105421] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83203.105428] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83203.105430] CPU: 17 PID: 33194 Comm: ldlm_bl_21 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83203.105430] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83203.105431] task: ffff88203d0b6eb0 ti: ffff882012eac000 task.ti: ffff882012eac000 [83203.105434] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83203.105434] RSP: 0018:ffff882012eafb70 EFLAGS: 00000246 [83203.105435] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000890000 [83203.105435] RDX: ffff88203c797880 RSI: 0000000000690000 RDI: ffff88102137981c [83203.105436] RBP: ffff882012eafb70 R08: ffff88203c817880 R09: 0000000000000000 [83203.105436] R10: ffff8820232ad600 R11: 0000000000000001 R12: 0000000000000000 [83203.105437] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c816cc0 [83203.105438] FS: 0000000000000000(0000) GS:ffff88203c800000(0000) knlGS:0000000000000000 [83203.105438] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83203.105439] CR2: 00007f7ce52f4550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83203.105439] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83203.105440] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83203.105440] Stack: [83203.105441] ffff882012eafb80 ffffffff8169e61f ffff882012eafb90 ffffffff816abb70 [83203.105442] ffff882012eafbd0 ffffffffc0b72198 0000000000000000 ffff8820232ad600 [83203.105444] ffff882012eafc18 ffff8820232ad660 ffff8820232ad600 ffff8810175b8000 [83203.105444] Call Trace: [83203.105446] [] queued_spin_lock_slowpath+0xb/0xf [83203.105447] [] _raw_spin_lock+0x20/0x30 [83203.105466] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83203.105487] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83203.105506] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83203.105523] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83203.105541] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83203.105559] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83203.105564] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83203.105582] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.105599] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.105605] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83203.105624] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83203.105641] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83203.105664] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83203.105665] [] ? __schedule+0x39d/0x8b0 [83203.105685] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83203.105687] [] kthread+0xcf/0xe0 [83203.105688] [] ? insert_kthread_work+0x40/0x40 [83203.105689] [] ret_from_fork+0x58/0x90 [83203.105690] [] ? insert_kthread_work+0x40/0x40 [83203.105704] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83203.111400] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ldlm_bl_18:33191] [83203.111421] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83203.111429] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83203.111430] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83203.111431] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83203.111432] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83203.111434] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83203.111435] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83203.111435] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83203.111436] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83203.111436] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83203.111437] R10: ffff8820232bf800 R11: 0000000000000001 R12: ffff88203d0b5f48 [83203.111437] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83203.111438] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83203.111439] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83203.111439] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83203.111440] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83203.111440] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83203.111441] Stack: [83203.111442] ffff881ffba83b80 ffffffff8169e61f ffff881ffba83b90 ffffffff816abb70 [83203.111443] ffff881ffba83bd0 ffffffffc0b72198 0000000000000000 ffff8820232bf800 [83203.111444] ffff881ffba83c18 ffff8820232bf860 ffff8820232bf800 ffff8810175b8000 [83203.111444] Call Trace: [83203.111446] [] queued_spin_lock_slowpath+0xb/0xf [83203.111448] [] _raw_spin_lock+0x20/0x30 [83203.111466] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83203.111487] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83203.111506] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83203.111524] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83203.111541] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83203.111559] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83203.111564] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83203.111582] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.111600] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.111605] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83203.111625] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83203.111642] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83203.111665] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83203.111666] [] ? __schedule+0x39d/0x8b0 [83203.111686] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83203.111688] [] kthread+0xcf/0xe0 [83203.111689] [] ? insert_kthread_work+0x40/0x40 [83203.111691] [] ret_from_fork+0x58/0x90 [83203.111692] [] ? insert_kthread_work+0x40/0x40 [83203.111705] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83203.117400] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ldlm_bl_20:33193] [83203.117418] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83203.117425] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83203.117426] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83203.117427] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83203.117427] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83203.117430] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83203.117430] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83203.117431] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83203.117431] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83203.117432] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83203.117432] R10: ffff8820232ae200 R11: 0000000000000000 R12: ffff88103ce96cc0 [83203.117432] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83203.117433] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83203.117434] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83203.117434] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83203.117435] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83203.117435] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83203.117435] Stack: [83203.117436] ffff881ff8e2fb80 ffffffff8169e61f ffff881ff8e2fb90 ffffffff816abb70 [83203.117437] ffff881ff8e2fbd0 ffffffffc0b72198 0000000000000000 ffff8820232ae200 [83203.117438] ffff881ff8e2fc18 ffff8820232ae260 ffff8820232ae200 ffff8810175b8000 [83203.117439] Call Trace: [83203.117441] [] queued_spin_lock_slowpath+0xb/0xf [83203.117442] [] _raw_spin_lock+0x20/0x30 [83203.117460] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83203.117478] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83203.117496] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83203.117513] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83203.117529] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83203.117546] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83203.117551] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83203.117568] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.117584] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.117590] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83203.117606] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83203.117623] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83203.117642] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83203.117643] [] ? __schedule+0x39d/0x8b0 [83203.117662] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83203.117664] [] kthread+0xcf/0xe0 [83203.117665] [] ? insert_kthread_work+0x40/0x40 [83203.117666] [] ret_from_fork+0x58/0x90 [83203.117667] [] ? insert_kthread_work+0x40/0x40 [83203.117678] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83203.123399] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ldlm_bl_04:16006] [83203.123421] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83203.123428] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83203.123429] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83203.123430] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83203.123430] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83203.123433] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83203.123433] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83203.123434] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83203.123434] RDX: ffff88103d057880 RSI: 0000000000910000 RDI: ffff88102137981c [83203.123435] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83203.123435] R10: ffff88202360d600 R11: 0000000000000000 R12: ffff88201c627c58 [83203.123436] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83203.123437] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83203.123437] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83203.123438] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83203.123439] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83203.123439] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83203.123439] Stack: [83203.123441] ffff88201c627b80 ffffffff8169e61f ffff88201c627b90 ffffffff816abb70 [83203.123442] ffff88201c627bd0 ffffffffc0b72198 0000000000000000 ffff88202360d600 [83203.123443] ffff88201c627c18 ffff88202360d660 ffff88202360d600 ffff8810175b8000 [83203.123443] Call Trace: [83203.123445] [] queued_spin_lock_slowpath+0xb/0xf [83203.123447] [] _raw_spin_lock+0x20/0x30 [83203.123467] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83203.123488] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83203.123507] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83203.123525] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83203.123543] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83203.123560] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83203.123565] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83203.123583] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.123601] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83203.123606] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83203.123624] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83203.123641] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83203.123663] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83203.123665] [] ? wake_up_state+0x20/0x20 [83203.123685] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83203.123687] [] kthread+0xcf/0xe0 [83203.123688] [] ? insert_kthread_work+0x40/0x40 [83203.123689] [] ret_from_fork+0x58/0x90 [83203.123691] [] ? insert_kthread_work+0x40/0x40 [83203.123704] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83206.223129] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83206.302595] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83206.331462] CPU: 9 PID: 15915 Comm: ldlm_bl_03 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83206.344174] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83206.352522] task: ffff88202c648000 ti: ffff88201763c000 task.ti: ffff88201763c000 [83206.360870] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83206.371942] RSP: 0018:ffff88201763fb70 EFLAGS: 00000246 [83206.377866] RAX: 0000000000000000 RBX: 0000000300000000 RCX: 0000000000490000 [83206.385826] RDX: ffff88103d097880 RSI: 0000000000a10000 RDI: ffff88102137981c [83206.393787] RBP: ffff88201763fb70 R08: ffff88203c717880 R09: 0000000000000000 [83206.401747] R10: ffff8820232c2e00 R11: 7fffffffffffffff R12: 0000000000000bf8 [83206.409707] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c656cc0 [83206.417667] FS: 0000000000000000(0000) GS:ffff88203c700000(0000) knlGS:0000000000000000 [83206.426693] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83206.433102] CR2: 00007ffa010a6000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83206.441062] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83206.449023] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83206.456982] Stack: [83206.459223] ffff88201763fb80 ffffffff8169e61f ffff88201763fb90 ffffffff816abb70 [83206.467509] ffff88201763fbd0 ffffffffc0b72198 0000000000000000 ffff8820232c2e00 [83206.475797] ffff88201763fc18 ffff8820232c2e60 ffff8820232c2e00 ffff8810175b8000 [83206.484084] Call Trace: [83206.486811] [] queued_spin_lock_slowpath+0xb/0xf [83206.493802] [] _raw_spin_lock+0x20/0x30 [83206.499938] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83206.508112] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83206.516189] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83206.523586] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83206.531371] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83206.539153] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83206.546535] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83206.554610] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83206.562490] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83206.570358] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83206.578435] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83206.586898] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83206.595076] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83206.602745] [] ? wake_up_state+0x20/0x20 [83206.608979] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83206.617231] [] kthread+0xcf/0xe0 [83206.622671] [] ? insert_kthread_work+0x40/0x40 [83206.629469] [] ret_from_fork+0x58/0x90 [83206.635491] [] ? insert_kthread_work+0x40/0x40 [83206.642288] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83207.063315] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ldlm_bl_16:33189] [83207.066316] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ldlm_bl_09:16011] [83207.066353] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83207.066365] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83207.066366] CPU: 4 PID: 16011 Comm: ldlm_bl_09 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83207.066367] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83207.066368] task: ffff88203ad88fd0 ti: ffff88201c7fc000 task.ti: ffff88201c7fc000 [83207.066373] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83207.066374] RSP: 0018:ffff88201c7ffb70 EFLAGS: 00000246 [83207.066374] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000210000 [83207.066375] RDX: ffff88103cf97880 RSI: 0000000000610001 RDI: ffff88102137981c [83207.066376] RBP: ffff88201c7ffb70 R08: ffff88103ce97880 R09: 0000000000000000 [83207.066376] R10: ffff88202370b000 R11: 0000000000000000 R12: ffff88201c7ffc58 [83207.066377] R13: 0000000000000001 R14: ffff88201c7ffb28 R15: ffffffff81322c35 [83207.066377] FS: 0000000000000000(0000) GS:ffff88103ce80000(0000) knlGS:0000000000000000 [83207.066378] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83207.066379] CR2: 00007f8905462248 CR3: 00000000019f2000 CR4: 00000000001407e0 [83207.066379] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83207.066380] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83207.066380] Stack: [83207.066381] ffff88201c7ffb80 ffffffff8169e61f ffff88201c7ffb90 ffffffff816abb70 [83207.066382] ffff88201c7ffbd0 ffffffffc0b72198 0000000000000000 ffff88202370b000 [83207.066384] ffff88201c7ffc18 ffff88202370b060 ffff88202370b000 ffff8810175b8000 [83207.066384] Call Trace: [83207.066388] [] queued_spin_lock_slowpath+0xb/0xf [83207.066390] [] _raw_spin_lock+0x20/0x30 [83207.066431] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83207.066454] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83207.066475] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83207.066493] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83207.066511] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83207.066528] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83207.066536] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83207.066554] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83207.066571] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83207.066577] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83207.066597] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83207.066615] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83207.066637] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83207.066639] [] ? wake_up_state+0x20/0x20 [83207.066660] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83207.066662] [] kthread+0xcf/0xe0 [83207.066663] [] ? insert_kthread_work+0x40/0x40 [83207.066665] [] ret_from_fork+0x58/0x90 [83207.066666] [] ? insert_kthread_work+0x40/0x40 [83207.066680] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83207.069315] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ldlm_bl_11:16013] [83207.069337] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83207.069345] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83207.069346] CPU: 5 PID: 16013 Comm: ldlm_bl_11 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83207.069347] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83207.069348] task: ffff88203ad88000 ti: ffff88201c804000 task.ti: ffff88201c804000 [83207.069350] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83207.069351] RSP: 0018:ffff88201c807b70 EFLAGS: 00000246 [83207.069352] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000290000 [83207.069352] RDX: ffff88103d017880 RSI: 0000000000810001 RDI: ffff88102137981c [83207.069353] RBP: ffff88201c807b70 R08: ffff88203c697880 R09: 0000000000000000 [83207.069353] R10: ffff882023610600 R11: 0000000000000000 R12: ffff88201c807c58 [83207.069354] R13: 0000000000000001 R14: ffff88201c807b28 R15: ffffffff81322c35 [83207.069355] FS: 0000000000000000(0000) GS:ffff88203c680000(0000) knlGS:0000000000000000 [83207.069355] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83207.069356] CR2: 00007fd7ddcd8090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83207.069356] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83207.069357] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83207.069357] Stack: [83207.069358] ffff88201c807b80 ffffffff8169e61f ffff88201c807b90 ffffffff816abb70 [83207.069359] ffff88201c807bd0 ffffffffc0b72198 0000000000000000 ffff882023610600 [83207.069361] ffff88201c807c18 ffff882023610660 ffff882023610600 ffff8810175b8000 [83207.069361] Call Trace: [83207.069363] [] queued_spin_lock_slowpath+0xb/0xf [83207.069364] [] _raw_spin_lock+0x20/0x30 [83207.069389] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83207.069412] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83207.069432] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83207.069450] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83207.069467] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83207.069485] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83207.069491] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83207.069510] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83207.069527] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83207.069533] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83207.069552] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83207.069570] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83207.069592] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83207.069594] [] ? wake_up_state+0x20/0x20 [83207.069615] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83207.069616] [] kthread+0xcf/0xe0 [83207.069617] [] ? insert_kthread_work+0x40/0x40 [83207.069619] [] ret_from_fork+0x58/0x90 [83207.069620] [] ? insert_kthread_work+0x40/0x40 [83207.069634] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83207.075314] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ldlm_bl_17:33190] [83207.075336] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83207.075343] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83207.075344] CPU: 7 PID: 33190 Comm: ldlm_bl_17 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83207.075345] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83207.075345] task: ffff88102d281fa0 ti: ffff881fffd34000 task.ti: ffff881fffd34000 [83207.075348] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83207.075349] RSP: 0018:ffff881fffd37b70 EFLAGS: 00000246 [83207.075349] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000390000 [83207.075350] RDX: ffff88103cf17880 RSI: 0000000000410000 RDI: ffff88102137981c [83207.075351] RBP: ffff881fffd37b70 R08: ffff88203c6d7880 R09: 0000000000000000 [83207.075351] R10: ffff8820232c4400 R11: 0000000000000000 R12: ffff88102d282008 [83207.075352] R13: ffff881fffd37b00 R14: ffff881f00000141 R15: ffff88207ffd8008 [83207.075352] FS: 0000000000000000(0000) GS:ffff88203c6c0000(0000) knlGS:0000000000000000 [83207.075353] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83207.075354] CR2: 00007f240470f978 CR3: 00000000019f2000 CR4: 00000000001407e0 [83207.075354] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83207.075355] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83207.075355] Stack: [83207.075356] ffff881fffd37b80 ffffffff8169e61f ffff881fffd37b90 ffffffff816abb70 [83207.075357] ffff881fffd37bd0 ffffffffc0b72198 0000000000000000 ffff8820232c4400 [83207.075358] ffff881fffd37c18 ffff8820232c4460 ffff8820232c4400 ffff8810175b8000 [83207.075359] Call Trace: [83207.075361] [] queued_spin_lock_slowpath+0xb/0xf [83207.075362] [] _raw_spin_lock+0x20/0x30 [83207.075380] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83207.075399] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83207.075418] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83207.075436] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83207.075454] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83207.075471] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83207.075477] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83207.075494] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83207.075512] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83207.075517] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83207.075535] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83207.075552] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83207.075575] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83207.075576] [] ? __schedule+0x39d/0x8b0 [83207.075596] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83207.075597] [] kthread+0xcf/0xe0 [83207.075599] [] ? insert_kthread_work+0x40/0x40 [83207.075600] [] ret_from_fork+0x58/0x90 [83207.075601] [] ? insert_kthread_work+0x40/0x40 [83207.075615] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83207.102314] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ldlm_bl_12:16014] [83207.102336] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83207.102344] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83207.102345] CPU: 16 PID: 16014 Comm: ldlm_bl_12 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83207.102346] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83207.102347] task: ffff88203d33af70 ti: ffff88201c808000 task.ti: ffff88201c808000 [83207.102349] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83207.102350] RSP: 0018:ffff88201c80bb70 EFLAGS: 00000246 [83207.102350] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000810000 [83207.102351] RDX: ffff88203c717880 RSI: 0000000000490000 RDI: ffff88102137981c [83207.102352] RBP: ffff88201c80bb70 R08: ffff88103d017880 R09: 0000000000000000 [83207.102352] R10: ffff8820236edc00 R11: 0000000000000005 R12: ffff88201c80bc58 [83207.102353] R13: 0000000000000001 R14: ffff88201c80bb28 R15: ffffffff81322c35 [83207.102354] FS: 0000000000000000(0000) GS:ffff88103d000000(0000) knlGS:0000000000000000 [83207.102354] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83207.102355] CR2: 00007f361a3bb4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83207.102355] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83207.102356] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83207.102356] Stack: [83207.102358] ffff88201c80bb80 ffffffff8169e61f ffff88201c80bb90 ffffffff816abb70 [83207.102359] ffff88201c80bbd0 ffffffffc0b72198 0000000000000000 ffff8820236edc00 [83207.102360] ffff88201c80bc18 ffff8820236edc60 ffff8820236edc00 ffff8810175b8000 [83207.102360] Call Trace: [83207.102362] [] queued_spin_lock_slowpath+0xb/0xf [83207.102363] [] _raw_spin_lock+0x20/0x30 [83207.102384] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83207.102405] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83207.102424] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83207.102442] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83207.102459] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83207.102477] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83207.102483] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83207.102500] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83207.102518] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83207.102523] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83207.102541] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83207.102559] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83207.102580] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83207.102582] [] ? wake_up_state+0x20/0x20 [83207.102602] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83207.102603] [] kthread+0xcf/0xe0 [83207.102605] [] ? insert_kthread_work+0x40/0x40 [83207.102606] [] ret_from_fork+0x58/0x90 [83207.102607] [] ? insert_kthread_work+0x40/0x40 [83207.102620] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83208.861962] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83208.941423] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83208.970294] CPU: 3 PID: 33189 Comm: ldlm_bl_16 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83208.983004] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83208.991352] task: ffff88102c376eb0 ti: ffff881fefb44000 task.ti: ffff881fefb44000 [83208.999700] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83209.010772] RSP: 0018:ffff881fefb47b70 EFLAGS: 00000246 [83209.016696] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000190000 [83209.024657] RDX: ffff88203c8d7880 RSI: 0000000000b90000 RDI: ffff88102137981c [83209.032617] RBP: ffff881fefb47b70 R08: ffff88203c657880 R09: 0000000000000000 [83209.040578] R10: ffff8820232b5800 R11: 0000000000000000 R12: ffff88102c376f18 [83209.048538] R13: ffff88203c2b92a0 R14: ffff881000000141 R15: ffff88207ffd8008 [83209.056499] FS: 0000000000000000(0000) GS:ffff88203c640000(0000) knlGS:0000000000000000 [83209.065525] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83209.071934] CR2: 000055ce548c9220 CR3: 00000000019f2000 CR4: 00000000001407e0 [83209.079894] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83209.087855] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83209.095815] Stack: [83209.098055] ffff881fefb47b80 ffffffff8169e61f ffff881fefb47b90 ffffffff816abb70 [83209.106340] ffff881fefb47bd0 ffffffffc0b72198 0000000000000000 ffff8820232b5800 [83209.114627] ffff881fefb47c18 ffff8820232b5860 ffff8820232b5800 ffff8810175b8000 [83209.122912] Call Trace: [83209.125639] [] queued_spin_lock_slowpath+0xb/0xf [83209.132630] [] _raw_spin_lock+0x20/0x30 [83209.138767] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83209.146944] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83209.155021] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83209.162416] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83209.170199] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83209.177982] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83209.185365] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83209.193440] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83209.201320] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83209.209188] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83209.217263] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83209.225728] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83209.233902] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83209.241573] [] ? __schedule+0x39d/0x8b0 [83209.247710] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83209.255962] [] kthread+0xcf/0xe0 [83209.261402] [] ? insert_kthread_work+0x40/0x40 [83209.268200] [] ret_from_fork+0x58/0x90 [83209.274223] [] ? insert_kthread_work+0x40/0x40 [83209.281020] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83219.056056] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ldlm_bl_08:16010] [83219.057056] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ldlm_bl_13:16015] [83219.057091] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.057102] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.057104] CPU: 0 PID: 16015 Comm: ldlm_bl_13 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.057105] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.057106] task: ffff88203d33bf40 ti: ffff88201c80c000 task.ti: ffff88201c80c000 [83219.057110] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83219.057111] RSP: 0018:ffff88201c80fb70 EFLAGS: 00000246 [83219.057111] RAX: 0000000000000000 RBX: 000000013ce56cc0 RCX: 0000000000010000 [83219.057112] RDX: ffff88103d0d7880 RSI: 0000000000b10000 RDI: ffff88102137981c [83219.057112] RBP: ffff88201c80fb70 R08: ffff88103ce17880 R09: 0000000000000000 [83219.057113] R10: ffff882023a34600 R11: 0000000000000005 R12: ffff88201c80fc58 [83219.057114] R13: 0000000000000001 R14: ffff88201c80fb28 R15: ffffffff81322c35 [83219.057114] FS: 0000000000000000(0000) GS:ffff88103ce00000(0000) knlGS:0000000000000000 [83219.057115] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.057116] CR2: 00007f17e6a8b248 CR3: 00000000019f2000 CR4: 00000000001407f0 [83219.057116] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.057117] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.057117] Stack: [83219.057118] ffff88201c80fb80 ffffffff8169e61f ffff88201c80fb90 ffffffff816abb70 [83219.057120] ffff88201c80fbd0 ffffffffc0b72198 0000000000000000 ffff882023a34600 [83219.057121] ffff88201c80fc18 ffff882023a34660 ffff882023a34600 ffff8810175b8000 [83219.057121] Call Trace: [83219.057125] [] queued_spin_lock_slowpath+0xb/0xf [83219.057127] [] _raw_spin_lock+0x20/0x30 [83219.057162] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.057184] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.057205] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.057223] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.057241] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.057258] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.057266] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.057285] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.057302] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.057308] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.057328] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.057346] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.057368] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.057371] [] ? wake_up_state+0x20/0x20 [83219.057391] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.057393] [] kthread+0xcf/0xe0 [83219.057394] [] ? insert_kthread_work+0x40/0x40 [83219.057396] [] ret_from_fork+0x58/0x90 [83219.057397] [] ? insert_kthread_work+0x40/0x40 [83219.057410] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83219.060056] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ldlm_bl_06:16008] [83219.060075] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.060080] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.060081] CPU: 2 PID: 16008 Comm: ldlm_bl_06 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.060081] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.060082] task: ffff8810247eeeb0 ti: ffff88201c7e8000 task.ti: ffff88201c7e8000 [83219.060085] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x110/0x1e0 [83219.060085] RSP: 0018:ffff88201c7ebb70 EFLAGS: 00000246 [83219.060086] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000110000 [83219.060086] RDX: ffff88203c897880 RSI: 0000000000a90000 RDI: ffff88102137981c [83219.060087] RBP: ffff88201c7ebb70 R08: ffff88103ce57880 R09: 0000000000000000 [83219.060087] R10: ffff882023a39800 R11: 0000000000000005 R12: ffff88201c7ebc58 [83219.060087] R13: 0000000000000001 R14: ffff88201c7ebb28 R15: ffffffff81322c35 [83219.060088] FS: 0000000000000000(0000) GS:ffff88103ce40000(0000) knlGS:0000000000000000 [83219.060089] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.060089] CR2: 0000000002577010 CR3: 00000000019f2000 CR4: 00000000001407e0 [83219.060089] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.060090] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.060090] Stack: [83219.060091] ffff88201c7ebb80 ffffffff8169e61f ffff88201c7ebb90 ffffffff816abb70 [83219.060092] ffff88201c7ebbd0 ffffffffc0b72198 0000000000000000 ffff882023a39800 [83219.060092] ffff88201c7ebc18 ffff882023a39860 ffff882023a39800 ffff8810175b8000 [83219.060093] Call Trace: [83219.060095] [] queued_spin_lock_slowpath+0xb/0xf [83219.060096] [] _raw_spin_lock+0x20/0x30 [83219.060111] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.060126] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.060141] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.060154] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.060167] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.060180] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.060185] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.060198] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.060211] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.060216] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.060229] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.060242] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.060258] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.060259] [] ? wake_up_state+0x20/0x20 [83219.060274] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.060275] [] kthread+0xcf/0xe0 [83219.060276] [] ? insert_kthread_work+0x40/0x40 [83219.060277] [] ret_from_fork+0x58/0x90 [83219.060278] [] ? insert_kthread_work+0x40/0x40 [83219.060286] Code: c1 e8 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 90 41 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 [83219.072056] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ldlm_bl_14:16016] [83219.072078] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.072085] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.072086] CPU: 6 PID: 16016 Comm: ldlm_bl_14 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.072087] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.072088] task: ffff88203d338000 ti: ffff88201c810000 task.ti: ffff88201c810000 [83219.072090] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83219.072091] RSP: 0018:ffff88201c813b70 EFLAGS: 00000246 [83219.072092] RAX: 0000000000000000 RBX: 000000011c813be8 RCX: 0000000000310000 [83219.072092] RDX: ffff88103ce57880 RSI: 0000000000110000 RDI: ffff88102137981c [83219.072093] RBP: ffff88201c813b70 R08: ffff88103ced7880 R09: 0000000000000000 [83219.072093] R10: ffff882023a3a400 R11: 0000000000000000 R12: ffff88201c813c58 [83219.072094] R13: 0000000000000001 R14: ffff88201c813b28 R15: ffffffff81322c35 [83219.072095] FS: 0000000000000000(0000) GS:ffff88103cec0000(0000) knlGS:0000000000000000 [83219.072095] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.072096] CR2: 0000000002079728 CR3: 00000000019f2000 CR4: 00000000001407e0 [83219.072096] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.072097] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.072097] Stack: [83219.072098] ffff88201c813b80 ffffffff8169e61f ffff88201c813b90 ffffffff816abb70 [83219.072100] ffff88201c813bd0 ffffffffc0b72198 0000000000000000 ffff882023a3a400 [83219.072101] ffff88201c813c18 ffff882023a3a460 ffff882023a3a400 ffff8810175b8000 [83219.072101] Call Trace: [83219.072103] [] queued_spin_lock_slowpath+0xb/0xf [83219.072105] [] _raw_spin_lock+0x20/0x30 [83219.072125] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.072147] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.072166] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.072184] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.072202] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.072219] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.072225] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.072242] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.072259] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.072265] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.072282] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.072300] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.072320] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.072321] [] ? wake_up_state+0x20/0x20 [83219.072342] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.072343] [] kthread+0xcf/0xe0 [83219.072344] [] ? insert_kthread_work+0x40/0x40 [83219.072346] [] ret_from_fork+0x58/0x90 [83219.072347] [] ? insert_kthread_work+0x40/0x40 [83219.072360] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83219.078055] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ldlm_bl_10:16012] [83219.078077] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.078084] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.078085] CPU: 8 PID: 16012 Comm: ldlm_bl_10 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.078086] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.078087] task: ffff88203ad89fa0 ti: ffff88201c800000 task.ti: ffff88201c800000 [83219.078089] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83219.078090] RSP: 0018:ffff88201c803b70 EFLAGS: 00000246 [83219.078090] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000410000 [83219.078091] RDX: ffff88203c617880 RSI: 0000000000090000 RDI: ffff88102137981c [83219.078091] RBP: ffff88201c803b70 R08: ffff88103cf17880 R09: 0000000000000000 [83219.078092] R10: ffff8820237fc000 R11: 0000000000000005 R12: ffff88201c803c58 [83219.078093] R13: 0000000000000001 R14: ffff88201c803b28 R15: ffffffff81322c35 [83219.078093] FS: 0000000000000000(0000) GS:ffff88103cf00000(0000) knlGS:0000000000000000 [83219.078094] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.078094] CR2: 00007fca10cf5000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83219.078095] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.078096] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.078096] Stack: [83219.078097] ffff88201c803b80 ffffffff8169e61f ffff88201c803b90 ffffffff816abb70 [83219.078098] ffff88201c803bd0 ffffffffc0b72198 0000000000000000 ffff8820237fc000 [83219.078099] ffff88201c803c18 ffff8820237fc060 ffff8820237fc000 ffff8810175b8000 [83219.078100] Call Trace: [83219.078102] [] queued_spin_lock_slowpath+0xb/0xf [83219.078103] [] _raw_spin_lock+0x20/0x30 [83219.078121] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.078142] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.078161] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.078179] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.078197] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.078214] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.078220] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.078237] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.078255] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.078260] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.078280] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.078298] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.078320] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.078322] [] ? wake_up_state+0x20/0x20 [83219.078342] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.078343] [] kthread+0xcf/0xe0 [83219.078345] [] ? insert_kthread_work+0x40/0x40 [83219.078346] [] ret_from_fork+0x58/0x90 [83219.078347] [] ? insert_kthread_work+0x40/0x40 [83219.078361] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83219.084056] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ldlm_bl_05:16007] [83219.084077] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.084084] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.084085] CPU: 10 PID: 16007 Comm: ldlm_bl_05 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.084086] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.084087] task: ffff8810247ecf10 ti: ffff88201c64c000 task.ti: ffff88201c64c000 [83219.084089] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83219.084090] RSP: 0018:ffff88201c64fb70 EFLAGS: 00000246 [83219.084090] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000510000 [83219.084091] RDX: ffff88103ce17880 RSI: 0000000000010000 RDI: ffff88102137981c [83219.084091] RBP: ffff88201c64fb70 R08: ffff88103cf57880 R09: 0000000000000000 [83219.084092] R10: ffff88202377d000 R11: 0000000000000005 R12: ffff88201c64fc58 [83219.084092] R13: 0000000000000001 R14: ffff88201c64fb28 R15: ffffffff81322c35 [83219.084093] FS: 0000000000000000(0000) GS:ffff88103cf40000(0000) knlGS:0000000000000000 [83219.084094] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.084094] CR2: 0000000001dd94b8 CR3: 00000000019f2000 CR4: 00000000001407e0 [83219.084095] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.084095] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.084096] Stack: [83219.084097] ffff88201c64fb80 ffffffff8169e61f ffff88201c64fb90 ffffffff816abb70 [83219.084098] ffff88201c64fbd0 ffffffffc0b72198 0000000000000000 ffff88202377d000 [83219.084099] ffff88201c64fc18 ffff88202377d060 ffff88202377d000 ffff8810175b8000 [83219.084099] Call Trace: [83219.084101] [] queued_spin_lock_slowpath+0xb/0xf [83219.084103] [] _raw_spin_lock+0x20/0x30 [83219.084123] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.084145] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.084164] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.084181] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.084199] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.084217] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.084222] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.084240] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.084257] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.084263] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.084280] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.084300] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.084322] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.084324] [] ? wake_up_state+0x20/0x20 [83219.084344] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.084345] [] kthread+0xcf/0xe0 [83219.084347] [] ? insert_kthread_work+0x40/0x40 [83219.084348] [] ret_from_fork+0x58/0x90 [83219.084349] [] ? insert_kthread_work+0x40/0x40 [83219.084363] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83219.090055] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ldlm_bl_15:16017] [83219.090076] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.090083] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.090085] CPU: 12 PID: 16017 Comm: ldlm_bl_15 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.090085] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.090086] task: ffff88203d338fd0 ti: ffff88201c814000 task.ti: ffff88201c814000 [83219.090088] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83219.090089] RSP: 0018:ffff88201c817b70 EFLAGS: 00000246 [83219.090089] RAX: 0000000000000000 RBX: 00000001f8c12595 RCX: 0000000000610000 [83219.090090] RDX: ffff88203c757880 RSI: 0000000000590000 RDI: ffff88102137981c [83219.090091] RBP: ffff88201c817b70 R08: ffff88103cf97880 R09: 0000000000000000 [83219.090091] R10: ffff8820237fc000 R11: 0000000000000005 R12: ffff88201c817c58 [83219.090092] R13: 0000000000000001 R14: ffff88201c817b28 R15: ffffffff81322c35 [83219.090092] FS: 0000000000000000(0000) GS:ffff88103cf80000(0000) knlGS:0000000000000000 [83219.090093] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.090093] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83219.090094] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.090095] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.090095] Stack: [83219.090096] ffff88201c817b80 ffffffff8169e61f ffff88201c817b90 ffffffff816abb70 [83219.090097] ffff88201c817bd0 ffffffffc0b72198 0000000000000000 ffff8820237fc000 [83219.090098] ffff88201c817c18 ffff8820237fc060 ffff8820237fc000 ffff8810175b8000 [83219.090098] Call Trace: [83219.090100] [] queued_spin_lock_slowpath+0xb/0xf [83219.090102] [] _raw_spin_lock+0x20/0x30 [83219.090120] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.090141] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.090160] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.090178] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.090195] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.090213] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.090218] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.090236] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.090253] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.090259] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.090279] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.090296] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.090318] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.090319] [] ? wake_up_state+0x20/0x20 [83219.090339] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.090341] [] kthread+0xcf/0xe0 [83219.090342] [] ? insert_kthread_work+0x40/0x40 [83219.090343] [] ret_from_fork+0x58/0x90 [83219.090344] [] ? insert_kthread_work+0x40/0x40 [83219.090358] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83219.108055] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ldlm_bl_02:15061] [83219.108076] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.108083] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.108085] CPU: 18 PID: 15061 Comm: ldlm_bl_02 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.108085] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.108086] task: ffff882012edeeb0 ti: ffff88203d3bc000 task.ti: ffff88203d3bc000 [83219.108088] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83219.108089] RSP: 0018:ffff88203d3bfb70 EFLAGS: 00000246 [83219.108089] RAX: 0000000000000000 RBX: 000000013cf16cc0 RCX: 0000000000910000 [83219.108090] RDX: ffff88203c7d7880 RSI: 0000000000790001 RDI: ffff88102137981c [83219.108091] RBP: ffff88203d3bfb70 R08: ffff88103d057880 R09: 0000000000000000 [83219.108091] R10: ffff8820237fc200 R11: 0000000000000005 R12: ffff88203d3bfc58 [83219.108092] R13: 0000000000000001 R14: ffff88203d3bfb28 R15: ffffffff81322c35 [83219.108092] FS: 0000000000000000(0000) GS:ffff88103d040000(0000) knlGS:0000000000000000 [83219.108093] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.108094] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83219.108094] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.108095] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.108095] Stack: [83219.108096] ffff88203d3bfb80 ffffffff8169e61f ffff88203d3bfb90 ffffffff816abb70 [83219.108097] ffff88203d3bfbd0 ffffffffc0b72198 0000000000000000 ffff8820237fc200 [83219.108098] ffff88203d3bfc18 ffff8820237fc260 ffff8820237fc200 ffff8810175b8000 [83219.108099] Call Trace: [83219.108101] [] queued_spin_lock_slowpath+0xb/0xf [83219.108102] [] _raw_spin_lock+0x20/0x30 [83219.108123] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.108144] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.108163] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.108180] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.108198] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.108216] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.108221] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.108239] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.108257] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.108262] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.108280] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.108297] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.108319] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.108321] [] ? wake_up_state+0x20/0x20 [83219.108341] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.108342] [] kthread+0xcf/0xe0 [83219.108344] [] ? insert_kthread_work+0x40/0x40 [83219.108345] [] ret_from_fork+0x58/0x90 [83219.108346] [] ? insert_kthread_work+0x40/0x40 [83219.108360] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83219.114055] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ldlm_bl_01:15060] [83219.114076] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.114083] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.114084] CPU: 20 PID: 15060 Comm: ldlm_bl_01 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.114085] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.114086] task: ffff882012eddee0 ti: ffff88203d3b8000 task.ti: ffff88203d3b8000 [83219.114088] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83219.114089] RSP: 0018:ffff88203d3bbb70 EFLAGS: 00000246 [83219.114089] RAX: 0000000000000000 RBX: 0000000180597933 RCX: 0000000000a10000 [83219.114090] RDX: ffff88203c857880 RSI: 0000000000990000 RDI: ffff88102137981c [83219.114091] RBP: ffff88203d3bbb70 R08: ffff88103d097880 R09: 0000000000000000 [83219.114091] R10: ffff882023a1ba00 R11: 0000000000000005 R12: ffff88203d3bbc58 [83219.114092] R13: 0000000000000001 R14: ffff88203d3bbb28 R15: ffffffff81322c35 [83219.114092] FS: 0000000000000000(0000) GS:ffff88103d080000(0000) knlGS:0000000000000000 [83219.114093] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.114094] CR2: 00007f02c314e4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83219.114094] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.114095] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.114095] Stack: [83219.114096] ffff88203d3bbb80 ffffffff8169e61f ffff88203d3bbb90 ffffffff816abb70 [83219.114097] ffff88203d3bbbd0 ffffffffc0b72198 0000000000000000 ffff882023a1ba00 [83219.114098] ffff88203d3bbc18 ffff882023a1ba60 ffff882023a1ba00 ffff8810175b8000 [83219.114099] Call Trace: [83219.114101] [] queued_spin_lock_slowpath+0xb/0xf [83219.114102] [] _raw_spin_lock+0x20/0x30 [83219.114122] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.114143] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.114163] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.114180] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.114198] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.114216] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.114221] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.114239] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.114256] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.114262] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.114279] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.114297] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.114319] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.114321] [] ? wake_up_state+0x20/0x20 [83219.114341] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.114342] [] kthread+0xcf/0xe0 [83219.114343] [] ? insert_kthread_work+0x40/0x40 [83219.114345] [] ret_from_fork+0x58/0x90 [83219.114346] [] ? insert_kthread_work+0x40/0x40 [83219.114359] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83219.120055] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ldlm_bl_07:16009] [83219.120076] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83219.120083] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83219.120085] CPU: 22 PID: 16009 Comm: ldlm_bl_07 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83219.120085] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83219.120086] task: ffff8810e9240fd0 ti: ffff88201c7ec000 task.ti: ffff88201c7ec000 [83219.120088] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83219.120089] RSP: 0018:ffff88201c7efb70 EFLAGS: 00000246 [83219.120089] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000b10000 [83219.120090] RDX: ffff88103ce97880 RSI: 0000000000210001 RDI: ffff88102137981c [83219.120091] RBP: ffff88201c7efb70 R08: ffff88103d0d7880 R09: 0000000000000000 [83219.120091] R10: ffff8820237fc200 R11: 0000000000000005 R12: ffff88201c7efc58 [83219.120092] R13: 0000000000000001 R14: ffff88201c7efb28 R15: ffffffff81322c35 [83219.120093] FS: 0000000000000000(0000) GS:ffff88103d0c0000(0000) knlGS:0000000000000000 [83219.120093] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83219.120094] CR2: 00007f262c4764f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83219.120094] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83219.120095] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83219.120095] Stack: [83219.120097] ffff88201c7efb80 ffffffff8169e61f ffff88201c7efb90 ffffffff816abb70 [83219.120098] ffff88201c7efbd0 ffffffffc0b72198 0000000000000000 ffff8820237fc200 [83219.120099] ffff88201c7efc18 ffff8820237fc260 ffff8820237fc200 ffff8810175b8000 [83219.120099] Call Trace: [83219.120101] [] queued_spin_lock_slowpath+0xb/0xf [83219.120103] [] _raw_spin_lock+0x20/0x30 [83219.120123] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83219.120144] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83219.120163] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83219.120181] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83219.120199] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83219.120216] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83219.120221] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83219.120239] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.120257] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83219.120262] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83219.120280] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83219.120297] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83219.120319] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83219.120321] [] ? wake_up_state+0x20/0x20 [83219.120341] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83219.120343] [] kthread+0xcf/0xe0 [83219.120344] [] ? insert_kthread_work+0x40/0x40 [83219.120346] [] ret_from_fork+0x58/0x90 [83219.120347] [] ? insert_kthread_work+0x40/0x40 [83219.120360] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83223.092947] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83223.172424] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83223.201292] CPU: 1 PID: 16010 Comm: ldlm_bl_08 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83223.214002] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83223.222350] task: ffff8810e9242f70 ti: ffff88201c7f8000 task.ti: ffff88201c7f8000 [83223.230698] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x148/0x1e0 [83223.241770] RSP: 0018:ffff88201c7fbb70 EFLAGS: 00000202 [83223.247694] RAX: 0000000000000001 RBX: 000000013d016cc0 RCX: 0000000000090000 [83223.255654] RDX: 0000000000010001 RSI: 0000000000510000 RDI: ffff88102137981c [83223.263614] RBP: ffff88201c7fbb70 R08: ffff88203c617880 R09: ffff88103cf17880 [83223.271574] R10: ffff8820236f9c00 R11: 0000000000000005 R12: ffff88201c7fbc58 [83223.279535] R13: 0000000000000001 R14: ffff88201c7fbb28 R15: ffffffff81322c35 [83223.287495] FS: 0000000000000000(0000) GS:ffff88203c600000(0000) knlGS:0000000000000000 [83223.296521] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83223.302930] CR2: 00007f11b0c8c000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83223.310890] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83223.318851] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83223.326810] Stack: [83223.329051] ffff88201c7fbb80 ffffffff8169e61f ffff88201c7fbb90 ffffffff816abb70 [83223.337338] ffff88201c7fbbd0 ffffffffc0b72198 0000000000000000 ffff8820236f9c00 [83223.345626] ffff88201c7fbc18 ffff8820236f9c60 ffff8820236f9c00 ffff8810175b8000 [83223.353913] Call Trace: [83223.356640] [] queued_spin_lock_slowpath+0xb/0xf [83223.363631] [] _raw_spin_lock+0x20/0x30 [83223.369772] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83223.377941] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83223.386009] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83223.393405] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83223.401187] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83223.408970] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83223.416354] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83223.424429] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83223.432309] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83223.440177] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83223.448252] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83223.456717] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83223.464894] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83223.472566] [] ? wake_up_state+0x20/0x20 [83223.478801] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83223.487052] [] kthread+0xcf/0xe0 [83223.492492] [] ? insert_kthread_work+0x40/0x40 [83223.499290] [] ret_from_fork+0x58/0x90 [83223.505313] [] ? insert_kthread_work+0x40/0x40 [83223.512110] Code: 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 8b 17 <0f> b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 00 00 [83231.080797] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [ldlm_bl_03:15915] [83231.086797] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [ldlm_bl_22:33195] [83231.086820] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83231.086829] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83231.086830] CPU: 11 PID: 33195 Comm: ldlm_bl_22 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83231.086831] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83231.086832] task: ffff88203d0b1fa0 ti: ffff88203d4b8000 task.ti: ffff88203d4b8000 [83231.086835] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83231.086836] RSP: 0018:ffff88203d4bbb70 EFLAGS: 00000246 [83231.086837] RAX: 0000000000000000 RBX: 5a5a5a5a5a5a5a5a RCX: 0000000000590000 [83231.086837] RDX: ffff88103ced7880 RSI: 0000000000310000 RDI: ffff88102137981c [83231.086838] RBP: ffff88203d4bbb70 R08: ffff88203c757880 R09: 0000000000000000 [83231.086838] R10: ffff882023734200 R11: 0000000000000001 R12: 5a5a5a5a5a5a5a5a [83231.086839] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83231.086840] FS: 0000000000000000(0000) GS:ffff88203c740000(0000) knlGS:0000000000000000 [83231.086840] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83231.086841] CR2: 00007fef29abd090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83231.086841] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83231.086842] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83231.086842] Stack: [83231.086843] ffff88203d4bbb80 ffffffff8169e61f ffff88203d4bbb90 ffffffff816abb70 [83231.086845] ffff88203d4bbbd0 ffffffffc0b72198 0000000000000000 ffff882023734200 [83231.086846] ffff88203d4bbc18 ffff882023734260 ffff882023734200 ffff8810175b8000 [83231.086846] Call Trace: [83231.086848] [] queued_spin_lock_slowpath+0xb/0xf [83231.086850] [] _raw_spin_lock+0x20/0x30 [83231.086879] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83231.086902] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83231.086922] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83231.086940] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83231.086959] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83231.086978] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83231.086984] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83231.087003] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.087022] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.087027] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83231.087046] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83231.087064] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83231.087085] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83231.087087] [] ? __schedule+0x39d/0x8b0 [83231.087108] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83231.087109] [] kthread+0xcf/0xe0 [83231.087111] [] ? insert_kthread_work+0x40/0x40 [83231.087112] [] ret_from_fork+0x58/0x90 [83231.087113] [] ? insert_kthread_work+0x40/0x40 [83231.087127] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83231.092796] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [ldlm_bl_19:33192] [83231.092819] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83231.092827] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83231.092828] CPU: 13 PID: 33192 Comm: ldlm_bl_19 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83231.092829] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83231.092830] task: ffff88203d0b2f70 ti: ffff881ff8e20000 task.ti: ffff881ff8e20000 [83231.092833] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83231.092834] RSP: 0018:ffff881ff8e23b70 EFLAGS: 00000246 [83231.092834] RAX: 0000000000000000 RBX: ffff881ff8e23be8 RCX: 0000000000690000 [83231.092835] RDX: ffff88203c697880 RSI: 0000000000290000 RDI: ffff88102137981c [83231.092836] RBP: ffff881ff8e23b70 R08: ffff88203c797880 R09: 0000000000000000 [83231.092836] R10: ffff882023737e00 R11: 0000000000000000 R12: ffff88103ce96cc0 [83231.092837] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83231.092837] FS: 0000000000000000(0000) GS:ffff88203c780000(0000) knlGS:0000000000000000 [83231.092838] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83231.092839] CR2: 00007f352c5974f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83231.092839] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83231.092840] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83231.092840] Stack: [83231.092841] ffff881ff8e23b80 ffffffff8169e61f ffff881ff8e23b90 ffffffff816abb70 [83231.092842] ffff881ff8e23bd0 ffffffffc0b72198 0000000000000000 ffff882023737e00 [83231.092844] ffff881ff8e23c18 ffff882023737e60 ffff882023737e00 ffff8810175b8000 [83231.092844] Call Trace: [83231.092846] [] queued_spin_lock_slowpath+0xb/0xf [83231.092847] [] _raw_spin_lock+0x20/0x30 [83231.092874] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83231.092896] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83231.092915] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83231.092933] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83231.092951] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83231.092968] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83231.092974] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83231.092992] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.093010] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.093015] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83231.093035] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83231.093053] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83231.093075] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83231.093076] [] ? __schedule+0x39d/0x8b0 [83231.093097] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83231.093099] [] kthread+0xcf/0xe0 [83231.093100] [] ? insert_kthread_work+0x40/0x40 [83231.093101] [] ret_from_fork+0x58/0x90 [83231.093102] [] ? insert_kthread_work+0x40/0x40 [83231.093116] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83231.098796] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ldlm_bl_23:33196] [83231.098817] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83231.098825] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83231.098826] CPU: 15 PID: 33196 Comm: ldlm_bl_23 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83231.098827] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83231.098828] task: ffff88203d0b0000 ti: ffff882015ae0000 task.ti: ffff882015ae0000 [83231.098830] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83231.098831] RSP: 0018:ffff882015ae3b70 EFLAGS: 00000246 [83231.098832] RAX: 0000000000000000 RBX: ffff88203c756cc0 RCX: 0000000000790000 [83231.098832] RDX: ffff88203c6d7880 RSI: 0000000000390000 RDI: ffff88102137981c [83231.098833] RBP: ffff882015ae3b70 R08: ffff88203c7d7880 R09: 0000000000000000 [83231.098833] R10: ffff882022e37200 R11: 0000000000000001 R12: ffff88203d0b0068 [83231.098834] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83231.098835] FS: 0000000000000000(0000) GS:ffff88203c7c0000(0000) knlGS:0000000000000000 [83231.098835] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83231.098836] CR2: 00007faebc9e03cc CR3: 00000000019f2000 CR4: 00000000001407e0 [83231.098836] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83231.098837] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83231.098837] Stack: [83231.098839] ffff882015ae3b80 ffffffff8169e61f ffff882015ae3b90 ffffffff816abb70 [83231.098840] ffff882015ae3bd0 ffffffffc0b72198 0000000000000000 ffff882022e37200 [83231.098841] ffff882015ae3c18 ffff882022e37260 ffff882022e37200 ffff8810175b8000 [83231.098841] Call Trace: [83231.098843] [] queued_spin_lock_slowpath+0xb/0xf [83231.098845] [] _raw_spin_lock+0x20/0x30 [83231.098863] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83231.098885] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83231.098904] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83231.098921] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83231.098938] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83231.098956] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83231.098961] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83231.098979] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.098997] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.099002] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83231.099020] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83231.099040] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83231.099060] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83231.099062] [] ? __schedule+0x39d/0x8b0 [83231.099082] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83231.099084] [] kthread+0xcf/0xe0 [83231.099085] [] ? insert_kthread_work+0x40/0x40 [83231.099086] [] ret_from_fork+0x58/0x90 [83231.099087] [] ? insert_kthread_work+0x40/0x40 [83231.099101] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83231.104796] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [ldlm_bl_21:33194] [83231.104818] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83231.104825] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83231.104826] CPU: 17 PID: 33194 Comm: ldlm_bl_21 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83231.104827] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83231.104828] task: ffff88203d0b6eb0 ti: ffff882012eac000 task.ti: ffff882012eac000 [83231.104830] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83231.104831] RSP: 0018:ffff882012eafb70 EFLAGS: 00000246 [83231.104832] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000890000 [83231.104832] RDX: ffff88203c797880 RSI: 0000000000690000 RDI: ffff88102137981c [83231.104833] RBP: ffff882012eafb70 R08: ffff88203c817880 R09: 0000000000000000 [83231.104833] R10: ffff882023756e00 R11: 0000000000000001 R12: 0000000000000000 [83231.104834] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c816cc0 [83231.104834] FS: 0000000000000000(0000) GS:ffff88203c800000(0000) knlGS:0000000000000000 [83231.104835] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83231.104836] CR2: 00007f7ce52f4550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83231.104836] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83231.104837] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83231.104837] Stack: [83231.104838] ffff882012eafb80 ffffffff8169e61f ffff882012eafb90 ffffffff816abb70 [83231.104839] ffff882012eafbd0 ffffffffc0b72198 0000000000000000 ffff882023756e00 [83231.104840] ffff882012eafc18 ffff882023756e60 ffff882023756e00 ffff8810175b8000 [83231.104841] Call Trace: [83231.104843] [] queued_spin_lock_slowpath+0xb/0xf [83231.104844] [] _raw_spin_lock+0x20/0x30 [83231.104862] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83231.104882] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83231.104901] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83231.104919] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83231.104936] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83231.104954] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83231.104959] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83231.104977] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.104995] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.105000] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83231.105020] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83231.105038] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83231.105060] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83231.105062] [] ? __schedule+0x39d/0x8b0 [83231.105082] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83231.105084] [] kthread+0xcf/0xe0 [83231.105085] [] ? insert_kthread_work+0x40/0x40 [83231.105086] [] ret_from_fork+0x58/0x90 [83231.105088] [] ? insert_kthread_work+0x40/0x40 [83231.105101] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83231.110796] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 23s! [ldlm_bl_18:33191] [83231.110817] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83231.110825] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83231.110826] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83231.110826] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83231.110827] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83231.110830] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83231.110830] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83231.110831] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83231.110832] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83231.110832] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83231.110833] R10: ffff882023772c00 R11: 0000000000000001 R12: ffff88203d0b5f48 [83231.110833] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83231.110834] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83231.110834] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83231.110835] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83231.110836] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83231.110836] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83231.110837] Stack: [83231.110838] ffff881ffba83b80 ffffffff8169e61f ffff881ffba83b90 ffffffff816abb70 [83231.110839] ffff881ffba83bd0 ffffffffc0b72198 0000000000000000 ffff882023772c00 [83231.110840] ffff881ffba83c18 ffff882023772c60 ffff882023772c00 ffff8810175b8000 [83231.110841] Call Trace: [83231.110843] [] queued_spin_lock_slowpath+0xb/0xf [83231.110844] [] _raw_spin_lock+0x20/0x30 [83231.110862] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83231.110884] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83231.110903] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83231.110921] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83231.110938] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83231.110956] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83231.110961] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83231.110979] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.110996] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.111002] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83231.111019] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83231.111037] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83231.111059] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83231.111060] [] ? __schedule+0x39d/0x8b0 [83231.111080] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83231.111082] [] kthread+0xcf/0xe0 [83231.111083] [] ? insert_kthread_work+0x40/0x40 [83231.111084] [] ret_from_fork+0x58/0x90 [83231.111086] [] ? insert_kthread_work+0x40/0x40 [83231.111099] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83231.116795] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [ldlm_bl_20:33193] [83231.116814] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83231.116820] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83231.116822] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83231.116822] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83231.116823] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83231.116825] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83231.116826] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83231.116826] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83231.116827] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83231.116827] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83231.116828] R10: ffff882023737400 R11: 0000000000000000 R12: ffff88103ce96cc0 [83231.116828] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83231.116829] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83231.116830] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83231.116831] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83231.116831] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83231.116832] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83231.116832] Stack: [83231.116833] ffff881ff8e2fb80 ffffffff8169e61f ffff881ff8e2fb90 ffffffff816abb70 [83231.116834] ffff881ff8e2fbd0 ffffffffc0b72198 0000000000000000 ffff882023737400 [83231.116835] ffff881ff8e2fc18 ffff882023737460 ffff882023737400 ffff8810175b8000 [83231.116835] Call Trace: [83231.116837] [] queued_spin_lock_slowpath+0xb/0xf [83231.116839] [] _raw_spin_lock+0x20/0x30 [83231.116856] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83231.116874] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83231.116892] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83231.116908] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83231.116925] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83231.116941] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83231.116946] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83231.116963] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.116979] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.116984] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83231.117001] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83231.117017] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83231.117036] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83231.117038] [] ? __schedule+0x39d/0x8b0 [83231.117057] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83231.117059] [] kthread+0xcf/0xe0 [83231.117060] [] ? insert_kthread_work+0x40/0x40 [83231.117061] [] ret_from_fork+0x58/0x90 [83231.117062] [] ? insert_kthread_work+0x40/0x40 [83231.117073] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83231.122796] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ldlm_bl_04:16006] [83231.122817] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83231.122824] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83231.122826] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83231.122826] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83231.122827] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83231.122830] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83231.122830] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83231.122831] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83231.122832] RDX: ffff88103d057880 RSI: 0000000000910000 RDI: ffff88102137981c [83231.122832] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83231.122833] R10: ffff882023a4b200 R11: 0000000000000000 R12: ffff88201c627c58 [83231.122833] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83231.122834] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83231.122835] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83231.122835] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83231.122836] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83231.122836] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83231.122837] Stack: [83231.122838] ffff88201c627b80 ffffffff8169e61f ffff88201c627b90 ffffffff816abb70 [83231.122839] ffff88201c627bd0 ffffffffc0b72198 0000000000000000 ffff882023a4b200 [83231.122840] ffff88201c627c18 ffff882023a4b260 ffff882023a4b200 ffff8810175b8000 [83231.122840] Call Trace: [83231.122843] [] queued_spin_lock_slowpath+0xb/0xf [83231.122844] [] _raw_spin_lock+0x20/0x30 [83231.122864] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83231.122885] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83231.122904] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83231.122922] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83231.122939] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83231.122957] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83231.122962] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83231.122980] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.122997] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83231.123002] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83231.123020] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83231.123038] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83231.123058] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83231.123059] [] ? wake_up_state+0x20/0x20 [83231.123079] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83231.123081] [] kthread+0xcf/0xe0 [83231.123082] [] ? insert_kthread_work+0x40/0x40 [83231.123083] [] ret_from_fork+0x58/0x90 [83231.123085] [] ? insert_kthread_work+0x40/0x40 [83231.123098] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83234.222543] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83234.302009] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83234.330881] CPU: 9 PID: 15915 Comm: ldlm_bl_03 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83234.343593] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83234.351941] task: ffff88202c648000 ti: ffff88201763c000 task.ti: ffff88201763c000 [83234.360289] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83234.371360] RSP: 0018:ffff88201763fb70 EFLAGS: 00000246 [83234.377285] RAX: 0000000000000000 RBX: 0000000300000000 RCX: 0000000000490000 [83234.385245] RDX: ffff88103d097880 RSI: 0000000000a10000 RDI: ffff88102137981c [83234.393205] RBP: ffff88201763fb70 R08: ffff88203c717880 R09: 0000000000000000 [83234.401165] R10: ffff88202374e000 R11: 7fffffffffffffff R12: 0000000000000bf8 [83234.409125] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c656cc0 [83234.417086] FS: 0000000000000000(0000) GS:ffff88203c700000(0000) knlGS:0000000000000000 [83234.426111] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83234.432521] CR2: 00007ffa010a6000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83234.440481] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83234.448441] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83234.456401] Stack: [83234.458641] ffff88201763fb80 ffffffff8169e61f ffff88201763fb90 ffffffff816abb70 [83234.466928] ffff88201763fbd0 ffffffffc0b72198 0000000000000000 ffff88202374e000 [83234.475215] ffff88201763fc18 ffff88202374e060 ffff88202374e000 ffff8810175b8000 [83234.483502] Call Trace: [83234.486229] [] queued_spin_lock_slowpath+0xb/0xf [83234.493220] [] _raw_spin_lock+0x20/0x30 [83234.499357] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83234.507533] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83234.515610] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83234.523005] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83234.530788] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83234.538571] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83234.545956] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83234.554030] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83234.561911] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83234.569779] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83234.577854] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83234.586317] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83234.594492] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83234.602163] [] ? wake_up_state+0x20/0x20 [83234.608398] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83234.616649] [] kthread+0xcf/0xe0 [83234.622089] [] ? insert_kthread_work+0x40/0x40 [83234.628887] [] ret_from_fork+0x58/0x90 [83234.634909] [] ? insert_kthread_work+0x40/0x40 [83234.641706] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83235.062712] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ldlm_bl_16:33189] [83235.065711] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ldlm_bl_09:16011] [83235.065740] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83235.065750] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83235.065751] CPU: 4 PID: 16011 Comm: ldlm_bl_09 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83235.065752] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83235.065753] task: ffff88203ad88fd0 ti: ffff88201c7fc000 task.ti: ffff88201c7fc000 [83235.065756] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83235.065756] RSP: 0018:ffff88201c7ffb70 EFLAGS: 00000246 [83235.065757] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000210000 [83235.065758] RDX: ffff88103cf97880 RSI: 0000000000610000 RDI: ffff88102137981c [83235.065758] RBP: ffff88201c7ffb70 R08: ffff88103ce97880 R09: 0000000000000000 [83235.065759] R10: ffff882023b52600 R11: 0000000000000000 R12: ffff88201c7ffc58 [83235.065759] R13: 0000000000000001 R14: ffff88201c7ffb28 R15: ffffffff81322c35 [83235.065760] FS: 0000000000000000(0000) GS:ffff88103ce80000(0000) knlGS:0000000000000000 [83235.065761] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83235.065761] CR2: 00007f8905462248 CR3: 00000000019f2000 CR4: 00000000001407e0 [83235.065762] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83235.065762] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83235.065763] Stack: [83235.065764] ffff88201c7ffb80 ffffffff8169e61f ffff88201c7ffb90 ffffffff816abb70 [83235.065765] ffff88201c7ffbd0 ffffffffc0b72198 0000000000000000 ffff882023b52600 [83235.065766] ffff88201c7ffc18 ffff882023b52660 ffff882023b52600 ffff8810175b8000 [83235.065766] Call Trace: [83235.065769] [] queued_spin_lock_slowpath+0xb/0xf [83235.065770] [] _raw_spin_lock+0x20/0x30 [83235.065803] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83235.065826] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83235.065846] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83235.065864] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83235.065881] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83235.065899] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83235.065905] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83235.065924] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83235.065944] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83235.065949] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83235.065969] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83235.065987] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83235.066009] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83235.066011] [] ? wake_up_state+0x20/0x20 [83235.066032] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83235.066033] [] kthread+0xcf/0xe0 [83235.066035] [] ? insert_kthread_work+0x40/0x40 [83235.066036] [] ret_from_fork+0x58/0x90 [83235.066038] [] ? insert_kthread_work+0x40/0x40 [83235.066052] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83235.068711] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ldlm_bl_11:16013] [83235.068733] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83235.068741] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83235.068743] CPU: 5 PID: 16013 Comm: ldlm_bl_11 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83235.068743] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83235.068744] task: ffff88203ad88000 ti: ffff88201c804000 task.ti: ffff88201c804000 [83235.068747] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83235.068748] RSP: 0018:ffff88201c807b70 EFLAGS: 00000246 [83235.068748] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000290000 [83235.068749] RDX: ffff88103d017880 RSI: 0000000000810000 RDI: ffff88102137981c [83235.068750] RBP: ffff88201c807b70 R08: ffff88203c697880 R09: 0000000000000000 [83235.068750] R10: ffff882023a59c00 R11: 0000000000000000 R12: ffff88201c807c58 [83235.068751] R13: 0000000000000001 R14: ffff88201c807b28 R15: ffffffff81322c35 [83235.068751] FS: 0000000000000000(0000) GS:ffff88203c680000(0000) knlGS:0000000000000000 [83235.068752] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83235.068753] CR2: 00007fd7ddcd8090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83235.068753] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83235.068754] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83235.068754] Stack: [83235.068755] ffff88201c807b80 ffffffff8169e61f ffff88201c807b90 ffffffff816abb70 [83235.068756] ffff88201c807bd0 ffffffffc0b72198 0000000000000000 ffff882023a59c00 [83235.068758] ffff88201c807c18 ffff882023a59c60 ffff882023a59c00 ffff8810175b8000 [83235.068758] Call Trace: [83235.068760] [] queued_spin_lock_slowpath+0xb/0xf [83235.068761] [] _raw_spin_lock+0x20/0x30 [83235.068786] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83235.068809] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83235.068830] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83235.068847] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83235.068865] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83235.068882] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83235.068888] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83235.068906] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83235.068923] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83235.068929] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83235.068948] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83235.068966] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83235.068988] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83235.068990] [] ? wake_up_state+0x20/0x20 [83235.069010] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83235.069012] [] kthread+0xcf/0xe0 [83235.069013] [] ? insert_kthread_work+0x40/0x40 [83235.069015] [] ret_from_fork+0x58/0x90 [83235.069016] [] ? insert_kthread_work+0x40/0x40 [83235.069029] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83235.074710] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ldlm_bl_17:33190] [83235.074732] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83235.074740] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83235.074741] CPU: 7 PID: 33190 Comm: ldlm_bl_17 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83235.074742] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83235.074743] task: ffff88102d281fa0 ti: ffff881fffd34000 task.ti: ffff881fffd34000 [83235.074746] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83235.074746] RSP: 0018:ffff881fffd37b70 EFLAGS: 00000246 [83235.074747] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000390000 [83235.074747] RDX: ffff88103cf17880 RSI: 0000000000410000 RDI: ffff88102137981c [83235.074748] RBP: ffff881fffd37b70 R08: ffff88203c6d7880 R09: 0000000000000000 [83235.074748] R10: ffff88202374ee00 R11: 0000000000000000 R12: ffff88102d282008 [83235.074749] R13: ffff881fffd37b00 R14: ffff881f00000141 R15: ffff88207ffd8008 [83235.074750] FS: 0000000000000000(0000) GS:ffff88203c6c0000(0000) knlGS:0000000000000000 [83235.074750] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83235.074751] CR2: 00007f240470f978 CR3: 00000000019f2000 CR4: 00000000001407e0 [83235.074752] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83235.074752] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83235.074752] Stack: [83235.074754] ffff881fffd37b80 ffffffff8169e61f ffff881fffd37b90 ffffffff816abb70 [83235.074755] ffff881fffd37bd0 ffffffffc0b72198 0000000000000000 ffff88202374ee00 [83235.074756] ffff881fffd37c18 ffff88202374ee60 ffff88202374ee00 ffff8810175b8000 [83235.074756] Call Trace: [83235.074758] [] queued_spin_lock_slowpath+0xb/0xf [83235.074760] [] _raw_spin_lock+0x20/0x30 [83235.074779] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83235.074799] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83235.074819] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83235.074837] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83235.074855] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83235.074873] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83235.074879] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83235.074897] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83235.074915] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83235.074921] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83235.074939] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83235.074957] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83235.074979] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83235.074980] [] ? __schedule+0x39d/0x8b0 [83235.075002] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83235.075003] [] kthread+0xcf/0xe0 [83235.075005] [] ? insert_kthread_work+0x40/0x40 [83235.075006] [] ret_from_fork+0x58/0x90 [83235.075007] [] ? insert_kthread_work+0x40/0x40 [83235.075021] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83235.101710] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ldlm_bl_12:16014] [83235.101732] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83235.101740] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83235.101741] CPU: 16 PID: 16014 Comm: ldlm_bl_12 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83235.101742] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83235.101743] task: ffff88203d33af70 ti: ffff88201c808000 task.ti: ffff88201c808000 [83235.101745] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83235.101746] RSP: 0018:ffff88201c80bb70 EFLAGS: 00000246 [83235.101747] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000810000 [83235.101747] RDX: ffff88203c717880 RSI: 0000000000490000 RDI: ffff88102137981c [83235.101748] RBP: ffff88201c80bb70 R08: ffff88103d017880 R09: 0000000000000000 [83235.101748] R10: ffff882023b24200 R11: 0000000000000005 R12: ffff88201c80bc58 [83235.101749] R13: 0000000000000001 R14: ffff88201c80bb28 R15: ffffffff81322c35 [83235.101749] FS: 0000000000000000(0000) GS:ffff88103d000000(0000) knlGS:0000000000000000 [83235.101750] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83235.101751] CR2: 00007f361a3bb4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83235.101751] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83235.101752] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83235.101752] Stack: [83235.101753] ffff88201c80bb80 ffffffff8169e61f ffff88201c80bb90 ffffffff816abb70 [83235.101754] ffff88201c80bbd0 ffffffffc0b72198 0000000000000000 ffff882023b24200 [83235.101755] ffff88201c80bc18 ffff882023b24260 ffff882023b24200 ffff8810175b8000 [83235.101756] Call Trace: [83235.101758] [] queued_spin_lock_slowpath+0xb/0xf [83235.101759] [] _raw_spin_lock+0x20/0x30 [83235.101778] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83235.101799] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83235.101818] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83235.101836] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83235.101853] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83235.101870] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83235.101875] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83235.101892] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83235.101910] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83235.101915] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83235.101935] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83235.101953] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83235.101974] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83235.101976] [] ? wake_up_state+0x20/0x20 [83235.101996] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83235.101998] [] kthread+0xcf/0xe0 [83235.101999] [] ? insert_kthread_work+0x40/0x40 [83235.102000] [] ret_from_fork+0x58/0x90 [83235.102001] [] ? insert_kthread_work+0x40/0x40 [83235.102015] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83236.861344] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83236.940817] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83236.969682] CPU: 3 PID: 33189 Comm: ldlm_bl_16 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83236.982394] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83236.990742] task: ffff88102c376eb0 ti: ffff881fefb44000 task.ti: ffff881fefb44000 [83236.999090] RIP: 0010:[] [] ldlm_process_plain_lock+0x357/0xb30 [ptlrpc] [83237.010181] RSP: 0018:ffff881fefb47be0 EFLAGS: 00000206 [83237.016106] RAX: ffff8820176bbfc8 RBX: ffff881fefb47b70 RCX: 0000000000000010 [83237.024066] RDX: 0000000000000000 RSI: ffff8820236fba00 RDI: ffff8810175b8000 [83237.032026] RBP: ffff881fefb47c58 R08: ffff881fefb47cd0 R09: ffff88203c857880 [83237.039987] R10: ffff8820236fba00 R11: 0000000000000000 R12: 0000000000000010 [83237.047947] R13: ffffffff810fa356 R14: ffffffffffffff10 R15: ffff88102137981c [83237.055908] FS: 0000000000000000(0000) GS:ffff88203c640000(0000) knlGS:0000000000000000 [83237.064934] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83237.071344] CR2: 000055ce548c9220 CR3: 00000000019f2000 CR4: 00000000001407e0 [83237.079304] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83237.087264] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83237.095224] Stack: [83237.097465] ffff881fefb47c7c ffff881fefb47cd0 ffff881fefb47c80 ffff881021379800 [83237.105752] ffff881021379820 0000001000000001 ffff881021379840 ffff881fefb47c18 [83237.114039] ffff881fefb47c18 000000007315cd91 0000000000000002 ffff881021379840 [83237.122327] Call Trace: [83237.125072] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83237.132467] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83237.140253] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83237.148036] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83237.155419] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83237.163493] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83237.171374] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83237.179242] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83237.187317] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83237.195783] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83237.203958] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83237.211628] [] ? __schedule+0x39d/0x8b0 [83237.217765] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83237.226016] [] kthread+0xcf/0xe0 [83237.231456] [] ? insert_kthread_work+0x40/0x40 [83237.238254] [] ret_from_fork+0x58/0x90 [83237.244277] [] ? insert_kthread_work+0x40/0x40 [83237.251074] Code: 8b 40 40 48 39 45 b8 4c 8d 78 a0 0f 84 e9 01 00 00 4c 39 fb ba 01 00 00 00 4d 89 fe 0f 84 cd 03 00 00 66 90 49 8b 86 d0 01 00 00 <8b> 4d b4 4c 8b a8 98 fe ff ff 41 8b 86 98 00 00 00 85 0c 85 60 [83247.055452] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ldlm_bl_08:16010] [83247.056452] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ldlm_bl_13:16015] [83247.056479] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.056488] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.056490] CPU: 0 PID: 16015 Comm: ldlm_bl_13 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.056490] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.056491] task: ffff88203d33bf40 ti: ffff88201c80c000 task.ti: ffff88201c80c000 [83247.056494] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83247.056495] RSP: 0018:ffff88201c80fb70 EFLAGS: 00000246 [83247.056496] RAX: 0000000000000000 RBX: 000000013ce56cc0 RCX: 0000000000010000 [83247.056496] RDX: ffff88103d0d7880 RSI: 0000000000b10001 RDI: ffff88102137981c [83247.056497] RBP: ffff88201c80fb70 R08: ffff88103ce17880 R09: 0000000000000000 [83247.056497] R10: ffff882023e12c00 R11: 0000000000000005 R12: ffff88201c80fc58 [83247.056498] R13: 0000000000000001 R14: ffff88201c80fb28 R15: ffffffff81322c35 [83247.056499] FS: 0000000000000000(0000) GS:ffff88103ce00000(0000) knlGS:0000000000000000 [83247.056500] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.056500] CR2: 00007f17e6a8b248 CR3: 00000000019f2000 CR4: 00000000001407f0 [83247.056501] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.056501] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.056502] Stack: [83247.056503] ffff88201c80fb80 ffffffff8169e61f ffff88201c80fb90 ffffffff816abb70 [83247.056504] ffff88201c80fbd0 ffffffffc0b72198 0000000000000000 ffff882023e12c00 [83247.056505] ffff88201c80fc18 ffff882023e12c60 ffff882023e12c00 ffff8810175b8000 [83247.056506] Call Trace: [83247.056508] [] queued_spin_lock_slowpath+0xb/0xf [83247.056509] [] _raw_spin_lock+0x20/0x30 [83247.056537] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.056559] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.056578] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.056596] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.056614] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.056632] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.056638] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.056656] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.056673] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.056679] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.056698] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.056716] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.056738] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.056740] [] ? wake_up_state+0x20/0x20 [83247.056760] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.056762] [] kthread+0xcf/0xe0 [83247.056763] [] ? insert_kthread_work+0x40/0x40 [83247.056765] [] ret_from_fork+0x58/0x90 [83247.056766] [] ? insert_kthread_work+0x40/0x40 [83247.056779] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83247.059452] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ldlm_bl_06:16008] [83247.059470] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.059475] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.059476] CPU: 2 PID: 16008 Comm: ldlm_bl_06 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.059477] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.059478] task: ffff8810247eeeb0 ti: ffff88201c7e8000 task.ti: ffff88201c7e8000 [83247.059481] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83247.059481] RSP: 0018:ffff88201c7ebb70 EFLAGS: 00000246 [83247.059482] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000110000 [83247.059482] RDX: ffff88203c897880 RSI: 0000000000a90000 RDI: ffff88102137981c [83247.059483] RBP: ffff88201c7ebb70 R08: ffff88103ce57880 R09: 0000000000000000 [83247.059483] R10: ffff882023e19a00 R11: 0000000000000005 R12: ffff88201c7ebc58 [83247.059484] R13: 0000000000000001 R14: ffff88201c7ebb28 R15: ffffffff81322c35 [83247.059484] FS: 0000000000000000(0000) GS:ffff88103ce40000(0000) knlGS:0000000000000000 [83247.059485] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.059485] CR2: 0000000002577010 CR3: 00000000019f2000 CR4: 00000000001407e0 [83247.059485] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.059486] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.059486] Stack: [83247.059487] ffff88201c7ebb80 ffffffff8169e61f ffff88201c7ebb90 ffffffff816abb70 [83247.059488] ffff88201c7ebbd0 ffffffffc0b72198 0000000000000000 ffff882023e19a00 [83247.059488] ffff88201c7ebc18 ffff882023e19a60 ffff882023e19a00 ffff8810175b8000 [83247.059488] Call Trace: [83247.059491] [] queued_spin_lock_slowpath+0xb/0xf [83247.059492] [] _raw_spin_lock+0x20/0x30 [83247.059508] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.059523] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.059538] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.059551] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.059564] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.059578] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.059582] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.059596] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.059608] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.059613] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.059626] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.059639] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.059655] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.059656] [] ? wake_up_state+0x20/0x20 [83247.059671] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.059672] [] kthread+0xcf/0xe0 [83247.059673] [] ? insert_kthread_work+0x40/0x40 [83247.059674] [] ret_from_fork+0x58/0x90 [83247.059675] [] ? insert_kthread_work+0x40/0x40 [83247.059683] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83247.071452] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ldlm_bl_14:16016] [83247.071474] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.071482] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.071484] CPU: 6 PID: 16016 Comm: ldlm_bl_14 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.071484] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.071485] task: ffff88203d338000 ti: ffff88201c810000 task.ti: ffff88201c810000 [83247.071488] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83247.071489] RSP: 0018:ffff88201c813b70 EFLAGS: 00000246 [83247.071489] RAX: 0000000000000000 RBX: 000000011c813be8 RCX: 0000000000310000 [83247.071490] RDX: ffff88103ce57880 RSI: 0000000000110000 RDI: ffff88102137981c [83247.071490] RBP: ffff88201c813b70 R08: ffff88103ced7880 R09: 0000000000000000 [83247.071491] R10: ffff882023e1a600 R11: 0000000000000000 R12: ffff88201c813c58 [83247.071492] R13: 0000000000000001 R14: ffff88201c813b28 R15: ffffffff81322c35 [83247.071492] FS: 0000000000000000(0000) GS:ffff88103cec0000(0000) knlGS:0000000000000000 [83247.071493] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.071493] CR2: 0000000002079728 CR3: 00000000019f2000 CR4: 00000000001407e0 [83247.071494] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.071495] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.071495] Stack: [83247.071496] ffff88201c813b80 ffffffff8169e61f ffff88201c813b90 ffffffff816abb70 [83247.071497] ffff88201c813bd0 ffffffffc0b72198 0000000000000000 ffff882023e1a600 [83247.071498] ffff88201c813c18 ffff882023e1a660 ffff882023e1a600 ffff8810175b8000 [83247.071499] Call Trace: [83247.071501] [] queued_spin_lock_slowpath+0xb/0xf [83247.071502] [] _raw_spin_lock+0x20/0x30 [83247.071526] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.071548] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.071567] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.071584] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.071601] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.071619] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.071625] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.071642] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.071660] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.071665] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.071683] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.071701] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.071723] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.071724] [] ? wake_up_state+0x20/0x20 [83247.071745] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.071746] [] kthread+0xcf/0xe0 [83247.071748] [] ? insert_kthread_work+0x40/0x40 [83247.071749] [] ret_from_fork+0x58/0x90 [83247.071750] [] ? insert_kthread_work+0x40/0x40 [83247.071764] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83247.077452] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ldlm_bl_10:16012] [83247.077473] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.077480] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.077482] CPU: 8 PID: 16012 Comm: ldlm_bl_10 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.077482] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.077483] task: ffff88203ad89fa0 ti: ffff88201c800000 task.ti: ffff88201c800000 [83247.077486] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83247.077486] RSP: 0018:ffff88201c803b70 EFLAGS: 00000246 [83247.077487] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000410000 [83247.077488] RDX: ffff88203c617880 RSI: 0000000000090000 RDI: ffff88102137981c [83247.077488] RBP: ffff88201c803b70 R08: ffff88103cf17880 R09: 0000000000000000 [83247.077489] R10: ffff882023e1a800 R11: 0000000000000005 R12: ffff88201c803c58 [83247.077489] R13: 0000000000000001 R14: ffff88201c803b28 R15: ffffffff81322c35 [83247.077490] FS: 0000000000000000(0000) GS:ffff88103cf00000(0000) knlGS:0000000000000000 [83247.077491] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.077491] CR2: 00007fca10cf5000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83247.077492] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.077492] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.077493] Stack: [83247.077494] ffff88201c803b80 ffffffff8169e61f ffff88201c803b90 ffffffff816abb70 [83247.077495] ffff88201c803bd0 ffffffffc0b72198 0000000000000000 ffff882023e1a800 [83247.077496] ffff88201c803c18 ffff882023e1a860 ffff882023e1a800 ffff8810175b8000 [83247.077496] Call Trace: [83247.077498] [] queued_spin_lock_slowpath+0xb/0xf [83247.077500] [] _raw_spin_lock+0x20/0x30 [83247.077518] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.077539] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.077558] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.077576] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.077593] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.077611] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.077617] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.077634] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.077652] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.077657] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.077677] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.077695] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.077717] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.077719] [] ? wake_up_state+0x20/0x20 [83247.077739] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.077741] [] kthread+0xcf/0xe0 [83247.077742] [] ? insert_kthread_work+0x40/0x40 [83247.077743] [] ret_from_fork+0x58/0x90 [83247.077744] [] ? insert_kthread_work+0x40/0x40 [83247.077758] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83247.083451] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ldlm_bl_05:16007] [83247.083472] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.083480] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.083481] CPU: 10 PID: 16007 Comm: ldlm_bl_05 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.083482] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.083483] task: ffff8810247ecf10 ti: ffff88201c64c000 task.ti: ffff88201c64c000 [83247.083485] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83247.083486] RSP: 0018:ffff88201c64fb70 EFLAGS: 00000246 [83247.083486] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000510000 [83247.083487] RDX: ffff88103ce17880 RSI: 0000000000010000 RDI: ffff88102137981c [83247.083487] RBP: ffff88201c64fb70 R08: ffff88103cf57880 R09: 0000000000000000 [83247.083488] R10: ffff882023be5a00 R11: 0000000000000005 R12: ffff88201c64fc58 [83247.083488] R13: 0000000000000001 R14: ffff88201c64fb28 R15: ffffffff81322c35 [83247.083489] FS: 0000000000000000(0000) GS:ffff88103cf40000(0000) knlGS:0000000000000000 [83247.083490] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.083490] CR2: 0000000001dd94b8 CR3: 00000000019f2000 CR4: 00000000001407e0 [83247.083491] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.083491] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.083492] Stack: [83247.083493] ffff88201c64fb80 ffffffff8169e61f ffff88201c64fb90 ffffffff816abb70 [83247.083494] ffff88201c64fbd0 ffffffffc0b72198 0000000000000000 ffff882023be5a00 [83247.083495] ffff88201c64fc18 ffff882023be5a60 ffff882023be5a00 ffff8810175b8000 [83247.083495] Call Trace: [83247.083497] [] queued_spin_lock_slowpath+0xb/0xf [83247.083499] [] _raw_spin_lock+0x20/0x30 [83247.083519] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.083538] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.083557] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.083575] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.083593] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.083610] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.083616] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.083634] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.083651] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.083656] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.083674] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.083692] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.083713] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.083715] [] ? wake_up_state+0x20/0x20 [83247.083735] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.083737] [] kthread+0xcf/0xe0 [83247.083738] [] ? insert_kthread_work+0x40/0x40 [83247.083739] [] ret_from_fork+0x58/0x90 [83247.083740] [] ? insert_kthread_work+0x40/0x40 [83247.083754] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83247.089451] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ldlm_bl_15:16017] [83247.089472] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.089480] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.089481] CPU: 12 PID: 16017 Comm: ldlm_bl_15 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.089481] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.089482] task: ffff88203d338fd0 ti: ffff88201c814000 task.ti: ffff88201c814000 [83247.089485] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83247.089485] RSP: 0018:ffff88201c817b70 EFLAGS: 00000246 [83247.089486] RAX: 0000000000000000 RBX: 00000001f8c12595 RCX: 0000000000610000 [83247.089486] RDX: ffff88203c757880 RSI: 0000000000590000 RDI: ffff88102137981c [83247.089487] RBP: ffff88201c817b70 R08: ffff88103cf97880 R09: 0000000000000000 [83247.089487] R10: ffff882023e1a800 R11: 0000000000000005 R12: ffff88201c817c58 [83247.089488] R13: 0000000000000001 R14: ffff88201c817b28 R15: ffffffff81322c35 [83247.089489] FS: 0000000000000000(0000) GS:ffff88103cf80000(0000) knlGS:0000000000000000 [83247.089489] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.089490] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83247.089490] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.089491] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.089491] Stack: [83247.089492] ffff88201c817b80 ffffffff8169e61f ffff88201c817b90 ffffffff816abb70 [83247.089493] ffff88201c817bd0 ffffffffc0b72198 0000000000000000 ffff882023e1a800 [83247.089494] ffff88201c817c18 ffff882023e1a860 ffff882023e1a800 ffff8810175b8000 [83247.089495] Call Trace: [83247.089497] [] queued_spin_lock_slowpath+0xb/0xf [83247.089498] [] _raw_spin_lock+0x20/0x30 [83247.089516] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.089537] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.089556] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.089574] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.089591] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.089609] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.089614] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.089632] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.089649] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.089654] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.089674] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.089692] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.089714] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.089715] [] ? wake_up_state+0x20/0x20 [83247.089735] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.089736] [] kthread+0xcf/0xe0 [83247.089738] [] ? insert_kthread_work+0x40/0x40 [83247.089739] [] ret_from_fork+0x58/0x90 [83247.089740] [] ? insert_kthread_work+0x40/0x40 [83247.089754] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83247.107451] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ldlm_bl_02:15061] [83247.107472] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.107479] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.107480] CPU: 18 PID: 15061 Comm: ldlm_bl_02 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.107481] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.107482] task: ffff882012edeeb0 ti: ffff88203d3bc000 task.ti: ffff88203d3bc000 [83247.107484] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83247.107485] RSP: 0018:ffff88203d3bfb70 EFLAGS: 00000246 [83247.107485] RAX: 0000000000000000 RBX: 000000013cf16cc0 RCX: 0000000000910000 [83247.107486] RDX: ffff88203c7d7880 RSI: 0000000000790000 RDI: ffff88102137981c [83247.107486] RBP: ffff88203d3bfb70 R08: ffff88103d057880 R09: 0000000000000000 [83247.107487] R10: ffff882023e1aa00 R11: 0000000000000005 R12: ffff88203d3bfc58 [83247.107487] R13: 0000000000000001 R14: ffff88203d3bfb28 R15: ffffffff81322c35 [83247.107488] FS: 0000000000000000(0000) GS:ffff88103d040000(0000) knlGS:0000000000000000 [83247.107489] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.107489] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83247.107490] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.107491] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.107491] Stack: [83247.107492] ffff88203d3bfb80 ffffffff8169e61f ffff88203d3bfb90 ffffffff816abb70 [83247.107493] ffff88203d3bfbd0 ffffffffc0b72198 0000000000000000 ffff882023e1aa00 [83247.107495] ffff88203d3bfc18 ffff882023e1aa60 ffff882023e1aa00 ffff8810175b8000 [83247.107495] Call Trace: [83247.107497] [] queued_spin_lock_slowpath+0xb/0xf [83247.107498] [] _raw_spin_lock+0x20/0x30 [83247.107516] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.107537] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.107556] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.107574] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.107591] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.107609] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.107614] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.107632] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.107649] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.107655] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.107673] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.107690] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.107711] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.107712] [] ? wake_up_state+0x20/0x20 [83247.107733] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.107734] [] kthread+0xcf/0xe0 [83247.107735] [] ? insert_kthread_work+0x40/0x40 [83247.107737] [] ret_from_fork+0x58/0x90 [83247.107738] [] ? insert_kthread_work+0x40/0x40 [83247.107751] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83247.113451] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ldlm_bl_01:15060] [83247.113472] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.113479] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.113481] CPU: 20 PID: 15060 Comm: ldlm_bl_01 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.113481] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.113482] task: ffff882012eddee0 ti: ffff88203d3b8000 task.ti: ffff88203d3b8000 [83247.113484] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83247.113485] RSP: 0018:ffff88203d3bbb70 EFLAGS: 00000246 [83247.113485] RAX: 0000000000000000 RBX: 0000000180597933 RCX: 0000000000a10000 [83247.113486] RDX: ffff88203c857880 RSI: 0000000000990000 RDI: ffff88102137981c [83247.113487] RBP: ffff88203d3bbb70 R08: ffff88103d097880 R09: 0000000000000000 [83247.113487] R10: ffff882023e10200 R11: 0000000000000005 R12: ffff88203d3bbc58 [83247.113488] R13: 0000000000000001 R14: ffff88203d3bbb28 R15: ffffffff81322c35 [83247.113489] FS: 0000000000000000(0000) GS:ffff88103d080000(0000) knlGS:0000000000000000 [83247.113489] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.113490] CR2: 00007f02c314e4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83247.113491] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.113491] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.113491] Stack: [83247.113493] ffff88203d3bbb80 ffffffff8169e61f ffff88203d3bbb90 ffffffff816abb70 [83247.113494] ffff88203d3bbbd0 ffffffffc0b72198 0000000000000000 ffff882023e10200 [83247.113495] ffff88203d3bbc18 ffff882023e10260 ffff882023e10200 ffff8810175b8000 [83247.113495] Call Trace: [83247.113497] [] queued_spin_lock_slowpath+0xb/0xf [83247.113499] [] _raw_spin_lock+0x20/0x30 [83247.113517] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.113538] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.113557] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.113575] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.113592] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.113610] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.113615] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.113633] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.113651] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.113656] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.113676] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.113694] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.113716] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.113717] [] ? wake_up_state+0x20/0x20 [83247.113738] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.113739] [] kthread+0xcf/0xe0 [83247.113740] [] ? insert_kthread_work+0x40/0x40 [83247.113742] [] ret_from_fork+0x58/0x90 [83247.113743] [] ? insert_kthread_work+0x40/0x40 [83247.113756] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83247.119451] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ldlm_bl_07:16009] [83247.119472] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83247.119479] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83247.119481] CPU: 22 PID: 16009 Comm: ldlm_bl_07 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83247.119481] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83247.119482] task: ffff8810e9240fd0 ti: ffff88201c7ec000 task.ti: ffff88201c7ec000 [83247.119484] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83247.119485] RSP: 0018:ffff88201c7efb70 EFLAGS: 00000246 [83247.119485] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000b10000 [83247.119486] RDX: ffff88103ce97880 RSI: 0000000000210001 RDI: ffff88102137981c [83247.119486] RBP: ffff88201c7efb70 R08: ffff88103d0d7880 R09: 0000000000000000 [83247.119487] R10: ffff882023e1aa00 R11: 0000000000000005 R12: ffff88201c7efc58 [83247.119488] R13: 0000000000000001 R14: ffff88201c7efb28 R15: ffffffff81322c35 [83247.119489] FS: 0000000000000000(0000) GS:ffff88103d0c0000(0000) knlGS:0000000000000000 [83247.119489] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83247.119490] CR2: 00007f262c4764f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83247.119491] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83247.119491] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83247.119491] Stack: [83247.119493] ffff88201c7efb80 ffffffff8169e61f ffff88201c7efb90 ffffffff816abb70 [83247.119494] ffff88201c7efbd0 ffffffffc0b72198 0000000000000000 ffff882023e1aa00 [83247.119495] ffff88201c7efc18 ffff882023e1aa60 ffff882023e1aa00 ffff8810175b8000 [83247.119495] Call Trace: [83247.119497] [] queued_spin_lock_slowpath+0xb/0xf [83247.119498] [] _raw_spin_lock+0x20/0x30 [83247.119519] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83247.119540] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83247.119559] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83247.119577] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83247.119594] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83247.119612] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83247.119617] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83247.119635] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.119652] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83247.119658] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83247.119675] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83247.119693] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83247.119715] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83247.119716] [] ? wake_up_state+0x20/0x20 [83247.119737] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83247.119738] [] kthread+0xcf/0xe0 [83247.119739] [] ? insert_kthread_work+0x40/0x40 [83247.119741] [] ret_from_fork+0x58/0x90 [83247.119742] [] ? insert_kthread_work+0x40/0x40 [83247.119755] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83251.092337] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83251.171802] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83251.200670] CPU: 1 PID: 16010 Comm: ldlm_bl_08 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83251.213382] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83251.221730] task: ffff8810e9242f70 ti: ffff88201c7f8000 task.ti: ffff88201c7f8000 [83251.230078] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83251.241150] RSP: 0018:ffff88201c7fbb70 EFLAGS: 00000246 [83251.247074] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000090000 [83251.255035] RDX: ffff88103cf57880 RSI: 0000000000510000 RDI: ffff88102137981c [83251.262995] RBP: ffff88201c7fbb70 R08: ffff88203c617880 R09: 0000000000000000 [83251.270955] R10: ffff882023bd9200 R11: 0000000000000005 R12: ffff88201c7fbc58 [83251.278915] R13: 0000000000000001 R14: ffff88201c7fbb28 R15: ffffffff81322c35 [83251.286876] FS: 0000000000000000(0000) GS:ffff88203c600000(0000) knlGS:0000000000000000 [83251.295902] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83251.302311] CR2: 00007f11b0c8c000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83251.310271] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83251.318231] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83251.326191] Stack: [83251.328431] ffff88201c7fbb80 ffffffff8169e61f ffff88201c7fbb90 ffffffff816abb70 [83251.336718] ffff88201c7fbbd0 ffffffffc0b72198 0000000000000000 ffff882023bd9200 [83251.345005] ffff88201c7fbc18 ffff882023bd9260 ffff882023bd9200 ffff8810175b8000 [83251.353292] Call Trace: [83251.356019] [] queued_spin_lock_slowpath+0xb/0xf [83251.363010] [] _raw_spin_lock+0x20/0x30 [83251.369155] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83251.377332] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83251.385410] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83251.392806] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83251.400589] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83251.408372] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83251.415756] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83251.423830] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83251.431711] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83251.439579] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83251.447656] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83251.456119] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83251.464295] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83251.471966] [] ? wake_up_state+0x20/0x20 [83251.478200] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83251.486451] [] kthread+0xcf/0xe0 [83251.491891] [] ? insert_kthread_work+0x40/0x40 [83251.498689] [] ret_from_fork+0x58/0x90 [83251.504712] [] ? insert_kthread_work+0x40/0x40 [83251.511509] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83259.080193] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [ldlm_bl_03:15915] [83259.086192] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ldlm_bl_22:33195] [83259.086216] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83259.086224] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83259.086226] CPU: 11 PID: 33195 Comm: ldlm_bl_22 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83259.086227] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83259.086228] task: ffff88203d0b1fa0 ti: ffff88203d4b8000 task.ti: ffff88203d4b8000 [83259.086231] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83259.086232] RSP: 0018:ffff88203d4bbb70 EFLAGS: 00000246 [83259.086232] RAX: 0000000000000000 RBX: 5a5a5a5a5a5a5a5a RCX: 0000000000590000 [83259.086233] RDX: ffff88103ced7880 RSI: 0000000000310000 RDI: ffff88102137981c [83259.086234] RBP: ffff88203d4bbb70 R08: ffff88203c757880 R09: 0000000000000000 [83259.086234] R10: ffff882023b74a00 R11: 0000000000000001 R12: 5a5a5a5a5a5a5a5a [83259.086235] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83259.086235] FS: 0000000000000000(0000) GS:ffff88203c740000(0000) knlGS:0000000000000000 [83259.086236] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83259.086237] CR2: 00007fef29abd090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83259.086237] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83259.086238] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83259.086238] Stack: [83259.086239] ffff88203d4bbb80 ffffffff8169e61f ffff88203d4bbb90 ffffffff816abb70 [83259.086240] ffff88203d4bbbd0 ffffffffc0b72198 0000000000000000 ffff882023b74a00 [83259.086241] ffff88203d4bbc18 ffff882023b74a60 ffff882023b74a00 ffff8810175b8000 [83259.086242] Call Trace: [83259.086244] [] queued_spin_lock_slowpath+0xb/0xf [83259.086245] [] _raw_spin_lock+0x20/0x30 [83259.086278] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83259.086301] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83259.086321] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83259.086338] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83259.086356] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83259.086374] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83259.086381] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83259.086400] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.086417] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.086423] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83259.086443] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83259.086460] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83259.086483] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83259.086484] [] ? __schedule+0x39d/0x8b0 [83259.086504] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83259.086506] [] kthread+0xcf/0xe0 [83259.086507] [] ? insert_kthread_work+0x40/0x40 [83259.086509] [] ret_from_fork+0x58/0x90 [83259.086510] [] ? insert_kthread_work+0x40/0x40 [83259.086524] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83259.092193] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [ldlm_bl_19:33192] [83259.092214] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83259.092222] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83259.092224] CPU: 13 PID: 33192 Comm: ldlm_bl_19 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83259.092224] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83259.092225] task: ffff88203d0b2f70 ti: ffff881ff8e20000 task.ti: ffff881ff8e20000 [83259.092228] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83259.092228] RSP: 0018:ffff881ff8e23b70 EFLAGS: 00000246 [83259.092229] RAX: 0000000000000000 RBX: ffff881ff8e23be8 RCX: 0000000000690000 [83259.092230] RDX: ffff88203c697880 RSI: 0000000000290000 RDI: ffff88102137981c [83259.092230] RBP: ffff881ff8e23b70 R08: ffff88203c797880 R09: 0000000000000000 [83259.092231] R10: ffff882023b3d800 R11: 0000000000000000 R12: ffff88103ce96cc0 [83259.092231] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83259.092232] FS: 0000000000000000(0000) GS:ffff88203c780000(0000) knlGS:0000000000000000 [83259.092233] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83259.092233] CR2: 00007f352c5974f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83259.092234] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83259.092234] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83259.092234] Stack: [83259.092236] ffff881ff8e23b80 ffffffff8169e61f ffff881ff8e23b90 ffffffff816abb70 [83259.092237] ffff881ff8e23bd0 ffffffffc0b72198 0000000000000000 ffff882023b3d800 [83259.092238] ffff881ff8e23c18 ffff882023b3d860 ffff882023b3d800 ffff8810175b8000 [83259.092238] Call Trace: [83259.092240] [] queued_spin_lock_slowpath+0xb/0xf [83259.092242] [] _raw_spin_lock+0x20/0x30 [83259.092262] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83259.092284] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83259.092303] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83259.092320] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83259.092338] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83259.092356] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83259.092362] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83259.092379] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.092397] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.092402] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83259.092422] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83259.092440] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83259.092462] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83259.092463] [] ? __schedule+0x39d/0x8b0 [83259.092484] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83259.092485] [] kthread+0xcf/0xe0 [83259.092487] [] ? insert_kthread_work+0x40/0x40 [83259.092488] [] ret_from_fork+0x58/0x90 [83259.092489] [] ? insert_kthread_work+0x40/0x40 [83259.092502] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83259.098192] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ldlm_bl_23:33196] [83259.098214] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83259.098221] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83259.098223] CPU: 15 PID: 33196 Comm: ldlm_bl_23 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83259.098223] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83259.098224] task: ffff88203d0b0000 ti: ffff882015ae0000 task.ti: ffff882015ae0000 [83259.098227] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83259.098227] RSP: 0018:ffff882015ae3b70 EFLAGS: 00000246 [83259.098228] RAX: 0000000000000000 RBX: ffff88203c756cc0 RCX: 0000000000790000 [83259.098229] RDX: ffff88203c6d7880 RSI: 0000000000390000 RDI: ffff88102137981c [83259.098229] RBP: ffff882015ae3b70 R08: ffff88203c7d7880 R09: 0000000000000000 [83259.098230] R10: ffff882023a96800 R11: 0000000000000001 R12: ffff88203d0b0068 [83259.098230] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83259.098231] FS: 0000000000000000(0000) GS:ffff88203c7c0000(0000) knlGS:0000000000000000 [83259.098232] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83259.098232] CR2: 00007faebc9e03cc CR3: 00000000019f2000 CR4: 00000000001407e0 [83259.098233] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83259.098233] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83259.098234] Stack: [83259.098235] ffff882015ae3b80 ffffffff8169e61f ffff882015ae3b90 ffffffff816abb70 [83259.098236] ffff882015ae3bd0 ffffffffc0b72198 0000000000000000 ffff882023a96800 [83259.098237] ffff882015ae3c18 ffff882023a96860 ffff882023a96800 ffff8810175b8000 [83259.098237] Call Trace: [83259.098240] [] queued_spin_lock_slowpath+0xb/0xf [83259.098241] [] _raw_spin_lock+0x20/0x30 [83259.098259] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83259.098280] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83259.098300] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83259.098317] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83259.098335] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83259.098353] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83259.098358] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83259.098376] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.098393] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.098399] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83259.098417] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83259.098435] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83259.098455] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83259.098456] [] ? __schedule+0x39d/0x8b0 [83259.098477] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83259.098479] [] kthread+0xcf/0xe0 [83259.098480] [] ? insert_kthread_work+0x40/0x40 [83259.098481] [] ret_from_fork+0x58/0x90 [83259.098482] [] ? insert_kthread_work+0x40/0x40 [83259.098496] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83259.104192] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [ldlm_bl_21:33194] [83259.104213] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83259.104221] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83259.104222] CPU: 17 PID: 33194 Comm: ldlm_bl_21 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83259.104222] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83259.104223] task: ffff88203d0b6eb0 ti: ffff882012eac000 task.ti: ffff882012eac000 [83259.104226] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83259.104226] RSP: 0018:ffff882012eafb70 EFLAGS: 00000246 [83259.104227] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000890000 [83259.104227] RDX: ffff88203c797880 RSI: 0000000000690000 RDI: ffff88102137981c [83259.104228] RBP: ffff882012eafb70 R08: ffff88203c817880 R09: 0000000000000000 [83259.104229] R10: ffff882023ba7e00 R11: 0000000000000001 R12: 0000000000000000 [83259.104229] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c816cc0 [83259.104230] FS: 0000000000000000(0000) GS:ffff88203c800000(0000) knlGS:0000000000000000 [83259.104231] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83259.104231] CR2: 00007f7ce52f4550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83259.104232] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83259.104232] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83259.104233] Stack: [83259.104234] ffff882012eafb80 ffffffff8169e61f ffff882012eafb90 ffffffff816abb70 [83259.104236] ffff882012eafbd0 ffffffffc0b72198 0000000000000000 ffff882023ba7e00 [83259.104237] ffff882012eafc18 ffff882023ba7e60 ffff882023ba7e00 ffff8810175b8000 [83259.104237] Call Trace: [83259.104239] [] queued_spin_lock_slowpath+0xb/0xf [83259.104241] [] _raw_spin_lock+0x20/0x30 [83259.104259] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83259.104280] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83259.104299] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83259.104317] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83259.104335] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83259.104352] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83259.104358] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83259.104375] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.104393] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.104399] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83259.104418] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83259.104436] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83259.104458] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83259.104460] [] ? __schedule+0x39d/0x8b0 [83259.104480] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83259.104481] [] kthread+0xcf/0xe0 [83259.104483] [] ? insert_kthread_work+0x40/0x40 [83259.104484] [] ret_from_fork+0x58/0x90 [83259.104485] [] ? insert_kthread_work+0x40/0x40 [83259.104499] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83259.110192] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 23s! [ldlm_bl_18:33191] [83259.110213] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83259.110220] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83259.110222] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83259.110222] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83259.110223] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83259.110225] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83259.110226] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83259.110227] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83259.110227] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83259.110228] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83259.110229] R10: ffff882023bbd400 R11: 0000000000000001 R12: ffff88203d0b5f48 [83259.110229] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83259.110230] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83259.110231] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83259.110232] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83259.110232] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83259.110233] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83259.110233] Stack: [83259.110234] ffff881ffba83b80 ffffffff8169e61f ffff881ffba83b90 ffffffff816abb70 [83259.110236] ffff881ffba83bd0 ffffffffc0b72198 0000000000000000 ffff882023bbd400 [83259.110237] ffff881ffba83c18 ffff882023bbd460 ffff882023bbd400 ffff8810175b8000 [83259.110237] Call Trace: [83259.110239] [] queued_spin_lock_slowpath+0xb/0xf [83259.110240] [] _raw_spin_lock+0x20/0x30 [83259.110261] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83259.110282] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83259.110301] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83259.110319] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83259.110336] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83259.110354] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83259.110360] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83259.110377] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.110395] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.110400] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83259.110418] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83259.110436] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83259.110458] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83259.110459] [] ? __schedule+0x39d/0x8b0 [83259.110480] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83259.110481] [] kthread+0xcf/0xe0 [83259.110482] [] ? insert_kthread_work+0x40/0x40 [83259.110484] [] ret_from_fork+0x58/0x90 [83259.110485] [] ? insert_kthread_work+0x40/0x40 [83259.110498] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83259.116192] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [ldlm_bl_20:33193] [83259.116210] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83259.116216] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83259.116217] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83259.116218] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83259.116218] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83259.116221] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x110/0x1e0 [83259.116221] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83259.116222] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83259.116222] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83259.116223] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83259.116223] R10: ffff882023baac00 R11: 0000000000000000 R12: ffff88103ce96cc0 [83259.116224] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83259.116225] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83259.116225] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83259.116226] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83259.116226] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83259.116227] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83259.116227] Stack: [83259.116228] ffff881ff8e2fb80 ffffffff8169e61f ffff881ff8e2fb90 ffffffff816abb70 [83259.116229] ffff881ff8e2fbd0 ffffffffc0b72198 0000000000000000 ffff882023baac00 [83259.116230] ffff881ff8e2fc18 ffff882023baac60 ffff882023baac00 ffff8810175b8000 [83259.116230] Call Trace: [83259.116232] [] queued_spin_lock_slowpath+0xb/0xf [83259.116233] [] _raw_spin_lock+0x20/0x30 [83259.116251] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83259.116269] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83259.116287] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83259.116303] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83259.116320] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83259.116336] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83259.116341] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83259.116358] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.116374] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.116379] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83259.116396] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83259.116412] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83259.116432] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83259.116433] [] ? __schedule+0x39d/0x8b0 [83259.116452] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83259.116454] [] kthread+0xcf/0xe0 [83259.116455] [] ? insert_kthread_work+0x40/0x40 [83259.116456] [] ret_from_fork+0x58/0x90 [83259.116457] [] ? insert_kthread_work+0x40/0x40 [83259.116468] Code: c1 e8 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 90 41 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 [83259.122192] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ldlm_bl_04:16006] [83259.122213] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83259.122220] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83259.122221] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83259.122222] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83259.122223] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83259.122225] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83259.122226] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83259.122226] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83259.122227] RDX: ffff88103d057880 RSI: 0000000000910000 RDI: ffff88102137981c [83259.122227] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83259.122228] R10: ffff882023e1b400 R11: 0000000000000000 R12: ffff88201c627c58 [83259.122228] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83259.122229] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83259.122230] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83259.122230] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83259.122231] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83259.122231] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83259.122231] Stack: [83259.122233] ffff88201c627b80 ffffffff8169e61f ffff88201c627b90 ffffffff816abb70 [83259.122234] ffff88201c627bd0 ffffffffc0b72198 0000000000000000 ffff882023e1b400 [83259.122235] ffff88201c627c18 ffff882023e1b460 ffff882023e1b400 ffff8810175b8000 [83259.122235] Call Trace: [83259.122237] [] queued_spin_lock_slowpath+0xb/0xf [83259.122239] [] _raw_spin_lock+0x20/0x30 [83259.122257] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83259.122278] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83259.122297] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83259.122315] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83259.122332] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83259.122350] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83259.122355] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83259.122373] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.122390] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83259.122395] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83259.122415] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83259.122433] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83259.122454] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83259.122456] [] ? wake_up_state+0x20/0x20 [83259.122476] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83259.122477] [] kthread+0xcf/0xe0 [83259.122478] [] ? insert_kthread_work+0x40/0x40 [83259.122480] [] ret_from_fork+0x58/0x90 [83259.122481] [] ? insert_kthread_work+0x40/0x40 [83259.122494] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83262.221916] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83262.301385] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83262.330249] CPU: 9 PID: 15915 Comm: ldlm_bl_03 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83262.342959] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83262.351307] task: ffff88202c648000 ti: ffff88201763c000 task.ti: ffff88201763c000 [83262.359654] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x140/0x1e0 [83262.370726] RSP: 0018:ffff88201763fb70 EFLAGS: 00000293 [83262.376650] RAX: 0000000000000001 RBX: 0000000300000000 RCX: 0000000000490000 [83262.384610] RDX: 0000000000990001 RSI: 0000000000a10000 RDI: ffff88102137981c [83262.392571] RBP: ffff88201763fb70 R08: ffff88203c717880 R09: ffff88103d017880 [83262.400531] R10: ffff882023bc5a00 R11: 7fffffffffffffff R12: 0000000000000bf8 [83262.408491] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c656cc0 [83262.416451] FS: 0000000000000000(0000) GS:ffff88203c700000(0000) knlGS:0000000000000000 [83262.425477] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83262.431887] CR2: 00007ffa010a6000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83262.439847] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83262.447807] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83262.455767] Stack: [83262.458007] ffff88201763fb80 ffffffff8169e61f ffff88201763fb90 ffffffff816abb70 [83262.466295] ffff88201763fbd0 ffffffffc0b72198 0000000000000000 ffff882023bc5a00 [83262.474581] ffff88201763fc18 ffff882023bc5a60 ffff882023bc5a00 ffff8810175b8000 [83262.482869] Call Trace: [83262.485595] [] queued_spin_lock_slowpath+0xb/0xf [83262.492586] [] _raw_spin_lock+0x20/0x30 [83262.498722] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83262.506896] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83262.514973] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83262.522368] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83262.530152] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83262.537935] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83262.545317] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83262.553392] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83262.561272] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83262.569140] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83262.577217] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83262.585680] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83262.593855] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83262.601526] [] ? wake_up_state+0x20/0x20 [83262.607761] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83262.616012] [] kthread+0xcf/0xe0 [83262.621452] [] ? insert_kthread_work+0x40/0x40 [83262.628250] [] ret_from_fork+0x58/0x90 [83262.634273] [] ? insert_kthread_work+0x40/0x40 [83262.641070] Code: 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 <85> c0 74 0c f3 90 8b 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 [83263.062107] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ldlm_bl_16:33189] [83263.065107] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ldlm_bl_09:16011] [83263.065141] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83263.065152] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83263.065154] CPU: 4 PID: 16011 Comm: ldlm_bl_09 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83263.065155] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83263.065156] task: ffff88203ad88fd0 ti: ffff88201c7fc000 task.ti: ffff88201c7fc000 [83263.065160] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x110/0x1e0 [83263.065160] RSP: 0018:ffff88201c7ffb70 EFLAGS: 00000246 [83263.065161] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000210000 [83263.065162] RDX: ffff88103cf97880 RSI: 0000000000610001 RDI: ffff88102137981c [83263.065162] RBP: ffff88201c7ffb70 R08: ffff88103ce97880 R09: 0000000000000000 [83263.065163] R10: ffff882023fa3200 R11: 0000000000000000 R12: ffff88201c7ffc58 [83263.065163] R13: 0000000000000001 R14: ffff88201c7ffb28 R15: ffffffff81322c35 [83263.065164] FS: 0000000000000000(0000) GS:ffff88103ce80000(0000) knlGS:0000000000000000 [83263.065165] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83263.065165] CR2: 00007f8905462248 CR3: 00000000019f2000 CR4: 00000000001407e0 [83263.065166] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83263.065166] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83263.065167] Stack: [83263.065168] ffff88201c7ffb80 ffffffff8169e61f ffff88201c7ffb90 ffffffff816abb70 [83263.065169] ffff88201c7ffbd0 ffffffffc0b72198 0000000000000000 ffff882023fa3200 [83263.065170] ffff88201c7ffc18 ffff882023fa3260 ffff882023fa3200 ffff8810175b8000 [83263.065170] Call Trace: [83263.065174] [] queued_spin_lock_slowpath+0xb/0xf [83263.065176] [] _raw_spin_lock+0x20/0x30 [83263.065211] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83263.065234] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83263.065254] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83263.065271] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83263.065289] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83263.065307] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83263.065315] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83263.065334] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83263.065351] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83263.065357] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83263.065377] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83263.065395] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83263.065417] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83263.065420] [] ? wake_up_state+0x20/0x20 [83263.065440] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83263.065442] [] kthread+0xcf/0xe0 [83263.065443] [] ? insert_kthread_work+0x40/0x40 [83263.065445] [] ret_from_fork+0x58/0x90 [83263.065446] [] ? insert_kthread_work+0x40/0x40 [83263.065460] Code: c1 e8 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 90 41 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 [83263.068107] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ldlm_bl_11:16013] [83263.068130] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83263.068138] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83263.068139] CPU: 5 PID: 16013 Comm: ldlm_bl_11 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83263.068140] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83263.068141] task: ffff88203ad88000 ti: ffff88201c804000 task.ti: ffff88201c804000 [83263.068144] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83263.068144] RSP: 0018:ffff88201c807b70 EFLAGS: 00000246 [83263.068145] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000290000 [83263.068146] RDX: ffff88103d017880 RSI: 0000000000810000 RDI: ffff88102137981c [83263.068146] RBP: ffff88201c807b70 R08: ffff88203c697880 R09: 0000000000000000 [83263.068147] R10: ffff882023e32400 R11: 0000000000000000 R12: ffff88201c807c58 [83263.068147] R13: 0000000000000001 R14: ffff88201c807b28 R15: ffffffff81322c35 [83263.068148] FS: 0000000000000000(0000) GS:ffff88203c680000(0000) knlGS:0000000000000000 [83263.068149] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83263.068149] CR2: 00007fd7ddcd8090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83263.068150] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83263.068150] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83263.068151] Stack: [83263.068152] ffff88201c807b80 ffffffff8169e61f ffff88201c807b90 ffffffff816abb70 [83263.068153] ffff88201c807bd0 ffffffffc0b72198 0000000000000000 ffff882023e32400 [83263.068154] ffff88201c807c18 ffff882023e32460 ffff882023e32400 ffff8810175b8000 [83263.068154] Call Trace: [83263.068156] [] queued_spin_lock_slowpath+0xb/0xf [83263.068158] [] _raw_spin_lock+0x20/0x30 [83263.068185] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83263.068208] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83263.068228] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83263.068246] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83263.068263] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83263.068281] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83263.068287] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83263.068306] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83263.068324] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83263.068329] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83263.068349] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83263.068366] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83263.068387] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83263.068388] [] ? wake_up_state+0x20/0x20 [83263.068409] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83263.068411] [] kthread+0xcf/0xe0 [83263.068412] [] ? insert_kthread_work+0x40/0x40 [83263.068414] [] ret_from_fork+0x58/0x90 [83263.068415] [] ? insert_kthread_work+0x40/0x40 [83263.068428] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83263.074107] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ldlm_bl_17:33190] [83263.074128] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83263.074136] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83263.074137] CPU: 7 PID: 33190 Comm: ldlm_bl_17 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83263.074138] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83263.074138] task: ffff88102d281fa0 ti: ffff881fffd34000 task.ti: ffff881fffd34000 [83263.074141] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83263.074142] RSP: 0018:ffff881fffd37b70 EFLAGS: 00000246 [83263.074142] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000390000 [83263.074143] RDX: ffff88103cf17880 RSI: 0000000000410000 RDI: ffff88102137981c [83263.074144] RBP: ffff881fffd37b70 R08: ffff88203c6d7880 R09: 0000000000000000 [83263.074144] R10: ffff882023b8ee00 R11: 0000000000000000 R12: ffff88102d282008 [83263.074145] R13: ffff881fffd37b00 R14: ffff881f00000141 R15: ffff88207ffd8008 [83263.074146] FS: 0000000000000000(0000) GS:ffff88203c6c0000(0000) knlGS:0000000000000000 [83263.074146] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83263.074147] CR2: 00007f240470f978 CR3: 00000000019f2000 CR4: 00000000001407e0 [83263.074147] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83263.074148] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83263.074148] Stack: [83263.074149] ffff881fffd37b80 ffffffff8169e61f ffff881fffd37b90 ffffffff816abb70 [83263.074151] ffff881fffd37bd0 ffffffffc0b72198 0000000000000000 ffff882023b8ee00 [83263.074152] ffff881fffd37c18 ffff882023b8ee60 ffff882023b8ee00 ffff8810175b8000 [83263.074152] Call Trace: [83263.074154] [] queued_spin_lock_slowpath+0xb/0xf [83263.074156] [] _raw_spin_lock+0x20/0x30 [83263.074174] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83263.074196] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83263.074215] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83263.074233] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83263.074250] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83263.074267] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83263.074273] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83263.074290] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83263.074308] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83263.074313] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83263.074331] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83263.074349] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83263.074371] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83263.074372] [] ? __schedule+0x39d/0x8b0 [83263.074392] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83263.074394] [] kthread+0xcf/0xe0 [83263.074395] [] ? insert_kthread_work+0x40/0x40 [83263.074396] [] ret_from_fork+0x58/0x90 [83263.074397] [] ? insert_kthread_work+0x40/0x40 [83263.074411] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83263.101106] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ldlm_bl_12:16014] [83263.101129] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83263.101136] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83263.101137] CPU: 16 PID: 16014 Comm: ldlm_bl_12 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83263.101138] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83263.101139] task: ffff88203d33af70 ti: ffff88201c808000 task.ti: ffff88201c808000 [83263.101141] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83263.101142] RSP: 0018:ffff88201c80bb70 EFLAGS: 00000246 [83263.101142] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000810000 [83263.101143] RDX: ffff88203c717880 RSI: 0000000000490000 RDI: ffff88102137981c [83263.101143] RBP: ffff88201c80bb70 R08: ffff88103d017880 R09: 0000000000000000 [83263.101144] R10: ffff882023f92a00 R11: 0000000000000005 R12: ffff88201c80bc58 [83263.101144] R13: 0000000000000001 R14: ffff88201c80bb28 R15: ffffffff81322c35 [83263.101145] FS: 0000000000000000(0000) GS:ffff88103d000000(0000) knlGS:0000000000000000 [83263.101146] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83263.101146] CR2: 00007f361a3bb4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83263.101147] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83263.101148] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83263.101148] Stack: [83263.101149] ffff88201c80bb80 ffffffff8169e61f ffff88201c80bb90 ffffffff816abb70 [83263.101151] ffff88201c80bbd0 ffffffffc0b72198 0000000000000000 ffff882023f92a00 [83263.101152] ffff88201c80bc18 ffff882023f92a60 ffff882023f92a00 ffff8810175b8000 [83263.101152] Call Trace: [83263.101154] [] queued_spin_lock_slowpath+0xb/0xf [83263.101155] [] _raw_spin_lock+0x20/0x30 [83263.101174] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83263.101195] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83263.101214] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83263.101232] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83263.101249] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83263.101267] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83263.101272] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83263.101290] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83263.101307] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83263.101313] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83263.101332] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83263.101350] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83263.101372] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83263.101373] [] ? wake_up_state+0x20/0x20 [83263.101393] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83263.101394] [] kthread+0xcf/0xe0 [83263.101396] [] ? insert_kthread_work+0x40/0x40 [83263.101397] [] ret_from_fork+0x58/0x90 [83263.101398] [] ? insert_kthread_work+0x40/0x40 [83263.101412] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83264.860743] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83264.940213] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83264.969079] CPU: 3 PID: 33189 Comm: ldlm_bl_16 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83264.981792] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83264.990140] task: ffff88102c376eb0 ti: ffff881fefb44000 task.ti: ffff881fefb44000 [83264.998488] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83265.009559] RSP: 0018:ffff881fefb47b70 EFLAGS: 00000246 [83265.015484] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000190000 [83265.023444] RDX: ffff88203c8d7880 RSI: 0000000000b90000 RDI: ffff88102137981c [83265.031405] RBP: ffff881fefb47b70 R08: ffff88203c657880 R09: 0000000000000000 [83265.039365] R10: ffff882023b8fc00 R11: 0000000000000000 R12: ffff88102c376f18 [83265.047326] R13: ffff88203c2b92a0 R14: ffff881000000141 R15: ffff88207ffd8008 [83265.055286] FS: 0000000000000000(0000) GS:ffff88203c640000(0000) knlGS:0000000000000000 [83265.064312] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83265.070722] CR2: 000055ce548c9220 CR3: 00000000019f2000 CR4: 00000000001407e0 [83265.078682] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83265.086643] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83265.094603] Stack: [83265.096843] ffff881fefb47b80 ffffffff8169e61f ffff881fefb47b90 ffffffff816abb70 [83265.105130] ffff881fefb47bd0 ffffffffc0b72198 0000000000000000 ffff882023b8fc00 [83265.113418] ffff881fefb47c18 ffff882023b8fc60 ffff882023b8fc00 ffff8810175b8000 [83265.121705] Call Trace: [83265.124430] [] queued_spin_lock_slowpath+0xb/0xf [83265.131421] [] _raw_spin_lock+0x20/0x30 [83265.137561] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83265.145735] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83265.153812] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83265.161208] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83265.168993] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83265.176779] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83265.184162] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83265.192237] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83265.200117] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83265.207985] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83265.216060] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83265.224525] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83265.232703] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83265.240364] [] ? __schedule+0x39d/0x8b0 [83265.246502] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83265.254753] [] kthread+0xcf/0xe0 [83265.260193] [] ? insert_kthread_work+0x40/0x40 [83265.266991] [] ret_from_fork+0x58/0x90 [83265.273014] [] ? insert_kthread_work+0x40/0x40 [83265.279811] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.054849] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [ldlm_bl_08:16010] [83275.055849] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ldlm_bl_13:16015] [83275.055876] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.055885] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.055887] CPU: 0 PID: 16015 Comm: ldlm_bl_13 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.055887] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.055888] task: ffff88203d33bf40 ti: ffff88201c80c000 task.ti: ffff88201c80c000 [83275.055891] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.055892] RSP: 0018:ffff88201c80fb70 EFLAGS: 00000246 [83275.055893] RAX: 0000000000000000 RBX: 000000013ce56cc0 RCX: 0000000000010000 [83275.055893] RDX: ffff88103d0d7880 RSI: 0000000000b10000 RDI: ffff88102137981c [83275.055894] RBP: ffff88201c80fb70 R08: ffff88103ce17880 R09: 0000000000000000 [83275.055895] R10: ffff88202568f000 R11: 0000000000000005 R12: ffff88201c80fc58 [83275.055895] R13: 0000000000000001 R14: ffff88201c80fb28 R15: ffffffff81322c35 [83275.055896] FS: 0000000000000000(0000) GS:ffff88103ce00000(0000) knlGS:0000000000000000 [83275.055896] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.055897] CR2: 00007f17e6a8b248 CR3: 00000000019f2000 CR4: 00000000001407f0 [83275.055898] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.055898] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.055898] Stack: [83275.055900] ffff88201c80fb80 ffffffff8169e61f ffff88201c80fb90 ffffffff816abb70 [83275.055901] ffff88201c80fbd0 ffffffffc0b72198 0000000000000000 ffff88202568f000 [83275.055902] ffff88201c80fc18 ffff88202568f060 ffff88202568f000 ffff8810175b8000 [83275.055902] Call Trace: [83275.055905] [] queued_spin_lock_slowpath+0xb/0xf [83275.055906] [] _raw_spin_lock+0x20/0x30 [83275.055936] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.055959] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.055979] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.055997] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.056014] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.056032] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.056038] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.056056] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.056074] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.056079] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.056097] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.056115] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.056138] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.056139] [] ? wake_up_state+0x20/0x20 [83275.056160] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.056162] [] kthread+0xcf/0xe0 [83275.056163] [] ? insert_kthread_work+0x40/0x40 [83275.056164] [] ret_from_fork+0x58/0x90 [83275.056166] [] ? insert_kthread_work+0x40/0x40 [83275.056179] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.058849] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ldlm_bl_06:16008] [83275.058867] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.058872] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.058874] CPU: 2 PID: 16008 Comm: ldlm_bl_06 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.058874] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.058875] task: ffff8810247eeeb0 ti: ffff88201c7e8000 task.ti: ffff88201c7e8000 [83275.058878] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.058878] RSP: 0018:ffff88201c7ebb70 EFLAGS: 00000246 [83275.058879] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000110000 [83275.058879] RDX: ffff88203c897880 RSI: 0000000000a90001 RDI: ffff88102137981c [83275.058879] RBP: ffff88201c7ebb70 R08: ffff88103ce57880 R09: 0000000000000000 [83275.058880] R10: ffff8820256a9e00 R11: 0000000000000005 R12: ffff88201c7ebc58 [83275.058880] R13: 0000000000000001 R14: ffff88201c7ebb28 R15: ffffffff81322c35 [83275.058881] FS: 0000000000000000(0000) GS:ffff88103ce40000(0000) knlGS:0000000000000000 [83275.058881] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.058881] CR2: 0000000002577010 CR3: 00000000019f2000 CR4: 00000000001407e0 [83275.058882] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.058882] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.058882] Stack: [83275.058883] ffff88201c7ebb80 ffffffff8169e61f ffff88201c7ebb90 ffffffff816abb70 [83275.058884] ffff88201c7ebbd0 ffffffffc0b72198 0000000000000000 ffff8820256a9e00 [83275.058885] ffff88201c7ebc18 ffff8820256a9e60 ffff8820256a9e00 ffff8810175b8000 [83275.058885] Call Trace: [83275.058887] [] queued_spin_lock_slowpath+0xb/0xf [83275.058888] [] _raw_spin_lock+0x20/0x30 [83275.058905] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.058920] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.058935] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.058948] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.058961] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.058974] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.058979] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.058992] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.059005] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.059010] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.059023] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.059036] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.059052] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.059053] [] ? wake_up_state+0x20/0x20 [83275.059068] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.059070] [] kthread+0xcf/0xe0 [83275.059070] [] ? insert_kthread_work+0x40/0x40 [83275.059071] [] ret_from_fork+0x58/0x90 [83275.059072] [] ? insert_kthread_work+0x40/0x40 [83275.059080] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.070849] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ldlm_bl_14:16016] [83275.070873] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.070880] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.070882] CPU: 6 PID: 16016 Comm: ldlm_bl_14 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.070883] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.070883] task: ffff88203d338000 ti: ffff88201c810000 task.ti: ffff88201c810000 [83275.070886] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.070887] RSP: 0018:ffff88201c813b70 EFLAGS: 00000246 [83275.070887] RAX: 0000000000000000 RBX: 000000011c813be8 RCX: 0000000000310000 [83275.070888] RDX: ffff88103ce57880 RSI: 0000000000110000 RDI: ffff88102137981c [83275.070889] RBP: ffff88201c813b70 R08: ffff88103ced7880 R09: 0000000000000000 [83275.070889] R10: ffff8820256aa800 R11: 0000000000000000 R12: ffff88201c813c58 [83275.070890] R13: 0000000000000001 R14: ffff88201c813b28 R15: ffffffff81322c35 [83275.070891] FS: 0000000000000000(0000) GS:ffff88103cec0000(0000) knlGS:0000000000000000 [83275.070891] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.070892] CR2: 0000000002079728 CR3: 00000000019f2000 CR4: 00000000001407e0 [83275.070892] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.070893] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.070893] Stack: [83275.070895] ffff88201c813b80 ffffffff8169e61f ffff88201c813b90 ffffffff816abb70 [83275.070896] ffff88201c813bd0 ffffffffc0b72198 0000000000000000 ffff8820256aa800 [83275.070897] ffff88201c813c18 ffff8820256aa860 ffff8820256aa800 ffff8810175b8000 [83275.070898] Call Trace: [83275.070900] [] queued_spin_lock_slowpath+0xb/0xf [83275.070901] [] _raw_spin_lock+0x20/0x30 [83275.070926] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.070948] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.070968] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.070986] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.071004] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.071021] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.071027] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.071045] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.071062] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.071068] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.071086] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.071103] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.071126] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.071127] [] ? wake_up_state+0x20/0x20 [83275.071147] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.071149] [] kthread+0xcf/0xe0 [83275.071150] [] ? insert_kthread_work+0x40/0x40 [83275.071152] [] ret_from_fork+0x58/0x90 [83275.071153] [] ? insert_kthread_work+0x40/0x40 [83275.071167] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.076848] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ldlm_bl_10:16012] [83275.076870] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.076877] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.076879] CPU: 8 PID: 16012 Comm: ldlm_bl_10 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.076879] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.076880] task: ffff88203ad89fa0 ti: ffff88201c800000 task.ti: ffff88201c800000 [83275.076883] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.076883] RSP: 0018:ffff88201c803b70 EFLAGS: 00000246 [83275.076884] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000410000 [83275.076884] RDX: ffff88203c617880 RSI: 0000000000090000 RDI: ffff88102137981c [83275.076885] RBP: ffff88201c803b70 R08: ffff88103cf17880 R09: 0000000000000000 [83275.076885] R10: ffff8820256aaa00 R11: 0000000000000005 R12: ffff88201c803c58 [83275.076886] R13: 0000000000000001 R14: ffff88201c803b28 R15: ffffffff81322c35 [83275.076887] FS: 0000000000000000(0000) GS:ffff88103cf00000(0000) knlGS:0000000000000000 [83275.076887] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.076888] CR2: 00007fca10cf5000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83275.076888] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.076889] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.076889] Stack: [83275.076890] ffff88201c803b80 ffffffff8169e61f ffff88201c803b90 ffffffff816abb70 [83275.076892] ffff88201c803bd0 ffffffffc0b72198 0000000000000000 ffff8820256aaa00 [83275.076893] ffff88201c803c18 ffff8820256aaa60 ffff8820256aaa00 ffff8810175b8000 [83275.076893] Call Trace: [83275.076895] [] queued_spin_lock_slowpath+0xb/0xf [83275.076896] [] _raw_spin_lock+0x20/0x30 [83275.076915] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.076936] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.076956] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.076973] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.076990] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.077007] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.077012] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.077030] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.077047] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.077053] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.077072] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.077090] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.077112] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.077113] [] ? wake_up_state+0x20/0x20 [83275.077133] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.077135] [] kthread+0xcf/0xe0 [83275.077136] [] ? insert_kthread_work+0x40/0x40 [83275.077137] [] ret_from_fork+0x58/0x90 [83275.077139] [] ? insert_kthread_work+0x40/0x40 [83275.077152] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.082848] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ldlm_bl_05:16007] [83275.082869] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.082876] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.082877] CPU: 10 PID: 16007 Comm: ldlm_bl_05 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.082878] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.082879] task: ffff8810247ecf10 ti: ffff88201c64c000 task.ti: ffff88201c64c000 [83275.082881] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.082882] RSP: 0018:ffff88201c64fb70 EFLAGS: 00000246 [83275.082883] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000510000 [83275.082883] RDX: ffff88103ce17880 RSI: 0000000000010000 RDI: ffff88102137981c [83275.082884] RBP: ffff88201c64fb70 R08: ffff88103cf57880 R09: 0000000000000000 [83275.082884] R10: ffff88202565f800 R11: 0000000000000005 R12: ffff88201c64fc58 [83275.082885] R13: 0000000000000001 R14: ffff88201c64fb28 R15: ffffffff81322c35 [83275.082885] FS: 0000000000000000(0000) GS:ffff88103cf40000(0000) knlGS:0000000000000000 [83275.082886] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.082887] CR2: 0000000001dd94b8 CR3: 00000000019f2000 CR4: 00000000001407e0 [83275.082887] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.082888] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.082888] Stack: [83275.082889] ffff88201c64fb80 ffffffff8169e61f ffff88201c64fb90 ffffffff816abb70 [83275.082890] ffff88201c64fbd0 ffffffffc0b72198 0000000000000000 ffff88202565f800 [83275.082891] ffff88201c64fc18 ffff88202565f860 ffff88202565f800 ffff8810175b8000 [83275.082892] Call Trace: [83275.082894] [] queued_spin_lock_slowpath+0xb/0xf [83275.082895] [] _raw_spin_lock+0x20/0x30 [83275.082914] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.082935] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.082954] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.082972] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.082989] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.083007] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.083012] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.083030] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.083048] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.083053] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.083071] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.083089] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.083110] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.083112] [] ? wake_up_state+0x20/0x20 [83275.083132] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.083134] [] kthread+0xcf/0xe0 [83275.083135] [] ? insert_kthread_work+0x40/0x40 [83275.083136] [] ret_from_fork+0x58/0x90 [83275.083137] [] ? insert_kthread_work+0x40/0x40 [83275.083151] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.088848] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ldlm_bl_15:16017] [83275.088869] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.088876] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.088878] CPU: 12 PID: 16017 Comm: ldlm_bl_15 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.088878] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.088879] task: ffff88203d338fd0 ti: ffff88201c814000 task.ti: ffff88201c814000 [83275.088881] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.088882] RSP: 0018:ffff88201c817b70 EFLAGS: 00000246 [83275.088883] RAX: 0000000000000000 RBX: 00000001f8c12595 RCX: 0000000000610000 [83275.088883] RDX: ffff88203c757880 RSI: 0000000000590000 RDI: ffff88102137981c [83275.088884] RBP: ffff88201c817b70 R08: ffff88103cf97880 R09: 0000000000000000 [83275.088884] R10: ffff8820256aaa00 R11: 0000000000000005 R12: ffff88201c817c58 [83275.088885] R13: 0000000000000001 R14: ffff88201c817b28 R15: ffffffff81322c35 [83275.088886] FS: 0000000000000000(0000) GS:ffff88103cf80000(0000) knlGS:0000000000000000 [83275.088886] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.088887] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83275.088887] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.088888] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.088888] Stack: [83275.088889] ffff88201c817b80 ffffffff8169e61f ffff88201c817b90 ffffffff816abb70 [83275.088891] ffff88201c817bd0 ffffffffc0b72198 0000000000000000 ffff8820256aaa00 [83275.088892] ffff88201c817c18 ffff8820256aaa60 ffff8820256aaa00 ffff8810175b8000 [83275.088893] Call Trace: [83275.088895] [] queued_spin_lock_slowpath+0xb/0xf [83275.088896] [] _raw_spin_lock+0x20/0x30 [83275.088915] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.088936] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.088955] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.088973] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.088990] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.089008] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.089013] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.089031] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.089048] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.089054] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.089073] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.089091] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.089113] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.089115] [] ? wake_up_state+0x20/0x20 [83275.089135] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.089136] [] kthread+0xcf/0xe0 [83275.089137] [] ? insert_kthread_work+0x40/0x40 [83275.089139] [] ret_from_fork+0x58/0x90 [83275.089140] [] ? insert_kthread_work+0x40/0x40 [83275.089153] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.106847] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ldlm_bl_02:15061] [83275.106868] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.106875] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.106876] CPU: 18 PID: 15061 Comm: ldlm_bl_02 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.106877] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.106878] task: ffff882012edeeb0 ti: ffff88203d3bc000 task.ti: ffff88203d3bc000 [83275.106880] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.106881] RSP: 0018:ffff88203d3bfb70 EFLAGS: 00000246 [83275.106881] RAX: 0000000000000000 RBX: 000000013cf16cc0 RCX: 0000000000910000 [83275.106882] RDX: ffff88203c7d7880 RSI: 0000000000790000 RDI: ffff88102137981c [83275.106882] RBP: ffff88203d3bfb70 R08: ffff88103d057880 R09: 0000000000000000 [83275.106883] R10: ffff8820256aac00 R11: 0000000000000005 R12: ffff88203d3bfc58 [83275.106884] R13: 0000000000000001 R14: ffff88203d3bfb28 R15: ffffffff81322c35 [83275.106884] FS: 0000000000000000(0000) GS:ffff88103d040000(0000) knlGS:0000000000000000 [83275.106885] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.106886] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83275.106886] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.106887] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.106887] Stack: [83275.106889] ffff88203d3bfb80 ffffffff8169e61f ffff88203d3bfb90 ffffffff816abb70 [83275.106890] ffff88203d3bfbd0 ffffffffc0b72198 0000000000000000 ffff8820256aac00 [83275.106891] ffff88203d3bfc18 ffff8820256aac60 ffff8820256aac00 ffff8810175b8000 [83275.106891] Call Trace: [83275.106893] [] queued_spin_lock_slowpath+0xb/0xf [83275.106894] [] _raw_spin_lock+0x20/0x30 [83275.106913] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.106932] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.106951] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.106969] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.106986] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.107004] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.107009] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.107027] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.107044] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.107049] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.107069] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.107087] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.107108] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.107110] [] ? wake_up_state+0x20/0x20 [83275.107130] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.107131] [] kthread+0xcf/0xe0 [83275.107133] [] ? insert_kthread_work+0x40/0x40 [83275.107134] [] ret_from_fork+0x58/0x90 [83275.107135] [] ? insert_kthread_work+0x40/0x40 [83275.107148] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.112847] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ldlm_bl_01:15060] [83275.112868] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.112875] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.112876] CPU: 20 PID: 15060 Comm: ldlm_bl_01 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.112877] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.112878] task: ffff882012eddee0 ti: ffff88203d3b8000 task.ti: ffff88203d3b8000 [83275.112880] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.112881] RSP: 0018:ffff88203d3bbb70 EFLAGS: 00000246 [83275.112881] RAX: 0000000000000000 RBX: 0000000180597933 RCX: 0000000000a10000 [83275.112882] RDX: ffff88203c857880 RSI: 0000000000990000 RDI: ffff88102137981c [83275.112882] RBP: ffff88203d3bbb70 R08: ffff88103d097880 R09: 0000000000000000 [83275.112883] R10: ffff88202569d400 R11: 0000000000000005 R12: ffff88203d3bbc58 [83275.112884] R13: 0000000000000001 R14: ffff88203d3bbb28 R15: ffffffff81322c35 [83275.112884] FS: 0000000000000000(0000) GS:ffff88103d080000(0000) knlGS:0000000000000000 [83275.112885] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.112885] CR2: 00007f02c314e4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83275.112886] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.112886] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.112887] Stack: [83275.112888] ffff88203d3bbb80 ffffffff8169e61f ffff88203d3bbb90 ffffffff816abb70 [83275.112889] ffff88203d3bbbd0 ffffffffc0b72198 0000000000000000 ffff88202569d400 [83275.112890] ffff88203d3bbc18 ffff88202569d460 ffff88202569d400 ffff8810175b8000 [83275.112890] Call Trace: [83275.112892] [] queued_spin_lock_slowpath+0xb/0xf [83275.112894] [] _raw_spin_lock+0x20/0x30 [83275.112912] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.112933] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.112952] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.112970] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.112988] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.113005] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.113011] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.113028] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.113046] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.113051] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.113069] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.113087] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.113109] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.113110] [] ? wake_up_state+0x20/0x20 [83275.113130] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.113132] [] kthread+0xcf/0xe0 [83275.113133] [] ? insert_kthread_work+0x40/0x40 [83275.113134] [] ret_from_fork+0x58/0x90 [83275.113135] [] ? insert_kthread_work+0x40/0x40 [83275.113149] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83275.118847] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ldlm_bl_07:16009] [83275.118867] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83275.118875] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83275.118876] CPU: 22 PID: 16009 Comm: ldlm_bl_07 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83275.118877] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83275.118878] task: ffff8810e9240fd0 ti: ffff88201c7ec000 task.ti: ffff88201c7ec000 [83275.118880] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83275.118880] RSP: 0018:ffff88201c7efb70 EFLAGS: 00000246 [83275.118881] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000b10000 [83275.118881] RDX: ffff88103ce97880 RSI: 0000000000210000 RDI: ffff88102137981c [83275.118882] RBP: ffff88201c7efb70 R08: ffff88103d0d7880 R09: 0000000000000000 [83275.118882] R10: ffff8820256aac00 R11: 0000000000000005 R12: ffff88201c7efc58 [83275.118883] R13: 0000000000000001 R14: ffff88201c7efb28 R15: ffffffff81322c35 [83275.118884] FS: 0000000000000000(0000) GS:ffff88103d0c0000(0000) knlGS:0000000000000000 [83275.118884] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83275.118885] CR2: 00007f262c4764f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83275.118885] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83275.118886] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83275.118886] Stack: [83275.118887] ffff88201c7efb80 ffffffff8169e61f ffff88201c7efb90 ffffffff816abb70 [83275.118888] ffff88201c7efbd0 ffffffffc0b72198 0000000000000000 ffff8820256aac00 [83275.118890] ffff88201c7efc18 ffff8820256aac60 ffff8820256aac00 ffff8810175b8000 [83275.118890] Call Trace: [83275.118892] [] queued_spin_lock_slowpath+0xb/0xf [83275.118893] [] _raw_spin_lock+0x20/0x30 [83275.118913] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83275.118933] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83275.118952] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83275.118969] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83275.118987] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83275.119004] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83275.119010] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83275.119027] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.119045] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83275.119050] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83275.119068] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83275.119085] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83275.119107] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83275.119109] [] ? wake_up_state+0x20/0x20 [83275.119128] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83275.119130] [] kthread+0xcf/0xe0 [83275.119131] [] ? insert_kthread_work+0x40/0x40 [83275.119132] [] ret_from_fork+0x58/0x90 [83275.119134] [] ? insert_kthread_work+0x40/0x40 [83275.119147] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83279.091719] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83279.171182] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83279.200053] CPU: 1 PID: 16010 Comm: ldlm_bl_08 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83279.212765] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83279.221113] task: ffff8810e9242f70 ti: ffff88201c7f8000 task.ti: ffff88201c7f8000 [83279.229461] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83279.240533] RSP: 0018:ffff88201c7fbb70 EFLAGS: 00000246 [83279.246457] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000090000 [83279.254418] RDX: ffff88103cf57880 RSI: 0000000000510000 RDI: ffff88102137981c [83279.262378] RBP: ffff88201c7fbb70 R08: ffff88203c617880 R09: 0000000000000000 [83279.270338] R10: ffff882025634c00 R11: 0000000000000005 R12: ffff88201c7fbc58 [83279.278298] R13: 0000000000000001 R14: ffff88201c7fbb28 R15: ffffffff81322c35 [83279.286258] FS: 0000000000000000(0000) GS:ffff88203c600000(0000) knlGS:0000000000000000 [83279.295284] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83279.301694] CR2: 00007f11b0c8c000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83279.309654] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83279.317614] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83279.325574] Stack: [83279.327814] ffff88201c7fbb80 ffffffff8169e61f ffff88201c7fbb90 ffffffff816abb70 [83279.336101] ffff88201c7fbbd0 ffffffffc0b72198 0000000000000000 ffff882025634c00 [83279.344386] ffff88201c7fbc18 ffff882025634c60 ffff882025634c00 ffff8810175b8000 [83279.352673] Call Trace: [83279.355400] [] queued_spin_lock_slowpath+0xb/0xf [83279.362391] [] _raw_spin_lock+0x20/0x30 [83279.368534] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83279.376710] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83279.384789] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83279.392185] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83279.399968] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83279.407753] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83279.415137] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83279.423212] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83279.431095] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83279.438963] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83279.447040] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83279.455503] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83279.463678] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83279.471349] [] ? wake_up_state+0x20/0x20 [83279.477584] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83279.485836] [] kthread+0xcf/0xe0 [83279.491276] [] ? insert_kthread_work+0x40/0x40 [83279.498073] [] ret_from_fork+0x58/0x90 [83279.504096] [] ? insert_kthread_work+0x40/0x40 [83279.510893] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83287.079590] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [ldlm_bl_03:15915] [83287.085590] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ldlm_bl_22:33195] [83287.085614] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83287.085623] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83287.085624] CPU: 11 PID: 33195 Comm: ldlm_bl_22 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83287.085625] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83287.085626] task: ffff88203d0b1fa0 ti: ffff88203d4b8000 task.ti: ffff88203d4b8000 [83287.085630] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83287.085630] RSP: 0018:ffff88203d4bbb70 EFLAGS: 00000246 [83287.085631] RAX: 0000000000000000 RBX: 5a5a5a5a5a5a5a5a RCX: 0000000000590000 [83287.085631] RDX: ffff88103ced7880 RSI: 0000000000310000 RDI: ffff88102137981c [83287.085632] RBP: ffff88203d4bbb70 R08: ffff88203c757880 R09: 0000000000000000 [83287.085633] R10: ffff882023f93600 R11: 0000000000000001 R12: 5a5a5a5a5a5a5a5a [83287.085633] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83287.085634] FS: 0000000000000000(0000) GS:ffff88203c740000(0000) knlGS:0000000000000000 [83287.085635] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83287.085635] CR2: 00007fef29abd090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83287.085636] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83287.085636] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83287.085637] Stack: [83287.085638] ffff88203d4bbb80 ffffffff8169e61f ffff88203d4bbb90 ffffffff816abb70 [83287.085639] ffff88203d4bbbd0 ffffffffc0b72198 0000000000000000 ffff882023f93600 [83287.085640] ffff88203d4bbc18 ffff882023f93660 ffff882023f93600 ffff8810175b8000 [83287.085640] Call Trace: [83287.085643] [] queued_spin_lock_slowpath+0xb/0xf [83287.085644] [] _raw_spin_lock+0x20/0x30 [83287.085678] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83287.085702] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83287.085722] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83287.085740] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83287.085757] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83287.085775] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83287.085782] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83287.085801] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.085819] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.085824] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83287.085842] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83287.085859] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83287.085881] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83287.085882] [] ? __schedule+0x39d/0x8b0 [83287.085902] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83287.085904] [] kthread+0xcf/0xe0 [83287.085906] [] ? insert_kthread_work+0x40/0x40 [83287.085907] [] ret_from_fork+0x58/0x90 [83287.085908] [] ? insert_kthread_work+0x40/0x40 [83287.085922] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83287.091590] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [ldlm_bl_19:33192] [83287.091613] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83287.091621] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83287.091622] CPU: 13 PID: 33192 Comm: ldlm_bl_19 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83287.091623] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83287.091624] task: ffff88203d0b2f70 ti: ffff881ff8e20000 task.ti: ffff881ff8e20000 [83287.091627] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83287.091628] RSP: 0018:ffff881ff8e23b70 EFLAGS: 00000246 [83287.091629] RAX: 0000000000000000 RBX: ffff881ff8e23be8 RCX: 0000000000690000 [83287.091629] RDX: ffff88203c697880 RSI: 0000000000290000 RDI: ffff88102137981c [83287.091630] RBP: ffff881ff8e23b70 R08: ffff88203c797880 R09: 0000000000000000 [83287.091630] R10: ffff882023fcd000 R11: 0000000000000000 R12: ffff88103ce96cc0 [83287.091631] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83287.091632] FS: 0000000000000000(0000) GS:ffff88203c780000(0000) knlGS:0000000000000000 [83287.091632] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83287.091633] CR2: 00007f352c5974f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83287.091633] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83287.091634] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83287.091634] Stack: [83287.091636] ffff881ff8e23b80 ffffffff8169e61f ffff881ff8e23b90 ffffffff816abb70 [83287.091637] ffff881ff8e23bd0 ffffffffc0b72198 0000000000000000 ffff882023fcd000 [83287.091638] ffff881ff8e23c18 ffff882023fcd060 ffff882023fcd000 ffff8810175b8000 [83287.091638] Call Trace: [83287.091640] [] queued_spin_lock_slowpath+0xb/0xf [83287.091642] [] _raw_spin_lock+0x20/0x30 [83287.091666] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83287.091687] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83287.091706] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83287.091724] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83287.091741] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83287.091759] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83287.091765] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83287.091783] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.091800] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.091806] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83287.091826] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83287.091843] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83287.091866] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83287.091867] [] ? __schedule+0x39d/0x8b0 [83287.091887] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83287.091889] [] kthread+0xcf/0xe0 [83287.091890] [] ? insert_kthread_work+0x40/0x40 [83287.091892] [] ret_from_fork+0x58/0x90 [83287.091893] [] ? insert_kthread_work+0x40/0x40 [83287.091906] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83287.097589] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ldlm_bl_23:33196] [83287.097612] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83287.097620] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83287.097622] CPU: 15 PID: 33196 Comm: ldlm_bl_23 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83287.097622] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83287.097623] task: ffff88203d0b0000 ti: ffff882015ae0000 task.ti: ffff882015ae0000 [83287.097626] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83287.097627] RSP: 0018:ffff882015ae3b70 EFLAGS: 00000246 [83287.097628] RAX: 0000000000000000 RBX: ffff88203c756cc0 RCX: 0000000000790000 [83287.097628] RDX: ffff88203c6d7880 RSI: 0000000000390000 RDI: ffff88102137981c [83287.097629] RBP: ffff882015ae3b70 R08: ffff88203c7d7880 R09: 0000000000000000 [83287.097629] R10: ffff882023f24000 R11: 0000000000000001 R12: ffff88203d0b0068 [83287.097630] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83287.097631] FS: 0000000000000000(0000) GS:ffff88203c7c0000(0000) knlGS:0000000000000000 [83287.097631] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83287.097632] CR2: 00007faebc9e03cc CR3: 00000000019f2000 CR4: 00000000001407e0 [83287.097632] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83287.097633] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83287.097633] Stack: [83287.097635] ffff882015ae3b80 ffffffff8169e61f ffff882015ae3b90 ffffffff816abb70 [83287.097636] ffff882015ae3bd0 ffffffffc0b72198 0000000000000000 ffff882023f24000 [83287.097637] ffff882015ae3c18 ffff882023f24060 ffff882023f24000 ffff8810175b8000 [83287.097638] Call Trace: [83287.097640] [] queued_spin_lock_slowpath+0xb/0xf [83287.097641] [] _raw_spin_lock+0x20/0x30 [83287.097671] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83287.097693] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83287.097712] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83287.097729] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83287.097747] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83287.097765] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83287.097772] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83287.097789] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.097807] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.097812] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83287.097832] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83287.097850] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83287.097872] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83287.097873] [] ? __schedule+0x39d/0x8b0 [83287.097894] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83287.097896] [] kthread+0xcf/0xe0 [83287.097897] [] ? insert_kthread_work+0x40/0x40 [83287.097898] [] ret_from_fork+0x58/0x90 [83287.097900] [] ? insert_kthread_work+0x40/0x40 [83287.097913] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83287.103589] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [ldlm_bl_21:33194] [83287.103611] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83287.103619] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83287.103620] CPU: 17 PID: 33194 Comm: ldlm_bl_21 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83287.103621] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83287.103622] task: ffff88203d0b6eb0 ti: ffff882012eac000 task.ti: ffff882012eac000 [83287.103624] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83287.103625] RSP: 0018:ffff882012eafb70 EFLAGS: 00000246 [83287.103626] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000890000 [83287.103626] RDX: ffff88203c797880 RSI: 0000000000690000 RDI: ffff88102137981c [83287.103627] RBP: ffff882012eafb70 R08: ffff88203c817880 R09: 0000000000000000 [83287.103627] R10: ffff882023fb7e00 R11: 0000000000000001 R12: 0000000000000000 [83287.103628] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c816cc0 [83287.103629] FS: 0000000000000000(0000) GS:ffff88203c800000(0000) knlGS:0000000000000000 [83287.103630] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83287.103631] CR2: 00007f7ce52f4550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83287.103631] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83287.103632] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83287.103632] Stack: [83287.103634] ffff882012eafb80 ffffffff8169e61f ffff882012eafb90 ffffffff816abb70 [83287.103635] ffff882012eafbd0 ffffffffc0b72198 0000000000000000 ffff882023fb7e00 [83287.103636] ffff882012eafc18 ffff882023fb7e60 ffff882023fb7e00 ffff8810175b8000 [83287.103636] Call Trace: [83287.103638] [] queued_spin_lock_slowpath+0xb/0xf [83287.103639] [] _raw_spin_lock+0x20/0x30 [83287.103658] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83287.103678] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83287.103697] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83287.103714] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83287.103732] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83287.103750] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83287.103755] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83287.103773] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.103790] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.103795] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83287.103815] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83287.103833] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83287.103855] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83287.103857] [] ? __schedule+0x39d/0x8b0 [83287.103877] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83287.103878] [] kthread+0xcf/0xe0 [83287.103880] [] ? insert_kthread_work+0x40/0x40 [83287.103881] [] ret_from_fork+0x58/0x90 [83287.103882] [] ? insert_kthread_work+0x40/0x40 [83287.103895] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83287.109589] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 23s! [ldlm_bl_18:33191] [83287.109610] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83287.109618] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83287.109619] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83287.109620] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83287.109620] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83287.109623] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83287.109623] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83287.109624] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83287.109625] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83287.109625] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83287.109626] R10: ffff882023ffc600 R11: 0000000000000001 R12: ffff88203d0b5f48 [83287.109627] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83287.109628] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83287.109628] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83287.109629] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83287.109629] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83287.109630] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83287.109630] Stack: [83287.109631] ffff881ffba83b80 ffffffff8169e61f ffff881ffba83b90 ffffffff816abb70 [83287.109632] ffff881ffba83bd0 ffffffffc0b72198 0000000000000000 ffff882023ffc600 [83287.109633] ffff881ffba83c18 ffff882023ffc660 ffff882023ffc600 ffff8810175b8000 [83287.109634] Call Trace: [83287.109636] [] queued_spin_lock_slowpath+0xb/0xf [83287.109637] [] _raw_spin_lock+0x20/0x30 [83287.109657] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83287.109679] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83287.109698] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83287.109715] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83287.109732] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83287.109750] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83287.109755] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83287.109773] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.109790] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.109796] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83287.109813] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83287.109831] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83287.109853] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83287.109854] [] ? __schedule+0x39d/0x8b0 [83287.109875] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83287.109876] [] kthread+0xcf/0xe0 [83287.109877] [] ? insert_kthread_work+0x40/0x40 [83287.109879] [] ret_from_fork+0x58/0x90 [83287.109880] [] ? insert_kthread_work+0x40/0x40 [83287.109893] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83287.115589] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [ldlm_bl_20:33193] [83287.115607] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83287.115613] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83287.115615] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83287.115615] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83287.115616] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83287.115618] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83287.115619] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83287.115619] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83287.115620] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83287.115620] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83287.115621] R10: ffff882023fdd200 R11: 0000000000000000 R12: ffff88103ce96cc0 [83287.115621] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83287.115622] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83287.115623] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83287.115623] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83287.115623] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83287.115624] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83287.115624] Stack: [83287.115625] ffff881ff8e2fb80 ffffffff8169e61f ffff881ff8e2fb90 ffffffff816abb70 [83287.115626] ffff881ff8e2fbd0 ffffffffc0b72198 0000000000000000 ffff882023fdd200 [83287.115627] ffff881ff8e2fc18 ffff882023fdd260 ffff882023fdd200 ffff8810175b8000 [83287.115627] Call Trace: [83287.115629] [] queued_spin_lock_slowpath+0xb/0xf [83287.115630] [] _raw_spin_lock+0x20/0x30 [83287.115648] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83287.115666] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83287.115684] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83287.115700] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83287.115716] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83287.115733] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83287.115738] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83287.115755] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.115771] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.115776] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83287.115792] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83287.115809] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83287.115828] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83287.115829] [] ? __schedule+0x39d/0x8b0 [83287.115848] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83287.115850] [] kthread+0xcf/0xe0 [83287.115851] [] ? insert_kthread_work+0x40/0x40 [83287.115852] [] ret_from_fork+0x58/0x90 [83287.115853] [] ? insert_kthread_work+0x40/0x40 [83287.115864] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83287.121588] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ldlm_bl_04:16006] [83287.121609] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83287.121617] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83287.121618] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83287.121619] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83287.121620] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83287.121622] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83287.121623] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83287.121623] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83287.121624] RDX: ffff88103d057880 RSI: 0000000000910000 RDI: ffff88102137981c [83287.121625] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83287.121625] R10: ffff8820256a0000 R11: 0000000000000000 R12: ffff88201c627c58 [83287.121626] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83287.121626] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83287.121627] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83287.121627] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83287.121628] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83287.121629] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83287.121629] Stack: [83287.121630] ffff88201c627b80 ffffffff8169e61f ffff88201c627b90 ffffffff816abb70 [83287.121631] ffff88201c627bd0 ffffffffc0b72198 0000000000000000 ffff8820256a0000 [83287.121632] ffff88201c627c18 ffff8820256a0060 ffff8820256a0000 ffff8810175b8000 [83287.121633] Call Trace: [83287.121635] [] queued_spin_lock_slowpath+0xb/0xf [83287.121636] [] _raw_spin_lock+0x20/0x30 [83287.121655] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83287.121676] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83287.121695] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83287.121712] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83287.121730] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83287.121748] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83287.121753] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83287.121770] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.121788] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83287.121793] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83287.121811] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83287.121829] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83287.121851] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83287.121852] [] ? wake_up_state+0x20/0x20 [83287.121873] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83287.121874] [] kthread+0xcf/0xe0 [83287.121876] [] ? insert_kthread_work+0x40/0x40 [83287.121877] [] ret_from_fork+0x58/0x90 [83287.121878] [] ? insert_kthread_work+0x40/0x40 [83287.121892] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83290.221335] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83290.300811] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83290.329674] CPU: 9 PID: 15915 Comm: ldlm_bl_03 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83290.342384] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83290.350732] task: ffff88202c648000 ti: ffff88201763c000 task.ti: ffff88201763c000 [83290.359080] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83290.370151] RSP: 0018:ffff88201763fb70 EFLAGS: 00000246 [83290.376076] RAX: 0000000000000000 RBX: 0000000300000000 RCX: 0000000000490000 [83290.384035] RDX: ffff88103d097880 RSI: 0000000000a10000 RDI: ffff88102137981c [83290.391996] RBP: ffff88201763fb70 R08: ffff88203c717880 R09: 0000000000000000 [83290.399956] R10: ffff882025610a00 R11: 7fffffffffffffff R12: 0000000000000bf8 [83290.407916] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c656cc0 [83290.415876] FS: 0000000000000000(0000) GS:ffff88203c700000(0000) knlGS:0000000000000000 [83290.424904] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83290.431314] CR2: 00007ffa010a6000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83290.439273] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83290.447234] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83290.455193] Stack: [83290.457433] ffff88201763fb80 ffffffff8169e61f ffff88201763fb90 ffffffff816abb70 [83290.465720] ffff88201763fbd0 ffffffffc0b72198 0000000000000000 ffff882025610a00 [83290.474007] ffff88201763fc18 ffff882025610a60 ffff882025610a00 ffff8810175b8000 [83290.482294] Call Trace: [83290.485021] [] queued_spin_lock_slowpath+0xb/0xf [83290.492011] [] _raw_spin_lock+0x20/0x30 [83290.498148] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83290.506320] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83290.514397] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83290.521792] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83290.529577] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83290.537359] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83290.544742] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83290.552817] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83290.560697] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83290.568564] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83290.576641] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83290.585105] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83290.593279] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83290.600951] [] ? wake_up_state+0x20/0x20 [83290.607186] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83290.615438] [] kthread+0xcf/0xe0 [83290.620878] [] ? insert_kthread_work+0x40/0x40 [83290.627676] [] ret_from_fork+0x58/0x90 [83290.633698] [] ? insert_kthread_work+0x40/0x40 [83290.640495] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83291.061504] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ldlm_bl_16:33189] [83291.064504] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ldlm_bl_09:16011] [83291.064532] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83291.064541] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83291.064542] CPU: 4 PID: 16011 Comm: ldlm_bl_09 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83291.064543] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83291.064544] task: ffff88203ad88fd0 ti: ffff88201c7fc000 task.ti: ffff88201c7fc000 [83291.064547] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83291.064548] RSP: 0018:ffff88201c7ffb70 EFLAGS: 00000246 [83291.064548] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000210000 [83291.064549] RDX: ffff88103cf97880 RSI: 0000000000610000 RDI: ffff88102137981c [83291.064549] RBP: ffff88201c7ffb70 R08: ffff88103ce97880 R09: 0000000000000000 [83291.064550] R10: ffff882029a3fe00 R11: 0000000000000000 R12: ffff88201c7ffc58 [83291.064551] R13: 0000000000000001 R14: ffff88201c7ffb28 R15: ffffffff81322c35 [83291.064552] FS: 0000000000000000(0000) GS:ffff88103ce80000(0000) knlGS:0000000000000000 [83291.064552] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83291.064553] CR2: 00007f8905462248 CR3: 00000000019f2000 CR4: 00000000001407e0 [83291.064553] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83291.064554] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83291.064554] Stack: [83291.064555] ffff88201c7ffb80 ffffffff8169e61f ffff88201c7ffb90 ffffffff816abb70 [83291.064556] ffff88201c7ffbd0 ffffffffc0b72198 0000000000000000 ffff882029a3fe00 [83291.064558] ffff88201c7ffc18 ffff882029a3fe60 ffff882029a3fe00 ffff8810175b8000 [83291.064558] Call Trace: [83291.064560] [] queued_spin_lock_slowpath+0xb/0xf [83291.064562] [] _raw_spin_lock+0x20/0x30 [83291.064591] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83291.064614] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83291.064633] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83291.064651] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83291.064668] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83291.064686] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83291.064692] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83291.064712] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83291.064729] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83291.064735] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83291.064755] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83291.064772] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83291.064795] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83291.064796] [] ? wake_up_state+0x20/0x20 [83291.064817] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83291.064818] [] kthread+0xcf/0xe0 [83291.064820] [] ? insert_kthread_work+0x40/0x40 [83291.064821] [] ret_from_fork+0x58/0x90 [83291.064822] [] ? insert_kthread_work+0x40/0x40 [83291.064836] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83291.067505] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ldlm_bl_11:16013] [83291.067528] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83291.067536] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83291.067538] CPU: 5 PID: 16013 Comm: ldlm_bl_11 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83291.067538] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83291.067539] task: ffff88203ad88000 ti: ffff88201c804000 task.ti: ffff88201c804000 [83291.067542] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83291.067543] RSP: 0018:ffff88201c807b70 EFLAGS: 00000246 [83291.067543] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000290000 [83291.067544] RDX: ffff88103d017880 RSI: 0000000000810000 RDI: ffff88102137981c [83291.067544] RBP: ffff88201c807b70 R08: ffff88203c697880 R09: 0000000000000000 [83291.067545] R10: ffff8820256b2e00 R11: 0000000000000000 R12: ffff88201c807c58 [83291.067545] R13: 0000000000000001 R14: ffff88201c807b28 R15: ffffffff81322c35 [83291.067546] FS: 0000000000000000(0000) GS:ffff88203c680000(0000) knlGS:0000000000000000 [83291.067547] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83291.067547] CR2: 00007fd7ddcd8090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83291.067548] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83291.067549] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83291.067549] Stack: [83291.067550] ffff88201c807b80 ffffffff8169e61f ffff88201c807b90 ffffffff816abb70 [83291.067551] ffff88201c807bd0 ffffffffc0b72198 0000000000000000 ffff8820256b2e00 [83291.067553] ffff88201c807c18 ffff8820256b2e60 ffff8820256b2e00 ffff8810175b8000 [83291.067553] Call Trace: [83291.067555] [] queued_spin_lock_slowpath+0xb/0xf [83291.067557] [] _raw_spin_lock+0x20/0x30 [83291.067585] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83291.067608] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83291.067628] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83291.067645] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83291.067663] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83291.067681] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83291.067687] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83291.067707] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83291.067724] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83291.067730] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83291.067747] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83291.067765] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83291.067787] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83291.067789] [] ? wake_up_state+0x20/0x20 [83291.067810] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83291.067811] [] kthread+0xcf/0xe0 [83291.067813] [] ? insert_kthread_work+0x40/0x40 [83291.067814] [] ret_from_fork+0x58/0x90 [83291.067815] [] ? insert_kthread_work+0x40/0x40 [83291.067829] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83291.073503] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ldlm_bl_17:33190] [83291.073525] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83291.073533] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83291.073535] CPU: 7 PID: 33190 Comm: ldlm_bl_17 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83291.073535] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83291.073536] task: ffff88102d281fa0 ti: ffff881fffd34000 task.ti: ffff881fffd34000 [83291.073539] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83291.073540] RSP: 0018:ffff881fffd37b70 EFLAGS: 00000246 [83291.073540] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000390000 [83291.073541] RDX: ffff88103cf17880 RSI: 0000000000410001 RDI: ffff88102137981c [83291.073541] RBP: ffff881fffd37b70 R08: ffff88203c6d7880 R09: 0000000000000000 [83291.073542] R10: ffff882025619200 R11: 0000000000000000 R12: ffff88102d282008 [83291.073543] R13: ffff881fffd37b00 R14: ffff881f00000141 R15: ffff88207ffd8008 [83291.073544] FS: 0000000000000000(0000) GS:ffff88203c6c0000(0000) knlGS:0000000000000000 [83291.073544] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83291.073545] CR2: 00007f240470f978 CR3: 00000000019f2000 CR4: 00000000001407e0 [83291.073546] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83291.073546] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83291.073547] Stack: [83291.073548] ffff881fffd37b80 ffffffff8169e61f ffff881fffd37b90 ffffffff816abb70 [83291.073549] ffff881fffd37bd0 ffffffffc0b72198 0000000000000000 ffff882025619200 [83291.073550] ffff881fffd37c18 ffff882025619260 ffff882025619200 ffff8810175b8000 [83291.073551] Call Trace: [83291.073553] [] queued_spin_lock_slowpath+0xb/0xf [83291.073555] [] _raw_spin_lock+0x20/0x30 [83291.073577] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83291.073598] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83291.073617] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83291.073635] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83291.073653] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83291.073670] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83291.073676] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83291.073693] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83291.073711] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83291.073716] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83291.073734] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83291.073751] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83291.073772] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83291.073773] [] ? __schedule+0x39d/0x8b0 [83291.073794] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83291.073795] [] kthread+0xcf/0xe0 [83291.073796] [] ? insert_kthread_work+0x40/0x40 [83291.073798] [] ret_from_fork+0x58/0x90 [83291.073799] [] ? insert_kthread_work+0x40/0x40 [83291.073812] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83291.100503] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ldlm_bl_12:16014] [83291.100527] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83291.100535] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83291.100537] CPU: 16 PID: 16014 Comm: ldlm_bl_12 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83291.100537] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83291.100538] task: ffff88203d33af70 ti: ffff88201c808000 task.ti: ffff88201c808000 [83291.100541] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83291.100541] RSP: 0018:ffff88201c80bb70 EFLAGS: 00000246 [83291.100542] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000810000 [83291.100543] RDX: ffff88203c717880 RSI: 0000000000490000 RDI: ffff88102137981c [83291.100543] RBP: ffff88201c80bb70 R08: ffff88103d017880 R09: 0000000000000000 [83291.100544] R10: ffff882029a43600 R11: 0000000000000005 R12: ffff88201c80bc58 [83291.100544] R13: 0000000000000001 R14: ffff88201c80bb28 R15: ffffffff81322c35 [83291.100545] FS: 0000000000000000(0000) GS:ffff88103d000000(0000) knlGS:0000000000000000 [83291.100546] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83291.100546] CR2: 00007f361a3bb4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83291.100547] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83291.100548] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83291.100548] Stack: [83291.100549] ffff88201c80bb80 ffffffff8169e61f ffff88201c80bb90 ffffffff816abb70 [83291.100550] ffff88201c80bbd0 ffffffffc0b72198 0000000000000000 ffff882029a43600 [83291.100551] ffff88201c80bc18 ffff882029a43660 ffff882029a43600 ffff8810175b8000 [83291.100552] Call Trace: [83291.100553] [] queued_spin_lock_slowpath+0xb/0xf [83291.100555] [] _raw_spin_lock+0x20/0x30 [83291.100579] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83291.100600] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83291.100620] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83291.100638] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83291.100656] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83291.100673] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83291.100679] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83291.100696] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83291.100714] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83291.100720] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83291.100739] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83291.100757] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83291.100779] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83291.100781] [] ? wake_up_state+0x20/0x20 [83291.100802] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83291.100803] [] kthread+0xcf/0xe0 [83291.100804] [] ? insert_kthread_work+0x40/0x40 [83291.100806] [] ret_from_fork+0x58/0x90 [83291.100807] [] ? insert_kthread_work+0x40/0x40 [83291.100820] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83292.860150] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83292.939615] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83292.968484] CPU: 3 PID: 33189 Comm: ldlm_bl_16 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83292.981196] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83292.989544] task: ffff88102c376eb0 ti: ffff881fefb44000 task.ti: ffff881fefb44000 [83292.997892] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83293.008965] RSP: 0018:ffff881fefb47b70 EFLAGS: 00000246 [83293.014889] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000190000 [83293.022849] RDX: ffff88203c8d7880 RSI: 0000000000b90000 RDI: ffff88102137981c [83293.030809] RBP: ffff881fefb47b70 R08: ffff88203c657880 R09: 0000000000000000 [83293.038769] R10: ffff88202561d800 R11: 0000000000000000 R12: ffff88102c376f18 [83293.046730] R13: ffff88203c2b92a0 R14: ffff881000000141 R15: ffff88207ffd8008 [83293.054690] FS: 0000000000000000(0000) GS:ffff88203c640000(0000) knlGS:0000000000000000 [83293.063716] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83293.070126] CR2: 000055ce548c9220 CR3: 00000000019f2000 CR4: 00000000001407e0 [83293.078086] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83293.086047] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83293.094006] Stack: [83293.096246] ffff881fefb47b80 ffffffff8169e61f ffff881fefb47b90 ffffffff816abb70 [83293.104534] ffff881fefb47bd0 ffffffffc0b72198 0000000000000000 ffff88202561d800 [83293.112819] ffff881fefb47c18 ffff88202561d860 ffff88202561d800 ffff8810175b8000 [83293.121106] Call Trace: [83293.123833] [] queued_spin_lock_slowpath+0xb/0xf [83293.130824] [] _raw_spin_lock+0x20/0x30 [83293.136966] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83293.145144] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83293.153221] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83293.160615] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83293.168398] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83293.176181] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83293.183565] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83293.191640] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83293.199521] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83293.207389] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83293.215464] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83293.223929] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83293.232103] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83293.239774] [] ? __schedule+0x39d/0x8b0 [83293.245911] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83293.254163] [] kthread+0xcf/0xe0 [83293.259603] [] ? insert_kthread_work+0x40/0x40 [83293.266401] [] ret_from_fork+0x58/0x90 [83293.272423] [] ? insert_kthread_work+0x40/0x40 [83293.279221] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83303.054246] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ldlm_bl_08:16010] [83303.055246] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ldlm_bl_13:16015] [83303.055273] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.055282] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.055283] CPU: 0 PID: 16015 Comm: ldlm_bl_13 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.055284] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.055285] task: ffff88203d33bf40 ti: ffff88201c80c000 task.ti: ffff88201c80c000 [83303.055289] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83303.055289] RSP: 0018:ffff88201c80fb70 EFLAGS: 00000246 [83303.055290] RAX: 0000000000000000 RBX: 000000013ce56cc0 RCX: 0000000000010000 [83303.055290] RDX: ffff88103d0d7880 RSI: 0000000000b10000 RDI: ffff88102137981c [83303.055291] RBP: ffff88201c80fb70 R08: ffff88103ce17880 R09: 0000000000000000 [83303.055291] R10: ffff882029b90a00 R11: 0000000000000005 R12: ffff88201c80fc58 [83303.055292] R13: 0000000000000001 R14: ffff88201c80fb28 R15: ffffffff81322c35 [83303.055293] FS: 0000000000000000(0000) GS:ffff88103ce00000(0000) knlGS:0000000000000000 [83303.055293] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.055294] CR2: 00007f17e6a8b248 CR3: 00000000019f2000 CR4: 00000000001407f0 [83303.055294] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.055295] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.055295] Stack: [83303.055296] ffff88201c80fb80 ffffffff8169e61f ffff88201c80fb90 ffffffff816abb70 [83303.055298] ffff88201c80fbd0 ffffffffc0b72198 0000000000000000 ffff882029b90a00 [83303.055299] ffff88201c80fc18 ffff882029b90a60 ffff882029b90a00 ffff8810175b8000 [83303.055299] Call Trace: [83303.055301] [] queued_spin_lock_slowpath+0xb/0xf [83303.055303] [] _raw_spin_lock+0x20/0x30 [83303.055332] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.055354] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.055374] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.055392] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.055409] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.055427] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.055433] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.055452] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.055470] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.055476] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.055496] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.055513] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.055534] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.055536] [] ? wake_up_state+0x20/0x20 [83303.055556] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.055557] [] kthread+0xcf/0xe0 [83303.055559] [] ? insert_kthread_work+0x40/0x40 [83303.055560] [] ret_from_fork+0x58/0x90 [83303.055561] [] ? insert_kthread_work+0x40/0x40 [83303.055575] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83303.058246] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ldlm_bl_06:16008] [83303.058264] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.058270] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.058271] CPU: 2 PID: 16008 Comm: ldlm_bl_06 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.058271] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.058272] task: ffff8810247eeeb0 ti: ffff88201c7e8000 task.ti: ffff88201c7e8000 [83303.058275] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83303.058275] RSP: 0018:ffff88201c7ebb70 EFLAGS: 00000246 [83303.058276] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000110000 [83303.058276] RDX: ffff88203c897880 RSI: 0000000000a90000 RDI: ffff88102137981c [83303.058277] RBP: ffff88201c7ebb70 R08: ffff88103ce57880 R09: 0000000000000000 [83303.058277] R10: ffff882029b99e00 R11: 0000000000000005 R12: ffff88201c7ebc58 [83303.058277] R13: 0000000000000001 R14: ffff88201c7ebb28 R15: ffffffff81322c35 [83303.058278] FS: 0000000000000000(0000) GS:ffff88103ce40000(0000) knlGS:0000000000000000 [83303.058278] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.058279] CR2: 0000000002577010 CR3: 00000000019f2000 CR4: 00000000001407e0 [83303.058279] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.058279] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.058280] Stack: [83303.058280] ffff88201c7ebb80 ffffffff8169e61f ffff88201c7ebb90 ffffffff816abb70 [83303.058281] ffff88201c7ebbd0 ffffffffc0b72198 0000000000000000 ffff882029b99e00 [83303.058282] ffff88201c7ebc18 ffff882029b99e60 ffff882029b99e00 ffff8810175b8000 [83303.058282] Call Trace: [83303.058284] [] queued_spin_lock_slowpath+0xb/0xf [83303.058285] [] _raw_spin_lock+0x20/0x30 [83303.058303] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.058318] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.058332] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.058345] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.058358] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.058371] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.058376] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.058389] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.058402] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.058406] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.058420] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.058433] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.058449] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.058450] [] ? wake_up_state+0x20/0x20 [83303.058465] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.058467] [] kthread+0xcf/0xe0 [83303.058468] [] ? insert_kthread_work+0x40/0x40 [83303.058469] [] ret_from_fork+0x58/0x90 [83303.058469] [] ? insert_kthread_work+0x40/0x40 [83303.058477] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83303.070246] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [ldlm_bl_14:16016] [83303.070270] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.070278] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.070280] CPU: 6 PID: 16016 Comm: ldlm_bl_14 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.070280] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.070281] task: ffff88203d338000 ti: ffff88201c810000 task.ti: ffff88201c810000 [83303.070284] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83303.070284] RSP: 0018:ffff88201c813b70 EFLAGS: 00000246 [83303.070285] RAX: 0000000000000000 RBX: 000000011c813be8 RCX: 0000000000310000 [83303.070285] RDX: ffff88103ce57880 RSI: 0000000000110000 RDI: ffff88102137981c [83303.070286] RBP: ffff88201c813b70 R08: ffff88103ced7880 R09: 0000000000000000 [83303.070287] R10: ffff882029b94600 R11: 0000000000000000 R12: ffff88201c813c58 [83303.070287] R13: 0000000000000001 R14: ffff88201c813b28 R15: ffffffff81322c35 [83303.070288] FS: 0000000000000000(0000) GS:ffff88103cec0000(0000) knlGS:0000000000000000 [83303.070288] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.070289] CR2: 0000000002079728 CR3: 00000000019f2000 CR4: 00000000001407e0 [83303.070289] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.070290] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.070290] Stack: [83303.070292] ffff88201c813b80 ffffffff8169e61f ffff88201c813b90 ffffffff816abb70 [83303.070293] ffff88201c813bd0 ffffffffc0b72198 0000000000000000 ffff882029b94600 [83303.070294] ffff88201c813c18 ffff882029b94660 ffff882029b94600 ffff8810175b8000 [83303.070294] Call Trace: [83303.070296] [] queued_spin_lock_slowpath+0xb/0xf [83303.070298] [] _raw_spin_lock+0x20/0x30 [83303.070323] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.070345] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.070366] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.070383] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.070402] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.070419] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.070425] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.070443] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.070461] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.070466] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.070484] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.070502] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.070524] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.070526] [] ? wake_up_state+0x20/0x20 [83303.070546] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.070547] [] kthread+0xcf/0xe0 [83303.070549] [] ? insert_kthread_work+0x40/0x40 [83303.070550] [] ret_from_fork+0x58/0x90 [83303.070551] [] ? insert_kthread_work+0x40/0x40 [83303.070565] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83303.076245] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [ldlm_bl_10:16012] [83303.076268] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.076276] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.076278] CPU: 8 PID: 16012 Comm: ldlm_bl_10 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.076278] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.076279] task: ffff88203ad89fa0 ti: ffff88201c800000 task.ti: ffff88201c800000 [83303.076282] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83303.076282] RSP: 0018:ffff88201c803b70 EFLAGS: 00000246 [83303.076283] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000410000 [83303.076283] RDX: ffff88203c617880 RSI: 0000000000090000 RDI: ffff88102137981c [83303.076284] RBP: ffff88201c803b70 R08: ffff88103cf17880 R09: 0000000000000000 [83303.076284] R10: ffff882029b9b600 R11: 0000000000000005 R12: ffff88201c803c58 [83303.076285] R13: 0000000000000001 R14: ffff88201c803b28 R15: ffffffff81322c35 [83303.076286] FS: 0000000000000000(0000) GS:ffff88103cf00000(0000) knlGS:0000000000000000 [83303.076286] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.076287] CR2: 00007fca10cf5000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83303.076287] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.076288] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.076288] Stack: [83303.076289] ffff88201c803b80 ffffffff8169e61f ffff88201c803b90 ffffffff816abb70 [83303.076290] ffff88201c803bd0 ffffffffc0b72198 0000000000000000 ffff882029b9b600 [83303.076292] ffff88201c803c18 ffff882029b9b660 ffff882029b9b600 ffff8810175b8000 [83303.076292] Call Trace: [83303.076294] [] queued_spin_lock_slowpath+0xb/0xf [83303.076296] [] _raw_spin_lock+0x20/0x30 [83303.076318] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.076337] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.076356] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.076374] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.076391] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.076409] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.076415] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.076432] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.076450] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.076456] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.076473] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.076491] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.076513] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.076515] [] ? wake_up_state+0x20/0x20 [83303.076535] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.076536] [] kthread+0xcf/0xe0 [83303.076538] [] ? insert_kthread_work+0x40/0x40 [83303.076539] [] ret_from_fork+0x58/0x90 [83303.076540] [] ? insert_kthread_work+0x40/0x40 [83303.076554] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83303.082245] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [ldlm_bl_05:16007] [83303.082267] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.082274] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.082276] CPU: 10 PID: 16007 Comm: ldlm_bl_05 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.082276] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.082277] task: ffff8810247ecf10 ti: ffff88201c64c000 task.ti: ffff88201c64c000 [83303.082280] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83303.082280] RSP: 0018:ffff88201c64fb70 EFLAGS: 00000246 [83303.082281] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000510000 [83303.082281] RDX: ffff88103ce17880 RSI: 0000000000010000 RDI: ffff88102137981c [83303.082282] RBP: ffff88201c64fb70 R08: ffff88103cf57880 R09: 0000000000000000 [83303.082282] R10: ffff882029b13800 R11: 0000000000000005 R12: ffff88201c64fc58 [83303.082283] R13: 0000000000000001 R14: ffff88201c64fb28 R15: ffffffff81322c35 [83303.082284] FS: 0000000000000000(0000) GS:ffff88103cf40000(0000) knlGS:0000000000000000 [83303.082284] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.082285] CR2: 0000000001dd94b8 CR3: 00000000019f2000 CR4: 00000000001407e0 [83303.082286] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.082286] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.082286] Stack: [83303.082288] ffff88201c64fb80 ffffffff8169e61f ffff88201c64fb90 ffffffff816abb70 [83303.082289] ffff88201c64fbd0 ffffffffc0b72198 0000000000000000 ffff882029b13800 [83303.082290] ffff88201c64fc18 ffff882029b13860 ffff882029b13800 ffff8810175b8000 [83303.082291] Call Trace: [83303.082292] [] queued_spin_lock_slowpath+0xb/0xf [83303.082294] [] _raw_spin_lock+0x20/0x30 [83303.082314] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.082336] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.082355] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.082373] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.082390] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.082408] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.082413] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.082431] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.082449] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.082454] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.082472] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.082489] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.082511] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.082513] [] ? wake_up_state+0x20/0x20 [83303.082534] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.082535] [] kthread+0xcf/0xe0 [83303.082536] [] ? insert_kthread_work+0x40/0x40 [83303.082538] [] ret_from_fork+0x58/0x90 [83303.082539] [] ? insert_kthread_work+0x40/0x40 [83303.082552] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83303.088245] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [ldlm_bl_15:16017] [83303.088266] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.088273] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.088275] CPU: 12 PID: 16017 Comm: ldlm_bl_15 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.088275] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.088276] task: ffff88203d338fd0 ti: ffff88201c814000 task.ti: ffff88201c814000 [83303.088278] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83303.088279] RSP: 0018:ffff88201c817b70 EFLAGS: 00000246 [83303.088279] RAX: 0000000000000000 RBX: 00000001f8c12595 RCX: 0000000000610000 [83303.088280] RDX: ffff88203c757880 RSI: 0000000000590000 RDI: ffff88102137981c [83303.088281] RBP: ffff88201c817b70 R08: ffff88103cf97880 R09: 0000000000000000 [83303.088282] R10: ffff882029b9b600 R11: 0000000000000005 R12: ffff88201c817c58 [83303.088282] R13: 0000000000000001 R14: ffff88201c817b28 R15: ffffffff81322c35 [83303.088283] FS: 0000000000000000(0000) GS:ffff88103cf80000(0000) knlGS:0000000000000000 [83303.088284] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.088284] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83303.088285] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.088285] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.088286] Stack: [83303.088287] ffff88201c817b80 ffffffff8169e61f ffff88201c817b90 ffffffff816abb70 [83303.088288] ffff88201c817bd0 ffffffffc0b72198 0000000000000000 ffff882029b9b600 [83303.088289] ffff88201c817c18 ffff882029b9b660 ffff882029b9b600 ffff8810175b8000 [83303.088289] Call Trace: [83303.088291] [] queued_spin_lock_slowpath+0xb/0xf [83303.088293] [] _raw_spin_lock+0x20/0x30 [83303.088311] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.088332] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.088351] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.088369] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.088386] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.088404] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.088409] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.088427] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.088444] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.088450] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.088470] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.088487] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.088509] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.088511] [] ? wake_up_state+0x20/0x20 [83303.088531] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.088532] [] kthread+0xcf/0xe0 [83303.088533] [] ? insert_kthread_work+0x40/0x40 [83303.088535] [] ret_from_fork+0x58/0x90 [83303.088536] [] ? insert_kthread_work+0x40/0x40 [83303.088550] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83303.106244] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 23s! [ldlm_bl_02:15061] [83303.106265] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.106272] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.106274] CPU: 18 PID: 15061 Comm: ldlm_bl_02 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.106274] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.106275] task: ffff882012edeeb0 ti: ffff88203d3bc000 task.ti: ffff88203d3bc000 [83303.106277] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83303.106278] RSP: 0018:ffff88203d3bfb70 EFLAGS: 00000246 [83303.106279] RAX: 0000000000000000 RBX: 000000013cf16cc0 RCX: 0000000000910000 [83303.106279] RDX: ffff88203c7d7880 RSI: 0000000000790000 RDI: ffff88102137981c [83303.106280] RBP: ffff88203d3bfb70 R08: ffff88103d057880 R09: 0000000000000000 [83303.106280] R10: ffff882029b9b800 R11: 0000000000000005 R12: ffff88203d3bfc58 [83303.106281] R13: 0000000000000001 R14: ffff88203d3bfb28 R15: ffffffff81322c35 [83303.106282] FS: 0000000000000000(0000) GS:ffff88103d040000(0000) knlGS:0000000000000000 [83303.106282] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.106283] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83303.106283] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.106284] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.106284] Stack: [83303.106285] ffff88203d3bfb80 ffffffff8169e61f ffff88203d3bfb90 ffffffff816abb70 [83303.106286] ffff88203d3bfbd0 ffffffffc0b72198 0000000000000000 ffff882029b9b800 [83303.106287] ffff88203d3bfc18 ffff882029b9b860 ffff882029b9b800 ffff8810175b8000 [83303.106288] Call Trace: [83303.106290] [] queued_spin_lock_slowpath+0xb/0xf [83303.106291] [] _raw_spin_lock+0x20/0x30 [83303.106312] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.106333] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.106352] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.106369] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.106387] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.106405] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.106410] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.106428] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.106445] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.106451] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.106468] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.106486] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.106506] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.106508] [] ? wake_up_state+0x20/0x20 [83303.106528] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.106530] [] kthread+0xcf/0xe0 [83303.106531] [] ? insert_kthread_work+0x40/0x40 [83303.106532] [] ret_from_fork+0x58/0x90 [83303.106534] [] ? insert_kthread_work+0x40/0x40 [83303.106547] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83303.112244] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ldlm_bl_01:15060] [83303.112265] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.112272] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.112273] CPU: 20 PID: 15060 Comm: ldlm_bl_01 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.112274] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.112275] task: ffff882012eddee0 ti: ffff88203d3b8000 task.ti: ffff88203d3b8000 [83303.112277] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83303.112277] RSP: 0018:ffff88203d3bbb70 EFLAGS: 00000246 [83303.112278] RAX: 0000000000000000 RBX: 0000000180597933 RCX: 0000000000a10000 [83303.112279] RDX: ffff88203c857880 RSI: 0000000000990000 RDI: ffff88102137981c [83303.112279] RBP: ffff88203d3bbb70 R08: ffff88103d097880 R09: 0000000000000000 [83303.112280] R10: ffff882029b72800 R11: 0000000000000005 R12: ffff88203d3bbc58 [83303.112280] R13: 0000000000000001 R14: ffff88203d3bbb28 R15: ffffffff81322c35 [83303.112281] FS: 0000000000000000(0000) GS:ffff88103d080000(0000) knlGS:0000000000000000 [83303.112282] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.112282] CR2: 00007f02c314e4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83303.112283] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.112283] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.112283] Stack: [83303.112285] ffff88203d3bbb80 ffffffff8169e61f ffff88203d3bbb90 ffffffff816abb70 [83303.112286] ffff88203d3bbbd0 ffffffffc0b72198 0000000000000000 ffff882029b72800 [83303.112287] ffff88203d3bbc18 ffff882029b72860 ffff882029b72800 ffff8810175b8000 [83303.112287] Call Trace: [83303.112289] [] queued_spin_lock_slowpath+0xb/0xf [83303.112290] [] _raw_spin_lock+0x20/0x30 [83303.112309] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.112330] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.112349] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.112367] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.112384] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.112402] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.112407] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.112425] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.112442] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.112448] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.112467] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.112485] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.112505] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.112507] [] ? wake_up_state+0x20/0x20 [83303.112527] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.112529] [] kthread+0xcf/0xe0 [83303.112530] [] ? insert_kthread_work+0x40/0x40 [83303.112531] [] ret_from_fork+0x58/0x90 [83303.112532] [] ? insert_kthread_work+0x40/0x40 [83303.112546] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83303.118244] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [ldlm_bl_07:16009] [83303.118265] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83303.118272] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83303.118273] CPU: 22 PID: 16009 Comm: ldlm_bl_07 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83303.118274] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83303.118274] task: ffff8810e9240fd0 ti: ffff88201c7ec000 task.ti: ffff88201c7ec000 [83303.118277] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83303.118277] RSP: 0018:ffff88201c7efb70 EFLAGS: 00000246 [83303.118278] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000b10000 [83303.118278] RDX: ffff88103ce97880 RSI: 0000000000210000 RDI: ffff88102137981c [83303.118279] RBP: ffff88201c7efb70 R08: ffff88103d0d7880 R09: 0000000000000000 [83303.118279] R10: ffff882029b9b800 R11: 0000000000000005 R12: ffff88201c7efc58 [83303.118280] R13: 0000000000000001 R14: ffff88201c7efb28 R15: ffffffff81322c35 [83303.118281] FS: 0000000000000000(0000) GS:ffff88103d0c0000(0000) knlGS:0000000000000000 [83303.118281] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83303.118282] CR2: 00007f262c4764f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83303.118282] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83303.118283] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83303.118283] Stack: [83303.118284] ffff88201c7efb80 ffffffff8169e61f ffff88201c7efb90 ffffffff816abb70 [83303.118286] ffff88201c7efbd0 ffffffffc0b72198 0000000000000000 ffff882029b9b800 [83303.118287] ffff88201c7efc18 ffff882029b9b860 ffff882029b9b800 ffff8810175b8000 [83303.118287] Call Trace: [83303.118289] [] queued_spin_lock_slowpath+0xb/0xf [83303.118290] [] _raw_spin_lock+0x20/0x30 [83303.118310] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83303.118331] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83303.118351] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83303.118368] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83303.118386] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83303.118403] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83303.118408] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83303.118426] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.118443] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83303.118448] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83303.118468] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83303.118486] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83303.118507] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83303.118509] [] ? wake_up_state+0x20/0x20 [83303.118529] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83303.118530] [] kthread+0xcf/0xe0 [83303.118532] [] ? insert_kthread_work+0x40/0x40 [83303.118533] [] ret_from_fork+0x58/0x90 [83303.118534] [] ? insert_kthread_work+0x40/0x40 [83303.118547] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83307.091126] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83307.170590] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83307.199458] CPU: 1 PID: 16010 Comm: ldlm_bl_08 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83307.212170] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83307.220518] task: ffff8810e9242f70 ti: ffff88201c7f8000 task.ti: ffff88201c7f8000 [83307.228866] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83307.239938] RSP: 0018:ffff88201c7fbb70 EFLAGS: 00000246 [83307.245862] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000090000 [83307.253823] RDX: ffff88103cf57880 RSI: 0000000000510000 RDI: ffff88102137981c [83307.261782] RBP: ffff88201c7fbb70 R08: ffff88203c617880 R09: 0000000000000000 [83307.269742] R10: ffff882029aca600 R11: 0000000000000005 R12: ffff88201c7fbc58 [83307.277702] R13: 0000000000000001 R14: ffff88201c7fbb28 R15: ffffffff81322c35 [83307.285663] FS: 0000000000000000(0000) GS:ffff88203c600000(0000) knlGS:0000000000000000 [83307.294689] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83307.301099] CR2: 00007f11b0c8c000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83307.309059] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83307.317019] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83307.324979] Stack: [83307.327219] ffff88201c7fbb80 ffffffff8169e61f ffff88201c7fbb90 ffffffff816abb70 [83307.335505] ffff88201c7fbbd0 ffffffffc0b72198 0000000000000000 ffff882029aca600 [83307.343792] ffff88201c7fbc18 ffff882029aca660 ffff882029aca600 ffff8810175b8000 [83307.352079] Call Trace: [83307.354807] [] queued_spin_lock_slowpath+0xb/0xf [83307.361798] [] _raw_spin_lock+0x20/0x30 [83307.367942] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83307.376121] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83307.384199] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83307.391594] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83307.399377] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83307.407160] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83307.414544] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83307.422621] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83307.430501] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83307.438370] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83307.446446] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83307.454911] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83307.463086] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83307.470758] [] ? wake_up_state+0x20/0x20 [83307.476993] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83307.485245] [] kthread+0xcf/0xe0 [83307.490685] [] ? insert_kthread_work+0x40/0x40 [83307.497483] [] ret_from_fork+0x58/0x90 [83307.503506] [] ? insert_kthread_work+0x40/0x40 [83307.510303] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83315.078987] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [ldlm_bl_03:15915] [83315.084987] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ldlm_bl_22:33195] [83315.085011] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83315.085020] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83315.085021] CPU: 11 PID: 33195 Comm: ldlm_bl_22 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83315.085022] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83315.085023] task: ffff88203d0b1fa0 ti: ffff88203d4b8000 task.ti: ffff88203d4b8000 [83315.085027] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83315.085027] RSP: 0018:ffff88203d4bbb70 EFLAGS: 00000246 [83315.085028] RAX: 0000000000000000 RBX: 5a5a5a5a5a5a5a5a RCX: 0000000000590000 [83315.085028] RDX: ffff88103ced7880 RSI: 0000000000310000 RDI: ffff88102137981c [83315.085029] RBP: ffff88203d4bbb70 R08: ffff88203c757880 R09: 0000000000000000 [83315.085030] R10: ffff8820257ef600 R11: 0000000000000001 R12: 5a5a5a5a5a5a5a5a [83315.085030] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83315.085031] FS: 0000000000000000(0000) GS:ffff88203c740000(0000) knlGS:0000000000000000 [83315.085032] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83315.085032] CR2: 00007fef29abd090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83315.085033] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83315.085033] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83315.085034] Stack: [83315.085035] ffff88203d4bbb80 ffffffff8169e61f ffff88203d4bbb90 ffffffff816abb70 [83315.085036] ffff88203d4bbbd0 ffffffffc0b72198 0000000000000000 ffff8820257ef600 [83315.085038] ffff88203d4bbc18 ffff8820257ef660 ffff8820257ef600 ffff8810175b8000 [83315.085038] Call Trace: [83315.085041] [] queued_spin_lock_slowpath+0xb/0xf [83315.085043] [] _raw_spin_lock+0x20/0x30 [83315.085080] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83315.085105] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83315.085125] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83315.085143] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83315.085160] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83315.085177] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83315.085185] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83315.085204] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.085221] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.085227] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83315.085244] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83315.085262] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83315.085284] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83315.085286] [] ? __schedule+0x39d/0x8b0 [83315.085306] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83315.085308] [] kthread+0xcf/0xe0 [83315.085309] [] ? insert_kthread_work+0x40/0x40 [83315.085311] [] ret_from_fork+0x58/0x90 [83315.085312] [] ? insert_kthread_work+0x40/0x40 [83315.085326] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83315.090986] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [ldlm_bl_19:33192] [83315.091011] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83315.091019] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83315.091021] CPU: 13 PID: 33192 Comm: ldlm_bl_19 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83315.091022] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83315.091022] task: ffff88203d0b2f70 ti: ffff881ff8e20000 task.ti: ffff881ff8e20000 [83315.091026] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83315.091027] RSP: 0018:ffff881ff8e23b70 EFLAGS: 00000246 [83315.091027] RAX: 0000000000000000 RBX: ffff881ff8e23be8 RCX: 0000000000690000 [83315.091028] RDX: ffff88203c697880 RSI: 0000000000290001 RDI: ffff88102137981c [83315.091028] RBP: ffff881ff8e23b70 R08: ffff88203c797880 R09: 0000000000000000 [83315.091029] R10: ffff882029a45c00 R11: 0000000000000000 R12: ffff88103ce96cc0 [83315.091030] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83315.091031] FS: 0000000000000000(0000) GS:ffff88203c780000(0000) knlGS:0000000000000000 [83315.091032] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83315.091032] CR2: 00007f352c5974f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83315.091033] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83315.091034] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83315.091034] Stack: [83315.091035] ffff881ff8e23b80 ffffffff8169e61f ffff881ff8e23b90 ffffffff816abb70 [83315.091036] ffff881ff8e23bd0 ffffffffc0b72198 0000000000000000 ffff882029a45c00 [83315.091037] ffff881ff8e23c18 ffff882029a45c60 ffff882029a45c00 ffff8810175b8000 [83315.091038] Call Trace: [83315.091040] [] queued_spin_lock_slowpath+0xb/0xf [83315.091042] [] _raw_spin_lock+0x20/0x30 [83315.091072] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83315.091095] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83315.091116] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83315.091134] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83315.091152] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83315.091169] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83315.091176] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83315.091193] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.091212] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.091217] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83315.091236] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83315.091253] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83315.091274] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83315.091276] [] ? __schedule+0x39d/0x8b0 [83315.091296] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83315.091298] [] kthread+0xcf/0xe0 [83315.091299] [] ? insert_kthread_work+0x40/0x40 [83315.091300] [] ret_from_fork+0x58/0x90 [83315.091302] [] ? insert_kthread_work+0x40/0x40 [83315.091316] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83315.096986] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ldlm_bl_23:33196] [83315.097011] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83315.097019] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83315.097020] CPU: 15 PID: 33196 Comm: ldlm_bl_23 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83315.097021] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83315.097022] task: ffff88203d0b0000 ti: ffff882015ae0000 task.ti: ffff882015ae0000 [83315.097025] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83315.097026] RSP: 0018:ffff882015ae3b70 EFLAGS: 00000246 [83315.097026] RAX: 0000000000000000 RBX: ffff88203c756cc0 RCX: 0000000000790000 [83315.097027] RDX: ffff88203c6d7880 RSI: 0000000000390000 RDI: ffff88102137981c [83315.097028] RBP: ffff882015ae3b70 R08: ffff88203c7d7880 R09: 0000000000000000 [83315.097028] R10: ffff8820256b0800 R11: 0000000000000001 R12: ffff88203d0b0068 [83315.097029] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83315.097030] FS: 0000000000000000(0000) GS:ffff88203c7c0000(0000) knlGS:0000000000000000 [83315.097030] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83315.097031] CR2: 00007faebc9e03cc CR3: 00000000019f2000 CR4: 00000000001407e0 [83315.097031] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83315.097032] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83315.097032] Stack: [83315.097034] ffff882015ae3b80 ffffffff8169e61f ffff882015ae3b90 ffffffff816abb70 [83315.097035] ffff882015ae3bd0 ffffffffc0b72198 0000000000000000 ffff8820256b0800 [83315.097036] ffff882015ae3c18 ffff8820256b0860 ffff8820256b0800 ffff8810175b8000 [83315.097036] Call Trace: [83315.097038] [] queued_spin_lock_slowpath+0xb/0xf [83315.097040] [] _raw_spin_lock+0x20/0x30 [83315.097069] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83315.097091] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83315.097111] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83315.097129] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83315.097146] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83315.097163] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83315.097170] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83315.097187] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.097205] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.097210] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83315.097227] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83315.097245] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83315.097266] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83315.097267] [] ? __schedule+0x39d/0x8b0 [83315.097287] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83315.097289] [] kthread+0xcf/0xe0 [83315.097290] [] ? insert_kthread_work+0x40/0x40 [83315.097292] [] ret_from_fork+0x58/0x90 [83315.097293] [] ? insert_kthread_work+0x40/0x40 [83315.097307] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83315.102987] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [ldlm_bl_21:33194] [83315.103009] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83315.103016] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83315.103017] CPU: 17 PID: 33194 Comm: ldlm_bl_21 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83315.103018] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83315.103019] task: ffff88203d0b6eb0 ti: ffff882012eac000 task.ti: ffff882012eac000 [83315.103022] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83315.103022] RSP: 0018:ffff882012eafb70 EFLAGS: 00000246 [83315.103023] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000890000 [83315.103023] RDX: ffff88203c797880 RSI: 0000000000690001 RDI: ffff88102137981c [83315.103024] RBP: ffff882012eafb70 R08: ffff88203c817880 R09: 0000000000000000 [83315.103024] R10: ffff882029a44600 R11: 0000000000000001 R12: 0000000000000000 [83315.103025] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c816cc0 [83315.103026] FS: 0000000000000000(0000) GS:ffff88203c800000(0000) knlGS:0000000000000000 [83315.103026] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83315.103027] CR2: 00007f7ce52f4550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83315.103027] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83315.103028] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83315.103028] Stack: [83315.103029] ffff882012eafb80 ffffffff8169e61f ffff882012eafb90 ffffffff816abb70 [83315.103030] ffff882012eafbd0 ffffffffc0b72198 0000000000000000 ffff882029a44600 [83315.103032] ffff882012eafc18 ffff882029a44660 ffff882029a44600 ffff8810175b8000 [83315.103032] Call Trace: [83315.103034] [] queued_spin_lock_slowpath+0xb/0xf [83315.103035] [] _raw_spin_lock+0x20/0x30 [83315.103054] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83315.103075] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83315.103094] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83315.103112] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83315.103130] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83315.103147] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83315.103153] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83315.103171] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.103188] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.103194] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83315.103213] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83315.103231] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83315.103253] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83315.103255] [] ? __schedule+0x39d/0x8b0 [83315.103275] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83315.103277] [] kthread+0xcf/0xe0 [83315.103278] [] ? insert_kthread_work+0x40/0x40 [83315.103279] [] ret_from_fork+0x58/0x90 [83315.103281] [] ? insert_kthread_work+0x40/0x40 [83315.103294] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83315.108986] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 23s! [ldlm_bl_18:33191] [83315.109008] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83315.109015] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83315.109017] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83315.109017] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83315.109018] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83315.109020] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83315.109021] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83315.109022] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83315.109022] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83315.109023] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83315.109023] R10: ffff882029a33400 R11: 0000000000000001 R12: ffff88203d0b5f48 [83315.109024] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83315.109025] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83315.109025] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83315.109026] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83315.109026] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83315.109027] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83315.109027] Stack: [83315.109028] ffff881ffba83b80 ffffffff8169e61f ffff881ffba83b90 ffffffff816abb70 [83315.109029] ffff881ffba83bd0 ffffffffc0b72198 0000000000000000 ffff882029a33400 [83315.109030] ffff881ffba83c18 ffff882029a33460 ffff882029a33400 ffff8810175b8000 [83315.109031] Call Trace: [83315.109033] [] queued_spin_lock_slowpath+0xb/0xf [83315.109034] [] _raw_spin_lock+0x20/0x30 [83315.109054] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83315.109075] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83315.109095] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83315.109112] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83315.109130] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83315.109147] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83315.109153] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83315.109171] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.109188] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.109193] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83315.109211] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83315.109229] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83315.109251] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83315.109252] [] ? __schedule+0x39d/0x8b0 [83315.109273] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83315.109274] [] kthread+0xcf/0xe0 [83315.109275] [] ? insert_kthread_work+0x40/0x40 [83315.109277] [] ret_from_fork+0x58/0x90 [83315.109278] [] ? insert_kthread_work+0x40/0x40 [83315.109291] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83315.114986] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [ldlm_bl_20:33193] [83315.115005] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83315.115011] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83315.115012] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83315.115013] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83315.115014] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83315.115016] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83315.115017] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83315.115018] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83315.115018] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83315.115019] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83315.115019] R10: ffff882029a45800 R11: 0000000000000000 R12: ffff88103ce96cc0 [83315.115020] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83315.115020] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83315.115021] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83315.115021] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83315.115022] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83315.115022] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83315.115022] Stack: [83315.115023] ffff881ff8e2fb80 ffffffff8169e61f ffff881ff8e2fb90 ffffffff816abb70 [83315.115024] ffff881ff8e2fbd0 ffffffffc0b72198 0000000000000000 ffff882029a45800 [83315.115025] ffff881ff8e2fc18 ffff882029a45860 ffff882029a45800 ffff8810175b8000 [83315.115026] Call Trace: [83315.115028] [] queued_spin_lock_slowpath+0xb/0xf [83315.115029] [] _raw_spin_lock+0x20/0x30 [83315.115046] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83315.115065] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83315.115083] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83315.115099] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83315.115115] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83315.115132] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83315.115137] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83315.115153] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.115170] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.115175] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83315.115191] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83315.115208] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83315.115227] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83315.115229] [] ? __schedule+0x39d/0x8b0 [83315.115247] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83315.115249] [] kthread+0xcf/0xe0 [83315.115250] [] ? insert_kthread_work+0x40/0x40 [83315.115251] [] ret_from_fork+0x58/0x90 [83315.115252] [] ? insert_kthread_work+0x40/0x40 [83315.115263] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83315.120986] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ldlm_bl_04:16006] [83315.121007] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83315.121014] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83315.121015] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83315.121016] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83315.121017] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83315.121019] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83315.121020] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83315.121020] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83315.121021] RDX: ffff88103d057880 RSI: 0000000000910000 RDI: ffff88102137981c [83315.121021] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83315.121022] R10: ffff882029b47a00 R11: 0000000000000000 R12: ffff88201c627c58 [83315.121022] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83315.121023] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83315.121024] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83315.121024] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83315.121025] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83315.121025] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83315.121026] Stack: [83315.121027] ffff88201c627b80 ffffffff8169e61f ffff88201c627b90 ffffffff816abb70 [83315.121028] ffff88201c627bd0 ffffffffc0b72198 0000000000000000 ffff882029b47a00 [83315.121029] ffff88201c627c18 ffff882029b47a60 ffff882029b47a00 ffff8810175b8000 [83315.121029] Call Trace: [83315.121032] [] queued_spin_lock_slowpath+0xb/0xf [83315.121033] [] _raw_spin_lock+0x20/0x30 [83315.121052] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83315.121073] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83315.121092] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83315.121109] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83315.121127] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83315.121144] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83315.121150] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83315.121167] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.121185] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83315.121190] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83315.121208] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83315.121226] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83315.121248] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83315.121250] [] ? wake_up_state+0x20/0x20 [83315.121270] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83315.121271] [] kthread+0xcf/0xe0 [83315.121272] [] ? insert_kthread_work+0x40/0x40 [83315.121274] [] ret_from_fork+0x58/0x90 [83315.121275] [] ? insert_kthread_work+0x40/0x40 [83315.121288] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83317.322940] INFO: rcu_sched self-detected stall on CPU [83317.322941] INFO: rcu_sched self-detected stall on CPU [83317.322942] INFO: rcu_sched self-detected stall on CPU [83317.322943] INFO: rcu_sched self-detected stall on CPU [83317.322943] INFO: rcu_sched self-detected stall on CPU [83317.322944] INFO: rcu_sched self-detected stall on CPU [83317.322945] { [83317.322946] { [83317.322947] { [83317.322947] { [83317.322947] { [83317.322948] { [83317.322950] 18 [83317.322951] 1 [83317.322951] 0 [83317.322952] 12 [83317.322952] 16 [83317.322953] 2 [83317.322954] } [83317.322954] } [83317.322955] } [83317.322955] } [83317.322955] } [83317.322965] } (t=420009 jiffies g=368094 c=368093 q=43231) [83317.322966] (t=420009 jiffies g=368094 c=368093 q=43231) [83317.322967] (t=420009 jiffies g=368094 c=368093 q=43231) [83317.322968] (t=420009 jiffies g=368094 c=368093 q=43231) [83317.322969] (t=420009 jiffies g=368094 c=368093 q=43231) [83317.322969] Task dump for CPU 0: [83317.322970] (t=420009 jiffies g=368094 c=368093 q=43231) [83317.322972] ldlm_bl_13 R running task 0 16015 2 0x00000088 [83317.322974] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.322974] ffff88203d33bf40 ffff88201c80fde8 ffffffffc0b72b3c ffff88200be50c00 [83317.322975] ffff88201c80fe18 ffffffffc0b737cc ffff88200be50c00 ffff882000000019 [83317.322975] Call Trace: [83317.322985] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323010] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323024] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323040] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323042] [] ? wake_up_state+0x20/0x20 [83317.323058] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323059] [] ? kthread+0xcf/0xe0 [83317.323060] [] ? insert_kthread_work+0x40/0x40 [83317.323061] [] ? ret_from_fork+0x58/0x90 [83317.323062] [] ? insert_kthread_work+0x40/0x40 [83317.323063] Task dump for CPU 1: [83317.323064] ldlm_bl_08 R running task 0 16010 2 0x00000088 [83317.323065] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.323066] ffff8810e9242f70 ffff88201c7fbde8 ffffffffc0b72b3c ffff881017a74c00 [83317.323066] ffff88201c7fbe18 ffffffffc0b737cc ffff881017a74c00 ffff882000000111 [83317.323067] Call Trace: [83317.323072] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323085] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323099] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323114] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323115] [] ? wake_up_state+0x20/0x20 [83317.323129] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323130] [] ? kthread+0xcf/0xe0 [83317.323131] [] ? insert_kthread_work+0x40/0x40 [83317.323132] [] ? ret_from_fork+0x58/0x90 [83317.323133] [] ? insert_kthread_work+0x40/0x40 [83317.323133] Task dump for CPU 2: [83317.323134] ldlm_bl_06 R running task 0 16008 2 0x00000088 [83317.323135] ffff8810247eeeb0 00000000bdec741d ffff88103ce43db8 ffffffff810c3a98 [83317.323136] 0000000000000002 ffffffff81a551c0 ffff88103ce43dd0 ffffffff810c7429 [83317.323136] 0000000000000003 ffff88103ce43e00 ffffffff81137b10 ffff88103ce50260 [83317.323136] Call Trace: [83317.323138] [] sched_show_task+0xa8/0x110 [83317.323139] [] dump_cpu_task+0x39/0x70 [83317.323141] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.323142] [] rcu_check_callbacks+0x442/0x730 [83317.323143] [] ? tick_sched_do_timer+0x50/0x50 [83317.323145] [] update_process_times+0x46/0x80 [83317.323146] [] tick_sched_handle+0x30/0x70 [83317.323147] [] tick_sched_timer+0x39/0x80 [83317.323148] [] __hrtimer_run_queues+0xd4/0x260 [83317.323150] [] hrtimer_interrupt+0xaf/0x1d0 [83317.323152] [] local_apic_timer_interrupt+0x35/0x60 [83317.323154] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.323155] [] apic_timer_interrupt+0x6d/0x80 [83317.323157] [] ? native_queued_spin_lock_slowpath+0x116/0x1e0 [83317.323158] [] queued_spin_lock_slowpath+0xb/0xf [83317.323160] [] _raw_spin_lock+0x20/0x30 [83317.323173] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.323188] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.323202] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.323215] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.323228] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.323241] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.323246] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.323259] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.323272] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.323276] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323289] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323302] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323317] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323318] [] ? wake_up_state+0x20/0x20 [83317.323333] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323334] [] kthread+0xcf/0xe0 [83317.323335] [] ? insert_kthread_work+0x40/0x40 [83317.323335] [] ret_from_fork+0x58/0x90 [83317.323336] [] ? insert_kthread_work+0x40/0x40 [83317.323337] Task dump for CPU 8: [83317.323337] ldlm_bl_10 R running task 0 16012 2 0x00000088 [83317.323338] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.323339] ffff88203ad89fa0 ffff88201c803de8 ffffffffc0b72b3c ffff882012e84400 [83317.323340] ffff88201c803e18 ffffffffc0b737cc ffff882012e84400 ffff88200000000d [83317.323340] Call Trace: [83317.323344] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323357] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323370] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323385] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323386] [] ? wake_up_state+0x20/0x20 [83317.323401] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323402] [] ? kthread+0xcf/0xe0 [83317.323403] [] ? insert_kthread_work+0x40/0x40 [83317.323404] [] ? ret_from_fork+0x58/0x90 [83317.323405] [] ? insert_kthread_work+0x40/0x40 [83317.323405] Task dump for CPU 10: [83317.323406] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.323406] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.323407] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83317.323408] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.323408] Call Trace: [83317.323412] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323425] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323439] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323453] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323454] [] ? wake_up_state+0x20/0x20 [83317.323469] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323470] [] ? kthread+0xcf/0xe0 [83317.323471] [] ? insert_kthread_work+0x40/0x40 [83317.323472] [] ? ret_from_fork+0x58/0x90 [83317.323473] [] ? insert_kthread_work+0x40/0x40 [83317.323473] Task dump for CPU 12: [83317.323474] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.323474] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.323475] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.323476] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.323476] Call Trace: [83317.323480] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323493] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323506] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323521] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323522] [] ? wake_up_state+0x20/0x20 [83317.323536] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323537] [] ? kthread+0xcf/0xe0 [83317.323538] [] ? insert_kthread_work+0x40/0x40 [83317.323539] [] ? ret_from_fork+0x58/0x90 [83317.323539] [] ? insert_kthread_work+0x40/0x40 [83317.323540] Task dump for CPU 16: [83317.323540] Task dump for CPU 0: [83317.323541] ldlm_bl_12 R [83317.323542] ldlm_bl_13 R [83317.323542] running task [83317.323543] 0 16014 2 0x00000088 [83317.323543] running task 0 16015 2 0x00000088 [83317.323544] ffffffffc084ad95 [83317.323545] ffffffffc084ad95 [83317.323545] 00000000c084afeb [83317.323546] 00000000c084afeb [83317.323546] 0000000000000000 [83317.323546] 0000000000000000 [83317.323547] ffff882011eb8c00 [83317.323547] [83317.323548] ffff882011eb8c00 [83317.323548] ffff88203d33af70 [83317.323549] ffff88203d33bf40 [83317.323549] ffff88201c80bde8 [83317.323550] ffff88201c80fde8 [83317.323550] ffffffffc0b72b3c [83317.323551] ffffffffc0b72b3c [83317.323551] ffff881017a75800 [83317.323551] [83317.323552] ffff88200be50c00 [83317.323553] ffff88201c80be18 [83317.323553] ffff88201c80fe18 [83317.323553] ffffffffc0b737cc [83317.323554] ffffffffc0b737cc [83317.323554] ffff881017a75800 [83317.323555] ffff88200be50c00 [83317.323555] ffff88200000000c [83317.323556] [83317.323556] ffff882000000019 [83317.323556] Call Trace: [83317.323557] Call Trace: [83317.323560] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323566] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323579] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323599] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323612] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323630] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323644] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323664] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323665] [] ? wake_up_state+0x20/0x20 [83317.323667] [] ? wake_up_state+0x20/0x20 [83317.323681] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323702] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323703] [] ? kthread+0xcf/0xe0 [83317.323705] [] ? kthread+0xcf/0xe0 [83317.323705] [] ? insert_kthread_work+0x40/0x40 [83317.323707] [] ? insert_kthread_work+0x40/0x40 [83317.323708] [] ? ret_from_fork+0x58/0x90 [83317.323709] [] ? ret_from_fork+0x58/0x90 [83317.323710] [] ? insert_kthread_work+0x40/0x40 [83317.323711] [] ? insert_kthread_work+0x40/0x40 [83317.323712] Task dump for CPU 18: [83317.323712] Task dump for CPU 1: [83317.323713] ldlm_bl_02 R running task 0 15061 2 0x00000088 [83317.323714] ldlm_bl_08 R [83317.323714] ffffffffc084ad95 [83317.323715] running task [83317.323716] 0 16010 2 0x00000088 [83317.323716] 00000000c084afeb 0000000000000000 [83317.323717] ffffffffc084ad95 [83317.323717] ffff882011eb8c00 [83317.323717] [83317.323718] 00000000c084afeb 0000000000000000 [83317.323719] ffff882012edeeb0 [83317.323719] ffff882011eb8c00 [83317.323719] [83317.323720] ffff88203d3bfde8 ffffffffc0b72b3c [83317.323720] ffff8810e9242f70 [83317.323721] ffff88200f638000 [83317.323721] [83317.323722] ffff88201c7fbde8 ffffffffc0b72b3c [83317.323722] ffff88203d3bfe18 [83317.323723] ffff881017a74c00 [83317.323723] [83317.323724] ffffffffc0b737cc ffff88200f638000 [83317.323724] ffff88201c7fbe18 [83317.323725] ffff88200000000d [83317.323725] [83317.323726] Call Trace: [83317.323726] ffffffffc0b737cc ffff881017a74c00 ffff882000000111 [83317.323726] [83317.323730] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323730] Call Trace: [83317.323744] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323749] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323762] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323782] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323796] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323814] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323815] [] ? wake_up_state+0x20/0x20 [83317.323837] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323851] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323852] [] ? wake_up_state+0x20/0x20 [83317.323853] [] ? kthread+0xcf/0xe0 [83317.323874] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.323875] [] ? insert_kthread_work+0x40/0x40 [83317.323876] [] ? kthread+0xcf/0xe0 [83317.323877] [] ? ret_from_fork+0x58/0x90 [83317.323879] [] ? insert_kthread_work+0x40/0x40 [83317.323880] [] ? insert_kthread_work+0x40/0x40 [83317.323881] [] ? ret_from_fork+0x58/0x90 [83317.323881] Task dump for CPU 20: [83317.323883] [] ? insert_kthread_work+0x40/0x40 [83317.323884] ldlm_bl_01 R running task 0 15060 2 0x00000088 [83317.323884] Task dump for CPU 2: [83317.323885] ffffffffc084ad95 [83317.323885] ldlm_bl_06 R [83317.323886] 00000000c084afeb [83317.323886] running task [83317.323887] 0 16008 2 0x00000088 [83317.323888] 0000000000000000 ffff882011eb8c00 [83317.323888] [83317.323889] ffffffffc084ad95 00000000c084afeb [83317.323889] ffff882012eddee0 [83317.323890] 0000000000000000 [83317.323890] ffff88203d3bbde8 [83317.323890] ffff882011eb8c00 [83317.323891] [83317.323891] ffffffffc0b72b3c ffff881039384c00 [83317.323892] [83317.323892] ffff8810247eeeb0 ffff88201c7ebde8 [83317.323893] ffff88203d3bbe18 [83317.323893] ffffffffc0b72b3c [83317.323894] ffffffffc0b737cc [83317.323894] ffff88102af4e400 [83317.323895] [83317.323895] ffff881039384c00 ffff88200000003d [83317.323896] [83317.323896] Call Trace: [83317.323897] ffff88201c7ebe18 ffffffffc0b737cc ffff88102af4e400 [83317.323901] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323901] ffff88200000000d [83317.323901] Call Trace: [83317.323915] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323920] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.323933] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323952] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.323953] INFO: rcu_sched self-detected stall on CPU [83317.323954] INFO: rcu_sched self-detected stall on CPU [83317.323954] INFO: rcu_sched self-detected stall on CPU [83317.323969] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.323969] INFO: rcu_sched self-detected stall on CPU [83317.323970] { [83317.323970] { [83317.323971] { [83317.323971] { [83317.323989] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.323990] 22 [83317.323990] 8 [83317.323991] 20 [83317.323992] [] ? wake_up_state+0x20/0x20 [83317.323992] 10 [83317.323993] } [83317.323993] } [83317.323994] } [83317.323994] } [83317.324016] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.324030] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.324031] (t=420010 jiffies g=368094 c=368093 q=43231) [83317.324032] (t=420010 jiffies g=368094 c=368093 q=43231) [83317.324033] (t=420010 jiffies g=368094 c=368093 q=43231) [83317.324033] (t=420010 jiffies g=368094 c=368093 q=43231) [83317.324035] [] ? wake_up_state+0x20/0x20 [83317.324036] [] ? kthread+0xcf/0xe0 [83317.324057] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.324057] [] ? insert_kthread_work+0x40/0x40 [83317.324059] [] ? kthread+0xcf/0xe0 [83317.324060] [] ? ret_from_fork+0x58/0x90 [83317.324061] [] ? insert_kthread_work+0x40/0x40 [83317.324062] [] ? insert_kthread_work+0x40/0x40 [83317.324063] [] ? ret_from_fork+0x58/0x90 [83317.324064] Task dump for CPU 22: [83317.324065] [] ? insert_kthread_work+0x40/0x40 [83317.324066] ldlm_bl_07 R running task 0 16009 2 0x00000088 [83317.324066] Task dump for CPU 8: [83317.324067] ffffffffc084ad95 [83317.324068] ldlm_bl_10 R [83317.324068] 00000000c084afeb [83317.324068] running task [83317.324069] 0 16012 2 0x00000088 [83317.324070] 0000000000000000 ffff882011eb8c00 [83317.324070] [83317.324071] ffffffffc084ad95 00000000c084afeb [83317.324071] ffff8810e9240fd0 [83317.324072] 0000000000000000 [83317.324072] ffff88201c7efde8 [83317.324073] ffff882011eb8c00 [83317.324073] [83317.324074] ffffffffc0b72b3c ffff88200f63f400 [83317.324074] [83317.324075] ffff88203ad89fa0 ffff88201c803de8 [83317.324075] ffff88201c7efe18 [83317.324076] ffffffffc0b72b3c [83317.324076] ffffffffc0b737cc [83317.324076] ffff882012e84400 [83317.324077] [83317.324077] ffff88200f63f400 ffff88200000000d [83317.324078] [83317.324078] Call Trace: [83317.324079] ffff88201c803e18 ffffffffc0b737cc ffff882012e84400 [83317.324083] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.324083] ffff88200000000d [83317.324084] Call Trace: [83317.324097] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.324102] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.324115] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.324132] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.324147] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.324165] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.324166] [] ? wake_up_state+0x20/0x20 [83317.324187] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.324201] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.324203] [] ? wake_up_state+0x20/0x20 [83317.324204] [] ? kthread+0xcf/0xe0 [83317.324224] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.324225] [] ? insert_kthread_work+0x40/0x40 [83317.324227] [] ? kthread+0xcf/0xe0 [83317.324228] [] ? ret_from_fork+0x58/0x90 [83317.324229] [] ? insert_kthread_work+0x40/0x40 [83317.324230] [] ? insert_kthread_work+0x40/0x40 [83317.324231] [] ? ret_from_fork+0x58/0x90 [83317.324232] [] ? insert_kthread_work+0x40/0x40 [83317.324233] Task dump for CPU 10: [83317.324234] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.324235] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.324237] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83317.324238] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.324238] Call Trace: [83317.324243] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.324261] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.324279] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.324300] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.324302] [] ? wake_up_state+0x20/0x20 [83317.324321] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.324323] [] ? kthread+0xcf/0xe0 [83317.324324] [] ? insert_kthread_work+0x40/0x40 [83317.324325] [] ? ret_from_fork+0x58/0x90 [83317.324326] [] ? insert_kthread_work+0x40/0x40 [83317.324327] Task dump for CPU 12: [83317.324328] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.324329] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.324330] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.324331] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.324332] Call Trace: [83317.324337] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.324357] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.324374] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.324396] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.324397] [] ? wake_up_state+0x20/0x20 [83317.324417] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.324418] [] ? kthread+0xcf/0xe0 [83317.324419] [] ? insert_kthread_work+0x40/0x40 [83317.324421] [] ? ret_from_fork+0x58/0x90 [83317.324422] [] ? insert_kthread_work+0x40/0x40 [83317.324422] Task dump for CPU 16: [83317.324423] Task dump for CPU 0: [83317.324424] ldlm_bl_12 R [83317.324424] ldlm_bl_13 R [83317.324425] running task [83317.324425] 0 16014 2 0x00000088 [83317.324426] running task 0 16015 2 0x00000088 [83317.324427] ffff88203d33af70 [83317.324428] ffff88203d33bf40 [83317.324428] 000000004f7e2e39 [83317.324429] 00000000dc8e5453 [83317.324429] ffff88103d003db8 [83317.324430] ffff88103ce03db8 [83317.324430] ffffffff810c3a98 [83317.324431] [83317.324431] ffffffff810c3a98 [83317.324432] 0000000000000010 [83317.324432] 0000000000000000 [83317.324433] ffffffff81a552c0 [83317.324433] ffffffff81a551c0 [83317.324434] ffff88103d003dd0 [83317.324434] ffff88103ce03dd0 [83317.324435] ffffffff810c7429 [83317.324435] [83317.324436] ffffffff810c7429 [83317.324436] 0000000000000001 [83317.324437] 0000000000000001 [83317.324437] ffff88103d003e00 [83317.324438] ffff88103ce03e00 [83317.324438] ffffffff81137b10 [83317.324439] ffffffff81137b10 [83317.324439] ffff88103d010260 [83317.324440] [83317.324440] ffff88103ce10260 [83317.324440] Call Trace: [83317.324441] Call Trace: [83317.324442] [83317.324442] [83317.324444] [] sched_show_task+0xa8/0x110 [83317.324446] [] sched_show_task+0xa8/0x110 [83317.324447] [] dump_cpu_task+0x39/0x70 [83317.324449] [] dump_cpu_task+0x39/0x70 [83317.324451] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.324453] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.324454] [] rcu_check_callbacks+0x442/0x730 [83317.324455] [] rcu_check_callbacks+0x442/0x730 [83317.324457] [] ? tick_sched_do_timer+0x50/0x50 [83317.324458] [] ? tick_sched_do_timer+0x50/0x50 [83317.324460] [] update_process_times+0x46/0x80 [83317.324461] [] update_process_times+0x46/0x80 [83317.324463] [] tick_sched_handle+0x30/0x70 [83317.324464] [] tick_sched_handle+0x30/0x70 [83317.324466] [] tick_sched_timer+0x39/0x80 [83317.324467] [] tick_sched_timer+0x39/0x80 [83317.324469] [] __hrtimer_run_queues+0xd4/0x260 [83317.324470] [] __hrtimer_run_queues+0xd4/0x260 [83317.324472] [] hrtimer_interrupt+0xaf/0x1d0 [83317.324474] [] hrtimer_interrupt+0xaf/0x1d0 [83317.324475] [] local_apic_timer_interrupt+0x35/0x60 [83317.324477] [] local_apic_timer_interrupt+0x35/0x60 [83317.324479] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.324481] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.324482] [] apic_timer_interrupt+0x6d/0x80 [83317.324484] [] apic_timer_interrupt+0x6d/0x80 [83317.324484] [83317.324485] [83317.324486] [] ? native_queued_spin_lock_slowpath+0x112/0x1e0 [83317.324488] [] ? native_queued_spin_lock_slowpath+0x112/0x1e0 [83317.324490] [] queued_spin_lock_slowpath+0xb/0xf [83317.324492] [] queued_spin_lock_slowpath+0xb/0xf [83317.324494] [] _raw_spin_lock+0x20/0x30 [83317.324495] [] _raw_spin_lock+0x20/0x30 [83317.324513] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.324532] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.324551] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.324571] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.324590] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.324610] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.324627] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.324645] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.324662] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.324680] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.324697] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.324715] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.324720] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.324726] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.324743] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.324762] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.324779] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.324797] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.324802] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.324808] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.324827] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.324846] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.324863] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.324882] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.324902] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.324922] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.324924] [] ? wake_up_state+0x20/0x20 [83317.324925] [] ? wake_up_state+0x20/0x20 [83317.324946] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.324966] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.324967] [] kthread+0xcf/0xe0 [83317.324969] [] kthread+0xcf/0xe0 [83317.324970] [] ? insert_kthread_work+0x40/0x40 [83317.324971] [] ? insert_kthread_work+0x40/0x40 [83317.324973] [] ret_from_fork+0x58/0x90 [83317.324974] [] ret_from_fork+0x58/0x90 [83317.324976] [] ? insert_kthread_work+0x40/0x40 [83317.324977] [] ? insert_kthread_work+0x40/0x40 [83317.324977] Task dump for CPU 18: [83317.324978] Task dump for CPU 1: [83317.324979] ldlm_bl_02 R [83317.324979] ldlm_bl_08 R [83317.324980] running task [83317.324980] 0 15061 2 0x00000088 [83317.324981] running task 0 16010 2 0x00000088 [83317.324982] ffffffffc084ad95 [83317.324982] ffffffffc084ad95 [83317.324983] 00000000c084afeb [83317.324983] 00000000c084afeb [83317.324984] 0000000000000000 [83317.324984] 0000000000000000 [83317.324985] ffff882011eb8c00 [83317.324985] [83317.324986] ffff882011eb8c00 [83317.324986] ffff882012edeeb0 [83317.324987] ffff8810e9242f70 [83317.324987] ffff88203d3bfde8 [83317.324988] ffff88201c7fbde8 [83317.324988] ffffffffc0b72b3c [83317.324989] ffffffffc0b72b3c [83317.324989] ffff88200f638000 [83317.324990] [83317.324990] ffff881017a74c00 [83317.324991] ffff88203d3bfe18 [83317.324991] ffff88201c7fbe18 [83317.324992] ffffffffc0b737cc [83317.324992] ffffffffc0b737cc [83317.324993] ffff88200f638000 [83317.324993] ffff881017a74c00 [83317.324994] ffff88200000000d [83317.324994] [83317.324995] ffff882000000111 [83317.324995] Call Trace: [83317.324995] Call Trace: [83317.325000] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325005] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325025] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325044] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325063] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325082] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325103] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325124] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325126] [] ? wake_up_state+0x20/0x20 [83317.325127] [] ? wake_up_state+0x20/0x20 [83317.325147] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325168] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325169] [] ? kthread+0xcf/0xe0 [83317.325171] [] ? kthread+0xcf/0xe0 [83317.325172] [] ? insert_kthread_work+0x40/0x40 [83317.325173] [] ? insert_kthread_work+0x40/0x40 [83317.325174] [] ? ret_from_fork+0x58/0x90 [83317.325176] [] ? ret_from_fork+0x58/0x90 [83317.325177] [] ? insert_kthread_work+0x40/0x40 [83317.325178] [] ? insert_kthread_work+0x40/0x40 [83317.325179] Task dump for CPU 20: [83317.325179] Task dump for CPU 2: [83317.325180] ldlm_bl_01 R [83317.325180] ldlm_bl_06 R [83317.325181] running task [83317.325181] 0 15060 2 0x00000088 [83317.325182] running task 0 16008 2 0x00000088 [83317.325183] ffffffffc084ad95 [83317.325183] ffffffffc084ad95 [83317.325184] 00000000c084afeb [83317.325184] 00000000c084afeb [83317.325185] 0000000000000000 [83317.325185] 0000000000000000 [83317.325186] ffff882011eb8c00 [83317.325186] [83317.325187] ffff882011eb8c00 [83317.325187] ffff882012eddee0 [83317.325188] ffff8810247eeeb0 [83317.325188] ffff88203d3bbde8 [83317.325189] ffff88201c7ebde8 [83317.325189] ffffffffc0b72b3c [83317.325190] ffffffffc0b72b3c [83317.325190] ffff881039384c00 [83317.325191] [83317.325191] ffff88102af4e400 [83317.325192] ffff88203d3bbe18 [83317.325192] ffff88201c7ebe18 [83317.325193] ffffffffc0b737cc [83317.325193] ffffffffc0b737cc [83317.325194] ffff881039384c00 [83317.325194] ffff88102af4e400 [83317.325195] ffff88200000003d [83317.325195] [83317.325196] ffff88200000000d [83317.325196] Call Trace: [83317.325196] Call Trace: [83317.325201] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325206] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325224] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325242] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325260] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325278] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325299] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325320] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325321] [] ? wake_up_state+0x20/0x20 [83317.325322] [] ? wake_up_state+0x20/0x20 [83317.325342] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325364] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325365] [] ? kthread+0xcf/0xe0 [83317.325366] [] ? kthread+0xcf/0xe0 [83317.325368] [] ? insert_kthread_work+0x40/0x40 [83317.325369] [] ? insert_kthread_work+0x40/0x40 [83317.325371] [] ? ret_from_fork+0x58/0x90 [83317.325372] [] ? ret_from_fork+0x58/0x90 [83317.325373] [] ? insert_kthread_work+0x40/0x40 [83317.325374] [] ? insert_kthread_work+0x40/0x40 [83317.325375] Task dump for CPU 22: [83317.325375] Task dump for CPU 8: [83317.325376] ldlm_bl_07 R [83317.325376] ldlm_bl_10 R [83317.325377] running task [83317.325377] 0 16009 2 0x00000088 [83317.325378] running task 0 16012 2 0x00000088 [83317.325379] ffffffffc084ad95 [83317.325379] ffffffffc084ad95 [83317.325380] 00000000c084afeb [83317.325380] 00000000c084afeb [83317.325381] 0000000000000000 [83317.325381] 0000000000000000 [83317.325382] ffff882011eb8c00 [83317.325382] [83317.325383] ffff882011eb8c00 [83317.325383] ffff8810e9240fd0 [83317.325384] ffff88203ad89fa0 [83317.325384] ffff88201c7efde8 [83317.325385] ffff88201c803de8 [83317.325385] ffffffffc0b72b3c [83317.325386] ffffffffc0b72b3c [83317.325386] ffff88200f63f400 [83317.325387] [83317.325387] ffff882012e84400 [83317.325388] ffff88201c7efe18 [83317.325388] ffff88201c803e18 [83317.325389] ffffffffc0b737cc [83317.325389] ffffffffc0b737cc [83317.325390] ffff88200f63f400 [83317.325390] ffff882012e84400 [83317.325391] ffff88200000000d [83317.325391] [83317.325392] ffff88200000000d [83317.325392] Call Trace: [83317.325392] Call Trace: [83317.325397] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325402] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325420] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325438] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325456] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325474] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325495] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325517] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325518] [] ? wake_up_state+0x20/0x20 [83317.325520] [] ? wake_up_state+0x20/0x20 [83317.325540] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325560] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325561] [] ? kthread+0xcf/0xe0 [83317.325563] [] ? kthread+0xcf/0xe0 [83317.325564] [] ? insert_kthread_work+0x40/0x40 [83317.325565] [] ? insert_kthread_work+0x40/0x40 [83317.325567] [] ? ret_from_fork+0x58/0x90 [83317.325568] [] ? ret_from_fork+0x58/0x90 [83317.325570] [] ? insert_kthread_work+0x40/0x40 [83317.325571] [] ? insert_kthread_work+0x40/0x40 [83317.325571] Task dump for CPU 10: [83317.325572] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.325574] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.325575] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83317.325576] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.325576] Call Trace: [83317.325582] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325600] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325619] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325639] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325641] [] ? wake_up_state+0x20/0x20 [83317.325661] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325662] [] ? kthread+0xcf/0xe0 [83317.325664] [] ? insert_kthread_work+0x40/0x40 [83317.325665] [] ? ret_from_fork+0x58/0x90 [83317.325666] [] ? insert_kthread_work+0x40/0x40 [83317.325667] Task dump for CPU 12: [83317.325668] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.325669] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.325670] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.325671] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.325672] Call Trace: [83317.325677] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325696] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325714] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325735] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325736] [] ? wake_up_state+0x20/0x20 [83317.325756] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325758] [] ? kthread+0xcf/0xe0 [83317.325759] [] ? insert_kthread_work+0x40/0x40 [83317.325760] [] ? ret_from_fork+0x58/0x90 [83317.325761] [] ? insert_kthread_work+0x40/0x40 [83317.325762] Task dump for CPU 16: [83317.325762] Task dump for CPU 0: [83317.325763] ldlm_bl_12 R [83317.325763] ldlm_bl_13 R [83317.325763] running task [83317.325764] 0 16014 2 0x00000088 [83317.325765] running task 0 16015 2 0x00000088 [83317.325765] ffffffffc084ad95 [83317.325765] ffffffffc084ad95 [83317.325766] 00000000c084afeb [83317.325766] 00000000c084afeb [83317.325766] 0000000000000000 [83317.325767] 0000000000000000 [83317.325767] ffff882011eb8c00 [83317.325767] [83317.325767] ffff882011eb8c00 [83317.325768] ffff88203d33af70 [83317.325768] ffff88203d33bf40 [83317.325769] ffff88201c80bde8 [83317.325769] ffff88201c80fde8 [83317.325769] ffffffffc0b72b3c [83317.325770] ffffffffc0b72b3c [83317.325770] ffff881017a75800 [83317.325770] [83317.325770] ffff88200be50c00 [83317.325771] ffff88201c80be18 [83317.325771] ffff88201c80fe18 [83317.325772] ffffffffc0b737cc [83317.325772] ffffffffc0b737cc [83317.325772] ffff881017a75800 [83317.325773] ffff88200be50c00 [83317.325773] ffff88200000000c [83317.325773] [83317.325774] ffff882000000019 [83317.325774] Call Trace: [83317.325774] Call Trace: [83317.325779] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325783] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325802] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325821] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.325840] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325859] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.325879] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325900] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.325901] [] ? wake_up_state+0x20/0x20 [83317.325902] [] ? wake_up_state+0x20/0x20 [83317.325922] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325943] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.325944] [] ? kthread+0xcf/0xe0 [83317.325945] [] ? kthread+0xcf/0xe0 [83317.325946] [] ? insert_kthread_work+0x40/0x40 [83317.325947] [] ? insert_kthread_work+0x40/0x40 [83317.325948] [] ? ret_from_fork+0x58/0x90 [83317.325949] [] ? ret_from_fork+0x58/0x90 [83317.325950] [] ? insert_kthread_work+0x40/0x40 [83317.325952] [] ? insert_kthread_work+0x40/0x40 [83317.325952] Task dump for CPU 18: [83317.325952] Task dump for CPU 1: [83317.325953] ldlm_bl_02 R [83317.325953] ldlm_bl_08 R [83317.325953] running task [83317.325954] 0 15061 2 0x00000088 [83317.325954] running task 0 16010 2 0x00000088 [83317.325955] ffffffffc084ad95 [83317.325955] ffffffffc084ad95 [83317.325955] 00000000c084afeb [83317.325956] 00000000c084afeb [83317.325956] 0000000000000000 [83317.325956] 0000000000000000 [83317.325957] ffff882011eb8c00 [83317.325957] [83317.325957] ffff882011eb8c00 [83317.325958] ffff882012edeeb0 [83317.325958] ffff8810e9242f70 [83317.325958] ffff88203d3bfde8 [83317.325958] ffff88201c7fbde8 [83317.325959] ffffffffc0b72b3c [83317.325959] ffffffffc0b72b3c [83317.325959] ffff88200f638000 [83317.325960] [83317.325960] ffff881017a74c00 [83317.325960] ffff88203d3bfe18 [83317.325961] ffff88201c7fbe18 [83317.325961] ffffffffc0b737cc [83317.325961] ffffffffc0b737cc [83317.325962] ffff88200f638000 [83317.325962] ffff881017a74c00 [83317.325962] ffff88200000000d [83317.325962] [83317.325963] ffff882000000111 [83317.325963] Call Trace: [83317.325963] Call Trace: [83317.325968] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325973] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.325992] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326010] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326029] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326047] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326068] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.326089] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.326090] [] ? wake_up_state+0x20/0x20 [83317.326091] [] ? wake_up_state+0x20/0x20 [83317.326112] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.326132] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.326133] [] ? kthread+0xcf/0xe0 [83317.326134] [] ? kthread+0xcf/0xe0 [83317.326136] [] ? insert_kthread_work+0x40/0x40 [83317.326137] [] ? insert_kthread_work+0x40/0x40 [83317.326138] [] ? ret_from_fork+0x58/0x90 [83317.326139] [] ? ret_from_fork+0x58/0x90 [83317.326140] [] ? insert_kthread_work+0x40/0x40 [83317.326141] [] ? insert_kthread_work+0x40/0x40 [83317.326141] Task dump for CPU 20: [83317.326142] Task dump for CPU 2: [83317.326142] ldlm_bl_01 R [83317.326142] ldlm_bl_06 R [83317.326143] running task [83317.326143] 0 15060 2 0x00000088 [83317.326144] running task 0 16008 2 0x00000088 [83317.326144] ffffffffc084ad95 [83317.326145] ffffffffc084ad95 [83317.326145] 00000000c084afeb [83317.326145] 00000000c084afeb [83317.326145] 0000000000000000 [83317.326146] 0000000000000000 [83317.326146] ffff882011eb8c00 [83317.326146] [83317.326146] ffff882011eb8c00 [83317.326147] ffff882012eddee0 [83317.326147] ffff8810247eeeb0 [83317.326148] ffff88203d3bbde8 [83317.326148] ffff88201c7ebde8 [83317.326148] ffffffffc0b72b3c [83317.326149] ffffffffc0b72b3c [83317.326149] ffff881039384c00 [83317.326149] [83317.326149] ffff88102af4e400 [83317.326150] ffff88203d3bbe18 [83317.326150] ffff88201c7ebe18 [83317.326151] ffffffffc0b737cc [83317.326151] ffffffffc0b737cc [83317.326151] ffff881039384c00 [83317.326151] ffff88102af4e400 [83317.326152] ffff88200000003d [83317.326152] [83317.326152] ffff88200000000d [83317.326152] Call Trace: [83317.326153] Call Trace: [83317.326158] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.326162] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.326181] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326199] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326218] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326237] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326257] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.326278] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.326279] [] ? wake_up_state+0x20/0x20 [83317.326280] [] ? wake_up_state+0x20/0x20 [83317.326301] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.326322] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.326323] [] ? kthread+0xcf/0xe0 [83317.326324] [] ? kthread+0xcf/0xe0 [83317.326325] [] ? insert_kthread_work+0x40/0x40 [83317.326326] [] ? insert_kthread_work+0x40/0x40 [83317.326327] [] ? ret_from_fork+0x58/0x90 [83317.326328] [] ? ret_from_fork+0x58/0x90 [83317.326329] [] ? insert_kthread_work+0x40/0x40 [83317.326330] [] ? insert_kthread_work+0x40/0x40 [83317.326331] Task dump for CPU 22: [83317.326331] Task dump for CPU 8: [83317.326332] ldlm_bl_07 R [83317.326332] ldlm_bl_10 R [83317.326332] running task [83317.326333] 0 16009 2 0x00000088 [83317.326333] running task 0 16012 2 0x00000088 [83317.326334] ffffffffc084ad95 [83317.326334] ffffffffc084ad95 [83317.326334] 00000000c084afeb [83317.326335] 00000000c084afeb [83317.326335] 0000000000000000 [83317.326335] 0000000000000000 [83317.326336] ffff882011eb8c00 [83317.326336] [83317.326336] ffff882011eb8c00 [83317.326337] ffff8810e9240fd0 [83317.326337] ffff88203ad89fa0 [83317.326337] ffff88201c7efde8 [83317.326338] ffff88201c803de8 [83317.326338] ffffffffc0b72b3c [83317.326338] ffffffffc0b72b3c [83317.326339] ffff88200f63f400 [83317.326339] [83317.326339] ffff882012e84400 [83317.326340] ffff88201c7efe18 [83317.326340] ffff88201c803e18 [83317.326340] ffffffffc0b737cc [83317.326341] ffffffffc0b737cc [83317.326341] ffff88200f63f400 [83317.326341] ffff882012e84400 [83317.326342] ffff88200000000d [83317.326342] [83317.326342] ffff88200000000d [83317.326342] Call Trace: [83317.326342] Call Trace: [83317.326347] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.326352] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.326370] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326389] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326408] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326427] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326447] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.326468] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.326470] [] ? wake_up_state+0x20/0x20 [83317.326471] [] ? wake_up_state+0x20/0x20 [83317.326491] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.326511] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.326513] [] ? kthread+0xcf/0xe0 [83317.326514] [] ? kthread+0xcf/0xe0 [83317.326515] [] ? insert_kthread_work+0x40/0x40 [83317.326516] [] ? insert_kthread_work+0x40/0x40 [83317.326517] [] ? ret_from_fork+0x58/0x90 [83317.326518] [] ? ret_from_fork+0x58/0x90 [83317.326519] [] ? insert_kthread_work+0x40/0x40 [83317.326520] [] ? insert_kthread_work+0x40/0x40 [83317.326521] Task dump for CPU 10: [83317.326522] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.326523] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.326524] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83317.326525] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.326526] Call Trace: [83317.326531] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.326549] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326566] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326588] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.326589] [] ? wake_up_state+0x20/0x20 [83317.326609] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.326610] [] ? kthread+0xcf/0xe0 [83317.326612] [] ? insert_kthread_work+0x40/0x40 [83317.326613] [] ? ret_from_fork+0x58/0x90 [83317.326614] [] ? insert_kthread_work+0x40/0x40 [83317.326615] Task dump for CPU 12: [83317.326616] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.326617] ffff88203d338fd0 00000000700731cb ffff88103cf83db8 ffffffff810c3a98 [83317.326618] 000000000000000c ffffffff81a551c0 ffff88103cf83dd0 ffffffff810c7429 [83317.326619] 000000000000000d ffff88103cf83e00 ffffffff81137b10 ffff88103cf90260 [83317.326620] Call Trace: [83317.326621] [] sched_show_task+0xa8/0x110 [83317.326623] [] dump_cpu_task+0x39/0x70 [83317.326624] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.326625] [] rcu_check_callbacks+0x442/0x730 [83317.326627] [] ? tick_sched_do_timer+0x50/0x50 [83317.326628] [] update_process_times+0x46/0x80 [83317.326629] [] tick_sched_handle+0x30/0x70 [83317.326630] [] tick_sched_timer+0x39/0x80 [83317.326632] [] __hrtimer_run_queues+0xd4/0x260 [83317.326633] [] hrtimer_interrupt+0xaf/0x1d0 [83317.326635] [] local_apic_timer_interrupt+0x35/0x60 [83317.326636] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.326638] [] apic_timer_interrupt+0x6d/0x80 [83317.326640] [] ? native_queued_spin_lock_slowpath+0x116/0x1e0 [83317.326642] [] queued_spin_lock_slowpath+0xb/0xf [83317.326643] [] _raw_spin_lock+0x20/0x30 [83317.326661] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.326680] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.326699] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.326717] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.326734] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.326752] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.326757] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.326775] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.326793] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.326798] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.326815] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326833] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326853] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.326855] [] ? wake_up_state+0x20/0x20 [83317.326874] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.326876] [] kthread+0xcf/0xe0 [83317.326877] [] ? insert_kthread_work+0x40/0x40 [83317.326878] [] ret_from_fork+0x58/0x90 [83317.326879] [] ? insert_kthread_work+0x40/0x40 [83317.326880] Task dump for CPU 16: [83317.326880] Task dump for CPU 0: [83317.326881] ldlm_bl_12 R [83317.326882] ldlm_bl_13 R [83317.326882] running task [83317.326883] 0 16014 2 0x00000088 [83317.326884] running task 0 16015 2 0x00000088 [83317.326885] ffffffffc084ad95 [83317.326885] ffffffffc084ad95 [83317.326886] 00000000c084afeb [83317.326886] 00000000c084afeb [83317.326887] 0000000000000000 [83317.326887] 0000000000000000 [83317.326888] ffff882011eb8c00 [83317.326888] [83317.326888] ffff882011eb8c00 [83317.326889] ffff88203d33af70 [83317.326890] ffff88203d33bf40 [83317.326890] ffff88201c80bde8 [83317.326891] ffff88201c80fde8 [83317.326891] ffffffffc0b72b3c [83317.326891] ffffffffc0b72b3c [83317.326892] ffff881017a75800 [83317.326892] [83317.326893] ffff88200be50c00 [83317.326893] ffff88201c80be18 [83317.326894] ffff88201c80fe18 [83317.326894] ffffffffc0b737cc [83317.326895] ffffffffc0b737cc [83317.326895] ffff881017a75800 [83317.326896] ffff88200be50c00 [83317.326896] ffff88200000000c [83317.326897] [83317.326897] ffff882000000019 [83317.326897] Call Trace: [83317.326898] Call Trace: [83317.326903] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.326909] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.326928] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326947] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.326966] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.326986] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327005] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327027] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327029] [] ? wake_up_state+0x20/0x20 [83317.327030] [] ? wake_up_state+0x20/0x20 [83317.327050] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327071] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327072] [] ? kthread+0xcf/0xe0 [83317.327074] [] ? kthread+0xcf/0xe0 [83317.327075] [] ? insert_kthread_work+0x40/0x40 [83317.327076] [] ? insert_kthread_work+0x40/0x40 [83317.327078] [] ? ret_from_fork+0x58/0x90 [83317.327079] [] ? ret_from_fork+0x58/0x90 [83317.327080] [] ? insert_kthread_work+0x40/0x40 [83317.327082] [] ? insert_kthread_work+0x40/0x40 [83317.327082] Task dump for CPU 18: [83317.327082] Task dump for CPU 1: [83317.327083] ldlm_bl_02 R [83317.327084] ldlm_bl_08 R [83317.327084] running task [83317.327085] 0 15061 2 0x00000088 [83317.327085] running task 0 16010 2 0x00000088 [83317.327086] ffffffffc084ad95 [83317.327087] ffffffffc084ad95 [83317.327087] 00000000c084afeb [83317.327088] 00000000c084afeb [83317.327088] 0000000000000000 [83317.327089] 0000000000000000 [83317.327089] ffff882011eb8c00 [83317.327090] [83317.327090] ffff882011eb8c00 [83317.327091] ffff882012edeeb0 [83317.327091] ffff8810e9242f70 [83317.327092] ffff88203d3bfde8 [83317.327092] ffff88201c7fbde8 [83317.327093] ffffffffc0b72b3c [83317.327093] ffffffffc0b72b3c [83317.327094] ffff88200f638000 [83317.327094] [83317.327094] ffff881017a74c00 [83317.327095] ffff88203d3bfe18 [83317.327095] ffff88201c7fbe18 [83317.327096] ffffffffc0b737cc [83317.327096] ffffffffc0b737cc [83317.327097] ffff88200f638000 [83317.327097] ffff881017a74c00 [83317.327098] ffff88200000000d [83317.327098] [83317.327099] ffff882000000111 [83317.327099] Call Trace: [83317.327099] Call Trace: [83317.327104] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327109] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327129] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327147] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327165] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327182] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327202] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327222] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327223] [] ? wake_up_state+0x20/0x20 [83317.327225] [] ? wake_up_state+0x20/0x20 [83317.327245] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327265] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327266] [] ? kthread+0xcf/0xe0 [83317.327268] [] ? kthread+0xcf/0xe0 [83317.327269] [] ? insert_kthread_work+0x40/0x40 [83317.327270] [] ? insert_kthread_work+0x40/0x40 [83317.327271] [] ? ret_from_fork+0x58/0x90 [83317.327273] [] ? ret_from_fork+0x58/0x90 [83317.327274] [] ? insert_kthread_work+0x40/0x40 [83317.327275] [] ? insert_kthread_work+0x40/0x40 [83317.327276] Task dump for CPU 20: [83317.327276] Task dump for CPU 2: [83317.327277] ldlm_bl_01 R [83317.327277] ldlm_bl_06 R [83317.327278] running task [83317.327278] 0 15060 2 0x00000088 [83317.327279] running task 0 16008 2 0x00000088 [83317.327280] ffffffffc084ad95 [83317.327280] ffffffffc084ad95 [83317.327281] 00000000c084afeb [83317.327281] 00000000c084afeb [83317.327282] 0000000000000000 [83317.327282] 0000000000000000 [83317.327283] ffff882011eb8c00 [83317.327283] [83317.327283] ffff882011eb8c00 [83317.327284] ffff882012eddee0 [83317.327285] ffff8810247eeeb0 [83317.327285] ffff88203d3bbde8 [83317.327286] ffff88201c7ebde8 [83317.327286] ffffffffc0b72b3c [83317.327287] ffffffffc0b72b3c [83317.327287] ffff881039384c00 [83317.327287] [83317.327288] ffff88102af4e400 [83317.327288] ffff88203d3bbe18 [83317.327289] ffff88201c7ebe18 [83317.327289] ffffffffc0b737cc [83317.327290] ffffffffc0b737cc [83317.327290] ffff881039384c00 [83317.327291] ffff88102af4e400 [83317.327291] ffff88200000003d [83317.327292] [83317.327292] ffff88200000000d [83317.327292] Call Trace: [83317.327293] Call Trace: [83317.327298] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327303] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327320] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327340] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327357] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327375] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327396] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327418] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327419] [] ? wake_up_state+0x20/0x20 [83317.327421] [] ? wake_up_state+0x20/0x20 [83317.327441] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327461] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327462] [] ? kthread+0xcf/0xe0 [83317.327463] [] ? kthread+0xcf/0xe0 [83317.327465] [] ? insert_kthread_work+0x40/0x40 [83317.327466] [] ? insert_kthread_work+0x40/0x40 [83317.327468] [] ? ret_from_fork+0x58/0x90 [83317.327469] [] ? ret_from_fork+0x58/0x90 [83317.327470] [] ? insert_kthread_work+0x40/0x40 [83317.327471] [] ? insert_kthread_work+0x40/0x40 [83317.327472] Task dump for CPU 22: [83317.327472] Task dump for CPU 8: [83317.327473] ldlm_bl_07 R [83317.327473] ldlm_bl_10 R [83317.327474] running task [83317.327474] 0 16009 2 0x00000088 [83317.327475] running task 0 16012 2 0x00000088 [83317.327476] ffffffffc084ad95 [83317.327476] ffffffffc084ad95 [83317.327477] 00000000c084afeb [83317.327477] 00000000c084afeb [83317.327478] 0000000000000000 [83317.327478] 0000000000000000 [83317.327479] ffff882011eb8c00 [83317.327479] [83317.327480] ffff882011eb8c00 [83317.327480] ffff8810e9240fd0 [83317.327481] ffff88203ad89fa0 [83317.327481] ffff88201c7efde8 [83317.327482] ffff88201c803de8 [83317.327482] ffffffffc0b72b3c [83317.327483] ffffffffc0b72b3c [83317.327483] ffff88200f63f400 [83317.327484] [83317.327484] ffff882012e84400 [83317.327485] ffff88201c7efe18 [83317.327485] ffff88201c803e18 [83317.327486] ffffffffc0b737cc [83317.327486] ffffffffc0b737cc [83317.327487] ffff88200f63f400 [83317.327487] ffff882012e84400 [83317.327488] ffff88200000000d [83317.327488] [83317.327488] ffff88200000000d [83317.327488] Call Trace: [83317.327489] Call Trace: [83317.327494] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327499] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327516] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327534] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327552] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327569] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327590] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327610] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327611] [] ? wake_up_state+0x20/0x20 [83317.327613] [] ? wake_up_state+0x20/0x20 [83317.327633] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327653] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327654] [] ? kthread+0xcf/0xe0 [83317.327655] [] ? kthread+0xcf/0xe0 [83317.327656] [] ? insert_kthread_work+0x40/0x40 [83317.327658] [] ? insert_kthread_work+0x40/0x40 [83317.327659] [] ? ret_from_fork+0x58/0x90 [83317.327661] [] ? ret_from_fork+0x58/0x90 [83317.327662] [] ? insert_kthread_work+0x40/0x40 [83317.327663] [] ? insert_kthread_work+0x40/0x40 [83317.327664] Task dump for CPU 10: [83317.327665] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.327666] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.327667] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83317.327668] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.327669] Call Trace: [83317.327674] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327692] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327709] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327730] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327732] [] ? wake_up_state+0x20/0x20 [83317.327752] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327753] [] ? kthread+0xcf/0xe0 [83317.327754] [] ? insert_kthread_work+0x40/0x40 [83317.327756] [] ? ret_from_fork+0x58/0x90 [83317.327757] [] ? insert_kthread_work+0x40/0x40 [83317.327757] Task dump for CPU 12: [83317.327758] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.327760] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.327761] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.327762] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.327762] Call Trace: [83317.327767] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327785] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327803] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327824] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.327826] [] ? wake_up_state+0x20/0x20 [83317.327845] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.327847] [] ? kthread+0xcf/0xe0 [83317.327848] [] ? insert_kthread_work+0x40/0x40 [83317.327849] [] ? ret_from_fork+0x58/0x90 [83317.327850] [] ? insert_kthread_work+0x40/0x40 [83317.327851] Task dump for CPU 16: [83317.327852] Task dump for CPU 0: [83317.327854] ldlm_bl_12 R [83317.327854] ldlm_bl_13 R [83317.327855] running task [83317.327857] 0 16014 2 0x00000088 [83317.327858] running task 0 16015 2 0x00000088 [83317.327860] ffffffffc084ad95 [83317.327861] ffffffffc084ad95 [83317.327862] 00000000c084afeb [83317.327863] 00000000c084afeb [83317.327864] 0000000000000000 [83317.327865] 0000000000000000 [83317.327866] ffff882011eb8c00 [83317.327867] [83317.327868] ffff882011eb8c00 [83317.327870] ffff88203d33af70 [83317.327871] ffff88203d33bf40 [83317.327872] ffff88201c80bde8 [83317.327873] ffff88201c80fde8 [83317.327874] ffffffffc0b72b3c [83317.327875] ffffffffc0b72b3c [83317.327875] ffff881017a75800 [83317.327876] [83317.327877] ffff88200be50c00 [83317.327879] ffff88201c80be18 [83317.327880] ffff88201c80fe18 [83317.327881] ffffffffc0b737cc [83317.327882] ffffffffc0b737cc [83317.327883] ffff881017a75800 [83317.327884] ffff88200be50c00 [83317.327885] ffff88200000000c [83317.327885] [83317.327886] ffff882000000019 [83317.327887] Call Trace: [83317.327888] Call Trace: [83317.327894] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327901] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.327918] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327940] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.327958] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327979] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.327999] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.328022] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.328024] [] ? wake_up_state+0x20/0x20 [83317.328026] [] ? wake_up_state+0x20/0x20 [83317.328047] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.328067] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.328070] [] ? kthread+0xcf/0xe0 [83317.328072] [] ? kthread+0xcf/0xe0 [83317.328074] [] ? insert_kthread_work+0x40/0x40 [83317.328076] [] ? insert_kthread_work+0x40/0x40 [83317.328078] [] ? ret_from_fork+0x58/0x90 [83317.328080] [] ? ret_from_fork+0x58/0x90 [83317.328083] [] ? insert_kthread_work+0x40/0x40 [83317.328084] [] ? insert_kthread_work+0x40/0x40 [83317.328085] Task dump for CPU 1: [83317.328086] Task dump for CPU 18: [83317.328088] ldlm_bl_08 R [83317.328088] ldlm_bl_02 R [83317.328089] running task [83317.328091] 0 16010 2 0x00000088 [83317.328092] running task 0 15061 2 0x00000088 [83317.328094] ffff8810e9242f70 [83317.328095] ffff882012edeeb0 [83317.328096] 00000000a6f785ac [83317.328097] 00000000a05c68c1 [83317.328098] ffff88203c603db8 [83317.328099] ffff88103d043db8 [83317.328100] ffffffff810c3a98 [83317.328101] [83317.328101] ffffffff810c3a98 [83317.328103] 0000000000000001 [83317.328104] 0000000000000012 [83317.328105] ffffffff81a551c0 [83317.328107] ffffffff81a552c0 [83317.328108] ffff88203c603dd0 [83317.328109] ffff88103d043dd0 [83317.328110] ffffffff810c7429 [83317.328111] [83317.328112] ffffffff810c7429 [83317.328114] 0000000000000002 [83317.328114] 0000000000000003 [83317.328116] ffff88203c603e00 [83317.328117] ffff88103d043e00 [83317.328118] ffffffff81137b10 [83317.328119] ffffffff81137b10 [83317.328120] ffff88203c610260 [83317.328121] [83317.328122] ffff88103d050260 [83317.328123] Call Trace: [83317.328123] Call Trace: [83317.328125] [83317.328126] [83317.328129] [] sched_show_task+0xa8/0x110 [83317.328131] [] sched_show_task+0xa8/0x110 [83317.328133] [] dump_cpu_task+0x39/0x70 [83317.328135] [] dump_cpu_task+0x39/0x70 [83317.328138] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.328141] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.328143] [] rcu_check_callbacks+0x442/0x730 [83317.328145] [] rcu_check_callbacks+0x442/0x730 [83317.328148] [] ? tick_sched_do_timer+0x50/0x50 [83317.328150] [] ? tick_sched_do_timer+0x50/0x50 [83317.328153] [] update_process_times+0x46/0x80 [83317.328155] [] update_process_times+0x46/0x80 [83317.328157] [] tick_sched_handle+0x30/0x70 [83317.328159] [] tick_sched_handle+0x30/0x70 [83317.328162] [] tick_sched_timer+0x39/0x80 [83317.328163] [] tick_sched_timer+0x39/0x80 [83317.328166] [] __hrtimer_run_queues+0xd4/0x260 [83317.328168] [] __hrtimer_run_queues+0xd4/0x260 [83317.328171] [] hrtimer_interrupt+0xaf/0x1d0 [83317.328173] [] hrtimer_interrupt+0xaf/0x1d0 [83317.328176] [] local_apic_timer_interrupt+0x35/0x60 [83317.328178] [] local_apic_timer_interrupt+0x35/0x60 [83317.328181] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.328183] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.328186] [] apic_timer_interrupt+0x6d/0x80 [83317.328188] [] apic_timer_interrupt+0x6d/0x80 [83317.328189] [83317.328190] [83317.328192] [] ? native_queued_spin_lock_slowpath+0x112/0x1e0 [83317.328194] [] ? native_queued_spin_lock_slowpath+0x116/0x1e0 [83317.328197] [] queued_spin_lock_slowpath+0xb/0xf [83317.328200] [] queued_spin_lock_slowpath+0xb/0xf [83317.328202] [] _raw_spin_lock+0x20/0x30 [83317.328204] [] _raw_spin_lock+0x20/0x30 [83317.328223] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.328240] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.328262] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.328283] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.328302] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.328321] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.328340] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.328358] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.328376] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.328394] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.328411] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.328417] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.328436] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.328454] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.328473] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.328478] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.328484] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.328502] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.328521] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.328539] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.328557] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.328580] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.328585] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.328588] [] ? wake_up_state+0x20/0x20 [83317.328605] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.328626] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.328628] [] kthread+0xcf/0xe0 [83317.328646] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.328649] [] ? insert_kthread_work+0x40/0x40 [83317.328671] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.328673] [] ret_from_fork+0x58/0x90 [83317.328675] [] ? wake_up_state+0x20/0x20 [83317.328677] [] ? insert_kthread_work+0x40/0x40 [83317.328678] Task dump for CPU 2: [83317.328699] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.328701] ldlm_bl_06 R [83317.328703] [] kthread+0xcf/0xe0 [83317.328704] running task 0 16008 2 0x00000088 [83317.328707] [] ? insert_kthread_work+0x40/0x40 [83317.328709] ffffffffc084ad95 00000000c084afeb [83317.328711] [] ret_from_fork+0x58/0x90 [83317.328713] 0000000000000000 ffff882011eb8c00 [83317.328714] [83317.328716] [] ? insert_kthread_work+0x40/0x40 [83317.328717] Task dump for CPU 20: [83317.328719] ffff8810247eeeb0 ffff88201c7ebde8 [83317.328720] ldlm_bl_01 R [83317.328721] ffffffffc0b72b3c [83317.328722] running task [83317.328723] 0 15060 2 0x00000088 [83317.328724] ffff88102af4e400 [83317.328726] ffffffffc084ad95 [83317.328727] ffff88201c7ebe18 [83317.328727] 00000000c084afeb [83317.328728] ffffffffc0b737cc [83317.328729] 0000000000000000 [83317.328730] ffff88102af4e400 [83317.328732] ffff882011eb8c00 [83317.328732] [83317.328733] ffff88200000000d [83317.328735] Call Trace: [83317.328736] ffff882012eddee0 ffff88203d3bbde8 [83317.328743] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.328744] ffffffffc0b72b3c ffff881039384c00 [83317.328745] [83317.328765] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.328767] ffff88203d3bbe18 [83317.328785] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.328787] ffffffffc0b737cc ffff881039384c00 [83317.328809] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.328810] ffff88200000003d [83317.328811] Call Trace: [83317.328813] [] ? wake_up_state+0x20/0x20 [83317.328819] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.328839] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.328858] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.328860] [] ? kthread+0xcf/0xe0 [83317.328862] [] ? insert_kthread_work+0x40/0x40 [83317.328880] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.328882] [] ? ret_from_fork+0x58/0x90 [83317.328884] [] ? insert_kthread_work+0x40/0x40 [83317.328904] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.328905] Task dump for CPU 8: [83317.328908] [] ? wake_up_state+0x20/0x20 [83317.328910] ldlm_bl_10 R running task [83317.328911] 0 16012 2 0x00000088 [83317.328931] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.328933] ffffffffc084ad95 [83317.328936] [] ? kthread+0xcf/0xe0 [83317.328937] 00000000c084afeb 0000000000000000 [83317.328939] [] ? insert_kthread_work+0x40/0x40 [83317.328940] ffff882011eb8c00 [83317.328943] [] ? ret_from_fork+0x58/0x90 [83317.328944] ffff88203ad89fa0 ffff88201c803de8 [83317.328947] [] ? insert_kthread_work+0x40/0x40 [83317.328948] Task dump for CPU 22: [83317.328949] ffffffffc0b72b3c ffff882012e84400 [83317.328951] ldlm_bl_07 R [83317.328952] ffff88201c803e18 [83317.328953] running task [83317.328954] 0 16009 2 0x00000088 [83317.328956] ffffffffc0b737cc ffff882012e84400 [83317.328957] ffffffffc084ad95 [83317.328958] ffff88200000000d [83317.328959] [83317.328960] Call Trace: [83317.328961] 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.328962] [83317.328968] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.328969] ffff8810e9240fd0 [83317.328990] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.328991] ffff88201c7efde8 ffffffffc0b72b3c [83317.329011] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329012] ffff88200f63f400 [83317.329034] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329035] ffff88201c7efe18 [83317.329038] [] ? wake_up_state+0x20/0x20 [83317.329039] ffffffffc0b737cc ffff88200f63f400 [83317.329059] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.329060] ffff88200000000d [83317.329062] Call Trace: [83317.329063] [] ? kthread+0xcf/0xe0 [83317.329065] [] ? insert_kthread_work+0x40/0x40 [83317.329071] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329073] [] ? ret_from_fork+0x58/0x90 [83317.329091] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329094] [] ? insert_kthread_work+0x40/0x40 [83317.329112] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329113] Task dump for CPU 10: [83317.329136] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329137] ldlm_bl_05 R running task [83317.329138] 0 16007 2 0x00000088 [83317.329140] [] ? wake_up_state+0x20/0x20 [83317.329141] ffffffffc084ad95 [83317.329163] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.329164] 00000000c084afeb 0000000000000000 [83317.329167] [] ? kthread+0xcf/0xe0 [83317.329168] ffff882011eb8c00 [83317.329170] [] ? insert_kthread_work+0x40/0x40 [83317.329172] ffff8810247ecf10 ffff88201c64fde8 [83317.329175] [] ? ret_from_fork+0x58/0x90 [83317.329176] ffffffffc0b72b3c ffff88202a732400 [83317.329177] [83317.329178] [] ? insert_kthread_work+0x40/0x40 [83317.329180] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.329180] Call Trace: [83317.329185] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329203] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329221] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329242] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329244] [] ? wake_up_state+0x20/0x20 [83317.329263] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.329265] [] ? kthread+0xcf/0xe0 [83317.329266] [] ? insert_kthread_work+0x40/0x40 [83317.329267] [] ? ret_from_fork+0x58/0x90 [83317.329268] [] ? insert_kthread_work+0x40/0x40 [83317.329269] Task dump for CPU 12: [83317.329271] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.329272] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.329273] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.329274] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.329274] Call Trace: [83317.329280] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329297] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329315] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329337] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329338] [] ? wake_up_state+0x20/0x20 [83317.329358] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.329359] [] ? kthread+0xcf/0xe0 [83317.329360] [] ? insert_kthread_work+0x40/0x40 [83317.329361] [] ? ret_from_fork+0x58/0x90 [83317.329363] [] ? insert_kthread_work+0x40/0x40 [83317.329363] Task dump for CPU 16: [83317.329364] Task dump for CPU 0: [83317.329366] ldlm_bl_12 R [83317.329367] ldlm_bl_13 R [83317.329368] running task [83317.329369] 0 16014 2 0x00000088 [83317.329370] running task 0 16015 2 0x00000088 [83317.329372] ffffffffc084ad95 [83317.329373] ffffffffc084ad95 [83317.329374] 00000000c084afeb [83317.329376] 00000000c084afeb [83317.329377] 0000000000000000 [83317.329378] 0000000000000000 [83317.329379] ffff882011eb8c00 [83317.329379] [83317.329380] ffff882011eb8c00 [83317.329382] ffff88203d33af70 [83317.329383] ffff88203d33bf40 [83317.329384] ffff88201c80bde8 [83317.329385] ffff88201c80fde8 [83317.329387] ffffffffc0b72b3c [83317.329387] ffffffffc0b72b3c [83317.329388] ffff881017a75800 [83317.329389] [83317.329390] ffff88200be50c00 [83317.329392] ffff88201c80be18 [83317.329393] ffff88201c80fe18 [83317.329394] ffffffffc0b737cc [83317.329394] ffffffffc0b737cc [83317.329395] ffff881017a75800 [83317.329396] ffff88200be50c00 [83317.329397] ffff88200000000c [83317.329398] [83317.329399] ffff882000000019 [83317.329400] Call Trace: [83317.329401] Call Trace: [83317.329407] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329414] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329434] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329454] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329472] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329492] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329513] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329536] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329538] [] ? wake_up_state+0x20/0x20 [83317.329541] [] ? wake_up_state+0x20/0x20 [83317.329561] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.329583] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.329584] [] ? kthread+0xcf/0xe0 [83317.329587] [] ? kthread+0xcf/0xe0 [83317.329588] [] ? insert_kthread_work+0x40/0x40 [83317.329591] [] ? insert_kthread_work+0x40/0x40 [83317.329592] [] ? ret_from_fork+0x58/0x90 [83317.329595] [] ? ret_from_fork+0x58/0x90 [83317.329597] [] ? insert_kthread_work+0x40/0x40 [83317.329598] Task dump for CPU 18: [83317.329599] [] ? insert_kthread_work+0x40/0x40 [83317.329601] Task dump for CPU 1: [83317.329603] ldlm_bl_02 R running task [83317.329604] 0 15061 2 0x00000088 [83317.329606] ldlm_bl_08 R running task [83317.329607] 0 16010 2 0x00000088 [83317.329609] ffffffffc084ad95 00000000c084afeb [83317.329610] ffffffffc084ad95 [83317.329611] 0000000000000000 [83317.329612] 00000000c084afeb [83317.329613] ffff882011eb8c00 [83317.329614] [83317.329616] 0000000000000000 ffff882011eb8c00 [83317.329616] [83317.329618] ffff882012edeeb0 ffff88203d3bfde8 [83317.329620] ffff8810e9242f70 [83317.329620] ffffffffc0b72b3c [83317.329622] ffff88201c7fbde8 [83317.329623] ffff88200f638000 [83317.329623] [83317.329625] ffffffffc0b72b3c ffff881017a74c00 [83317.329626] [83317.329627] ffff88203d3bfe18 ffffffffc0b737cc [83317.329628] ffff88201c7fbe18 [83317.329629] ffff88200f638000 [83317.329630] ffffffffc0b737cc [83317.329631] ffff88200000000d [83317.329632] [83317.329634] Call Trace: [83317.329635] ffff881017a74c00 ffff882000000111 [83317.329641] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329641] Call Trace: [83317.329661] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329666] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329685] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329705] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329727] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329746] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329749] [] ? wake_up_state+0x20/0x20 [83317.329770] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329791] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.329793] [] ? wake_up_state+0x20/0x20 [83317.329795] [] ? kthread+0xcf/0xe0 [83317.329816] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.329819] [] ? insert_kthread_work+0x40/0x40 [83317.329821] [] ? kthread+0xcf/0xe0 [83317.329824] [] ? ret_from_fork+0x58/0x90 [83317.329825] [] ? insert_kthread_work+0x40/0x40 [83317.329828] [] ? insert_kthread_work+0x40/0x40 [83317.329830] [] ? ret_from_fork+0x58/0x90 [83317.329830] Task dump for CPU 20: [83317.329833] [] ? insert_kthread_work+0x40/0x40 [83317.329835] Task dump for CPU 2: [83317.329836] ldlm_bl_01 R running task 0 15060 2 0x00000088 [83317.329838] ldlm_bl_06 R [83317.329839] ffffffffc084ad95 [83317.329840] running task [83317.329841] 0 16008 2 0x00000088 [83317.329843] 00000000c084afeb 0000000000000000 [83317.329844] ffffffffc084ad95 [83317.329845] ffff882011eb8c00 [83317.329846] [83317.329848] 00000000c084afeb 0000000000000000 [83317.329848] ffff882012eddee0 [83317.329849] ffff882011eb8c00 [83317.329850] [83317.329852] ffff88203d3bbde8 ffffffffc0b72b3c [83317.329853] ffff8810247eeeb0 [83317.329854] ffff881039384c00 [83317.329854] [83317.329856] ffff88201c7ebde8 ffffffffc0b72b3c [83317.329857] ffff88203d3bbe18 [83317.329858] ffff88102af4e400 [83317.329859] [83317.329861] ffffffffc0b737cc ffff881039384c00 [83317.329862] ffff88201c7ebe18 [83317.329863] ffff88200000003d [83317.329864] [83317.329865] Call Trace: [83317.329867] ffffffffc0b737cc ffff88102af4e400 ffff88200000000d [83317.329868] [83317.329873] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329874] Call Trace: [83317.329895] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329900] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.329919] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329938] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.329961] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.329980] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.329983] [] ? wake_up_state+0x20/0x20 [83317.330003] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330005] [] ? wake_up_state+0x20/0x20 [83317.330025] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.330047] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.330049] [] ? kthread+0xcf/0xe0 [83317.330052] [] ? insert_kthread_work+0x40/0x40 [83317.330053] [] ? kthread+0xcf/0xe0 [83317.330056] [] ? ret_from_fork+0x58/0x90 [83317.330057] [] ? insert_kthread_work+0x40/0x40 [83317.330060] [] ? insert_kthread_work+0x40/0x40 [83317.330061] [] ? ret_from_fork+0x58/0x90 [83317.330062] Task dump for CPU 22: [83317.330065] [] ? insert_kthread_work+0x40/0x40 [83317.330067] Task dump for CPU 8: [83317.330068] ldlm_bl_07 R running task 0 16009 2 0x00000088 [83317.330070] ldlm_bl_10 R [83317.330070] ffffffffc084ad95 [83317.330072] running task [83317.330073] 0 16012 2 0x00000088 [83317.330075] 00000000c084afeb 0000000000000000 [83317.330076] ffffffffc084ad95 [83317.330077] ffff882011eb8c00 [83317.330078] [83317.330080] 00000000c084afeb 0000000000000000 [83317.330081] ffff8810e9240fd0 [83317.330082] ffff882011eb8c00 [83317.330082] [83317.330084] ffff88201c7efde8 ffffffffc0b72b3c [83317.330085] ffff88203ad89fa0 [83317.330086] ffff88200f63f400 [83317.330087] [83317.330089] ffff88201c803de8 ffffffffc0b72b3c [83317.330090] ffff88201c7efe18 [83317.330091] ffff882012e84400 [83317.330092] [83317.330094] ffffffffc0b737cc ffff88200f63f400 [83317.330095] ffff88201c803e18 [83317.330096] ffff88200000000d [83317.330097] [83317.330099] Call Trace: [83317.330100] ffffffffc0b737cc ffff882012e84400 ffff88200000000d [83317.330101] [83317.330106] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.330107] Call Trace: [83317.330126] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.330131] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.330150] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.330169] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.330190] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330208] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.330210] [] ? wake_up_state+0x20/0x20 [83317.330232] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.330252] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330255] [] ? kthread+0xcf/0xe0 [83317.330257] [] ? wake_up_state+0x20/0x20 [83317.330259] [] ? insert_kthread_work+0x40/0x40 [83317.330280] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.330283] [] ? ret_from_fork+0x58/0x90 [83317.330285] [] ? kthread+0xcf/0xe0 [83317.330287] [] ? insert_kthread_work+0x40/0x40 [83317.330289] [] ? insert_kthread_work+0x40/0x40 [83317.330291] [] ? ret_from_fork+0x58/0x90 [83317.330292] [] ? insert_kthread_work+0x40/0x40 [83317.330293] Task dump for CPU 10: [83317.330294] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.330296] ffff8810247ecf10 0000000011cf3248 ffff88103cf43db8 ffffffff810c3a98 [83317.330297] 000000000000000a ffffffff81a551c0 ffff88103cf43dd0 ffffffff810c7429 [83317.330298] 000000000000000b ffff88103cf43e00 ffffffff81137b10 ffff88103cf50260 [83317.330298] Call Trace: [83317.330300] [] sched_show_task+0xa8/0x110 [83317.330302] [] dump_cpu_task+0x39/0x70 [83317.330303] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.330304] [] rcu_check_callbacks+0x442/0x730 [83317.330306] [] ? tick_sched_do_timer+0x50/0x50 [83317.330307] [] update_process_times+0x46/0x80 [83317.330308] [] tick_sched_handle+0x30/0x70 [83317.330310] [] tick_sched_timer+0x39/0x80 [83317.330311] [] __hrtimer_run_queues+0xd4/0x260 [83317.330313] [] hrtimer_interrupt+0xaf/0x1d0 [83317.330314] [] local_apic_timer_interrupt+0x35/0x60 [83317.330316] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.330317] [] apic_timer_interrupt+0x6d/0x80 [83317.330320] [] ? native_queued_spin_lock_slowpath+0x112/0x1e0 [83317.330321] [] queued_spin_lock_slowpath+0xb/0xf [83317.330323] [] _raw_spin_lock+0x20/0x30 [83317.330342] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.330361] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.330381] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.330399] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.330417] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.330436] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.330441] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.330459] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.330477] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.330482] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.330500] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.330519] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.330539] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330541] [] ? wake_up_state+0x20/0x20 [83317.330561] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.330562] [] kthread+0xcf/0xe0 [83317.330564] [] ? insert_kthread_work+0x40/0x40 [83317.330565] [] ret_from_fork+0x58/0x90 [83317.330566] [] ? insert_kthread_work+0x40/0x40 [83317.330567] Task dump for CPU 12: [83317.330568] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.330569] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.330571] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.330572] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.330572] Call Trace: [83317.330578] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.330596] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.330614] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.330635] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330636] [] ? wake_up_state+0x20/0x20 [83317.330656] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.330658] [] ? kthread+0xcf/0xe0 [83317.330659] [] ? insert_kthread_work+0x40/0x40 [83317.330660] [] ? ret_from_fork+0x58/0x90 [83317.330662] [] ? insert_kthread_work+0x40/0x40 [83317.330662] Task dump for CPU 16: [83317.330663] Task dump for CPU 0: [83317.330663] ldlm_bl_12 R [83317.330663] ldlm_bl_13 R [83317.330664] running task [83317.330664] 0 16014 2 0x00000088 [83317.330665] running task 0 16015 2 0x00000088 [83317.330665] ffffffffc084ad95 [83317.330666] ffffffffc084ad95 [83317.330666] 00000000c084afeb [83317.330666] 00000000c084afeb [83317.330666] 0000000000000000 [83317.330667] 0000000000000000 [83317.330667] ffff882011eb8c00 [83317.330667] [83317.330668] ffff882011eb8c00 [83317.330668] ffff88203d33af70 [83317.330668] ffff88203d33bf40 [83317.330669] ffff88201c80bde8 [83317.330669] ffff88201c80fde8 [83317.330669] ffffffffc0b72b3c [83317.330670] ffffffffc0b72b3c [83317.330670] ffff881017a75800 [83317.330670] [83317.330671] ffff88200be50c00 [83317.330671] ffff88201c80be18 [83317.330671] ffff88201c80fe18 [83317.330672] ffffffffc0b737cc [83317.330672] ffffffffc0b737cc [83317.330672] ffff881017a75800 [83317.330673] ffff88200be50c00 [83317.330673] ffff88200000000c [83317.330673] [83317.330673] ffff882000000019 [83317.330674] Call Trace: [83317.330674] Call Trace: [83317.330679] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.330684] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.330702] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.330720] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.330739] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.330757] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.330778] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330799] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330800] [] ? wake_up_state+0x20/0x20 [83317.330801] [] ? wake_up_state+0x20/0x20 [83317.330822] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.330841] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.330843] [] ? kthread+0xcf/0xe0 [83317.330844] [] ? kthread+0xcf/0xe0 [83317.330845] [] ? insert_kthread_work+0x40/0x40 [83317.330846] [] ? insert_kthread_work+0x40/0x40 [83317.330847] [] ? ret_from_fork+0x58/0x90 [83317.330848] [] ? ret_from_fork+0x58/0x90 [83317.330849] [] ? insert_kthread_work+0x40/0x40 [83317.330850] [] ? insert_kthread_work+0x40/0x40 [83317.330851] Task dump for CPU 18: [83317.330851] Task dump for CPU 1: [83317.330851] ldlm_bl_02 R [83317.330852] ldlm_bl_08 R [83317.330852] running task [83317.330852] 0 15061 2 0x00000088 [83317.330853] running task 0 16010 2 0x00000088 [83317.330853] ffffffffc084ad95 [83317.330854] ffffffffc084ad95 [83317.330854] 00000000c084afeb [83317.330854] 00000000c084afeb [83317.330855] 0000000000000000 [83317.330855] 0000000000000000 [83317.330855] ffff882011eb8c00 [83317.330855] [83317.330856] ffff882011eb8c00 [83317.330856] ffff882012edeeb0 [83317.330856] ffff8810e9242f70 [83317.330857] ffff88203d3bfde8 [83317.330857] ffff88201c7fbde8 [83317.330857] ffffffffc0b72b3c [83317.330858] ffffffffc0b72b3c [83317.330858] ffff88200f638000 [83317.330858] [83317.330859] ffff881017a74c00 [83317.330859] ffff88203d3bfe18 [83317.330859] ffff88201c7fbe18 [83317.330860] ffffffffc0b737cc [83317.330860] ffffffffc0b737cc [83317.330860] ffff88200f638000 [83317.330861] ffff881017a74c00 [83317.330861] ffff88200000000d [83317.330861] [83317.330861] ffff882000000111 [83317.330862] Call Trace: [83317.330862] Call Trace: [83317.330867] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.330872] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.330890] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.330909] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.330928] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.330947] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.330967] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330988] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.330990] [] ? wake_up_state+0x20/0x20 [83317.330991] [] ? wake_up_state+0x20/0x20 [83317.331011] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331032] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331033] [] ? kthread+0xcf/0xe0 [83317.331034] [] ? kthread+0xcf/0xe0 [83317.331035] [] ? insert_kthread_work+0x40/0x40 [83317.331037] [] ? insert_kthread_work+0x40/0x40 [83317.331038] [] ? ret_from_fork+0x58/0x90 [83317.331039] [] ? ret_from_fork+0x58/0x90 [83317.331040] [] ? insert_kthread_work+0x40/0x40 [83317.331041] [] ? insert_kthread_work+0x40/0x40 [83317.331041] Task dump for CPU 20: [83317.331041] Task dump for CPU 2: [83317.331042] ldlm_bl_01 R [83317.331042] ldlm_bl_06 R [83317.331042] running task [83317.331043] 0 15060 2 0x00000088 [83317.331043] running task 0 16008 2 0x00000088 [83317.331044] ffffffffc084ad95 [83317.331044] ffffffffc084ad95 [83317.331045] 00000000c084afeb [83317.331045] 00000000c084afeb [83317.331045] 0000000000000000 [83317.331045] 0000000000000000 [83317.331046] ffff882011eb8c00 [83317.331046] [83317.331046] ffff882011eb8c00 [83317.331047] ffff882012eddee0 [83317.331047] ffff8810247eeeb0 [83317.331047] ffff88203d3bbde8 [83317.331048] ffff88201c7ebde8 [83317.331048] ffffffffc0b72b3c [83317.331048] ffffffffc0b72b3c [83317.331049] ffff881039384c00 [83317.331049] [83317.331049] ffff88102af4e400 [83317.331050] ffff88203d3bbe18 [83317.331050] ffff88201c7ebe18 [83317.331050] ffffffffc0b737cc [83317.331051] ffffffffc0b737cc [83317.331051] ffff881039384c00 [83317.331051] ffff88102af4e400 [83317.331052] ffff88200000003d [83317.331052] [83317.331052] ffff88200000000d [83317.331052] Call Trace: [83317.331052] Call Trace: [83317.331058] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331062] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331080] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331099] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331118] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331136] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331157] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331178] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331179] [] ? wake_up_state+0x20/0x20 [83317.331180] [] ? wake_up_state+0x20/0x20 [83317.331201] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331222] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331223] [] ? kthread+0xcf/0xe0 [83317.331224] [] ? kthread+0xcf/0xe0 [83317.331225] [] ? insert_kthread_work+0x40/0x40 [83317.331227] [] ? insert_kthread_work+0x40/0x40 [83317.331228] [] ? ret_from_fork+0x58/0x90 [83317.331229] [] ? ret_from_fork+0x58/0x90 [83317.331230] [] ? insert_kthread_work+0x40/0x40 [83317.331231] [] ? insert_kthread_work+0x40/0x40 [83317.331231] Task dump for CPU 22: [83317.331232] Task dump for CPU 8: [83317.331232] ldlm_bl_07 R [83317.331232] ldlm_bl_10 R [83317.331233] running task [83317.331233] 0 16009 2 0x00000088 [83317.331234] running task 0 16012 2 0x00000088 [83317.331234] ffffffffc084ad95 [83317.331234] ffffffffc084ad95 [83317.331235] 00000000c084afeb [83317.331235] 00000000c084afeb [83317.331235] 0000000000000000 [83317.331236] 0000000000000000 [83317.331236] ffff882011eb8c00 [83317.331236] [83317.331236] ffff882011eb8c00 [83317.331237] ffff8810e9240fd0 [83317.331237] ffff88203ad89fa0 [83317.331237] ffff88201c7efde8 [83317.331238] ffff88201c803de8 [83317.331238] ffffffffc0b72b3c [83317.331238] ffffffffc0b72b3c [83317.331239] ffff88200f63f400 [83317.331239] [83317.331239] ffff882012e84400 [83317.331239] ffff88201c7efe18 [83317.331240] ffff88201c803e18 [83317.331240] ffffffffc0b737cc [83317.331240] ffffffffc0b737cc [83317.331241] ffff88200f63f400 [83317.331241] ffff882012e84400 [83317.331241] ffff88200000000d [83317.331242] [83317.331242] ffff88200000000d [83317.331242] Call Trace: [83317.331242] Call Trace: [83317.331247] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331252] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331270] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331289] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331308] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331327] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331348] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331368] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331370] [] ? wake_up_state+0x20/0x20 [83317.331371] [] ? wake_up_state+0x20/0x20 [83317.331391] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331412] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331413] [] ? kthread+0xcf/0xe0 [83317.331414] [] ? kthread+0xcf/0xe0 [83317.331415] [] ? insert_kthread_work+0x40/0x40 [83317.331416] [] ? insert_kthread_work+0x40/0x40 [83317.331417] [] ? ret_from_fork+0x58/0x90 [83317.331418] [] ? ret_from_fork+0x58/0x90 [83317.331419] [] ? insert_kthread_work+0x40/0x40 [83317.331420] [] ? insert_kthread_work+0x40/0x40 [83317.331421] Task dump for CPU 10: [83317.331422] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.331423] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.331424] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83317.331425] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.331426] Call Trace: [83317.331431] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331449] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331467] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331488] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331489] [] ? wake_up_state+0x20/0x20 [83317.331509] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331510] [] ? kthread+0xcf/0xe0 [83317.331512] [] ? insert_kthread_work+0x40/0x40 [83317.331513] [] ? ret_from_fork+0x58/0x90 [83317.331514] [] ? insert_kthread_work+0x40/0x40 [83317.331515] Task dump for CPU 12: [83317.331516] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.331517] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.331518] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.331519] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.331519] Call Trace: [83317.331525] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331543] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331560] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331582] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331583] [] ? wake_up_state+0x20/0x20 [83317.331603] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331605] [] ? kthread+0xcf/0xe0 [83317.331606] [] ? insert_kthread_work+0x40/0x40 [83317.331607] [] ? ret_from_fork+0x58/0x90 [83317.331608] [] ? insert_kthread_work+0x40/0x40 [83317.331609] Task dump for CPU 16: [83317.331609] Task dump for CPU 0: [83317.331610] ldlm_bl_12 R [83317.331611] ldlm_bl_13 R [83317.331611] running task [83317.331612] 0 16014 2 0x00000088 [83317.331613] running task 0 16015 2 0x00000088 [83317.331614] ffffffffc084ad95 [83317.331614] ffffffffc084ad95 [83317.331615] 00000000c084afeb [83317.331615] 00000000c084afeb [83317.331616] 0000000000000000 [83317.331616] 0000000000000000 [83317.331617] ffff882011eb8c00 [83317.331617] [83317.331617] ffff882011eb8c00 [83317.331618] ffff88203d33af70 [83317.331619] ffff88203d33bf40 [83317.331619] ffff88201c80bde8 [83317.331620] ffff88201c80fde8 [83317.331620] ffffffffc0b72b3c [83317.331621] ffffffffc0b72b3c [83317.331621] ffff881017a75800 [83317.331621] [83317.331622] ffff88200be50c00 [83317.331623] ffff88201c80be18 [83317.331623] ffff88201c80fe18 [83317.331624] ffffffffc0b737cc [83317.331624] ffffffffc0b737cc [83317.331624] ffff881017a75800 [83317.331625] ffff88200be50c00 [83317.331625] ffff88200000000c [83317.331626] [83317.331626] ffff882000000019 [83317.331626] Call Trace: [83317.331627] Call Trace: [83317.331632] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331637] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331655] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331673] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331691] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331709] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331731] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331752] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331753] [] ? wake_up_state+0x20/0x20 [83317.331755] [] ? wake_up_state+0x20/0x20 [83317.331774] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331796] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331797] [] ? kthread+0xcf/0xe0 [83317.331798] [] ? kthread+0xcf/0xe0 [83317.331800] [] ? insert_kthread_work+0x40/0x40 [83317.331801] [] ? insert_kthread_work+0x40/0x40 [83317.331802] [] ? ret_from_fork+0x58/0x90 [83317.331804] [] ? ret_from_fork+0x58/0x90 [83317.331805] [] ? insert_kthread_work+0x40/0x40 [83317.331806] [] ? insert_kthread_work+0x40/0x40 [83317.331807] Task dump for CPU 18: [83317.331807] Task dump for CPU 1: [83317.331808] ldlm_bl_02 R [83317.331808] ldlm_bl_08 R [83317.331809] running task [83317.331809] 0 15061 2 0x00000088 [83317.331810] running task 0 16010 2 0x00000088 [83317.331811] ffffffffc084ad95 [83317.331811] ffffffffc084ad95 [83317.331812] 00000000c084afeb [83317.331812] 00000000c084afeb [83317.331813] 0000000000000000 [83317.331813] 0000000000000000 [83317.331814] ffff882011eb8c00 [83317.331814] [83317.331815] ffff882011eb8c00 [83317.331815] ffff882012edeeb0 [83317.331816] ffff8810e9242f70 [83317.331816] ffff88203d3bfde8 [83317.331817] ffff88201c7fbde8 [83317.331817] ffffffffc0b72b3c [83317.331818] ffffffffc0b72b3c [83317.331818] ffff88200f638000 [83317.331819] [83317.331819] ffff881017a74c00 [83317.331820] ffff88203d3bfe18 [83317.331820] ffff88201c7fbe18 [83317.331821] ffffffffc0b737cc [83317.331821] ffffffffc0b737cc [83317.331822] ffff88200f638000 [83317.331822] ffff881017a74c00 [83317.331823] ffff88200000000d [83317.331823] [83317.331823] ffff882000000111 [83317.331824] Call Trace: [83317.331824] Call Trace: [83317.331829] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331834] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.331852] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331871] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.331888] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331907] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.331926] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331948] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.331949] [] ? wake_up_state+0x20/0x20 [83317.331950] [] ? wake_up_state+0x20/0x20 [83317.331970] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331991] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.331992] [] ? kthread+0xcf/0xe0 [83317.331994] [] ? kthread+0xcf/0xe0 [83317.331995] [] ? insert_kthread_work+0x40/0x40 [83317.331996] [] ? insert_kthread_work+0x40/0x40 [83317.331998] [] ? ret_from_fork+0x58/0x90 [83317.331999] [] ? ret_from_fork+0x58/0x90 [83317.332000] [] ? insert_kthread_work+0x40/0x40 [83317.332001] [] ? insert_kthread_work+0x40/0x40 [83317.332002] Task dump for CPU 20: [83317.332002] Task dump for CPU 2: [83317.332003] ldlm_bl_01 R [83317.332003] ldlm_bl_06 R [83317.332004] running task [83317.332005] 0 15060 2 0x00000088 [83317.332005] running task 0 16008 2 0x00000088 [83317.332006] ffffffffc084ad95 [83317.332007] ffffffffc084ad95 [83317.332007] 00000000c084afeb [83317.332008] 00000000c084afeb [83317.332008] 0000000000000000 [83317.332008] 0000000000000000 [83317.332009] ffff882011eb8c00 [83317.332009] [83317.332010] ffff882011eb8c00 [83317.332010] ffff882012eddee0 [83317.332011] ffff8810247eeeb0 [83317.332011] ffff88203d3bbde8 [83317.332012] ffff88201c7ebde8 [83317.332012] ffffffffc0b72b3c [83317.332013] ffffffffc0b72b3c [83317.332013] ffff881039384c00 [83317.332013] [83317.332014] ffff88102af4e400 [83317.332015] ffff88203d3bbe18 [83317.332015] ffff88201c7ebe18 [83317.332016] ffffffffc0b737cc [83317.332016] ffffffffc0b737cc [83317.332017] ffff881039384c00 [83317.332017] ffff88102af4e400 [83317.332018] ffff88200000003d [83317.332018] [83317.332018] ffff88200000000d [83317.332019] Call Trace: [83317.332019] Call Trace: [83317.332024] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.332029] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.332046] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.332065] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.332082] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.332101] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.332120] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.332141] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.332142] [] ? wake_up_state+0x20/0x20 [83317.332144] [] ? wake_up_state+0x20/0x20 [83317.332164] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.332184] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.332185] [] ? kthread+0xcf/0xe0 [83317.332187] [] ? kthread+0xcf/0xe0 [83317.332188] [] ? insert_kthread_work+0x40/0x40 [83317.332189] [] ? insert_kthread_work+0x40/0x40 [83317.332191] [] ? ret_from_fork+0x58/0x90 [83317.332192] [] ? ret_from_fork+0x58/0x90 [83317.332193] [] ? insert_kthread_work+0x40/0x40 [83317.332195] [] ? insert_kthread_work+0x40/0x40 [83317.332195] Task dump for CPU 22: [83317.332195] Task dump for CPU 8: [83317.332196] ldlm_bl_07 R [83317.332197] ldlm_bl_10 R [83317.332197] running task [83317.332198] 0 16009 2 0x00000088 [83317.332198] running task 0 16012 2 0x00000088 [83317.332199] ffff8810e9240fd0 [83317.332200] ffff88203ad89fa0 [83317.332200] 0000000068242ff6 [83317.332201] 00000000c388efa9 [83317.332201] ffff88103d0c3db8 [83317.332202] ffff88103cf03db8 [83317.332202] ffffffff810c3a98 [83317.332202] [83317.332203] ffffffff810c3a98 [83317.332203] 0000000000000016 [83317.332204] 0000000000000008 [83317.332204] ffffffff81a552c0 [83317.332205] ffffffff81a551c0 [83317.332205] ffff88103d0c3dd0 [83317.332206] ffff88103cf03dd0 [83317.332206] ffffffff810c7429 [83317.332207] [83317.332207] ffffffff810c7429 [83317.332208] 0000000000000007 [83317.332208] 0000000000000009 [83317.332209] ffff88103d0c3e00 [83317.332209] ffff88103cf03e00 [83317.332210] ffffffff81137b10 [83317.332210] ffffffff81137b10 [83317.332211] ffff88103d0d0260 [83317.332211] [83317.332211] ffff88103cf10260 [83317.332212] Call Trace: [83317.332212] Call Trace: [83317.332213] [83317.332213] [83317.332215] [] sched_show_task+0xa8/0x110 [83317.332216] [] sched_show_task+0xa8/0x110 [83317.332218] [] dump_cpu_task+0x39/0x70 [83317.332219] [] dump_cpu_task+0x39/0x70 [83317.332221] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.332222] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.332224] [] rcu_check_callbacks+0x442/0x730 [83317.332225] [] rcu_check_callbacks+0x442/0x730 [83317.332227] [] ? tick_sched_do_timer+0x50/0x50 [83317.332228] [] ? tick_sched_do_timer+0x50/0x50 [83317.332230] [] update_process_times+0x46/0x80 [83317.332231] [] update_process_times+0x46/0x80 [83317.332232] [] tick_sched_handle+0x30/0x70 [83317.332233] [] tick_sched_handle+0x30/0x70 [83317.332235] [] tick_sched_timer+0x39/0x80 [83317.332236] [] tick_sched_timer+0x39/0x80 [83317.332238] [] __hrtimer_run_queues+0xd4/0x260 [83317.332240] [] __hrtimer_run_queues+0xd4/0x260 [83317.332241] [] hrtimer_interrupt+0xaf/0x1d0 [83317.332243] [] hrtimer_interrupt+0xaf/0x1d0 [83317.332245] [] local_apic_timer_interrupt+0x35/0x60 [83317.332246] [] local_apic_timer_interrupt+0x35/0x60 [83317.332248] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.332250] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.332251] [] apic_timer_interrupt+0x6d/0x80 [83317.332253] [] apic_timer_interrupt+0x6d/0x80 [83317.332253] [83317.332254] [83317.332256] [] ? native_queued_spin_lock_slowpath+0x112/0x1e0 [83317.332257] [] ? native_queued_spin_lock_slowpath+0x112/0x1e0 [83317.332259] [] queued_spin_lock_slowpath+0xb/0xf [83317.332261] [] queued_spin_lock_slowpath+0xb/0xf [83317.332262] [] _raw_spin_lock+0x20/0x30 [83317.332264] [] _raw_spin_lock+0x20/0x30 [83317.332281] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.332299] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.332319] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.332339] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.332358] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.332377] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.332395] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.332413] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.332430] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.332448] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.332465] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.332484] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.332489] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.332494] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.332512] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.332530] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.332548] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.332566] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.332571] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.332576] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.332596] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.332614] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.332631] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.332649] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.332671] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.332693] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.332694] [] ? wake_up_state+0x20/0x20 [83317.332695] [] ? wake_up_state+0x20/0x20 [83317.332715] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.332737] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.332738] [] kthread+0xcf/0xe0 [83317.332739] [] kthread+0xcf/0xe0 [83317.332741] [] ? insert_kthread_work+0x40/0x40 [83317.332742] [] ? insert_kthread_work+0x40/0x40 [83317.332743] [] ret_from_fork+0x58/0x90 [83317.332745] [] ret_from_fork+0x58/0x90 [83317.332746] [] ? insert_kthread_work+0x40/0x40 [83317.332747] [] ? insert_kthread_work+0x40/0x40 [83317.332748] Task dump for CPU 10: [83317.332749] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.332750] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.332752] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83317.332753] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.332753] Call Trace: [83317.332759] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.332777] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.332796] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.332816] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.332818] [] ? wake_up_state+0x20/0x20 [83317.332838] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.332840] [] ? kthread+0xcf/0xe0 [83317.332841] [] ? insert_kthread_work+0x40/0x40 [83317.332843] [] ? ret_from_fork+0x58/0x90 [83317.332844] [] ? insert_kthread_work+0x40/0x40 [83317.332844] Task dump for CPU 12: [83317.332845] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.332847] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.332848] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.332849] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.332849] Call Trace: [83317.332854] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.332873] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.332891] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.332912] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.332913] [] ? wake_up_state+0x20/0x20 [83317.332934] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.332935] [] ? kthread+0xcf/0xe0 [83317.332937] [] ? insert_kthread_work+0x40/0x40 [83317.332938] [] ? ret_from_fork+0x58/0x90 [83317.332939] [] ? insert_kthread_work+0x40/0x40 [83317.332940] Task dump for CPU 16: [83317.332940] Task dump for CPU 0: [83317.332941] ldlm_bl_12 R [83317.332941] ldlm_bl_13 R [83317.332941] running task [83317.332942] 0 16014 2 0x00000088 [83317.332942] running task 0 16015 2 0x00000088 [83317.332943] ffffffffc084ad95 [83317.332943] ffffffffc084ad95 [83317.332943] 00000000c084afeb [83317.332944] 00000000c084afeb [83317.332944] 0000000000000000 [83317.332944] 0000000000000000 [83317.332945] ffff882011eb8c00 [83317.332945] [83317.332945] ffff882011eb8c00 [83317.332946] ffff88203d33af70 [83317.332946] ffff88203d33bf40 [83317.332946] ffff88201c80bde8 [83317.332947] ffff88201c80fde8 [83317.332947] ffffffffc0b72b3c [83317.332948] ffffffffc0b72b3c [83317.332948] ffff881017a75800 [83317.332948] [83317.332948] ffff88200be50c00 [83317.332949] ffff88201c80be18 [83317.332949] ffff88201c80fe18 [83317.332950] ffffffffc0b737cc [83317.332950] ffffffffc0b737cc [83317.332950] ffff881017a75800 [83317.332951] ffff88200be50c00 [83317.332951] ffff88200000000c [83317.332951] [83317.332951] ffff882000000019 [83317.332951] Call Trace: [83317.332952] Call Trace: [83317.332957] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.332962] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.332980] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.332999] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333017] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333036] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333057] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333077] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333079] [] ? wake_up_state+0x20/0x20 [83317.333080] [] ? wake_up_state+0x20/0x20 [83317.333101] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333121] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333123] [] ? kthread+0xcf/0xe0 [83317.333123] [] ? kthread+0xcf/0xe0 [83317.333125] [] ? insert_kthread_work+0x40/0x40 [83317.333126] [] ? insert_kthread_work+0x40/0x40 [83317.333127] [] ? ret_from_fork+0x58/0x90 [83317.333128] [] ? ret_from_fork+0x58/0x90 [83317.333129] [] ? insert_kthread_work+0x40/0x40 [83317.333130] [] ? insert_kthread_work+0x40/0x40 [83317.333130] Task dump for CPU 18: [83317.333131] Task dump for CPU 1: [83317.333131] ldlm_bl_02 R [83317.333132] ldlm_bl_08 R [83317.333132] running task [83317.333132] 0 15061 2 0x00000088 [83317.333133] running task 0 16010 2 0x00000088 [83317.333134] ffffffffc084ad95 [83317.333134] ffffffffc084ad95 [83317.333134] 00000000c084afeb [83317.333134] 00000000c084afeb [83317.333135] 0000000000000000 [83317.333135] 0000000000000000 [83317.333135] ffff882011eb8c00 [83317.333136] [83317.333136] ffff882011eb8c00 [83317.333136] ffff882012edeeb0 [83317.333137] ffff8810e9242f70 [83317.333137] ffff88203d3bfde8 [83317.333137] ffff88201c7fbde8 [83317.333137] ffffffffc0b72b3c [83317.333138] ffffffffc0b72b3c [83317.333138] ffff88200f638000 [83317.333138] [83317.333139] ffff881017a74c00 [83317.333139] ffff88203d3bfe18 [83317.333139] ffff88201c7fbe18 [83317.333140] ffffffffc0b737cc [83317.333140] ffffffffc0b737cc [83317.333140] ffff88200f638000 [83317.333140] ffff881017a74c00 [83317.333141] ffff88200000000d [83317.333141] [83317.333141] ffff882000000111 [83317.333141] Call Trace: [83317.333142] Call Trace: [83317.333147] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333151] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333170] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333188] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333207] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333225] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333247] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333267] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333268] [] ? wake_up_state+0x20/0x20 [83317.333269] [] ? wake_up_state+0x20/0x20 [83317.333290] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333311] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333312] [] ? kthread+0xcf/0xe0 [83317.333313] [] ? kthread+0xcf/0xe0 [83317.333314] [] ? insert_kthread_work+0x40/0x40 [83317.333315] [] ? insert_kthread_work+0x40/0x40 [83317.333316] [] ? ret_from_fork+0x58/0x90 [83317.333317] [] ? ret_from_fork+0x58/0x90 [83317.333318] [] ? insert_kthread_work+0x40/0x40 [83317.333319] [] ? insert_kthread_work+0x40/0x40 [83317.333320] Task dump for CPU 20: [83317.333320] Task dump for CPU 2: [83317.333321] ldlm_bl_01 R [83317.333321] ldlm_bl_06 R [83317.333321] running task [83317.333322] 0 15060 2 0x00000088 [83317.333322] running task 0 16008 2 0x00000088 [83317.333323] ffffffffc084ad95 [83317.333323] ffffffffc084ad95 [83317.333323] 00000000c084afeb [83317.333324] 00000000c084afeb [83317.333324] 0000000000000000 [83317.333324] 0000000000000000 [83317.333324] ffff882011eb8c00 [83317.333325] [83317.333325] ffff882011eb8c00 [83317.333325] ffff882012eddee0 [83317.333326] ffff8810247eeeb0 [83317.333326] ffff88203d3bbde8 [83317.333326] ffff88201c7ebde8 [83317.333327] ffffffffc0b72b3c [83317.333327] ffffffffc0b72b3c [83317.333327] ffff881039384c00 [83317.333327] [83317.333328] ffff88102af4e400 [83317.333328] ffff88203d3bbe18 [83317.333328] ffff88201c7ebe18 [83317.333329] ffffffffc0b737cc [83317.333329] ffffffffc0b737cc [83317.333329] ffff881039384c00 [83317.333330] ffff88102af4e400 [83317.333330] ffff88200000003d [83317.333330] [83317.333330] ffff88200000000d [83317.333331] Call Trace: [83317.333331] Call Trace: [83317.333336] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333340] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333359] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333378] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333396] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333414] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333435] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333457] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333458] [] ? wake_up_state+0x20/0x20 [83317.333459] [] ? wake_up_state+0x20/0x20 [83317.333479] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333500] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333501] [] ? kthread+0xcf/0xe0 [83317.333502] [] ? kthread+0xcf/0xe0 [83317.333503] [] ? insert_kthread_work+0x40/0x40 [83317.333504] [] ? insert_kthread_work+0x40/0x40 [83317.333506] [] ? ret_from_fork+0x58/0x90 [83317.333507] [] ? ret_from_fork+0x58/0x90 [83317.333508] [] ? insert_kthread_work+0x40/0x40 [83317.333509] [] ? insert_kthread_work+0x40/0x40 [83317.333509] Task dump for CPU 22: [83317.333510] Task dump for CPU 8: [83317.333510] ldlm_bl_07 R [83317.333510] ldlm_bl_10 R [83317.333511] running task [83317.333511] 0 16009 2 0x00000088 [83317.333512] running task 0 16012 2 0x00000088 [83317.333512] ffffffffc084ad95 [83317.333513] ffffffffc084ad95 [83317.333513] 00000000c084afeb [83317.333513] 00000000c084afeb [83317.333514] 0000000000000000 [83317.333514] 0000000000000000 [83317.333514] ffff882011eb8c00 [83317.333514] [83317.333515] ffff882011eb8c00 [83317.333515] ffff8810e9240fd0 [83317.333515] ffff88203ad89fa0 [83317.333516] ffff88201c7efde8 [83317.333516] ffff88201c803de8 [83317.333516] ffffffffc0b72b3c [83317.333517] ffffffffc0b72b3c [83317.333517] ffff88200f63f400 [83317.333517] [83317.333517] ffff882012e84400 [83317.333518] ffff88201c7efe18 [83317.333518] ffff88201c803e18 [83317.333518] ffffffffc0b737cc [83317.333519] ffffffffc0b737cc [83317.333519] ffff88200f63f400 [83317.333519] ffff882012e84400 [83317.333520] ffff88200000000d [83317.333520] [83317.333520] ffff88200000000d [83317.333520] Call Trace: [83317.333520] Call Trace: [83317.333526] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333530] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333548] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333567] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333586] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333605] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333626] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333646] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333647] [] ? wake_up_state+0x20/0x20 [83317.333648] [] ? wake_up_state+0x20/0x20 [83317.333669] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333689] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333690] [] ? kthread+0xcf/0xe0 [83317.333691] [] ? kthread+0xcf/0xe0 [83317.333692] [] ? insert_kthread_work+0x40/0x40 [83317.333693] [] ? insert_kthread_work+0x40/0x40 [83317.333694] [] ? ret_from_fork+0x58/0x90 [83317.333696] [] ? ret_from_fork+0x58/0x90 [83317.333697] [] ? insert_kthread_work+0x40/0x40 [83317.333698] [] ? insert_kthread_work+0x40/0x40 [83317.333698] Task dump for CPU 10: [83317.333699] ldlm_bl_05 R running task 0 16007 2 0x00000088 [83317.333701] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.333702] ffff8810247ecf10 ffff88201c64fde8 ffffffffc0b72b3c ffff88202a732400 [83317.333703] ffff88201c64fe18 ffffffffc0b737cc ffff88202a732400 ffff882000000105 [83317.333703] Call Trace: [83317.333709] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333727] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333744] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333765] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333766] [] ? wake_up_state+0x20/0x20 [83317.333786] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333787] [] ? kthread+0xcf/0xe0 [83317.333789] [] ? insert_kthread_work+0x40/0x40 [83317.333790] [] ? ret_from_fork+0x58/0x90 [83317.333791] [] ? insert_kthread_work+0x40/0x40 [83317.333792] Task dump for CPU 12: [83317.333793] ldlm_bl_15 R running task 0 16017 2 0x00000088 [83317.333794] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.333795] ffff88203d338fd0 ffff88201c817de8 ffffffffc0b72b3c ffff880ffef55400 [83317.333796] ffff88201c817e18 ffffffffc0b737cc ffff880ffef55400 ffff88200000000d [83317.333796] Call Trace: [83317.333802] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333820] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333837] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333859] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333860] [] ? wake_up_state+0x20/0x20 [83317.333881] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333882] [] ? kthread+0xcf/0xe0 [83317.333883] [] ? insert_kthread_work+0x40/0x40 [83317.333884] [] ? ret_from_fork+0x58/0x90 [83317.333885] [] ? insert_kthread_work+0x40/0x40 [83317.333886] Task dump for CPU 16: [83317.333887] ldlm_bl_12 R running task 0 16014 2 0x00000088 [83317.333888] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.333889] ffff88203d33af70 ffff88201c80bde8 ffffffffc0b72b3c ffff881017a75800 [83317.333891] ffff88201c80be18 ffffffffc0b737cc ffff881017a75800 ffff88200000000c [83317.333891] Call Trace: [83317.333896] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.333914] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.333931] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.333951] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.333952] [] ? wake_up_state+0x20/0x20 [83317.333973] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.333974] [] ? kthread+0xcf/0xe0 [83317.333975] [] ? insert_kthread_work+0x40/0x40 [83317.333976] [] ? ret_from_fork+0x58/0x90 [83317.333977] [] ? insert_kthread_work+0x40/0x40 [83317.333978] Task dump for CPU 18: [83317.333979] ldlm_bl_02 R running task 0 15061 2 0x00000088 [83317.333980] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.333982] ffff882012edeeb0 ffff88203d3bfde8 ffffffffc0b72b3c ffff88200f638000 [83317.333983] ffff88203d3bfe18 ffffffffc0b737cc ffff88200f638000 ffff88200000000d [83317.333983] Call Trace: [83317.333988] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.334008] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.334025] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.334045] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.334046] [] ? wake_up_state+0x20/0x20 [83317.334066] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.334067] [] ? kthread+0xcf/0xe0 [83317.334069] [] ? insert_kthread_work+0x40/0x40 [83317.334070] [] ? ret_from_fork+0x58/0x90 [83317.334071] [] ? insert_kthread_work+0x40/0x40 [83317.334071] Task dump for CPU 20: [83317.334072] ldlm_bl_01 R running task 0 15060 2 0x00000088 [83317.334074] ffff882012eddee0 00000000059359c0 ffff88103d083db8 ffffffff810c3a98 [83317.334075] 0000000000000014 ffffffff81a552c0 ffff88103d083dd0 ffffffff810c7429 [83317.334076] 0000000000000005 ffff88103d083e00 ffffffff81137b10 ffff88103d090260 [83317.334076] Call Trace: [83317.334078] [] sched_show_task+0xa8/0x110 [83317.334079] [] dump_cpu_task+0x39/0x70 [83317.334081] [] rcu_dump_cpu_stacks+0x90/0xd0 [83317.334082] [] rcu_check_callbacks+0x442/0x730 [83317.334084] [] ? tick_sched_do_timer+0x50/0x50 [83317.334085] [] update_process_times+0x46/0x80 [83317.334086] [] tick_sched_handle+0x30/0x70 [83317.334087] [] tick_sched_timer+0x39/0x80 [83317.334089] [] __hrtimer_run_queues+0xd4/0x260 [83317.334090] [] hrtimer_interrupt+0xaf/0x1d0 [83317.334092] [] local_apic_timer_interrupt+0x35/0x60 [83317.334094] [] smp_apic_timer_interrupt+0x3d/0x50 [83317.334095] [] apic_timer_interrupt+0x6d/0x80 [83317.334097] [] ? native_queued_spin_lock_slowpath+0x116/0x1e0 [83317.334099] [] queued_spin_lock_slowpath+0xb/0xf [83317.334100] [] _raw_spin_lock+0x20/0x30 [83317.334118] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83317.334137] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83317.334156] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83317.334174] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83317.334191] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83317.334209] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83317.334214] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83317.334232] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.334250] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83317.334256] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.334275] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.334293] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.334315] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.334316] [] ? wake_up_state+0x20/0x20 [83317.334337] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.334338] [] kthread+0xcf/0xe0 [83317.334339] [] ? insert_kthread_work+0x40/0x40 [83317.334341] [] ret_from_fork+0x58/0x90 [83317.334342] [] ? insert_kthread_work+0x40/0x40 [83317.334342] Task dump for CPU 22: [83317.334343] ldlm_bl_07 R running task 0 16009 2 0x00000088 [83317.334345] ffffffffc084ad95 00000000c084afeb 0000000000000000 ffff882011eb8c00 [83317.334346] ffff8810e9240fd0 ffff88201c7efde8 ffffffffc0b72b3c ffff88200f63f400 [83317.334347] ffff88201c7efe18 ffffffffc0b737cc ffff88200f63f400 ffff88200000000d [83317.334347] Call Trace: [83317.334352] [] ? cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83317.334370] [] ? ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83317.334388] [] ? ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83317.334409] [] ? ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83317.334411] [] ? wake_up_state+0x20/0x20 [83317.334431] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83317.334433] [] ? kthread+0xcf/0xe0 [83317.334434] [] ? insert_kthread_work+0x40/0x40 [83317.334435] [] ? ret_from_fork+0x58/0x90 [83317.334436] [] ? insert_kthread_work+0x40/0x40 [83319.060901] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ldlm_bl_16:33189] [83319.060924] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83319.060931] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83319.060932] CPU: 3 PID: 33189 Comm: ldlm_bl_16 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83319.060933] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83319.060933] task: ffff88102c376eb0 ti: ffff881fefb44000 task.ti: ffff881fefb44000 [83319.060936] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83319.060936] RSP: 0018:ffff881fefb47b70 EFLAGS: 00000246 [83319.060937] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000190000 [83319.060937] RDX: ffff88203c8d7880 RSI: 0000000000b90001 RDI: ffff88102137981c [83319.060938] RBP: ffff881fefb47b70 R08: ffff88203c657880 R09: 0000000000000000 [83319.060938] R10: ffff882029a58800 R11: 0000000000000000 R12: ffff88102c376f18 [83319.060939] R13: ffff88203c2b92a0 R14: ffff881000000141 R15: ffff88207ffd8008 [83319.060940] FS: 0000000000000000(0000) GS:ffff88203c640000(0000) knlGS:0000000000000000 [83319.060941] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83319.060941] CR2: 000055ce548c9220 CR3: 00000000019f2000 CR4: 00000000001407e0 [83319.060942] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83319.060943] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83319.060943] Stack: [83319.060944] ffff881fefb47b80 ffffffff8169e61f ffff881fefb47b90 ffffffff816abb70 [83319.060946] ffff881fefb47bd0 ffffffffc0b72198 0000000000000000 ffff882029a58800 [83319.060947] ffff881fefb47c18 ffff882029a58860 ffff882029a58800 ffff8810175b8000 [83319.060947] Call Trace: [83319.060949] [] queued_spin_lock_slowpath+0xb/0xf [83319.060950] [] _raw_spin_lock+0x20/0x30 [83319.060969] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83319.060990] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83319.061009] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83319.061027] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83319.061044] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83319.061062] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83319.061067] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83319.061085] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83319.061102] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83319.061108] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83319.061128] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83319.061146] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83319.061168] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83319.061169] [] ? __schedule+0x39d/0x8b0 [83319.061189] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83319.061190] [] kthread+0xcf/0xe0 [83319.061192] [] ? insert_kthread_work+0x40/0x40 [83319.061193] [] ret_from_fork+0x58/0x90 [83319.061194] [] ? insert_kthread_work+0x40/0x40 [83319.061208] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83319.063901] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ldlm_bl_09:16011] [83319.063928] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83319.063937] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83319.063938] CPU: 4 PID: 16011 Comm: ldlm_bl_09 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83319.063939] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83319.063940] task: ffff88203ad88fd0 ti: ffff88201c7fc000 task.ti: ffff88201c7fc000 [83319.063942] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83319.063943] RSP: 0018:ffff88201c7ffb70 EFLAGS: 00000246 [83319.063943] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000210000 [83319.063944] RDX: ffff88103cf97880 RSI: 0000000000610000 RDI: ffff88102137981c [83319.063945] RBP: ffff88201c7ffb70 R08: ffff88103ce97880 R09: 0000000000000000 [83319.063945] R10: ffff88202b227800 R11: 0000000000000000 R12: ffff88201c7ffc58 [83319.063946] R13: 0000000000000001 R14: ffff88201c7ffb28 R15: ffffffff81322c35 [83319.063947] FS: 0000000000000000(0000) GS:ffff88103ce80000(0000) knlGS:0000000000000000 [83319.063947] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83319.063948] CR2: 00007f8905462248 CR3: 00000000019f2000 CR4: 00000000001407e0 [83319.063948] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83319.063949] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83319.063949] Stack: [83319.063951] ffff88201c7ffb80 ffffffff8169e61f ffff88201c7ffb90 ffffffff816abb70 [83319.063952] ffff88201c7ffbd0 ffffffffc0b72198 0000000000000000 ffff88202b227800 [83319.063953] ffff88201c7ffc18 ffff88202b227860 ffff88202b227800 ffff8810175b8000 [83319.063953] Call Trace: [83319.063955] [] queued_spin_lock_slowpath+0xb/0xf [83319.063957] [] _raw_spin_lock+0x20/0x30 [83319.063977] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83319.063999] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83319.064018] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83319.064035] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83319.064053] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83319.064070] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83319.064076] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83319.064094] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83319.064111] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83319.064116] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83319.064136] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83319.064154] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83319.064176] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83319.064178] [] ? wake_up_state+0x20/0x20 [83319.064198] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83319.064199] [] kthread+0xcf/0xe0 [83319.064201] [] ? insert_kthread_work+0x40/0x40 [83319.064202] [] ret_from_fork+0x58/0x90 [83319.064203] [] ? insert_kthread_work+0x40/0x40 [83319.064217] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83319.066901] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [ldlm_bl_11:16013] [83319.066924] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83319.066931] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83319.066932] CPU: 5 PID: 16013 Comm: ldlm_bl_11 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83319.066932] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83319.066933] task: ffff88203ad88000 ti: ffff88201c804000 task.ti: ffff88201c804000 [83319.066936] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83319.066936] RSP: 0018:ffff88201c807b70 EFLAGS: 00000246 [83319.066937] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000290000 [83319.066938] RDX: ffff88103d017880 RSI: 0000000000810001 RDI: ffff88102137981c [83319.066938] RBP: ffff88201c807b70 R08: ffff88203c697880 R09: 0000000000000000 [83319.066939] R10: ffff882029b7b000 R11: 0000000000000000 R12: ffff88201c807c58 [83319.066939] R13: 0000000000000001 R14: ffff88201c807b28 R15: ffffffff81322c35 [83319.066940] FS: 0000000000000000(0000) GS:ffff88203c680000(0000) knlGS:0000000000000000 [83319.066941] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83319.066941] CR2: 00007fd7ddcd8090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83319.066942] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83319.066942] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83319.066942] Stack: [83319.066944] ffff88201c807b80 ffffffff8169e61f ffff88201c807b90 ffffffff816abb70 [83319.066945] ffff88201c807bd0 ffffffffc0b72198 0000000000000000 ffff882029b7b000 [83319.066946] ffff88201c807c18 ffff882029b7b060 ffff882029b7b000 ffff8810175b8000 [83319.066946] Call Trace: [83319.066948] [] queued_spin_lock_slowpath+0xb/0xf [83319.066950] [] _raw_spin_lock+0x20/0x30 [83319.066970] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83319.066991] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83319.067010] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83319.067028] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83319.067045] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83319.067063] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83319.067069] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83319.067086] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83319.067104] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83319.067109] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83319.067129] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83319.067147] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83319.067169] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83319.067171] [] ? wake_up_state+0x20/0x20 [83319.067191] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83319.067192] [] kthread+0xcf/0xe0 [83319.067194] [] ? insert_kthread_work+0x40/0x40 [83319.067195] [] ret_from_fork+0x58/0x90 [83319.067196] [] ? insert_kthread_work+0x40/0x40 [83319.067210] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83319.072901] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ldlm_bl_17:33190] [83319.072923] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83319.072930] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83319.072931] CPU: 7 PID: 33190 Comm: ldlm_bl_17 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83319.072932] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83319.072933] task: ffff88102d281fa0 ti: ffff881fffd34000 task.ti: ffff881fffd34000 [83319.072935] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83319.072935] RSP: 0018:ffff881fffd37b70 EFLAGS: 00000246 [83319.072936] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000390000 [83319.072937] RDX: ffff88103cf17880 RSI: 0000000000410001 RDI: ffff88102137981c [83319.072937] RBP: ffff881fffd37b70 R08: ffff88203c6d7880 R09: 0000000000000000 [83319.072938] R10: ffff882029a34400 R11: 0000000000000000 R12: ffff88102d282008 [83319.072938] R13: ffff881fffd37b00 R14: ffff881f00000141 R15: ffff88207ffd8008 [83319.072939] FS: 0000000000000000(0000) GS:ffff88203c6c0000(0000) knlGS:0000000000000000 [83319.072940] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83319.072940] CR2: 00007f240470f978 CR3: 00000000019f2000 CR4: 00000000001407e0 [83319.072941] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83319.072941] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83319.072942] Stack: [83319.072943] ffff881fffd37b80 ffffffff8169e61f ffff881fffd37b90 ffffffff816abb70 [83319.072944] ffff881fffd37bd0 ffffffffc0b72198 0000000000000000 ffff882029a34400 [83319.072945] ffff881fffd37c18 ffff882029a34460 ffff882029a34400 ffff8810175b8000 [83319.072945] Call Trace: [83319.072947] [] queued_spin_lock_slowpath+0xb/0xf [83319.072949] [] _raw_spin_lock+0x20/0x30 [83319.072969] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83319.072991] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83319.073010] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83319.073027] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83319.073045] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83319.073063] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83319.073068] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83319.073085] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83319.073103] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83319.073108] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83319.073128] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83319.073146] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83319.073167] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83319.073168] [] ? __schedule+0x39d/0x8b0 [83319.073188] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83319.073189] [] kthread+0xcf/0xe0 [83319.073191] [] ? insert_kthread_work+0x40/0x40 [83319.073192] [] ret_from_fork+0x58/0x90 [83319.073193] [] ? insert_kthread_work+0x40/0x40 [83319.073207] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83331.069642] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [ldlm_bl_14:16016] [83331.069666] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83331.069673] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83331.069675] CPU: 6 PID: 16016 Comm: ldlm_bl_14 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83331.069675] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83331.069676] task: ffff88203d338000 ti: ffff88201c810000 task.ti: ffff88201c810000 [83331.069678] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83331.069679] RSP: 0018:ffff88201c813b70 EFLAGS: 00000246 [83331.069680] RAX: 0000000000000000 RBX: 000000011c813be8 RCX: 0000000000310000 [83331.069680] RDX: ffff88103ce57880 RSI: 0000000000110000 RDI: ffff88102137981c [83331.069681] RBP: ffff88201c813b70 R08: ffff88103ced7880 R09: 0000000000000000 [83331.069681] R10: ffff88202b20cc00 R11: 0000000000000000 R12: ffff88201c813c58 [83331.069682] R13: 0000000000000001 R14: ffff88201c813b28 R15: ffffffff81322c35 [83331.069683] FS: 0000000000000000(0000) GS:ffff88103cec0000(0000) knlGS:0000000000000000 [83331.069684] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83331.069684] CR2: 0000000002079728 CR3: 00000000019f2000 CR4: 00000000001407e0 [83331.069685] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83331.069685] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83331.069685] Stack: [83331.069687] ffff88201c813b80 ffffffff8169e61f ffff88201c813b90 ffffffff816abb70 [83331.069688] ffff88201c813bd0 ffffffffc0b72198 0000000000000000 ffff88202b20cc00 [83331.069689] ffff88201c813c18 ffff88202b20cc60 ffff88202b20cc00 ffff8810175b8000 [83331.069689] Call Trace: [83331.069691] [] queued_spin_lock_slowpath+0xb/0xf [83331.069693] [] _raw_spin_lock+0x20/0x30 [83331.069715] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83331.069736] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83331.069756] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83331.069773] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83331.069791] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83331.069808] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83331.069814] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83331.069832] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83331.069849] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83331.069854] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83331.069871] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83331.069889] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83331.069911] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83331.069913] [] ? wake_up_state+0x20/0x20 [83331.069933] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83331.069934] [] kthread+0xcf/0xe0 [83331.069936] [] ? insert_kthread_work+0x40/0x40 [83331.069937] [] ret_from_fork+0x58/0x90 [83331.069938] [] ? insert_kthread_work+0x40/0x40 [83331.069952] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83334.992178] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83335.071640] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83335.100509] CPU: 9 PID: 15915 Comm: ldlm_bl_03 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83335.113221] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83335.121569] task: ffff88202c648000 ti: ffff88201763c000 task.ti: ffff88201763c000 [83335.129917] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83335.140989] RSP: 0018:ffff88201763fb70 EFLAGS: 00000246 [83335.146913] RAX: 0000000000000000 RBX: 0000000300000000 RCX: 0000000000490000 [83335.154873] RDX: ffff88103d097880 RSI: 0000000000a10000 RDI: ffff88102137981c [83335.162833] RBP: ffff88201763fb70 R08: ffff88203c717880 R09: 0000000000000000 [83335.170793] R10: ffff882029a5aa00 R11: 7fffffffffffffff R12: 0000000000000bf8 [83335.178753] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c656cc0 [83335.186713] FS: 0000000000000000(0000) GS:ffff88203c700000(0000) knlGS:0000000000000000 [83335.195739] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83335.202148] CR2: 00007ffa010a6000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83335.210108] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83335.218068] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83335.226028] Stack: [83335.228268] ffff88201763fb80 ffffffff8169e61f ffff88201763fb90 ffffffff816abb70 [83335.236555] ffff88201763fbd0 ffffffffc0b72198 0000000000000000 ffff882029a5aa00 [83335.244842] ffff88201763fc18 ffff882029a5aa60 ffff882029a5aa00 ffff8810175b8000 [83335.253129] Call Trace: [83335.255856] [] queued_spin_lock_slowpath+0xb/0xf [83335.262846] [] _raw_spin_lock+0x20/0x30 [83335.268985] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83335.277162] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83335.285238] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83335.292634] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83335.300417] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83335.308202] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83335.315587] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83335.323661] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83335.331542] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83335.339409] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83335.347486] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83335.355949] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83335.364125] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83335.371797] [] ? wake_up_state+0x20/0x20 [83335.378031] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83335.386283] [] kthread+0xcf/0xe0 [83335.391722] [] ? insert_kthread_work+0x40/0x40 [83335.398520] [] ret_from_fork+0x58/0x90 [83335.404543] [] ? insert_kthread_work+0x40/0x40 [83335.411340] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83343.053386] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [ldlm_bl_08:16010] [83343.054385] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ldlm_bl_13:16015] [83343.054417] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.054427] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.054428] CPU: 0 PID: 16015 Comm: ldlm_bl_13 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.054429] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.054430] task: ffff88203d33bf40 ti: ffff88201c80c000 task.ti: ffff88201c80c000 [83343.054433] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.054434] RSP: 0018:ffff88201c80fb70 EFLAGS: 00000246 [83343.054435] RAX: 0000000000000000 RBX: 000000013ce56cc0 RCX: 0000000000010000 [83343.054435] RDX: ffff88103d0d7880 RSI: 0000000000b10000 RDI: ffff88102137981c [83343.054436] RBP: ffff88201c80fb70 R08: ffff88103ce17880 R09: 0000000000000000 [83343.054436] R10: ffff882029b09a00 R11: 0000000000000005 R12: ffff88201c80fc58 [83343.054437] R13: 0000000000000001 R14: ffff88201c80fb28 R15: ffffffff81322c35 [83343.054438] FS: 0000000000000000(0000) GS:ffff88103ce00000(0000) knlGS:0000000000000000 [83343.054438] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.054439] CR2: 00007f17e6a8b248 CR3: 00000000019f2000 CR4: 00000000001407f0 [83343.054440] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.054440] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.054440] Stack: [83343.054442] ffff88201c80fb80 ffffffff8169e61f ffff88201c80fb90 ffffffff816abb70 [83343.054443] ffff88201c80fbd0 ffffffffc0b72198 0000000000000000 ffff882029b09a00 [83343.054444] ffff88201c80fc18 ffff882029b09a60 ffff882029b09a00 ffff8810175b8000 [83343.054445] Call Trace: [83343.054447] [] queued_spin_lock_slowpath+0xb/0xf [83343.054449] [] _raw_spin_lock+0x20/0x30 [83343.054481] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.054504] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.054523] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.054541] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.054558] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.054577] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.054584] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.054602] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.054620] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.054625] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.054644] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.054662] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.054684] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.054686] [] ? wake_up_state+0x20/0x20 [83343.054707] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.054708] [] kthread+0xcf/0xe0 [83343.054710] [] ? insert_kthread_work+0x40/0x40 [83343.054711] [] ret_from_fork+0x58/0x90 [83343.054712] [] ? insert_kthread_work+0x40/0x40 [83343.054726] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.057385] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ldlm_bl_06:16008] [83343.057405] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.057410] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.057411] CPU: 2 PID: 16008 Comm: ldlm_bl_06 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.057411] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.057412] task: ffff8810247eeeb0 ti: ffff88201c7e8000 task.ti: ffff88201c7e8000 [83343.057415] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.057416] RSP: 0018:ffff88201c7ebb70 EFLAGS: 00000246 [83343.057416] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000110000 [83343.057417] RDX: ffff88203c897880 RSI: 0000000000a90001 RDI: ffff88102137981c [83343.057417] RBP: ffff88201c7ebb70 R08: ffff88103ce57880 R09: 0000000000000000 [83343.057418] R10: ffff88202b328200 R11: 0000000000000005 R12: ffff88201c7ebc58 [83343.057418] R13: 0000000000000001 R14: ffff88201c7ebb28 R15: ffffffff81322c35 [83343.057419] FS: 0000000000000000(0000) GS:ffff88103ce40000(0000) knlGS:0000000000000000 [83343.057419] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.057420] CR2: 0000000002577010 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.057420] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.057420] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.057421] Stack: [83343.057422] ffff88201c7ebb80 ffffffff8169e61f ffff88201c7ebb90 ffffffff816abb70 [83343.057422] ffff88201c7ebbd0 ffffffffc0b72198 0000000000000000 ffff88202b328200 [83343.057423] ffff88201c7ebc18 ffff88202b328260 ffff88202b328200 ffff8810175b8000 [83343.057423] Call Trace: [83343.057426] [] queued_spin_lock_slowpath+0xb/0xf [83343.057427] [] _raw_spin_lock+0x20/0x30 [83343.057444] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.057459] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.057474] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.057487] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.057500] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.057513] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.057518] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.057531] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.057544] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.057549] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.057562] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.057575] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.057591] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.057592] [] ? wake_up_state+0x20/0x20 [83343.057607] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.057608] [] kthread+0xcf/0xe0 [83343.057609] [] ? insert_kthread_work+0x40/0x40 [83343.057610] [] ret_from_fork+0x58/0x90 [83343.057611] [] ? insert_kthread_work+0x40/0x40 [83343.057619] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.075384] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [ldlm_bl_10:16012] [83343.075409] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.075417] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.075418] CPU: 8 PID: 16012 Comm: ldlm_bl_10 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.075419] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.075420] task: ffff88203ad89fa0 ti: ffff88201c800000 task.ti: ffff88201c800000 [83343.075423] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.075423] RSP: 0018:ffff88201c803b70 EFLAGS: 00000246 [83343.075424] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000410000 [83343.075425] RDX: ffff88203c617880 RSI: 0000000000090000 RDI: ffff88102137981c [83343.075425] RBP: ffff88201c803b70 R08: ffff88103cf17880 R09: 0000000000000000 [83343.075426] R10: ffff88202b32b000 R11: 0000000000000005 R12: ffff88201c803c58 [83343.075427] R13: 0000000000000001 R14: ffff88201c803b28 R15: ffffffff81322c35 [83343.075428] FS: 0000000000000000(0000) GS:ffff88103cf00000(0000) knlGS:0000000000000000 [83343.075428] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.075429] CR2: 00007fca10cf5000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.075430] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.075430] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.075430] Stack: [83343.075432] ffff88201c803b80 ffffffff8169e61f ffff88201c803b90 ffffffff816abb70 [83343.075433] ffff88201c803bd0 ffffffffc0b72198 0000000000000000 ffff88202b32b000 [83343.075434] ffff88201c803c18 ffff88202b32b060 ffff88202b32b000 ffff8810175b8000 [83343.075435] Call Trace: [83343.075437] [] queued_spin_lock_slowpath+0xb/0xf [83343.075438] [] _raw_spin_lock+0x20/0x30 [83343.075464] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.075486] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.075506] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.075523] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.075541] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.075559] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.075565] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.075582] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.075600] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.075606] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.075626] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.075645] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.075667] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.075669] [] ? wake_up_state+0x20/0x20 [83343.075690] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.075691] [] kthread+0xcf/0xe0 [83343.075693] [] ? insert_kthread_work+0x40/0x40 [83343.075694] [] ret_from_fork+0x58/0x90 [83343.075695] [] ? insert_kthread_work+0x40/0x40 [83343.075709] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.081384] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [ldlm_bl_05:16007] [83343.081406] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.081413] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.081415] CPU: 10 PID: 16007 Comm: ldlm_bl_05 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.081415] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.081416] task: ffff8810247ecf10 ti: ffff88201c64c000 task.ti: ffff88201c64c000 [83343.081418] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.081419] RSP: 0018:ffff88201c64fb70 EFLAGS: 00000246 [83343.081420] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000510000 [83343.081420] RDX: ffff88103ce17880 RSI: 0000000000010000 RDI: ffff88102137981c [83343.081421] RBP: ffff88201c64fb70 R08: ffff88103cf57880 R09: 0000000000000000 [83343.081422] R10: ffff88202b24a200 R11: 0000000000000005 R12: ffff88201c64fc58 [83343.081422] R13: 0000000000000001 R14: ffff88201c64fb28 R15: ffffffff81322c35 [83343.081423] FS: 0000000000000000(0000) GS:ffff88103cf40000(0000) knlGS:0000000000000000 [83343.081424] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.081424] CR2: 0000000001dd94b8 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.081425] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.081425] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.081425] Stack: [83343.081427] ffff88201c64fb80 ffffffff8169e61f ffff88201c64fb90 ffffffff816abb70 [83343.081428] ffff88201c64fbd0 ffffffffc0b72198 0000000000000000 ffff88202b24a200 [83343.081429] ffff88201c64fc18 ffff88202b24a260 ffff88202b24a200 ffff8810175b8000 [83343.081429] Call Trace: [83343.081431] [] queued_spin_lock_slowpath+0xb/0xf [83343.081433] [] _raw_spin_lock+0x20/0x30 [83343.081454] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.081475] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.081495] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.081512] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.081530] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.081547] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.081553] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.081571] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.081588] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.081593] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.081613] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.081631] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.081653] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.081655] [] ? wake_up_state+0x20/0x20 [83343.081675] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.081676] [] kthread+0xcf/0xe0 [83343.081678] [] ? insert_kthread_work+0x40/0x40 [83343.081679] [] ret_from_fork+0x58/0x90 [83343.081680] [] ? insert_kthread_work+0x40/0x40 [83343.081694] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.084384] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [ldlm_bl_22:33195] [83343.084408] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.084416] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.084418] CPU: 11 PID: 33195 Comm: ldlm_bl_22 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.084418] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.084419] task: ffff88203d0b1fa0 ti: ffff88203d4b8000 task.ti: ffff88203d4b8000 [83343.084422] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.084422] RSP: 0018:ffff88203d4bbb70 EFLAGS: 00000246 [83343.084423] RAX: 0000000000000000 RBX: 5a5a5a5a5a5a5a5a RCX: 0000000000590000 [83343.084424] RDX: ffff88103ced7880 RSI: 0000000000310000 RDI: ffff88102137981c [83343.084424] RBP: ffff88203d4bbb70 R08: ffff88203c757880 R09: 0000000000000000 [83343.084425] R10: ffff882029aec800 R11: 0000000000000001 R12: 5a5a5a5a5a5a5a5a [83343.084425] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83343.084426] FS: 0000000000000000(0000) GS:ffff88203c740000(0000) knlGS:0000000000000000 [83343.084427] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.084427] CR2: 00007fef29abd090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.084428] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.084428] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.084429] Stack: [83343.084430] ffff88203d4bbb80 ffffffff8169e61f ffff88203d4bbb90 ffffffff816abb70 [83343.084431] ffff88203d4bbbd0 ffffffffc0b72198 0000000000000000 ffff882029aec800 [83343.084432] ffff88203d4bbc18 ffff882029aec860 ffff882029aec800 ffff8810175b8000 [83343.084433] Call Trace: [83343.084435] [] queued_spin_lock_slowpath+0xb/0xf [83343.084436] [] _raw_spin_lock+0x20/0x30 [83343.084466] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.084489] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.084510] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.084527] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.084545] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.084562] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.084569] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.084588] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.084606] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.084611] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.084631] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.084649] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.084671] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.084672] [] ? __schedule+0x39d/0x8b0 [83343.084692] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.084694] [] kthread+0xcf/0xe0 [83343.084695] [] ? insert_kthread_work+0x40/0x40 [83343.084697] [] ret_from_fork+0x58/0x90 [83343.084698] [] ? insert_kthread_work+0x40/0x40 [83343.084712] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.087384] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [ldlm_bl_15:16017] [83343.087407] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.087414] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.087415] CPU: 12 PID: 16017 Comm: ldlm_bl_15 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.087415] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.087416] task: ffff88203d338fd0 ti: ffff88201c814000 task.ti: ffff88201c814000 [83343.087419] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83343.087419] RSP: 0018:ffff88201c817b70 EFLAGS: 00000246 [83343.087420] RAX: 0000000000000000 RBX: 00000001f8c12595 RCX: 0000000000610000 [83343.087420] RDX: ffff88203c757880 RSI: 0000000000590000 RDI: ffff88102137981c [83343.087421] RBP: ffff88201c817b70 R08: ffff88103cf97880 R09: 0000000000000000 [83343.087422] R10: ffff88202b32b000 R11: 0000000000000005 R12: ffff88201c817c58 [83343.087423] R13: 0000000000000001 R14: ffff88201c817b28 R15: ffffffff81322c35 [83343.087423] FS: 0000000000000000(0000) GS:ffff88103cf80000(0000) knlGS:0000000000000000 [83343.087424] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.087424] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.087425] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.087426] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.087426] Stack: [83343.087427] ffff88201c817b80 ffffffff8169e61f ffff88201c817b90 ffffffff816abb70 [83343.087428] ffff88201c817bd0 ffffffffc0b72198 0000000000000000 ffff88202b32b000 [83343.087429] ffff88201c817c18 ffff88202b32b060 ffff88202b32b000 ffff8810175b8000 [83343.087430] Call Trace: [83343.087432] [] queued_spin_lock_slowpath+0xb/0xf [83343.087433] [] _raw_spin_lock+0x20/0x30 [83343.087451] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.087473] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.087492] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.087509] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.087527] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.087545] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.087550] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.087568] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.087585] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.087590] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.087610] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.087628] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.087650] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.087652] [] ? wake_up_state+0x20/0x20 [83343.087672] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.087674] [] kthread+0xcf/0xe0 [83343.087675] [] ? insert_kthread_work+0x40/0x40 [83343.087676] [] ret_from_fork+0x58/0x90 [83343.087677] [] ? insert_kthread_work+0x40/0x40 [83343.087691] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83343.090385] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [ldlm_bl_19:33192] [83343.090405] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.090412] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.090413] CPU: 13 PID: 33192 Comm: ldlm_bl_19 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.090414] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.090414] task: ffff88203d0b2f70 ti: ffff881ff8e20000 task.ti: ffff881ff8e20000 [83343.090417] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83343.090417] RSP: 0018:ffff881ff8e23b70 EFLAGS: 00000246 [83343.090418] RAX: 0000000000000000 RBX: ffff881ff8e23be8 RCX: 0000000000690000 [83343.090419] RDX: ffff88203c697880 RSI: 0000000000290000 RDI: ffff88102137981c [83343.090419] RBP: ffff881ff8e23b70 R08: ffff88203c797880 R09: 0000000000000000 [83343.090420] R10: ffff882029b2e000 R11: 0000000000000000 R12: ffff88103ce96cc0 [83343.090420] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83343.090421] FS: 0000000000000000(0000) GS:ffff88203c780000(0000) knlGS:0000000000000000 [83343.090422] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.090422] CR2: 00007f352c5974f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.090422] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.090423] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.090423] Stack: [83343.090424] ffff881ff8e23b80 ffffffff8169e61f ffff881ff8e23b90 ffffffff816abb70 [83343.090425] ffff881ff8e23bd0 ffffffffc0b72198 0000000000000000 ffff882029b2e000 [83343.090426] ffff881ff8e23c18 ffff882029b2e060 ffff882029b2e000 ffff8810175b8000 [83343.090427] Call Trace: [83343.090429] [] queued_spin_lock_slowpath+0xb/0xf [83343.090430] [] _raw_spin_lock+0x20/0x30 [83343.090447] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.090465] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.090483] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.090500] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.090516] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.090533] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.090538] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.090554] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.090571] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.090576] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.090592] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.090608] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.090628] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.090629] [] ? __schedule+0x39d/0x8b0 [83343.090648] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.090649] [] kthread+0xcf/0xe0 [83343.090651] [] ? insert_kthread_work+0x40/0x40 [83343.090652] [] ret_from_fork+0x58/0x90 [83343.090653] [] ? insert_kthread_work+0x40/0x40 [83343.090664] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83343.096384] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [ldlm_bl_23:33196] [83343.096405] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.096413] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.096414] CPU: 15 PID: 33196 Comm: ldlm_bl_23 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.096415] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.096415] task: ffff88203d0b0000 ti: ffff882015ae0000 task.ti: ffff882015ae0000 [83343.096418] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.096418] RSP: 0018:ffff882015ae3b70 EFLAGS: 00000246 [83343.096419] RAX: 0000000000000000 RBX: ffff88203c756cc0 RCX: 0000000000790000 [83343.096420] RDX: ffff88203c6d7880 RSI: 0000000000390000 RDI: ffff88102137981c [83343.096420] RBP: ffff882015ae3b70 R08: ffff88203c7d7880 R09: 0000000000000000 [83343.096421] R10: ffff882029a1d600 R11: 0000000000000001 R12: ffff88203d0b0068 [83343.096421] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83343.096422] FS: 0000000000000000(0000) GS:ffff88203c7c0000(0000) knlGS:0000000000000000 [83343.096423] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.096423] CR2: 00007faebc9e03cc CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.096424] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.096424] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.096424] Stack: [83343.096426] ffff882015ae3b80 ffffffff8169e61f ffff882015ae3b90 ffffffff816abb70 [83343.096427] ffff882015ae3bd0 ffffffffc0b72198 0000000000000000 ffff882029a1d600 [83343.096428] ffff882015ae3c18 ffff882029a1d660 ffff882029a1d600 ffff8810175b8000 [83343.096428] Call Trace: [83343.096430] [] queued_spin_lock_slowpath+0xb/0xf [83343.096432] [] _raw_spin_lock+0x20/0x30 [83343.096452] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.096473] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.096492] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.096510] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.096527] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.096545] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.096551] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.096568] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.096586] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.096591] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.096609] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.096626] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.096647] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.096648] [] ? __schedule+0x39d/0x8b0 [83343.096668] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.096670] [] kthread+0xcf/0xe0 [83343.096671] [] ? insert_kthread_work+0x40/0x40 [83343.096672] [] ret_from_fork+0x58/0x90 [83343.096673] [] ? insert_kthread_work+0x40/0x40 [83343.096687] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.099384] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [ldlm_bl_12:16014] [83343.099406] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.099413] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.099414] CPU: 16 PID: 16014 Comm: ldlm_bl_12 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.099414] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.099415] task: ffff88203d33af70 ti: ffff88201c808000 task.ti: ffff88201c808000 [83343.099418] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.099418] RSP: 0018:ffff88201c80bb70 EFLAGS: 00000246 [83343.099419] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000810000 [83343.099419] RDX: ffff88203c717880 RSI: 0000000000490000 RDI: ffff88102137981c [83343.099420] RBP: ffff88201c80bb70 R08: ffff88103d017880 R09: 0000000000000000 [83343.099420] R10: ffff88202b32bc00 R11: 0000000000000005 R12: ffff88201c80bc58 [83343.099421] R13: 0000000000000001 R14: ffff88201c80bb28 R15: ffffffff81322c35 [83343.099422] FS: 0000000000000000(0000) GS:ffff88103d000000(0000) knlGS:0000000000000000 [83343.099422] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.099423] CR2: 00007f361a3bb4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.099424] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.099424] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.099424] Stack: [83343.099426] ffff88201c80bb80 ffffffff8169e61f ffff88201c80bb90 ffffffff816abb70 [83343.099427] ffff88201c80bbd0 ffffffffc0b72198 0000000000000000 ffff88202b32bc00 [83343.099428] ffff88201c80bc18 ffff88202b32bc60 ffff88202b32bc00 ffff8810175b8000 [83343.099428] Call Trace: [83343.099430] [] queued_spin_lock_slowpath+0xb/0xf [83343.099432] [] _raw_spin_lock+0x20/0x30 [83343.099452] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.099474] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.099493] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.099511] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.099528] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.099546] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.099552] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.099569] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.099587] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.099592] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.099610] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.099628] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.099650] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.099651] [] ? wake_up_state+0x20/0x20 [83343.099671] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.099673] [] kthread+0xcf/0xe0 [83343.099674] [] ? insert_kthread_work+0x40/0x40 [83343.099675] [] ret_from_fork+0x58/0x90 [83343.099677] [] ? insert_kthread_work+0x40/0x40 [83343.099690] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.102384] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [ldlm_bl_21:33194] [83343.102406] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.102413] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.102415] CPU: 17 PID: 33194 Comm: ldlm_bl_21 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.102415] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.102416] task: ffff88203d0b6eb0 ti: ffff882012eac000 task.ti: ffff882012eac000 [83343.102419] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.102419] RSP: 0018:ffff882012eafb70 EFLAGS: 00000246 [83343.102420] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000890000 [83343.102421] RDX: ffff88203c797880 RSI: 0000000000690000 RDI: ffff88102137981c [83343.102422] RBP: ffff882012eafb70 R08: ffff88203c817880 R09: 0000000000000000 [83343.102422] R10: ffff882029b38200 R11: 0000000000000001 R12: 0000000000000000 [83343.102423] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c816cc0 [83343.102424] FS: 0000000000000000(0000) GS:ffff88203c800000(0000) knlGS:0000000000000000 [83343.102425] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.102425] CR2: 00007f7ce52f4550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.102426] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.102426] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.102427] Stack: [83343.102428] ffff882012eafb80 ffffffff8169e61f ffff882012eafb90 ffffffff816abb70 [83343.102429] ffff882012eafbd0 ffffffffc0b72198 0000000000000000 ffff882029b38200 [83343.102430] ffff882012eafc18 ffff882029b38260 ffff882029b38200 ffff8810175b8000 [83343.102431] Call Trace: [83343.102433] [] queued_spin_lock_slowpath+0xb/0xf [83343.102434] [] _raw_spin_lock+0x20/0x30 [83343.102452] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.102473] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.102492] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.102510] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.102528] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.102545] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.102551] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.102568] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.102586] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.102591] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.102611] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.102629] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.102651] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.102653] [] ? __schedule+0x39d/0x8b0 [83343.102673] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.102675] [] kthread+0xcf/0xe0 [83343.102676] [] ? insert_kthread_work+0x40/0x40 [83343.102677] [] ret_from_fork+0x58/0x90 [83343.102679] [] ? insert_kthread_work+0x40/0x40 [83343.102692] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.105384] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 23s! [ldlm_bl_02:15061] [83343.105407] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.105414] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.105415] CPU: 18 PID: 15061 Comm: ldlm_bl_02 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.105415] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.105416] task: ffff882012edeeb0 ti: ffff88203d3bc000 task.ti: ffff88203d3bc000 [83343.105419] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.105419] RSP: 0018:ffff88203d3bfb70 EFLAGS: 00000246 [83343.105420] RAX: 0000000000000000 RBX: 000000013cf16cc0 RCX: 0000000000910000 [83343.105421] RDX: ffff88203c7d7880 RSI: 0000000000790000 RDI: ffff88102137981c [83343.105421] RBP: ffff88203d3bfb70 R08: ffff88103d057880 R09: 0000000000000000 [83343.105422] R10: ffff88202b32b200 R11: 0000000000000005 R12: ffff88203d3bfc58 [83343.105422] R13: 0000000000000001 R14: ffff88203d3bfb28 R15: ffffffff81322c35 [83343.105423] FS: 0000000000000000(0000) GS:ffff88103d040000(0000) knlGS:0000000000000000 [83343.105424] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.105424] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.105425] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.105425] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.105426] Stack: [83343.105427] ffff88203d3bfb80 ffffffff8169e61f ffff88203d3bfb90 ffffffff816abb70 [83343.105428] ffff88203d3bfbd0 ffffffffc0b72198 0000000000000000 ffff88202b32b200 [83343.105429] ffff88203d3bfc18 ffff88202b32b260 ffff88202b32b200 ffff8810175b8000 [83343.105429] Call Trace: [83343.105432] [] queued_spin_lock_slowpath+0xb/0xf [83343.105433] [] _raw_spin_lock+0x20/0x30 [83343.105452] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.105473] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.105492] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.105510] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.105527] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.105545] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.105550] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.105568] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.105585] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.105591] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.105608] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.105626] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.105648] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.105649] [] ? wake_up_state+0x20/0x20 [83343.105669] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.105671] [] kthread+0xcf/0xe0 [83343.105672] [] ? insert_kthread_work+0x40/0x40 [83343.105674] [] ret_from_fork+0x58/0x90 [83343.105675] [] ? insert_kthread_work+0x40/0x40 [83343.105688] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.108383] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 23s! [ldlm_bl_18:33191] [83343.108405] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.108413] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.108414] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.108414] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.108415] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83343.108417] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.108418] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83343.108419] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83343.108419] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83343.108420] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83343.108420] R10: ffff882029b33c00 R11: 0000000000000001 R12: ffff88203d0b5f48 [83343.108421] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83343.108422] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83343.108422] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.108423] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.108423] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.108424] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.108424] Stack: [83343.108425] ffff881ffba83b80 ffffffff8169e61f ffff881ffba83b90 ffffffff816abb70 [83343.108426] ffff881ffba83bd0 ffffffffc0b72198 0000000000000000 ffff882029b33c00 [83343.108428] ffff881ffba83c18 ffff882029b33c60 ffff882029b33c00 ffff8810175b8000 [83343.108428] Call Trace: [83343.108430] [] queued_spin_lock_slowpath+0xb/0xf [83343.108431] [] _raw_spin_lock+0x20/0x30 [83343.108452] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.108473] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.108492] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.108510] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.108527] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.108544] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.108550] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.108567] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.108584] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.108590] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.108607] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.108625] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.108647] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.108648] [] ? __schedule+0x39d/0x8b0 [83343.108668] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.108670] [] kthread+0xcf/0xe0 [83343.108671] [] ? insert_kthread_work+0x40/0x40 [83343.108672] [] ret_from_fork+0x58/0x90 [83343.108673] [] ? insert_kthread_work+0x40/0x40 [83343.108687] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.111383] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 23s! [ldlm_bl_01:15060] [83343.111405] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.111412] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.111413] CPU: 20 PID: 15060 Comm: ldlm_bl_01 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.111414] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.111415] task: ffff882012eddee0 ti: ffff88203d3b8000 task.ti: ffff88203d3b8000 [83343.111417] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83343.111418] RSP: 0018:ffff88203d3bbb70 EFLAGS: 00000246 [83343.111418] RAX: 0000000000000000 RBX: 0000000180597933 RCX: 0000000000a10000 [83343.111419] RDX: ffff88203c857880 RSI: 0000000000990000 RDI: ffff88102137981c [83343.111419] RBP: ffff88203d3bbb70 R08: ffff88103d097880 R09: 0000000000000000 [83343.111420] R10: ffff88202b2d3a00 R11: 0000000000000005 R12: ffff88203d3bbc58 [83343.111420] R13: 0000000000000001 R14: ffff88203d3bbb28 R15: ffffffff81322c35 [83343.111421] FS: 0000000000000000(0000) GS:ffff88103d080000(0000) knlGS:0000000000000000 [83343.111422] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.111422] CR2: 00007f02c314e4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.111423] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.111423] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.111424] Stack: [83343.111425] ffff88203d3bbb80 ffffffff8169e61f ffff88203d3bbb90 ffffffff816abb70 [83343.111426] ffff88203d3bbbd0 ffffffffc0b72198 0000000000000000 ffff88202b2d3a00 [83343.111427] ffff88203d3bbc18 ffff88202b2d3a60 ffff88202b2d3a00 ffff8810175b8000 [83343.111427] Call Trace: [83343.111429] [] queued_spin_lock_slowpath+0xb/0xf [83343.111431] [] _raw_spin_lock+0x20/0x30 [83343.111449] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.111471] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.111490] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.111507] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.111525] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.111543] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.111548] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.111566] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.111583] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.111588] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.111608] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.111626] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.111648] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.111650] [] ? wake_up_state+0x20/0x20 [83343.111670] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.111671] [] kthread+0xcf/0xe0 [83343.111672] [] ? insert_kthread_work+0x40/0x40 [83343.111674] [] ret_from_fork+0x58/0x90 [83343.111675] [] ? insert_kthread_work+0x40/0x40 [83343.111688] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83343.114383] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 23s! [ldlm_bl_20:33193] [83343.114406] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.114413] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.114415] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.114415] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.114416] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83343.114418] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.114419] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83343.114420] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83343.114420] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83343.114421] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83343.114421] R10: ffff882029b3b400 R11: 0000000000000000 R12: ffff88103ce96cc0 [83343.114422] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83343.114423] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83343.114423] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.114424] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.114424] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.114425] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.114425] Stack: [83343.114426] ffff881ff8e2fb80 ffffffff8169e61f ffff881ff8e2fb90 ffffffff816abb70 [83343.114427] ffff881ff8e2fbd0 ffffffffc0b72198 0000000000000000 ffff882029b3b400 [83343.114429] ffff881ff8e2fc18 ffff882029b3b460 ffff882029b3b400 ffff8810175b8000 [83343.114429] Call Trace: [83343.114431] [] queued_spin_lock_slowpath+0xb/0xf [83343.114432] [] _raw_spin_lock+0x20/0x30 [83343.114451] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.114472] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.114491] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.114508] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.114526] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.114544] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.114549] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.114567] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.114584] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.114589] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.114609] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.114627] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.114649] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.114651] [] ? __schedule+0x39d/0x8b0 [83343.114671] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.114672] [] kthread+0xcf/0xe0 [83343.114673] [] ? insert_kthread_work+0x40/0x40 [83343.114675] [] ret_from_fork+0x58/0x90 [83343.114676] [] ? insert_kthread_work+0x40/0x40 [83343.114689] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.117383] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [ldlm_bl_07:16009] [83343.117405] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.117412] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.117414] CPU: 22 PID: 16009 Comm: ldlm_bl_07 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.117414] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.117415] task: ffff8810e9240fd0 ti: ffff88201c7ec000 task.ti: ffff88201c7ec000 [83343.117417] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.117418] RSP: 0018:ffff88201c7efb70 EFLAGS: 00000246 [83343.117419] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000b10000 [83343.117419] RDX: ffff88103ce97880 RSI: 0000000000210000 RDI: ffff88102137981c [83343.117420] RBP: ffff88201c7efb70 R08: ffff88103d0d7880 R09: 0000000000000000 [83343.117420] R10: ffff88202b32b200 R11: 0000000000000005 R12: ffff88201c7efc58 [83343.117421] R13: 0000000000000001 R14: ffff88201c7efb28 R15: ffffffff81322c35 [83343.117422] FS: 0000000000000000(0000) GS:ffff88103d0c0000(0000) knlGS:0000000000000000 [83343.117422] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.117423] CR2: 00007f262c4764f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.117423] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.117424] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.117424] Stack: [83343.117425] ffff88201c7efb80 ffffffff8169e61f ffff88201c7efb90 ffffffff816abb70 [83343.117427] ffff88201c7efbd0 ffffffffc0b72198 0000000000000000 ffff88202b32b200 [83343.117428] ffff88201c7efc18 ffff88202b32b260 ffff88202b32b200 ffff8810175b8000 [83343.117428] Call Trace: [83343.117430] [] queued_spin_lock_slowpath+0xb/0xf [83343.117431] [] _raw_spin_lock+0x20/0x30 [83343.117450] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.117471] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.117490] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.117508] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.117525] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.117543] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.117548] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.117566] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.117584] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.117589] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.117607] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.117625] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.117647] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.117648] [] ? wake_up_state+0x20/0x20 [83343.117669] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.117670] [] kthread+0xcf/0xe0 [83343.117671] [] ? insert_kthread_work+0x40/0x40 [83343.117673] [] ret_from_fork+0x58/0x90 [83343.117674] [] ? insert_kthread_work+0x40/0x40 [83343.117687] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83343.120383] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [ldlm_bl_04:16006] [83343.120406] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83343.120413] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83343.120414] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83343.120415] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83343.120415] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83343.120418] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83343.120418] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83343.120419] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83343.120419] RDX: ffff88103d057880 RSI: 0000000000910000 RDI: ffff88102137981c [83343.120420] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83343.120421] R10: ffff882029bede00 R11: 0000000000000000 R12: ffff88201c627c58 [83343.120421] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83343.120422] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83343.120423] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83343.120423] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83343.120424] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83343.120424] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83343.120424] Stack: [83343.120426] ffff88201c627b80 ffffffff8169e61f ffff88201c627b90 ffffffff816abb70 [83343.120427] ffff88201c627bd0 ffffffffc0b72198 0000000000000000 ffff882029bede00 [83343.120428] ffff88201c627c18 ffff882029bede60 ffff882029bede00 ffff8810175b8000 [83343.120428] Call Trace: [83343.120431] [] queued_spin_lock_slowpath+0xb/0xf [83343.120432] [] _raw_spin_lock+0x20/0x30 [83343.120452] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83343.120473] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83343.120492] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83343.120510] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83343.120527] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83343.120545] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83343.120551] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83343.120568] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.120586] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83343.120591] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83343.120608] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83343.120626] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83343.120648] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83343.120650] [] ? wake_up_state+0x20/0x20 [83343.120670] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83343.120671] [] kthread+0xcf/0xe0 [83343.120673] [] ? insert_kthread_work+0x40/0x40 [83343.120674] [] ret_from_fork+0x58/0x90 [83343.120675] [] ? insert_kthread_work+0x40/0x40 [83343.120689] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83347.060298] NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ldlm_bl_16:33189] [83347.060320] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83347.060327] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83347.060328] CPU: 3 PID: 33189 Comm: ldlm_bl_16 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83347.060328] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83347.060329] task: ffff88102c376eb0 ti: ffff881fefb44000 task.ti: ffff881fefb44000 [83347.060331] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83347.060332] RSP: 0018:ffff881fefb47b70 EFLAGS: 00000246 [83347.060333] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000190000 [83347.060333] RDX: ffff88203c8d7880 RSI: 0000000000b90001 RDI: ffff88102137981c [83347.060334] RBP: ffff881fefb47b70 R08: ffff88203c657880 R09: 0000000000000000 [83347.060335] R10: ffff882029b36600 R11: 0000000000000000 R12: ffff88102c376f18 [83347.060335] R13: ffff88203c2b92a0 R14: ffff881000000141 R15: ffff88207ffd8008 [83347.060336] FS: 0000000000000000(0000) GS:ffff88203c640000(0000) knlGS:0000000000000000 [83347.060337] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83347.060337] CR2: 000055ce548c9220 CR3: 00000000019f2000 CR4: 00000000001407e0 [83347.060338] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83347.060339] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83347.060339] Stack: [83347.060340] ffff881fefb47b80 ffffffff8169e61f ffff881fefb47b90 ffffffff816abb70 [83347.060342] ffff881fefb47bd0 ffffffffc0b72198 0000000000000000 ffff882029b36600 [83347.060343] ffff881fefb47c18 ffff882029b36660 ffff882029b36600 ffff8810175b8000 [83347.060343] Call Trace: [83347.060345] [] queued_spin_lock_slowpath+0xb/0xf [83347.060346] [] _raw_spin_lock+0x20/0x30 [83347.060365] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83347.060386] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83347.060405] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83347.060422] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83347.060440] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83347.060457] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83347.060463] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83347.060480] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83347.060498] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83347.060503] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83347.060521] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83347.060541] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83347.060563] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83347.060564] [] ? __schedule+0x39d/0x8b0 [83347.060584] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83347.060585] [] kthread+0xcf/0xe0 [83347.060587] [] ? insert_kthread_work+0x40/0x40 [83347.060588] [] ret_from_fork+0x58/0x90 [83347.060589] [] ? insert_kthread_work+0x40/0x40 [83347.060602] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83347.063298] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [ldlm_bl_09:16011] [83347.063321] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83347.063328] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83347.063329] CPU: 4 PID: 16011 Comm: ldlm_bl_09 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83347.063329] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83347.063330] task: ffff88203ad88fd0 ti: ffff88201c7fc000 task.ti: ffff88201c7fc000 [83347.063332] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83347.063333] RSP: 0018:ffff88201c7ffb70 EFLAGS: 00000246 [83347.063333] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000210000 [83347.063334] RDX: ffff88103cf97880 RSI: 0000000000610001 RDI: ffff88102137981c [83347.063335] RBP: ffff88201c7ffb70 R08: ffff88103ce97880 R09: 0000000000000000 [83347.063335] R10: ffff88202b336e00 R11: 0000000000000000 R12: ffff88201c7ffc58 [83347.063336] R13: 0000000000000001 R14: ffff88201c7ffb28 R15: ffffffff81322c35 [83347.063337] FS: 0000000000000000(0000) GS:ffff88103ce80000(0000) knlGS:0000000000000000 [83347.063337] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83347.063338] CR2: 00007f8905462248 CR3: 00000000019f2000 CR4: 00000000001407e0 [83347.063338] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83347.063339] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83347.063339] Stack: [83347.063340] ffff88201c7ffb80 ffffffff8169e61f ffff88201c7ffb90 ffffffff816abb70 [83347.063342] ffff88201c7ffbd0 ffffffffc0b72198 0000000000000000 ffff88202b336e00 [83347.063343] ffff88201c7ffc18 ffff88202b336e60 ffff88202b336e00 ffff8810175b8000 [83347.063343] Call Trace: [83347.063345] [] queued_spin_lock_slowpath+0xb/0xf [83347.063346] [] _raw_spin_lock+0x20/0x30 [83347.063366] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83347.063387] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83347.063406] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83347.063424] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83347.063442] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83347.063459] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83347.063465] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83347.063482] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83347.063500] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83347.063505] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83347.063525] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83347.063542] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83347.063562] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83347.063564] [] ? wake_up_state+0x20/0x20 [83347.063584] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83347.063585] [] kthread+0xcf/0xe0 [83347.063587] [] ? insert_kthread_work+0x40/0x40 [83347.063588] [] ret_from_fork+0x58/0x90 [83347.063589] [] ? insert_kthread_work+0x40/0x40 [83347.063603] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83347.066299] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [ldlm_bl_11:16013] [83347.066322] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83347.066330] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83347.066331] CPU: 5 PID: 16013 Comm: ldlm_bl_11 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83347.066332] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83347.066333] task: ffff88203ad88000 ti: ffff88201c804000 task.ti: ffff88201c804000 [83347.066335] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83347.066335] RSP: 0018:ffff88201c807b70 EFLAGS: 00000246 [83347.066336] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000290000 [83347.066337] RDX: ffff88103d017880 RSI: 0000000000810001 RDI: ffff88102137981c [83347.066337] RBP: ffff88201c807b70 R08: ffff88203c697880 R09: 0000000000000000 [83347.066338] R10: ffff88202b216a00 R11: 0000000000000000 R12: ffff88201c807c58 [83347.066338] R13: 0000000000000001 R14: ffff88201c807b28 R15: ffffffff81322c35 [83347.066339] FS: 0000000000000000(0000) GS:ffff88203c680000(0000) knlGS:0000000000000000 [83347.066340] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83347.066340] CR2: 00007fd7ddcd8090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83347.066341] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83347.066341] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83347.066341] Stack: [83347.066343] ffff88201c807b80 ffffffff8169e61f ffff88201c807b90 ffffffff816abb70 [83347.066344] ffff88201c807bd0 ffffffffc0b72198 0000000000000000 ffff88202b216a00 [83347.066345] ffff88201c807c18 ffff88202b216a60 ffff88202b216a00 ffff8810175b8000 [83347.066345] Call Trace: [83347.066347] [] queued_spin_lock_slowpath+0xb/0xf [83347.066349] [] _raw_spin_lock+0x20/0x30 [83347.066367] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83347.066388] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83347.066407] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83347.066425] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83347.066443] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83347.066460] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83347.066466] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83347.066483] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83347.066501] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83347.066506] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83347.066526] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83347.066544] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83347.066566] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83347.066567] [] ? wake_up_state+0x20/0x20 [83347.066588] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83347.066589] [] kthread+0xcf/0xe0 [83347.066590] [] ? insert_kthread_work+0x40/0x40 [83347.066592] [] ret_from_fork+0x58/0x90 [83347.066593] [] ? insert_kthread_work+0x40/0x40 [83347.066606] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83347.072298] NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [ldlm_bl_17:33190] [83347.072319] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83347.072326] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83347.072327] CPU: 7 PID: 33190 Comm: ldlm_bl_17 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83347.072328] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83347.072329] task: ffff88102d281fa0 ti: ffff881fffd34000 task.ti: ffff881fffd34000 [83347.072331] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83347.072332] RSP: 0018:ffff881fffd37b70 EFLAGS: 00000246 [83347.072332] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000390000 [83347.072333] RDX: ffff88103cf17880 RSI: 0000000000410000 RDI: ffff88102137981c [83347.072334] RBP: ffff881fffd37b70 R08: ffff88203c6d7880 R09: 0000000000000000 [83347.072334] R10: ffff882029b71e00 R11: 0000000000000000 R12: ffff88102d282008 [83347.072335] R13: ffff881fffd37b00 R14: ffff881f00000141 R15: ffff88207ffd8008 [83347.072336] FS: 0000000000000000(0000) GS:ffff88203c6c0000(0000) knlGS:0000000000000000 [83347.072336] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83347.072337] CR2: 00007f240470f978 CR3: 00000000019f2000 CR4: 00000000001407e0 [83347.072337] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83347.072338] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83347.072338] Stack: [83347.072339] ffff881fffd37b80 ffffffff8169e61f ffff881fffd37b90 ffffffff816abb70 [83347.072340] ffff881fffd37bd0 ffffffffc0b72198 0000000000000000 ffff882029b71e00 [83347.072341] ffff881fffd37c18 ffff882029b71e60 ffff882029b71e00 ffff8810175b8000 [83347.072342] Call Trace: [83347.072344] [] queued_spin_lock_slowpath+0xb/0xf [83347.072345] [] _raw_spin_lock+0x20/0x30 [83347.072363] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83347.072385] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83347.072403] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83347.072421] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83347.072438] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83347.072456] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83347.072461] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83347.072479] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83347.072497] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83347.072502] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83347.072520] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83347.072537] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83347.072559] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83347.072561] [] ? __schedule+0x39d/0x8b0 [83347.072581] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83347.072582] [] kthread+0xcf/0xe0 [83347.072583] [] ? insert_kthread_work+0x40/0x40 [83347.072585] [] ret_from_fork+0x58/0x90 [83347.072586] [] ? insert_kthread_work+0x40/0x40 [83347.072599] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83352.013271] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83352.092746] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83352.121616] CPU: 1 PID: 16010 Comm: ldlm_bl_08 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83352.134326] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83352.142674] task: ffff8810e9242f70 ti: ffff88201c7f8000 task.ti: ffff88201c7f8000 [83352.151022] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83352.162093] RSP: 0018:ffff88201c7fbb70 EFLAGS: 00000246 [83352.168018] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000090000 [83352.175977] RDX: ffff88103cf57880 RSI: 0000000000510000 RDI: ffff88102137981c [83352.183938] RBP: ffff88201c7fbb70 R08: ffff88203c617880 R09: 0000000000000000 [83352.191897] R10: ffff88202b236e00 R11: 0000000000000005 R12: ffff88201c7fbc58 [83352.199857] R13: 0000000000000001 R14: ffff88201c7fbb28 R15: ffffffff81322c35 [83352.207818] FS: 0000000000000000(0000) GS:ffff88203c600000(0000) knlGS:0000000000000000 [83352.216843] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83352.223253] CR2: 00007f11b0c8c000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83352.231213] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83352.239174] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83352.247134] Stack: [83352.249374] ffff88201c7fbb80 ffffffff8169e61f ffff88201c7fbb90 ffffffff816abb70 [83352.257659] ffff88201c7fbbd0 ffffffffc0b72198 0000000000000000 ffff88202b236e00 [83352.265946] ffff88201c7fbc18 ffff88202b236e60 ffff88202b236e00 ffff8810175b8000 [83352.274234] Call Trace: [83352.276961] [] queued_spin_lock_slowpath+0xb/0xf [83352.283952] [] _raw_spin_lock+0x20/0x30 [83352.290089] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83352.298265] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83352.306343] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83352.313738] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83352.321521] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83352.329304] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83352.336687] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83352.344762] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83352.352642] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83352.360510] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83352.368584] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83352.377049] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83352.385222] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83352.392894] [] ? wake_up_state+0x20/0x20 [83352.399129] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83352.407380] [] kthread+0xcf/0xe0 [83352.412820] [] ? insert_kthread_work+0x40/0x40 [83352.419619] [] ret_from_fork+0x58/0x90 [83352.425641] [] ? insert_kthread_work+0x40/0x40 [83352.432438] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83359.069040] NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [ldlm_bl_14:16016] [83359.078040] NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [ldlm_bl_03:15915] [83359.077682] Modules linked in: [83359.078041] Modules linked in: [83359.078042] mgs(OE) [83359.078042] mgc(OE) [83359.078043] osd_ldiskfs(OE) [83359.078043] lquota(OE) [83359.078044] ldiskfs(OE) [83359.078044] lustre(OE) [83359.078044] lmv(OE) [83359.078045] mdc(OE) [83359.078045] lov(OE) [83359.078046] fid(OE) [83359.078046] fld(OE) [83359.078047] ko2iblnd(OE) [83359.078047] ptlrpc(OE) [83359.078047] obdclass(OE) [83359.078048] lnet(OE) [83359.078048] libcfs(OE) [83359.078048] vfat [83359.078049] fat [83359.078049] uas [83359.078050] usb_storage [83359.078050] mpt2sas [83359.078050] mptctl [83359.078051] mptbase [83359.078051] rpcsec_gss_krb5 [83359.078051] dell_rbu [83359.078052] nfsv4 [83359.078052] dns_resolver [83359.078053] nfs [83359.078053] fscache [83359.078053] ib_ipoib [83359.078054] rdma_ucm [83359.078054] ib_ucm [83359.078054] ib_uverbs [83359.078055] ib_umad [83359.078055] rdma_cm [83359.078055] ib_cm [83359.078056] iw_cm [83359.078056] mlx4_ib [83359.078056] ib_core [83359.078057] sb_edac [83359.078057] edac_core [83359.078058] intel_powerclamp [83359.078058] coretemp [83359.078058] intel_rapl [83359.078059] iosf_mbi [83359.078059] dm_service_time [83359.078060] kvm_intel [83359.078060] kvm [83359.078060] irqbypass [83359.078061] crc32_pclmul [83359.078061] ghash_clmulni_intel [83359.078061] aesni_intel [83359.078062] lrw [83359.078062] gf128mul [83359.078062] glue_helper [83359.078063] ablk_helper [83359.078063] cryptd [83359.078064] iTCO_wdt [83359.078064] dcdbas [83359.078064] iTCO_vendor_support [83359.078065] ipmi_si [83359.078065] ipmi_devintf [83359.078065] mxm_wmi [83359.078066] dm_round_robin [83359.078066] pcspkr [83359.078066] sg [83359.078067] ipmi_msghandler [83359.078067] acpi_power_meter [83359.078068] wmi [83359.078068] mei_me [83359.078068] mei [83359.078069] shpchp [83359.078069] lpc_ich [83359.078069] nfsd [83359.078070] auth_rpcgss [83359.078070] dm_multipath [83359.078070] dm_mod [83359.078071] nfs_acl [83359.078071] lockd [83359.078071] grace [83359.078072] sunrpc [83359.078072] ip_tables [83359.078072] ext4 [83359.078073] mbcache [83359.078073] jbd2 [83359.078074] sd_mod [83359.078074] crc_t10dif [83359.078074] crct10dif_generic [83359.078075] mlx4_en [83359.078075] i2c_algo_bit [83359.078076] drm_kms_helper [83359.078076] syscopyarea [83359.078076] sysfillrect [83359.078077] sysimgblt [83359.078077] fb_sys_fops [83359.078077] ttm [83359.078078] tg3 [83359.078078] ahci [83359.078079] crct10dif_pclmul [83359.078079] crct10dif_common [83359.078079] mlx4_core [83359.078080] mpt3sas [83359.078080] drm [83359.078080] libahci [83359.078081] crc32c_intel [83359.078081] ptp [83359.078081] raid_class [83359.078082] libata [83359.078082] megaraid_sas [83359.078082] devlink [83359.078083] i2c_core [83359.078084] scsi_transport_sas [83359.078084] pps_core [83359.078084] [83359.078086] CPU: 9 PID: 15915 Comm: ldlm_bl_03 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83359.078086] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83359.078087] task: ffff88202c648000 ti: ffff88201763c000 task.ti: ffff88201763c000 [83359.078088] RIP: 0010:[] [83359.078092] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83359.078092] RSP: 0018:ffff88201763fb70 EFLAGS: 00000246 [83359.078093] RAX: 0000000000000000 RBX: 0000000300000000 RCX: 0000000000490000 [83359.078094] RDX: ffff88103d097880 RSI: 0000000000a10000 RDI: ffff88102137981c [83359.078094] RBP: ffff88201763fb70 R08: ffff88203c717880 R09: 0000000000000000 [83359.078095] R10: ffff88202b202000 R11: 7fffffffffffffff R12: 0000000000000bf8 [83359.078095] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c656cc0 [83359.078096] FS: 0000000000000000(0000) GS:ffff88203c700000(0000) knlGS:0000000000000000 [83359.078097] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83359.078098] CR2: 00007ffa010a6000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83359.078098] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83359.078099] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83359.078099] Stack: [83359.078099] ffff88201763fb80 [83359.078100] ffffffff8169e61f [83359.078100] ffff88201763fb90 [83359.078100] ffffffff816abb70 [83359.078100] [83359.078101] ffff88201763fbd0 [83359.078101] ffffffffc0b72198 [83359.078101] 0000000000000000 [83359.078102] ffff88202b202000 [83359.078102] [83359.078102] ffff88201763fc18 [83359.078102] ffff88202b202060 [83359.078103] ffff88202b202000 [83359.078103] ffff8810175b8000 [83359.078103] [83359.078103] Call Trace: [83359.078104] [83359.078106] [] queued_spin_lock_slowpath+0xb/0xf [83359.078106] [83359.078107] [] _raw_spin_lock+0x20/0x30 [83359.078108] [83359.078144] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83359.078144] [83359.078167] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83359.078167] [83359.078186] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83359.078187] [83359.078204] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83359.078205] [83359.078222] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83359.078222] [83359.078240] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83359.078240] [83359.078247] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83359.078247] [83359.078265] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83359.078265] [83359.078282] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83359.078283] [83359.078288] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83359.078288] [83359.078308] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83359.078309] [83359.078326] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83359.078326] [83359.078348] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83359.078349] [83359.078350] [] ? wake_up_state+0x20/0x20 [83359.078351] [83359.078371] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83359.078371] [83359.078373] [] kthread+0xcf/0xe0 [83359.078373] [83359.078374] [] ? insert_kthread_work+0x40/0x40 [83359.078374] [83359.078376] [] ret_from_fork+0x58/0x90 [83359.078376] [83359.078377] [] ? insert_kthread_work+0x40/0x40 [83359.078377] Code: [83359.078378] 0d [83359.078378] 48 [83359.078378] 98 [83359.078379] 83 [83359.078379] e2 [83359.078379] 30 [83359.078379] 48 [83359.078380] 81 [83359.078380] c2 [83359.078380] 80 [83359.078380] 78 [83359.078381] 01 [83359.078381] 00 [83359.078381] 48 [83359.078381] 03 [83359.078382] 14 [83359.078382] c5 [83359.078382] e0 [83359.078383] fd [83359.078383] b0 [83359.078383] 81 [83359.078383] 4c [83359.078384] 89 [83359.078384] 02 [83359.078384] 41 [83359.078384] 8b [83359.078385] 40 [83359.078385] 08 [83359.078385] 85 [83359.078385] c0 [83359.078386] 75 [83359.078386] 0f [83359.078386] 0f [83359.078386] 1f [83359.078387] 44 [83359.078387] 00 [83359.078387] 00 [83359.078387] f3 [83359.078388] 90 [83359.078388] 41 [83359.078388] 8b [83359.078389] 40 [83359.078389] 08 [83359.078389] <85> [83359.078389] c0 [83359.078390] 74 [83359.078390] f6 [83359.078390] 4d [83359.078391] 8b [83359.078391] 08 [83359.078391] 4d [83359.078391] 85 [83359.078392] c9 [83359.078392] 74 [83359.078392] 04 [83359.078392] 41 [83359.078393] 0f [83359.078393] 18 [83359.078393] 09 [83359.078393] 8b [83359.078394] 17 [83359.078394] 0f [83359.078394] b7 [83359.078394] c2 [83359.078395] [83359.530117] mgs(OE) [83359.532561] mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif [83359.610305] crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83359.636910] CPU: 6 PID: 16016 Comm: ldlm_bl_14 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83359.649622] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83359.657969] task: ffff88203d338000 ti: ffff88201c810000 task.ti: ffff88201c810000 [83359.666315] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83359.677387] RSP: 0018:ffff88201c813b70 EFLAGS: 00000246 [83359.683310] RAX: 0000000000000000 RBX: 000000011c813be8 RCX: 0000000000310000 [83359.691271] RDX: ffff88103ce57880 RSI: 0000000000110000 RDI: ffff88102137981c [83359.699231] RBP: ffff88201c813b70 R08: ffff88103ced7880 R09: 0000000000000000 [83359.707190] R10: ffff88202b393c00 R11: 0000000000000000 R12: ffff88201c813c58 [83359.715151] R13: 0000000000000001 R14: ffff88201c813b28 R15: ffffffff81322c35 [83359.723112] FS: 0000000000000000(0000) GS:ffff88103cec0000(0000) knlGS:0000000000000000 [83359.732137] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83359.738546] CR2: 0000000002079728 CR3: 00000000019f2000 CR4: 00000000001407e0 [83359.746506] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83359.754467] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83359.762427] Stack: [83359.764667] ffff88201c813b80 ffffffff8169e61f ffff88201c813b90 ffffffff816abb70 [83359.772952] ffff88201c813bd0 ffffffffc0b72198 0000000000000000 ffff88202b393c00 [83359.781232] ffff88201c813c18 ffff88202b393c60 ffff88202b393c00 ffff8810175b8000 [83359.789514] Call Trace: [83359.792239] [] queued_spin_lock_slowpath+0xb/0xf [83359.799230] [] _raw_spin_lock+0x20/0x30 [83359.805379] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83359.813555] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83359.821631] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83359.829027] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83359.836810] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83359.844593] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83359.851977] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83359.860051] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83359.867932] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83359.875801] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83359.883874] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83359.892338] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83359.900514] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83359.908186] [] ? wake_up_state+0x20/0x20 [83359.914420] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83359.922670] [] kthread+0xcf/0xe0 [83359.928111] [] ? insert_kthread_work+0x40/0x40 [83359.934909] [] ret_from_fork+0x58/0x90 [83359.940930] [] ? insert_kthread_work+0x40/0x40 [83359.947726] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83371.053782] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ldlm_bl_13:16015] [83371.056783] NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [ldlm_bl_06:16008] [83371.056812] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.056820] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.056821] CPU: 2 PID: 16008 Comm: ldlm_bl_06 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.056822] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.056823] task: ffff8810247eeeb0 ti: ffff88201c7e8000 task.ti: ffff88201c7e8000 [83371.056827] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83371.056827] RSP: 0018:ffff88201c7ebb70 EFLAGS: 00000246 [83371.056828] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000110000 [83371.056828] RDX: ffff88203c897880 RSI: 0000000000a90000 RDI: ffff88102137981c [83371.056829] RBP: ffff88201c7ebb70 R08: ffff88103ce57880 R09: 0000000000000000 [83371.056829] R10: ffff88202ea7e400 R11: 0000000000000005 R12: ffff88201c7ebc58 [83371.056829] R13: 0000000000000001 R14: ffff88201c7ebb28 R15: ffffffff81322c35 [83371.056830] FS: 0000000000000000(0000) GS:ffff88103ce40000(0000) knlGS:0000000000000000 [83371.056830] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.056831] CR2: 0000000002577010 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.056831] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.056831] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.056832] Stack: [83371.056833] ffff88201c7ebb80 ffffffff8169e61f ffff88201c7ebb90 ffffffff816abb70 [83371.056833] ffff88201c7ebbd0 ffffffffc0b72198 0000000000000000 ffff88202ea7e400 [83371.056834] ffff88201c7ebc18 ffff88202ea7e460 ffff88202ea7e400 ffff8810175b8000 [83371.056834] Call Trace: [83371.056838] [] queued_spin_lock_slowpath+0xb/0xf [83371.056839] [] _raw_spin_lock+0x20/0x30 [83371.056869] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.056885] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.056900] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.056913] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.056927] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.056940] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.056946] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.056960] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.056973] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.056978] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.056991] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.057005] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.057021] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.057022] [] ? wake_up_state+0x20/0x20 [83371.057038] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.057039] [] kthread+0xcf/0xe0 [83371.057040] [] ? insert_kthread_work+0x40/0x40 [83371.057042] [] ret_from_fork+0x58/0x90 [83371.057042] [] ? insert_kthread_work+0x40/0x40 [83371.057051] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83371.074782] NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ldlm_bl_10:16012] [83371.074807] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.074816] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.074817] CPU: 8 PID: 16012 Comm: ldlm_bl_10 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.074817] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.074818] task: ffff88203ad89fa0 ti: ffff88201c800000 task.ti: ffff88201c800000 [83371.074822] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83371.074822] RSP: 0018:ffff88201c803b70 EFLAGS: 00000246 [83371.074823] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000410000 [83371.074823] RDX: ffff88203c617880 RSI: 0000000000090000 RDI: ffff88102137981c [83371.074824] RBP: ffff88201c803b70 R08: ffff88103cf17880 R09: 0000000000000000 [83371.074824] R10: ffff88202ea7f800 R11: 0000000000000005 R12: ffff88201c803c58 [83371.074825] R13: 0000000000000001 R14: ffff88201c803b28 R15: ffffffff81322c35 [83371.074826] FS: 0000000000000000(0000) GS:ffff88103cf00000(0000) knlGS:0000000000000000 [83371.074826] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.074827] CR2: 00007fca10cf5000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.074828] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.074828] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.074829] Stack: [83371.074830] ffff88201c803b80 ffffffff8169e61f ffff88201c803b90 ffffffff816abb70 [83371.074831] ffff88201c803bd0 ffffffffc0b72198 0000000000000000 ffff88202ea7f800 [83371.074832] ffff88201c803c18 ffff88202ea7f860 ffff88202ea7f800 ffff8810175b8000 [83371.074833] Call Trace: [83371.074835] [] queued_spin_lock_slowpath+0xb/0xf [83371.074836] [] _raw_spin_lock+0x20/0x30 [83371.074862] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.074885] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.074905] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.074922] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.074940] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.074958] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.074964] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.074981] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.074999] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.075004] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.075025] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.075042] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.075064] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.075066] [] ? wake_up_state+0x20/0x20 [83371.075086] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.075088] [] kthread+0xcf/0xe0 [83371.075089] [] ? insert_kthread_work+0x40/0x40 [83371.075090] [] ret_from_fork+0x58/0x90 [83371.075092] [] ? insert_kthread_work+0x40/0x40 [83371.075105] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83371.080781] NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [ldlm_bl_05:16007] [83371.080803] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.080810] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.080812] CPU: 10 PID: 16007 Comm: ldlm_bl_05 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.080813] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.080813] task: ffff8810247ecf10 ti: ffff88201c64c000 task.ti: ffff88201c64c000 [83371.080816] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83371.080817] RSP: 0018:ffff88201c64fb70 EFLAGS: 00000246 [83371.080817] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000510000 [83371.080818] RDX: ffff88103ce17880 RSI: 0000000000010000 RDI: ffff88102137981c [83371.080818] RBP: ffff88201c64fb70 R08: ffff88103cf57880 R09: 0000000000000000 [83371.080819] R10: ffff88202ea37000 R11: 0000000000000005 R12: ffff88201c64fc58 [83371.080820] R13: 0000000000000001 R14: ffff88201c64fb28 R15: ffffffff81322c35 [83371.080821] FS: 0000000000000000(0000) GS:ffff88103cf40000(0000) knlGS:0000000000000000 [83371.080821] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.080822] CR2: 0000000001dd94b8 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.080822] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.080823] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.080823] Stack: [83371.080825] ffff88201c64fb80 ffffffff8169e61f ffff88201c64fb90 ffffffff816abb70 [83371.080826] ffff88201c64fbd0 ffffffffc0b72198 0000000000000000 ffff88202ea37000 [83371.080827] ffff88201c64fc18 ffff88202ea37060 ffff88202ea37000 ffff8810175b8000 [83371.080827] Call Trace: [83371.080829] [] queued_spin_lock_slowpath+0xb/0xf [83371.080831] [] _raw_spin_lock+0x20/0x30 [83371.080852] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.080872] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.080891] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.080908] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.080926] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.080944] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.080949] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.080967] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.080984] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.080990] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.081007] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.081025] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.081047] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.081048] [] ? wake_up_state+0x20/0x20 [83371.081068] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.081070] [] kthread+0xcf/0xe0 [83371.081071] [] ? insert_kthread_work+0x40/0x40 [83371.081072] [] ret_from_fork+0x58/0x90 [83371.081074] [] ? insert_kthread_work+0x40/0x40 [83371.081087] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83371.083781] NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [ldlm_bl_22:33195] [83371.083806] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.083814] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.083816] CPU: 11 PID: 33195 Comm: ldlm_bl_22 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.083817] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.083818] task: ffff88203d0b1fa0 ti: ffff88203d4b8000 task.ti: ffff88203d4b8000 [83371.083821] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83371.083821] RSP: 0018:ffff88203d4bbb70 EFLAGS: 00000246 [83371.083822] RAX: 0000000000000000 RBX: 5a5a5a5a5a5a5a5a RCX: 0000000000590000 [83371.083822] RDX: ffff88103ced7880 RSI: 0000000000310000 RDI: ffff88102137981c [83371.083823] RBP: ffff88203d4bbb70 R08: ffff88203c757880 R09: 0000000000000000 [83371.083824] R10: ffff88202b246600 R11: 0000000000000001 R12: 5a5a5a5a5a5a5a5a [83371.083824] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83371.083825] FS: 0000000000000000(0000) GS:ffff88203c740000(0000) knlGS:0000000000000000 [83371.083826] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.083826] CR2: 00007fef29abd090 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.083827] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.083827] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.083827] Stack: [83371.083829] ffff88203d4bbb80 ffffffff8169e61f ffff88203d4bbb90 ffffffff816abb70 [83371.083830] ffff88203d4bbbd0 ffffffffc0b72198 0000000000000000 ffff88202b246600 [83371.083831] ffff88203d4bbc18 ffff88202b246660 ffff88202b246600 ffff8810175b8000 [83371.083831] Call Trace: [83371.083833] [] queued_spin_lock_slowpath+0xb/0xf [83371.083835] [] _raw_spin_lock+0x20/0x30 [83371.083864] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.083887] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.083907] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.083924] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.083942] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.083960] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.083967] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.083984] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.084002] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.084007] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.084025] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.084043] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.084065] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.084066] [] ? __schedule+0x39d/0x8b0 [83371.084087] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.084089] [] kthread+0xcf/0xe0 [83371.084090] [] ? insert_kthread_work+0x40/0x40 [83371.084091] [] ret_from_fork+0x58/0x90 [83371.084093] [] ? insert_kthread_work+0x40/0x40 [83371.084106] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83371.086781] NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [ldlm_bl_15:16017] [83371.086800] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.086807] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.086808] CPU: 12 PID: 16017 Comm: ldlm_bl_15 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.086808] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.086809] task: ffff88203d338fd0 ti: ffff88201c814000 task.ti: ffff88201c814000 [83371.086811] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83371.086812] RSP: 0018:ffff88201c817b70 EFLAGS: 00000246 [83371.086812] RAX: 0000000000000000 RBX: 00000001f8c12595 RCX: 0000000000610000 [83371.086813] RDX: ffff88203c757880 RSI: 0000000000590001 RDI: ffff88102137981c [83371.086813] RBP: ffff88201c817b70 R08: ffff88103cf97880 R09: 0000000000000000 [83371.086814] R10: ffff88202ea7f800 R11: 0000000000000005 R12: ffff88201c817c58 [83371.086814] R13: 0000000000000001 R14: ffff88201c817b28 R15: ffffffff81322c35 [83371.086815] FS: 0000000000000000(0000) GS:ffff88103cf80000(0000) knlGS:0000000000000000 [83371.086815] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.086816] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.086816] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.086817] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.086817] Stack: [83371.086818] ffff88201c817b80 ffffffff8169e61f ffff88201c817b90 ffffffff816abb70 [83371.086819] ffff88201c817bd0 ffffffffc0b72198 0000000000000000 ffff88202ea7f800 [83371.086820] ffff88201c817c18 ffff88202ea7f860 ffff88202ea7f800 ffff8810175b8000 [83371.086820] Call Trace: [83371.086822] [] queued_spin_lock_slowpath+0xb/0xf [83371.086823] [] _raw_spin_lock+0x20/0x30 [83371.086841] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.086859] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.086877] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.086893] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.086909] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.086926] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.086931] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.086947] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.086963] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.086968] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.086985] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.087001] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.087020] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.087022] [] ? wake_up_state+0x20/0x20 [83371.087041] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.087042] [] kthread+0xcf/0xe0 [83371.087043] [] ? insert_kthread_work+0x40/0x40 [83371.087044] [] ret_from_fork+0x58/0x90 [83371.087045] [] ? insert_kthread_work+0x40/0x40 [83371.087057] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83371.089781] NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [ldlm_bl_19:33192] [83371.089805] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.089812] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.089814] CPU: 13 PID: 33192 Comm: ldlm_bl_19 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.089814] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.089815] task: ffff88203d0b2f70 ti: ffff881ff8e20000 task.ti: ffff881ff8e20000 [83371.089818] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83371.089819] RSP: 0018:ffff881ff8e23b70 EFLAGS: 00000246 [83371.089819] RAX: 0000000000000000 RBX: ffff881ff8e23be8 RCX: 0000000000690000 [83371.089820] RDX: ffff88203c697880 RSI: 0000000000290000 RDI: ffff88102137981c [83371.089821] RBP: ffff881ff8e23b70 R08: ffff88203c797880 R09: 0000000000000000 [83371.089822] R10: ffff88202b2d2c00 R11: 0000000000000000 R12: ffff88103ce96cc0 [83371.089823] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83371.089823] FS: 0000000000000000(0000) GS:ffff88203c780000(0000) knlGS:0000000000000000 [83371.089824] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.089825] CR2: 00007f352c5974f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.089825] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.089826] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.089826] Stack: [83371.089827] ffff881ff8e23b80 ffffffff8169e61f ffff881ff8e23b90 ffffffff816abb70 [83371.089828] ffff881ff8e23bd0 ffffffffc0b72198 0000000000000000 ffff88202b2d2c00 [83371.089830] ffff881ff8e23c18 ffff88202b2d2c60 ffff88202b2d2c00 ffff8810175b8000 [83371.089830] Call Trace: [83371.089832] [] queued_spin_lock_slowpath+0xb/0xf [83371.089833] [] _raw_spin_lock+0x20/0x30 [83371.089851] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.089871] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.089890] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.089907] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.089925] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.089943] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.089948] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.089966] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.089983] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.089988] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.090008] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.090026] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.090048] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.090050] [] ? __schedule+0x39d/0x8b0 [83371.090070] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.090071] [] kthread+0xcf/0xe0 [83371.090072] [] ? insert_kthread_work+0x40/0x40 [83371.090074] [] ret_from_fork+0x58/0x90 [83371.090075] [] ? insert_kthread_work+0x40/0x40 [83371.090088] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83371.095780] NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [ldlm_bl_23:33196] [83371.095803] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.095810] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.095811] CPU: 15 PID: 33196 Comm: ldlm_bl_23 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.095812] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.095813] task: ffff88203d0b0000 ti: ffff882015ae0000 task.ti: ffff882015ae0000 [83371.095815] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83371.095816] RSP: 0018:ffff882015ae3b70 EFLAGS: 00000246 [83371.095817] RAX: 0000000000000000 RBX: ffff88203c756cc0 RCX: 0000000000790000 [83371.095817] RDX: ffff88203c6d7880 RSI: 0000000000390000 RDI: ffff88102137981c [83371.095818] RBP: ffff882015ae3b70 R08: ffff88203c7d7880 R09: 0000000000000000 [83371.095819] R10: ffff882029bcea00 R11: 0000000000000001 R12: ffff88203d0b0068 [83371.095819] R13: 5a5a5a5a5a5a5a5a R14: 5a5a5a5a5a5a5a5a R15: 5a5a5a5a5a5a5a5a [83371.095820] FS: 0000000000000000(0000) GS:ffff88203c7c0000(0000) knlGS:0000000000000000 [83371.095821] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.095821] CR2: 00007faebc9e03cc CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.095822] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.095822] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.095823] Stack: [83371.095824] ffff882015ae3b80 ffffffff8169e61f ffff882015ae3b90 ffffffff816abb70 [83371.095825] ffff882015ae3bd0 ffffffffc0b72198 0000000000000000 ffff882029bcea00 [83371.095826] ffff882015ae3c18 ffff882029bcea60 ffff882029bcea00 ffff8810175b8000 [83371.095826] Call Trace: [83371.095828] [] queued_spin_lock_slowpath+0xb/0xf [83371.095830] [] _raw_spin_lock+0x20/0x30 [83371.095848] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.095870] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.095889] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.095906] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.095924] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.095942] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.095948] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.095965] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.095983] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.095988] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.096006] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.096023] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.096045] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.096046] [] ? __schedule+0x39d/0x8b0 [83371.096066] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.096068] [] kthread+0xcf/0xe0 [83371.096069] [] ? insert_kthread_work+0x40/0x40 [83371.096070] [] ret_from_fork+0x58/0x90 [83371.096071] [] ? insert_kthread_work+0x40/0x40 [83371.096085] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83371.098781] NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [ldlm_bl_12:16014] [83371.098803] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.098810] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.098812] CPU: 16 PID: 16014 Comm: ldlm_bl_12 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.098812] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.098813] task: ffff88203d33af70 ti: ffff88201c808000 task.ti: ffff88201c808000 [83371.098815] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83371.098816] RSP: 0018:ffff88201c80bb70 EFLAGS: 00000246 [83371.098816] RAX: 0000000000000000 RBX: 000000013c796cc0 RCX: 0000000000810000 [83371.098817] RDX: ffff88203c717880 RSI: 0000000000490000 RDI: ffff88102137981c [83371.098818] RBP: ffff88201c80bb70 R08: ffff88103d017880 R09: 0000000000000000 [83371.098818] R10: ffff88202ea99400 R11: 0000000000000005 R12: ffff88201c80bc58 [83371.098819] R13: 0000000000000001 R14: ffff88201c80bb28 R15: ffffffff81322c35 [83371.098820] FS: 0000000000000000(0000) GS:ffff88103d000000(0000) knlGS:0000000000000000 [83371.098820] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.098821] CR2: 00007f361a3bb4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.098821] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.098822] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.098822] Stack: [83371.098823] ffff88201c80bb80 ffffffff8169e61f ffff88201c80bb90 ffffffff816abb70 [83371.098825] ffff88201c80bbd0 ffffffffc0b72198 0000000000000000 ffff88202ea99400 [83371.098826] ffff88201c80bc18 ffff88202ea99460 ffff88202ea99400 ffff8810175b8000 [83371.098826] Call Trace: [83371.098828] [] queued_spin_lock_slowpath+0xb/0xf [83371.098829] [] _raw_spin_lock+0x20/0x30 [83371.098848] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.098867] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.098886] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.098903] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.098921] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.098939] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.098944] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.098962] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.098979] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.098985] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.099004] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.099022] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.099044] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.099046] [] ? wake_up_state+0x20/0x20 [83371.099066] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.099068] [] kthread+0xcf/0xe0 [83371.099069] [] ? insert_kthread_work+0x40/0x40 [83371.099070] [] ret_from_fork+0x58/0x90 [83371.099071] [] ? insert_kthread_work+0x40/0x40 [83371.099085] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83371.101781] NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [ldlm_bl_21:33194] [83371.101802] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.101810] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.101811] CPU: 17 PID: 33194 Comm: ldlm_bl_21 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.101812] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.101812] task: ffff88203d0b6eb0 ti: ffff882012eac000 task.ti: ffff882012eac000 [83371.101815] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83371.101815] RSP: 0018:ffff882012eafb70 EFLAGS: 00000246 [83371.101816] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000890000 [83371.101816] RDX: ffff88203c797880 RSI: 0000000000690000 RDI: ffff88102137981c [83371.101817] RBP: ffff882012eafb70 R08: ffff88203c817880 R09: 0000000000000000 [83371.101817] R10: ffff88202b307a00 R11: 0000000000000001 R12: 0000000000000000 [83371.101818] R13: 0000000200000000 R14: 0000000000000000 R15: ffff88203c816cc0 [83371.101819] FS: 0000000000000000(0000) GS:ffff88203c800000(0000) knlGS:0000000000000000 [83371.101819] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.101820] CR2: 00007f7ce52f4550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.101821] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.101821] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.101822] Stack: [83371.101823] ffff882012eafb80 ffffffff8169e61f ffff882012eafb90 ffffffff816abb70 [83371.101824] ffff882012eafbd0 ffffffffc0b72198 0000000000000000 ffff88202b307a00 [83371.101825] ffff882012eafc18 ffff88202b307a60 ffff88202b307a00 ffff8810175b8000 [83371.101826] Call Trace: [83371.101828] [] queued_spin_lock_slowpath+0xb/0xf [83371.101829] [] _raw_spin_lock+0x20/0x30 [83371.101847] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.101869] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.101888] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.101905] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.101923] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.101941] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.101946] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.101964] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.101981] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.101986] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.102006] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.102024] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.102046] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.102047] [] ? __schedule+0x39d/0x8b0 [83371.102067] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.102069] [] kthread+0xcf/0xe0 [83371.102070] [] ? insert_kthread_work+0x40/0x40 [83371.102071] [] ret_from_fork+0x58/0x90 [83371.102073] [] ? insert_kthread_work+0x40/0x40 [83371.102086] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83371.104781] NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [ldlm_bl_02:15061] [83371.104803] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.104810] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.104811] CPU: 18 PID: 15061 Comm: ldlm_bl_02 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.104812] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.104813] task: ffff882012edeeb0 ti: ffff88203d3bc000 task.ti: ffff88203d3bc000 [83371.104815] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83371.104816] RSP: 0018:ffff88203d3bfb70 EFLAGS: 00000246 [83371.104817] RAX: 0000000000000000 RBX: 000000013cf16cc0 RCX: 0000000000910000 [83371.104817] RDX: ffff88203c7d7880 RSI: 0000000000790000 RDI: ffff88102137981c [83371.104818] RBP: ffff88203d3bfb70 R08: ffff88103d057880 R09: 0000000000000000 [83371.104818] R10: ffff88202ea98200 R11: 0000000000000005 R12: ffff88203d3bfc58 [83371.104819] R13: 0000000000000001 R14: ffff88203d3bfb28 R15: ffffffff81322c35 [83371.104820] FS: 0000000000000000(0000) GS:ffff88103d040000(0000) knlGS:0000000000000000 [83371.104821] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.104821] CR2: 00007f24044f7000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.104822] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.104823] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.104823] Stack: [83371.104824] ffff88203d3bfb80 ffffffff8169e61f ffff88203d3bfb90 ffffffff816abb70 [83371.104825] ffff88203d3bfbd0 ffffffffc0b72198 0000000000000000 ffff88202ea98200 [83371.104826] ffff88203d3bfc18 ffff88202ea98260 ffff88202ea98200 ffff8810175b8000 [83371.104827] Call Trace: [83371.104829] [] queued_spin_lock_slowpath+0xb/0xf [83371.104830] [] _raw_spin_lock+0x20/0x30 [83371.104849] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.104868] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.104887] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.104905] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.104922] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.104940] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.104945] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.104963] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.104980] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.104986] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.105003] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.105021] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.105041] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.105043] [] ? wake_up_state+0x20/0x20 [83371.105063] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.105064] [] kthread+0xcf/0xe0 [83371.105066] [] ? insert_kthread_work+0x40/0x40 [83371.105067] [] ret_from_fork+0x58/0x90 [83371.105068] [] ? insert_kthread_work+0x40/0x40 [83371.105081] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83371.107780] NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [ldlm_bl_18:33191] [83371.107802] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.107809] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.107811] CPU: 19 PID: 33191 Comm: ldlm_bl_18 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.107811] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.107812] task: ffff88203d0b5ee0 ti: ffff881ffba80000 task.ti: ffff881ffba80000 [83371.107814] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83371.107815] RSP: 0018:ffff881ffba83b70 EFLAGS: 00000246 [83371.107816] RAX: 0000000000000000 RBX: ffff88203c856cc0 RCX: 0000000000990000 [83371.107816] RDX: ffff88203c657880 RSI: 0000000000190000 RDI: ffff88102137981c [83371.107817] RBP: ffff881ffba83b70 R08: ffff88203c857880 R09: 0000000000000000 [83371.107817] R10: ffff88202b32ec00 R11: 0000000000000001 R12: ffff88203d0b5f48 [83371.107818] R13: ffff881ffba83ae8 R14: ffff880f00000141 R15: ffff88107ffdb008 [83371.107819] FS: 0000000000000000(0000) GS:ffff88203c840000(0000) knlGS:0000000000000000 [83371.107819] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.107820] CR2: 00007fb0fc86d550 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.107820] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.107821] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.107821] Stack: [83371.107822] ffff881ffba83b80 ffffffff8169e61f ffff881ffba83b90 ffffffff816abb70 [83371.107824] ffff881ffba83bd0 ffffffffc0b72198 0000000000000000 ffff88202b32ec00 [83371.107825] ffff881ffba83c18 ffff88202b32ec60 ffff88202b32ec00 ffff8810175b8000 [83371.107825] Call Trace: [83371.107827] [] queued_spin_lock_slowpath+0xb/0xf [83371.107828] [] _raw_spin_lock+0x20/0x30 [83371.107849] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.107870] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.107889] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.107906] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.107924] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.107942] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.107947] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.107965] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.107982] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.107987] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.108005] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.108023] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.108045] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.108046] [] ? __schedule+0x39d/0x8b0 [83371.108067] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.108068] [] kthread+0xcf/0xe0 [83371.108069] [] ? insert_kthread_work+0x40/0x40 [83371.108071] [] ret_from_fork+0x58/0x90 [83371.108072] [] ? insert_kthread_work+0x40/0x40 [83371.108085] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83371.110781] NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [ldlm_bl_01:15060] [83371.110803] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.110811] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.110812] CPU: 20 PID: 15060 Comm: ldlm_bl_01 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.110813] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.110813] task: ffff882012eddee0 ti: ffff88203d3b8000 task.ti: ffff88203d3b8000 [83371.110816] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83371.110816] RSP: 0018:ffff88203d3bbb70 EFLAGS: 00000246 [83371.110817] RAX: 0000000000000000 RBX: 0000000180597933 RCX: 0000000000a10000 [83371.110818] RDX: ffff88203c857880 RSI: 0000000000990000 RDI: ffff88102137981c [83371.110818] RBP: ffff88203d3bbb70 R08: ffff88103d097880 R09: 0000000000000000 [83371.110819] R10: ffff88202ea5de00 R11: 0000000000000005 R12: ffff88203d3bbc58 [83371.110819] R13: 0000000000000001 R14: ffff88203d3bbb28 R15: ffffffff81322c35 [83371.110820] FS: 0000000000000000(0000) GS:ffff88103d080000(0000) knlGS:0000000000000000 [83371.110821] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.110821] CR2: 00007f02c314e4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.110822] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.110822] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.110822] Stack: [83371.110824] ffff88203d3bbb80 ffffffff8169e61f ffff88203d3bbb90 ffffffff816abb70 [83371.110825] ffff88203d3bbbd0 ffffffffc0b72198 0000000000000000 ffff88202ea5de00 [83371.110826] ffff88203d3bbc18 ffff88202ea5de60 ffff88202ea5de00 ffff8810175b8000 [83371.110826] Call Trace: [83371.110828] [] queued_spin_lock_slowpath+0xb/0xf [83371.110829] [] _raw_spin_lock+0x20/0x30 [83371.110847] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.110867] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.110886] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.110903] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.110921] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.110939] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.110944] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.110962] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.110979] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.110985] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.111004] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.111022] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.111042] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.111044] [] ? wake_up_state+0x20/0x20 [83371.111064] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.111066] [] kthread+0xcf/0xe0 [83371.111067] [] ? insert_kthread_work+0x40/0x40 [83371.111068] [] ret_from_fork+0x58/0x90 [83371.111069] [] ? insert_kthread_work+0x40/0x40 [83371.111083] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83371.113780] NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [ldlm_bl_20:33193] [83371.113803] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.113810] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.113811] CPU: 21 PID: 33193 Comm: ldlm_bl_20 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.113811] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.113812] task: ffff88203d0b0fd0 ti: ffff881ff8e2c000 task.ti: ffff881ff8e2c000 [83371.113815] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83371.113815] RSP: 0018:ffff881ff8e2fb70 EFLAGS: 00000246 [83371.113816] RAX: 0000000000000000 RBX: ffff881ff8e2fbe8 RCX: 0000000000a90000 [83371.113817] RDX: ffff88203c817880 RSI: 0000000000890000 RDI: ffff88102137981c [83371.113817] RBP: ffff881ff8e2fb70 R08: ffff88203c897880 R09: 0000000000000000 [83371.113818] R10: ffff88202b30cc00 R11: 0000000000000000 R12: ffff88103ce96cc0 [83371.113818] R13: ffff88203ad89060 R14: ffff88103ce96cc0 R15: ffff88203ad89060 [83371.113819] FS: 0000000000000000(0000) GS:ffff88203c880000(0000) knlGS:0000000000000000 [83371.113820] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.113820] CR2: 00007fef9fa7a4f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.113821] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.113821] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.113821] Stack: [83371.113823] ffff881ff8e2fb80 ffffffff8169e61f ffff881ff8e2fb90 ffffffff816abb70 [83371.113824] ffff881ff8e2fbd0 ffffffffc0b72198 0000000000000000 ffff88202b30cc00 [83371.113825] ffff881ff8e2fc18 ffff88202b30cc60 ffff88202b30cc00 ffff8810175b8000 [83371.113825] Call Trace: [83371.113827] [] queued_spin_lock_slowpath+0xb/0xf [83371.113829] [] _raw_spin_lock+0x20/0x30 [83371.113847] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.113867] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.113886] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.113903] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.113920] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.113938] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.113944] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.113961] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.113979] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.113985] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.114002] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.114020] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.114042] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.114043] [] ? __schedule+0x39d/0x8b0 [83371.114064] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.114065] [] kthread+0xcf/0xe0 [83371.114067] [] ? insert_kthread_work+0x40/0x40 [83371.114068] [] ret_from_fork+0x58/0x90 [83371.114069] [] ? insert_kthread_work+0x40/0x40 [83371.114082] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83371.116780] NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [ldlm_bl_07:16009] [83371.116802] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.116809] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.116810] CPU: 22 PID: 16009 Comm: ldlm_bl_07 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.116811] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.116811] task: ffff8810e9240fd0 ti: ffff88201c7ec000 task.ti: ffff88201c7ec000 [83371.116813] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x116/0x1e0 [83371.116814] RSP: 0018:ffff88201c7efb70 EFLAGS: 00000246 [83371.116815] RAX: 0000000000000000 RBX: 000000013d016cc0 RCX: 0000000000b10000 [83371.116815] RDX: ffff88103ce97880 RSI: 0000000000210000 RDI: ffff88102137981c [83371.116816] RBP: ffff88201c7efb70 R08: ffff88103d0d7880 R09: 0000000000000000 [83371.116816] R10: ffff88202ea98200 R11: 0000000000000005 R12: ffff88201c7efc58 [83371.116817] R13: 0000000000000001 R14: ffff88201c7efb28 R15: ffffffff81322c35 [83371.116817] FS: 0000000000000000(0000) GS:ffff88103d0c0000(0000) knlGS:0000000000000000 [83371.116818] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.116819] CR2: 00007f262c4764f0 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.116819] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.116820] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.116820] Stack: [83371.116821] ffff88201c7efb80 ffffffff8169e61f ffff88201c7efb90 ffffffff816abb70 [83371.116822] ffff88201c7efbd0 ffffffffc0b72198 0000000000000000 ffff88202ea98200 [83371.116823] ffff88201c7efc18 ffff88202ea98260 ffff88202ea98200 ffff8810175b8000 [83371.116824] Call Trace: [83371.116826] [] queued_spin_lock_slowpath+0xb/0xf [83371.116827] [] _raw_spin_lock+0x20/0x30 [83371.116847] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.116868] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.116887] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.116905] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.116922] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.116940] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.116945] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.116963] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.116980] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.116986] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.117003] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.117021] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.117043] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.117045] [] ? wake_up_state+0x20/0x20 [83371.117065] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.117066] [] kthread+0xcf/0xe0 [83371.117067] [] ? insert_kthread_work+0x40/0x40 [83371.117069] [] ret_from_fork+0x58/0x90 [83371.117070] [] ? insert_kthread_work+0x40/0x40 [83371.117083] Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 [83371.119780] NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [ldlm_bl_04:16006] [83371.119802] Modules linked in: mgs(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) ldiskfs(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) vfat fat uas usb_storage mpt2sas mptctl mptbase rpcsec_gss_krb5 dell_rbu nfsv4 dns_resolver nfs fscache ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dm_service_time kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt dcdbas iTCO_vendor_support ipmi_si ipmi_devintf mxm_wmi dm_round_robin pcspkr sg ipmi_msghandler acpi_power_meter wmi mei_me mei shpchp lpc_ich nfsd auth_rpcgss dm_multipath dm_mod nfs_acl lockd grace sunrpc ip_tables ext4 mbcache [83371.119809] jbd2 sd_mod crc_t10dif crct10dif_generic mlx4_en i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm tg3 ahci crct10dif_pclmul crct10dif_common mlx4_core mpt3sas drm libahci crc32c_intel ptp raid_class libata megaraid_sas devlink i2c_core scsi_transport_sas pps_core [83371.119810] CPU: 23 PID: 16006 Comm: ldlm_bl_04 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83371.119811] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83371.119811] task: ffff8810247e9fa0 ti: ffff88201c624000 task.ti: ffff88201c624000 [83371.119814] RIP: 0010:[] [] native_queued_spin_lock_slowpath+0x112/0x1e0 [83371.119814] RSP: 0018:ffff88201c627b70 EFLAGS: 00000246 [83371.119815] RAX: 0000000000000000 RBX: 000000013c7d6cc0 RCX: 0000000000b90000 [83371.119815] RDX: ffff88103d057880 RSI: 0000000000910000 RDI: ffff88102137981c [83371.119816] RBP: ffff88201c627b70 R08: ffff88203c8d7880 R09: 0000000000000000 [83371.119817] R10: ffff88202b39e400 R11: 0000000000000000 R12: ffff88201c627c58 [83371.119817] R13: 0000000000000001 R14: ffff88201c627b28 R15: ffffffff81322c35 [83371.119818] FS: 0000000000000000(0000) GS:ffff88203c8c0000(0000) knlGS:0000000000000000 [83371.119818] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [83371.119819] CR2: 00007f5e72caa000 CR3: 00000000019f2000 CR4: 00000000001407e0 [83371.119820] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [83371.119820] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [83371.119820] Stack: [83371.119822] ffff88201c627b80 ffffffff8169e61f ffff88201c627b90 ffffffff816abb70 [83371.119823] ffff88201c627bd0 ffffffffc0b72198 0000000000000000 ffff88202b39e400 [83371.119824] ffff88201c627c18 ffff88202b39e460 ffff88202b39e400 ffff8810175b8000 [83371.119824] Call Trace: [83371.119826] [] queued_spin_lock_slowpath+0xb/0xf [83371.119827] [] _raw_spin_lock+0x20/0x30 [83371.119846] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83371.119867] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83371.119887] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83371.119904] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83371.119922] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83371.119939] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83371.119945] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83371.119962] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.119980] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83371.119986] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83371.120003] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83371.120021] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83371.120043] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83371.120045] [] ? wake_up_state+0x20/0x20 [83371.120066] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83371.120067] [] kthread+0xcf/0xe0 [83371.120068] [] ? insert_kthread_work+0x40/0x40 [83371.120070] [] ret_from_fork+0x58/0x90 [83371.120071] [] ? insert_kthread_work+0x40/0x40 [83371.120084] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b [83374.954350] Uhhuh. NMI received for unknown reason 39 on CPU 0. [83374.954350] Do you have a strange power saving mode enabled? [83374.954351] Kernel panic - not syncing: NMI: Not continuing [83374.954353] CPU: 0 PID: 16015 Comm: ldlm_bl_13 Tainted: G OEL ------------ 3.10.0-693.2.2.el7_lustre.pl2.x86_64 #1 [83374.954353] Hardware name: Dell Inc. PowerEdge R630/0CNCJW, BIOS 2.6.0 10/26/2017 [83374.954355] ffff88103ce05e00 00000000dc8e5453 ffff88103ce05e18 ffffffff816a3db1 [83374.954356] ffff88103ce05e98 ffffffff8169dc74 ffff882000000010 ffff88103ce05ea8 [83374.954357] ffff88103ce05e48 00000000dc8e5453 ffff88103ce05ea8 ffffffff818e7858 [83374.954357] Call Trace: [83374.954360] [] dump_stack+0x19/0x1b [83374.954362] [] panic+0xe8/0x20d [83374.954365] [] nmi_panic+0x3f/0x40 [83374.954367] [] do_nmi+0x3e6/0x450 [83374.954369] [] end_repeat_nmi+0x1e/0x2e [83374.954371] [] ? io_serial_in+0x15/0x20 [83374.954373] [] ? io_serial_in+0x15/0x20 [83374.954374] [] ? io_serial_in+0x15/0x20 [83374.954376] <> [] wait_for_xmitr+0x40/0xa0 [83374.954378] [] serial8250_console_putchar+0x1c/0x30 [83374.954379] [] ? serial8250_console_write+0x140/0x140 [83374.954381] [] uart_console_write+0x3a/0x80 [83374.954382] [] serial8250_console_write+0xae/0x140 [83374.954385] [] call_console_drivers.constprop.17+0x90/0xf0 [83374.954386] [] console_unlock+0x468/0x4b0 [83374.954388] [] vprintk_emit+0x3c4/0x510 [83374.954390] [] vprintk_default+0x29/0x40 [83374.954391] [] printk+0x5d/0x74 [83374.954394] [] watchdog_timer_fn+0x1cc/0x230 [83374.954396] [] ? watchdog+0x40/0x40 [83374.954398] [] __hrtimer_run_queues+0xd4/0x260 [83374.954400] [] hrtimer_interrupt+0xaf/0x1d0 [83374.954402] [] local_apic_timer_interrupt+0x35/0x60 [83374.954404] [] smp_apic_timer_interrupt+0x3d/0x50 [83374.954406] [] apic_timer_interrupt+0x6d/0x80 [83374.954408] [] ? native_queued_spin_lock_slowpath+0x112/0x1e0 [83374.954410] [] queued_spin_lock_slowpath+0xb/0xf [83374.954411] [] _raw_spin_lock+0x20/0x30 [83374.954433] [] ldlm_handle_conflict_lock+0xd8/0x330 [ptlrpc] [83374.954455] [] ldlm_process_plain_lock+0x435/0xb30 [ptlrpc] [83374.954474] [] ? ldlm_errno2error+0x60/0x60 [ptlrpc] [83374.954491] [] ldlm_reprocess_queue+0x13b/0x2a0 [ptlrpc] [83374.954509] [] __ldlm_reprocess_all+0x14d/0x3a0 [ptlrpc] [83374.954527] [] ldlm_reprocess_res+0x20/0x30 [ptlrpc] [83374.954533] [] cfs_hash_for_each_relax+0x21f/0x400 [libcfs] [83374.954550] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83374.954568] [] ? ldlm_lock_downgrade+0x320/0x320 [ptlrpc] [83374.954573] [] cfs_hash_for_each_nolock+0x75/0x1c0 [libcfs] [83374.954593] [] ldlm_reprocess_recovery_done+0x3c/0x110 [ptlrpc] [83374.954611] [] ldlm_export_cancel_locks+0x11c/0x130 [ptlrpc] [83374.954633] [] ldlm_bl_thread_main+0x4c8/0x700 [ptlrpc] [83374.954634] [] ? wake_up_state+0x20/0x20 [83374.954655] [] ? ldlm_handle_bl_callback+0x410/0x410 [ptlrpc] [83374.954656] [] kthread+0xcf/0xe0 [83374.954657] [] ? insert_kthread_work+0x40/0x40 [83374.954659] [] ret_from_fork+0x58/0x90 [83374.954660] [] ? insert_kthread_work+0x40/0x40